def _read_files(self):
"""
如果没有找到可以解析的文件, 程序退出
否则
生成器迭代文件
"""
try:
_files = self._get_local_file(self.time_args)
if len(_files) == 0:
etllog.warning(u'没有可以解析的文件,程序退出!')
from dwetl.petl.p_decorator import sys_exit1
sys_exit1()
for onefile in _files:
etllog.lfinfo(
'========== ETL-LOG-FILE: Read File %s ==========' %
onefile)
etllog.info(
'========== ETL-LOG-FILE: Read File %s ==========' %
onefile)
with open(onefile, 'rb') as onef:
for line in onef:
yield line
except Exception, e:
etllog.error('[consume_log.py] ReadFile [_read_files]' + str(e))
def _put(self):
try:
self.load_datas.bulk_put()
except Exception, e:
etllog.error('[etl_task.py] EtlTask [_put]' + str(e))
from dwetl.petl.p_decorator import sys_exit1
sys_exit1()
def _get(self):
try:
if len(self._get_local_file(self.time_args)) != 0:
# 如果有当前的文件, 属于执行过的文件
etllog.error(' 文件已经执行过, 请在本地删除在执行! [* 不删除DB中相应的数据会有主键错误!]')
from dwetl.petl.p_decorator import sys_exit1
sys_exit1()
for _server_info in self.server: # 多台服务器
self._conn_server(_server_info)
etllog.info(
'========== ETL-LOG-FILE:Conn Server %s ==========' %
(json.dumps(_server_info.get('host'))))
etllog.lfinfo('========== ETL-LOG-FILE:Server %s ==========' %
(json.dumps(_server_info.get('host'))))
for f in self._get_remote_file(self.time_args): # 多个目录文件
# 放文件
# 一对一的
fname = f.split('/')[-1]
fpath = '/'.join(f.split('/')[:-1]) + '/'
index = self._remote_path.index(fpath)
self.sftp.get(f, self._local_path[index] + fname)
etllog.lfinfo(
'========== ETL-LOG-FILE: Get File remote %s local %s=========='
% (f, self._local_path[index] + fname))
etllog.info(
'========== ETL-LOG-FILE: Get File remote %s local %s=========='
% (f, self._local_path[index] + fname))
self._close()
except Exception, e:
etllog.error('[sftp_get_file.py] SftpGFile [_get]' + str(e))
def _get(self):
try:
res = self.extract_datas._bulk_get()
return res
except Exception, e:
etllog.error('[etl_task.py] EtlTask [_get]' + str(e))
from dwetl.petl.p_decorator import sys_exit1
sys_exit1()
def global_setting():
global CONFIG, DB_INFO, DB_TABLE_STRUCTURE, FILE_SERVER, OTHERS_PARAMS, REFRESH_VIEW
if not CONFIG:
with open(__package_json__, 'rb') as js:
CONFIG = json.loads(js.read())
DB_INFO = CONFIG.get('DB_INFO')
DB_TABLE_STRUCTURE = CONFIG.get('DB_TABLE_STRUCTURE')
FILE_SERVER = CONFIG.get('FILE_SERVER')
OTHERS_PARAMS = CONFIG.get('OTHERS_PARAMS')
REFRESH_VIEW = CONFIG.get('REFRESH_VIEW')
try:
# from dev import *
from prod import *
# 不使用settings.json配置文件
# try:
# # global_setting()
# pass
# except:
# # from prod import *
# pass
except:
from dwetl.petl.p_decorator import sys_exit1
sys_exit1()