def main_db(table):
start_time = time.time()
etllog.info('========== [ETL] START TIME: %s ==========' % clean_datetime._today())
etllog.info('========== [ETL] READ DETAL TABLE ==========')
t_delta_conf._detal_caches()
etllog.info('========== [ETL] START TASK ==========')
etl_work = EtlWork(table)
etl_work.start()
etllog.info('========== [ETL] FINISH TASK ==========')
etllog.info('==========[ETL] SAVE DETAL TABLE==========')
etllog.info('========== [ETL] END TIME: %s ==========' % clean_datetime._today())
etllog.info('========== [ETL] TIME ELAPSED %s s==========' % (time.time()-start_time))
def subject():
"""
@邮件主题
"""
res_msg = "ETL-Lawson结果:OK"
if len(self._log_message['ERROR']) > 0:
res_msg = "ETL-Lawson结果:NG"
SUBJECT = res_msg + " - 表|视图： '%s' （周期：1日/1次、执行时间： '%s'）" % (
self._log_message['TABLE_OR_VIEW'], clean_datetime._today())
return SUBJECT
def _html_plain():
try:
msg = '<h3> [ETL] 程序出现异常 (%s: %s)</h3>' % EMAIL_SEND.items(
)[0]
except:
msg = '<h3> [ETL] 程序出现异常 </h3>'
msg += '<h3> [ETL] 时间: %s </h3>' % clean_datetime._today()
errmsg = msg + '\r\n'.join(['<p>' + _ + '</p>' for _ in self.err])
return errmsg
def location_log(self):
# 取location相应的数据
def _match_line(line):
p = re.compile(r'\[.*\]')
# p.search(line).group()
new_line = json.loads(p.search(line).group())
return new_line
def _rank_replace(line):
clean_user_id = _user_id(line['userId'])
if clean_user_id == 0:
return
else:
return (
clean_user_id,
line['time'],
line['latitude'],
line['longitude'],
)
lines = self._get_rows()
try:
match_datas = map(_match_line, lines)
# match_datas = [_match_line(line) for line in lines]
_dict_list_datas = [
_ for detail_datas in match_datas for _ in detail_datas
]
# _datas_clean_none = [_rank_replace(_) for _ in _dict_list_datas]
_datas_clean_none = map(_rank_replace, _dict_list_datas)
self.lineno += len(match_datas)
if len(match_datas) == 0:
etllog.lfinfo(
'========== ETL-LOG-FILE: TOTAL LINENUMBER: %s, TOTAL Data =========='
% (self.lineno, ))
etllog.info(
'========== ETL-LOG-FILE: Read File END ==========')
etllog.lfinfo('========== ETL-LOG-FILE END %s ==========' %
clean_datetime._today())
return 0
else:
DATAS.put(_datas_clean_none)
return DATAS.qsize()
except Exception, e:
etllog.error(
'[consume_log.py] ReadFile [_get_file_datas_to_queue]' +
str(e))
def access_log(self):
# 取相应的数据
def _rank_replace(line):
new_line = line.replace('\r\n', '').split(',')
clean_user_id = _user_id(new_line[3])
if clean_user_id == 0:
return
else:
return (
clean_user_id,
new_line[0],
5 if new_line[2] == 2 else _interaction_type(new_line[5]),
_object_id(new_line[5]),
_object_id_2(),
new_line[2] if new_line[2] else 0,
new_line[5],
)
lines = self._get_rows()
try:
# _datas = [_rank_replace(line) for line in lines]
_datas = map(_rank_replace, lines)
_datas_clean_none = [_ for _ in _datas if _ != None]
self.lineno += len(_datas)
if len(_datas) == 0:
etllog.lfinfo(
'========== ETL-LOG-FILE: TOTAL LINENUMBER: %s. ignore UUID: %s =========='
% (self.lineno, _datas.count(None)))
etllog.info(
'========== ETL-LOG-FILE: Read File END ==========')
etllog.lfinfo('========== ETL-LOG-FILE END %s ==========' %
clean_datetime._today())
return 0
else:
DATAS.put(_datas_clean_none)
return DATAS.qsize()
except Exception, e:
etllog.error(
'[consume_log.py] ReadFile [_get_file_datas_to_queue]' +
str(e))
def _refresh_view():
start_time = time.time()
etllog.info('========== [ETL] REFRESH MATERIALIZED VIEW START TIME: %s ==========' % clean_datetime._today())
rmv = RefreshMaterializedView('user_age_group')
rmv._refresh_start()
etllog.info('========== [ETL] REFRESH MATERIALIZED VIEW END TIME: %s ==========' % clean_datetime._today())
etllog.info('========== [ETL] TIME ELAPSED %s s ==========' % (time.time()-start_time))
def warning(self, msg):
self.ilogger.warning(msg)
self.blogger.warning(msg)
self._log_message['WARNING'].append(clean_datetime._today() + msg)
def info(self, msg):
self.ilogger.info(msg)
self.blogger.info(msg)
self._log_message['INFO'].append(clean_datetime._today() + msg)
def error(self, msg):
self.elogger.error(msg)
self.ilogger.error(msg)
self.blogger.error(msg)
self._log_message['ERROR'].append(clean_datetime._today() + msg)