def download_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id):
start_date_dt = datetime.datetime.strptime(start_date,'%Y%m%d')
end_date_dt = datetime.datetime.strptime(end_date,'%Y%m%d')
# get stock ids which is_download_success=N
chk_sql = '''
select t.biz_date,
t.stock_id
from (
select
biz_date,
stock_id,
is_download_success,
row_number() over(partition by biz_date, stock_id order by download_end_time desc nulls last) rankid
from dw.log_stock_transaction
where biz_date between '{start_date}' and '{end_date}'
) t where t.rankid = 1
and t.is_download_success = 'N' '''.format(start_date=start_date_dt, end_date=end_date_dt)
if not stock_id is None: chk_sql = chk_sql + ' and t.stock_id = \'' + stock_id + '\''
cur = get_cur(conn)
cur.execute(chk_sql)
rows = list(cur)
if len(rows) == 0:
print_log('All the stocks have been downloaded successfully.')
else:
for row in rows:
error_log(str(row['biz_date']) + ':' + row['stock_id'] + ' failed to download.')
return len(rows)
def load_log_checker(conn,
start_date=options.start_date,
end_date=options.end_date,
stock_id=options.stock_id):
start_date_dt = datetime.datetime.strptime(start_date, '%Y%m%d')
end_date_dt = datetime.datetime.strptime(end_date, '%Y%m%d')
chk_sql = '''
select biz_date, stock_id
from dw.log_stock_transaction
where biz_date between '{start_date}' and '{end_date}'
and is_download_success = 'Y'
and (is_load_success = 'N' or is_load_success is null)
'''.format(start_date=start_date_dt, end_date=end_date_dt)
if not stock_id is None:
chk_sql = chk_sql + ' and stock_id = \'' + stock_id + '\''
cur = get_cur(conn)
cur.execute(chk_sql)
rows = list(cur)
if len(rows) == 0:
print_log('All the stocks have been loaded successfully.')
else:
for row in rows:
error_log(
str(row['biz_date']) + ':' + row['stock_id'] +
' failed to load.')
return len(rows)
def load_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id):
start_date_dt = datetime.datetime.strptime(start_date,'%Y%m%d')
end_date_dt = datetime.datetime.strptime(end_date,'%Y%m%d')
chk_sql = '''
select biz_date, stock_id
from dw.log_stock_transaction
where biz_date between '{start_date}' and '{end_date}'
and is_download_success = 'Y'
and (is_load_success = 'N' or is_load_success is null)
'''.format(start_date=start_date_dt, end_date=end_date_dt)
if not stock_id is None: chk_sql = chk_sql + ' and stock_id = \'' + stock_id + '\''
cur = get_cur(conn)
cur.execute(chk_sql)
rows = list(cur)
if len(rows) == 0:
print_log('All the stocks have been loaded successfully.')
else:
for row in rows:
error_log(str(row['biz_date']) + ':' + row['stock_id'] + ' failed to load.')
return len(rows)
def download_log_checker(conn,
start_date=options.start_date,
end_date=options.end_date,
stock_id=options.stock_id):
start_date_dt = datetime.datetime.strptime(start_date, '%Y%m%d')
end_date_dt = datetime.datetime.strptime(end_date, '%Y%m%d')
# get stock ids which is_download_success=N
chk_sql = '''
select t.biz_date,
t.stock_id
from (
select
biz_date,
stock_id,
is_download_success,
row_number() over(partition by biz_date, stock_id order by download_end_time desc nulls last) rankid
from dw.log_stock_transaction
where biz_date between '{start_date}' and '{end_date}'
) t where t.rankid = 1
and t.is_download_success = 'N' '''.format(start_date=start_date_dt,
end_date=end_date_dt)
if not stock_id is None:
chk_sql = chk_sql + ' and t.stock_id = \'' + stock_id + '\''
cur = get_cur(conn)
cur.execute(chk_sql)
rows = list(cur)
if len(rows) == 0:
print_log('All the stocks have been downloaded successfully.')
else:
for row in rows:
error_log(
str(row['biz_date']) + ':' + row['stock_id'] +
' failed to download.')
return len(rows)
parser.add_option("--type", "-t", dest="type", action="store", type="string", help="bankuai|stock_bankuai")
parser.add_option("--in_file", "-f", dest="in_file", action="store", type="string", help="To recon a specific file")
(options, args) = parser.parse_args()
#-- function
def exit_process():
os.system("python " + FILE_NAME + " -h")
sys.exit()
#-- verify param
if options.type is None:
types_to_recon = file_db_recon.keys()
if not options.in_file is None:
error_log("type can't be null if in_file specified!")
exit_process()
elif options.type in file_db_recon.keys():
types_to_recon.append(options.type)
else:
error_log("type is not correct! [" + options.type + "]")
exit_process()
#-- fetch DB info
db_dict = get_yaml(DB_YML)
#-- open db connection
conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"])
def exit_error(msg):
error_log(msg)
raise RuntimeError(msg)
def exit_error(msg):
error_log(msg)
raise RuntimeError(msg)
type="string",
help="To recon a specific file")
(options, args) = parser.parse_args()
#-- function
def exit_process():
os.system("python " + FILE_NAME + " -h")
sys.exit()
#-- verify param
if options.type is None:
types_to_recon = file_db_recon.keys()
if not options.in_file is None:
error_log("type can't be null if in_file specified!")
exit_process()
elif options.type in file_db_recon.keys():
types_to_recon.append(options.type)
else:
error_log("type is not correct! [" + options.type + "]")
exit_process()
#-- fetch DB info
db_dict = get_yaml(DB_YML)
#-- open db connection
conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"],
db_dict["Host"], db_dict["Port"])
for type in types_to_recon:
'recon_stock_bankuai': Task_recon_stock_bankuai('recon_stock_bankuai'),
'download_stock_eod': Task_download_stock_eod('download_stock_eod'),
'download_stock_transaction': Task_download_stock_transaction('download_stock_transaction'),
}
#job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod', 'download_stock_transaction']
job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod']
job_to_run = []
# determine which jobs need to run
for i, job in enumerate(job_run_seq):
status = check_job_status(conn, job)
print_log(job + ' ====> ' + status)
if status == 'N': # one job failed, itself and the jobs depend on it will be added to to-run list
job_to_run = job_run_seq[i:]
break
# add to flow
flow = linear_flow.Flow('Eod loading')
for job in job_to_run:
flow.add(jobs[job])
engine = taskflow.engines.load(flow)
engine.notifier.register('*', flow_watch)
engine.task_notifier.register('*', task_watch)
try:
engine.run()
except Exception as ex:
error_log(ex.message)
'recon_stock_bankuai': Task_recon_stock_bankuai('recon_stock_bankuai'),
'download_stock_eod': Task_download_stock_eod('download_stock_eod'),
'download_stock_transaction': Task_download_stock_transaction('download_stock_transaction'),
}
#job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod', 'download_stock_transaction']
job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod']
job_to_run = []
# determine which jobs need to run
for i, job in enumerate(job_run_seq):
status = check_job_status(conn, job)
print_log(job + ' ====> ' + status)
if status == 'N': # one job failed, itself and the jobs depend on it will be added to to-run list
job_to_run = job_run_seq[i:]
break
# add to flow
flow = linear_flow.Flow('Eod loading')
for job in job_to_run:
flow.add(jobs[job])
engine = taskflow.engines.load(flow)
engine.notifier.register('*', flow_watch)
#engine.task_notifier.register('*', task_watch)
try:
engine.run()
except Exception as ex:
error_log(ex.message)
def revert(self, *args, **kwargs):
error_log('[FAILED] ' + self.cmd)
