def download_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id): start_date_dt = datetime.datetime.strptime(start_date,'%Y%m%d') end_date_dt = datetime.datetime.strptime(end_date,'%Y%m%d') # get stock ids which is_download_success=N chk_sql = ''' select t.biz_date, t.stock_id from ( select biz_date, stock_id, is_download_success, row_number() over(partition by biz_date, stock_id order by download_end_time desc nulls last) rankid from dw.log_stock_transaction where biz_date between '{start_date}' and '{end_date}' ) t where t.rankid = 1 and t.is_download_success = 'N' '''.format(start_date=start_date_dt, end_date=end_date_dt) if not stock_id is None: chk_sql = chk_sql + ' and t.stock_id = \'' + stock_id + '\'' cur = get_cur(conn) cur.execute(chk_sql) rows = list(cur) if len(rows) == 0: print_log('All the stocks have been downloaded successfully.') else: for row in rows: error_log(str(row['biz_date']) + ':' + row['stock_id'] + ' failed to download.') return len(rows)
def load_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id): start_date_dt = datetime.datetime.strptime(start_date, '%Y%m%d') end_date_dt = datetime.datetime.strptime(end_date, '%Y%m%d') chk_sql = ''' select biz_date, stock_id from dw.log_stock_transaction where biz_date between '{start_date}' and '{end_date}' and is_download_success = 'Y' and (is_load_success = 'N' or is_load_success is null) '''.format(start_date=start_date_dt, end_date=end_date_dt) if not stock_id is None: chk_sql = chk_sql + ' and stock_id = \'' + stock_id + '\'' cur = get_cur(conn) cur.execute(chk_sql) rows = list(cur) if len(rows) == 0: print_log('All the stocks have been loaded successfully.') else: for row in rows: error_log( str(row['biz_date']) + ':' + row['stock_id'] + ' failed to load.') return len(rows)
def load_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id): start_date_dt = datetime.datetime.strptime(start_date,'%Y%m%d') end_date_dt = datetime.datetime.strptime(end_date,'%Y%m%d') chk_sql = ''' select biz_date, stock_id from dw.log_stock_transaction where biz_date between '{start_date}' and '{end_date}' and is_download_success = 'Y' and (is_load_success = 'N' or is_load_success is null) '''.format(start_date=start_date_dt, end_date=end_date_dt) if not stock_id is None: chk_sql = chk_sql + ' and stock_id = \'' + stock_id + '\'' cur = get_cur(conn) cur.execute(chk_sql) rows = list(cur) if len(rows) == 0: print_log('All the stocks have been loaded successfully.') else: for row in rows: error_log(str(row['biz_date']) + ':' + row['stock_id'] + ' failed to load.') return len(rows)
def download_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id): start_date_dt = datetime.datetime.strptime(start_date, '%Y%m%d') end_date_dt = datetime.datetime.strptime(end_date, '%Y%m%d') # get stock ids which is_download_success=N chk_sql = ''' select t.biz_date, t.stock_id from ( select biz_date, stock_id, is_download_success, row_number() over(partition by biz_date, stock_id order by download_end_time desc nulls last) rankid from dw.log_stock_transaction where biz_date between '{start_date}' and '{end_date}' ) t where t.rankid = 1 and t.is_download_success = 'N' '''.format(start_date=start_date_dt, end_date=end_date_dt) if not stock_id is None: chk_sql = chk_sql + ' and t.stock_id = \'' + stock_id + '\'' cur = get_cur(conn) cur.execute(chk_sql) rows = list(cur) if len(rows) == 0: print_log('All the stocks have been downloaded successfully.') else: for row in rows: error_log( str(row['biz_date']) + ':' + row['stock_id'] + ' failed to download.') return len(rows)
parser.add_option("--type", "-t", dest="type", action="store", type="string", help="bankuai|stock_bankuai") parser.add_option("--in_file", "-f", dest="in_file", action="store", type="string", help="To recon a specific file") (options, args) = parser.parse_args() #-- function def exit_process(): os.system("python " + FILE_NAME + " -h") sys.exit() #-- verify param if options.type is None: types_to_recon = file_db_recon.keys() if not options.in_file is None: error_log("type can't be null if in_file specified!") exit_process() elif options.type in file_db_recon.keys(): types_to_recon.append(options.type) else: error_log("type is not correct! [" + options.type + "]") exit_process() #-- fetch DB info db_dict = get_yaml(DB_YML) #-- open db connection conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"])
def exit_error(msg): error_log(msg) raise RuntimeError(msg)
type="string", help="To recon a specific file") (options, args) = parser.parse_args() #-- function def exit_process(): os.system("python " + FILE_NAME + " -h") sys.exit() #-- verify param if options.type is None: types_to_recon = file_db_recon.keys() if not options.in_file is None: error_log("type can't be null if in_file specified!") exit_process() elif options.type in file_db_recon.keys(): types_to_recon.append(options.type) else: error_log("type is not correct! [" + options.type + "]") exit_process() #-- fetch DB info db_dict = get_yaml(DB_YML) #-- open db connection conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"]) for type in types_to_recon:
'recon_stock_bankuai': Task_recon_stock_bankuai('recon_stock_bankuai'), 'download_stock_eod': Task_download_stock_eod('download_stock_eod'), 'download_stock_transaction': Task_download_stock_transaction('download_stock_transaction'), } #job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod', 'download_stock_transaction'] job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod'] job_to_run = [] # determine which jobs need to run for i, job in enumerate(job_run_seq): status = check_job_status(conn, job) print_log(job + ' ====> ' + status) if status == 'N': # one job failed, itself and the jobs depend on it will be added to to-run list job_to_run = job_run_seq[i:] break # add to flow flow = linear_flow.Flow('Eod loading') for job in job_to_run: flow.add(jobs[job]) engine = taskflow.engines.load(flow) engine.notifier.register('*', flow_watch) engine.task_notifier.register('*', task_watch) try: engine.run() except Exception as ex: error_log(ex.message)
'recon_stock_bankuai': Task_recon_stock_bankuai('recon_stock_bankuai'), 'download_stock_eod': Task_download_stock_eod('download_stock_eod'), 'download_stock_transaction': Task_download_stock_transaction('download_stock_transaction'), } #job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod', 'download_stock_transaction'] job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod'] job_to_run = [] # determine which jobs need to run for i, job in enumerate(job_run_seq): status = check_job_status(conn, job) print_log(job + ' ====> ' + status) if status == 'N': # one job failed, itself and the jobs depend on it will be added to to-run list job_to_run = job_run_seq[i:] break # add to flow flow = linear_flow.Flow('Eod loading') for job in job_to_run: flow.add(jobs[job]) engine = taskflow.engines.load(flow) engine.notifier.register('*', flow_watch) #engine.task_notifier.register('*', task_watch) try: engine.run() except Exception as ex: error_log(ex.message)
def revert(self, *args, **kwargs): error_log('[FAILED] ' + self.cmd)