def download_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id):
    start_date_dt = datetime.datetime.strptime(start_date,'%Y%m%d')
    end_date_dt = datetime.datetime.strptime(end_date,'%Y%m%d')
    
    # get stock ids which is_download_success=N
    chk_sql = '''
    select t.biz_date, 
      t.stock_id
    from (
    select 
      biz_date, 
      stock_id, 
      is_download_success, 
      row_number() over(partition by biz_date, stock_id order by download_end_time desc nulls last) rankid
    from dw.log_stock_transaction
    where biz_date between '{start_date}' and '{end_date}' 
    ) t where t.rankid = 1
    and t.is_download_success = 'N' '''.format(start_date=start_date_dt, end_date=end_date_dt)
    if not stock_id is None: chk_sql = chk_sql + ' and t.stock_id = \'' + stock_id + '\''

    cur = get_cur(conn)
    cur.execute(chk_sql)
    rows = list(cur)
    if len(rows) == 0:
        print_log('All the stocks have been downloaded successfully.')
    else:
        for row in rows:
            error_log(str(row['biz_date']) + ':' + row['stock_id'] + ' failed to download.')
    return len(rows)
def load_log_checker(conn,
                     start_date=options.start_date,
                     end_date=options.end_date,
                     stock_id=options.stock_id):
    start_date_dt = datetime.datetime.strptime(start_date, '%Y%m%d')
    end_date_dt = datetime.datetime.strptime(end_date, '%Y%m%d')

    chk_sql = '''
    select biz_date, stock_id
    from dw.log_stock_transaction
    where biz_date between '{start_date}' and '{end_date}'
    and is_download_success = 'Y'
    and (is_load_success = 'N' or is_load_success is null)
    '''.format(start_date=start_date_dt, end_date=end_date_dt)
    if not stock_id is None:
        chk_sql = chk_sql + ' and stock_id = \'' + stock_id + '\''

    cur = get_cur(conn)
    cur.execute(chk_sql)
    rows = list(cur)
    if len(rows) == 0:
        print_log('All the stocks have been loaded successfully.')
    else:
        for row in rows:
            error_log(
                str(row['biz_date']) + ':' + row['stock_id'] +
                ' failed to load.')
    return len(rows)
def load_log_checker(conn, start_date=options.start_date, end_date=options.end_date, stock_id=options.stock_id):
    start_date_dt = datetime.datetime.strptime(start_date,'%Y%m%d')
    end_date_dt = datetime.datetime.strptime(end_date,'%Y%m%d')

    chk_sql = '''
    select biz_date, stock_id
    from dw.log_stock_transaction
    where biz_date between '{start_date}' and '{end_date}'
    and is_download_success = 'Y'
    and (is_load_success = 'N' or is_load_success is null)
    '''.format(start_date=start_date_dt, end_date=end_date_dt)
    if not stock_id is None: chk_sql = chk_sql + ' and stock_id = \'' + stock_id + '\''

    cur = get_cur(conn)
    cur.execute(chk_sql)
    rows = list(cur)
    if len(rows) == 0:
        print_log('All the stocks have been loaded successfully.')
    else:
        for row in rows:
            error_log(str(row['biz_date']) + ':' + row['stock_id'] + ' failed to load.')
    return len(rows)
def download_log_checker(conn,
                         start_date=options.start_date,
                         end_date=options.end_date,
                         stock_id=options.stock_id):
    start_date_dt = datetime.datetime.strptime(start_date, '%Y%m%d')
    end_date_dt = datetime.datetime.strptime(end_date, '%Y%m%d')

    # get stock ids which is_download_success=N
    chk_sql = '''
    select t.biz_date, 
      t.stock_id
    from (
    select 
      biz_date, 
      stock_id, 
      is_download_success, 
      row_number() over(partition by biz_date, stock_id order by download_end_time desc nulls last) rankid
    from dw.log_stock_transaction
    where biz_date between '{start_date}' and '{end_date}' 
    ) t where t.rankid = 1
    and t.is_download_success = 'N' '''.format(start_date=start_date_dt,
                                               end_date=end_date_dt)
    if not stock_id is None:
        chk_sql = chk_sql + ' and t.stock_id = \'' + stock_id + '\''

    cur = get_cur(conn)
    cur.execute(chk_sql)
    rows = list(cur)
    if len(rows) == 0:
        print_log('All the stocks have been downloaded successfully.')
    else:
        for row in rows:
            error_log(
                str(row['biz_date']) + ':' + row['stock_id'] +
                ' failed to download.')
    return len(rows)
parser.add_option("--type", "-t", dest="type", action="store", type="string", help="bankuai|stock_bankuai")
parser.add_option("--in_file", "-f", dest="in_file", action="store", type="string", help="To recon a specific file")
(options, args) = parser.parse_args()


#-- function
def exit_process():
	os.system("python " + FILE_NAME + " -h")
	sys.exit()
	

#-- verify param
if options.type is None:
	types_to_recon = file_db_recon.keys()
	if not options.in_file is None:
		error_log("type can't be null if in_file specified!")
		exit_process()
elif options.type in file_db_recon.keys():
	types_to_recon.append(options.type)
else:
	error_log("type is not correct! [" + options.type + "]")
	exit_process()
		
#-- fetch DB info
db_dict = get_yaml(DB_YML)

#-- open db connection
conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"])


def exit_error(msg):
    error_log(msg)
    raise RuntimeError(msg)
def exit_error(msg):
    error_log(msg)
    raise RuntimeError(msg)
                  type="string",
                  help="To recon a specific file")
(options, args) = parser.parse_args()


#-- function
def exit_process():
    os.system("python " + FILE_NAME + " -h")
    sys.exit()


#-- verify param
if options.type is None:
    types_to_recon = file_db_recon.keys()
    if not options.in_file is None:
        error_log("type can't be null if in_file specified!")
        exit_process()
elif options.type in file_db_recon.keys():
    types_to_recon.append(options.type)
else:
    error_log("type is not correct! [" + options.type + "]")
    exit_process()

#-- fetch DB info
db_dict = get_yaml(DB_YML)

#-- open db connection
conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"],
                db_dict["Host"], db_dict["Port"])

for type in types_to_recon:
Esempio n. 9
0
    'recon_stock_bankuai': Task_recon_stock_bankuai('recon_stock_bankuai'),
    'download_stock_eod': Task_download_stock_eod('download_stock_eod'),
    'download_stock_transaction': Task_download_stock_transaction('download_stock_transaction'),
}

#job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod', 'download_stock_transaction']
job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod']
job_to_run = []

# determine which jobs need to run
for i, job in enumerate(job_run_seq):
    status = check_job_status(conn, job)
    print_log(job + ' ====> ' + status)
    if status == 'N': # one job failed, itself and the jobs depend on it will be added to to-run list
        job_to_run = job_run_seq[i:]
        break

# add to flow
flow = linear_flow.Flow('Eod loading')
for job in job_to_run:
    flow.add(jobs[job])

engine  = taskflow.engines.load(flow)
engine.notifier.register('*', flow_watch)
engine.task_notifier.register('*', task_watch)
try:
    engine.run()
except Exception as ex:
    error_log(ex.message)
    
Esempio n. 10
0
    'recon_stock_bankuai': Task_recon_stock_bankuai('recon_stock_bankuai'),
    'download_stock_eod': Task_download_stock_eod('download_stock_eod'),
    'download_stock_transaction': Task_download_stock_transaction('download_stock_transaction'),
}

#job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod', 'download_stock_transaction']
job_run_seq = ['download_stock_bankuai', 'recon_stock_bankuai', 'download_stock_eod']
job_to_run = []

# determine which jobs need to run
for i, job in enumerate(job_run_seq):
    status = check_job_status(conn, job)
    print_log(job + ' ====> ' + status)
    if status == 'N': # one job failed, itself and the jobs depend on it will be added to to-run list
        job_to_run = job_run_seq[i:]
        break

# add to flow
flow = linear_flow.Flow('Eod loading')
for job in job_to_run:
    flow.add(jobs[job])

engine  = taskflow.engines.load(flow)
engine.notifier.register('*', flow_watch)
#engine.task_notifier.register('*', task_watch)
try:
    engine.run()
except Exception as ex:
    error_log(ex.message)
    
Esempio n. 11
0
 def revert(self, *args, **kwargs):
     error_log('[FAILED] ' + self.cmd)