def sync(t0, t1):
WHERE = "WHERE update_time BETWEEN '{t0}' AND '{t1}'".format(
t0=t0.strftime("%Y%m%d%H%M%S"), t1=t1.strftime("%Y%m%d%H%M%S"))
for TASK_NO in range(0, len(tasks)):
print("task_no: {no}; task: {src} --> {tgt}".format(
no=TASK_NO,
src=tasks[TASK_NO][0]._name,
tgt=tasks[TASK_NO][1]._name))
try:
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE)
if tasks[TASK_NO][0]._name == "fund_announcement":
j.update(chunksize=10)
elif tasks[TASK_NO][0]._name in {
"fund_nv", "fund_yield", "fund_dividend_split"
}:
j.sync()
else:
j.update()
except Exception as e:
print(TASK_NO, e)
def sync(t0, t1):
WHERE = "WHERE update_time BETWEEN '{t0}' AND '{t1}'".format(
t0=t0.strftime("%Y%m%d%H%M%S"), t1=t1.strftime("%Y%m%d%H%M%S"))
for TASK_NO in range(0, len(tasks)):
print("task_no: {no}; task: {src} --> {tgt}".format(
no=TASK_NO,
src=tasks[TASK_NO][0]._name,
tgt=tasks[TASK_NO][1]._name))
try:
if tasks[TASK_NO][0]._name in (
"fund_weekly_return", "fund_weekly_risk",
"fund_subsidiary_weekly_index", "fund_month_return",
"fund_month_risk", "fund_subsidiary_month_index",
"fund_month_risk2", "fund_weekly_risk2",
"fund_subsidiary_weekly_index2",
"fund_subsidiary_weekly_index3",
"fund_subsidiary_month_index2",
"fund_subsidiary_month_index3", "fund_weekly_subsidiary",
"fund_weekly_subsidiary2", "fund_monthly_subsidiary",
"fund_monthly_subsidiary2"):
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
where_del="WHERE fund_id LIKE 'JR%%'")
j.update()
elif tasks[TASK_NO][0]._name in ("fund_type_mapping", ):
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE + "AND flag = 1",
where_del="WHERE fund_id LIKE 'JR%%'")
j.sync()
else:
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE)
j.sync()
except Exception as e:
print(TASK_NO, e)
def main():
WHERE = "WHERE update_time BETWEEN '{t0}' AND '{t1}'".format(
t0=(dt.datetime.now() -
relativedelta(hours=1, minutes=5)).strftime("%Y%m%d%H%M%S"),
t1=dt.datetime.now().strftime("%Y%m%d%H%M%S"))
for TASK_NO in range(0, len(tasks)):
print("task_no: {no}; task: {src} --> {tgt}".format(
no=TASK_NO,
src=tasks[TASK_NO][0]._name,
tgt=tasks[TASK_NO][1]._name))
try:
if tasks[TASK_NO][0]._name in (
"fund_weekly_return",
"fund_weekly_risk",
"fund_subsidiary_weekly_index",
"fund_month_return",
"fund_month_risk",
"fund_subsidiary_month_index",
"fund_month_risk2",
"fund_weekly_risk2",
"fund_subsidiary_weekly_index2",
"fund_subsidiary_weekly_index3",
"fund_subsidiary_month_index2",
"fund_subsidiary_month_index3",
):
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
where_del="WHERE fund_id LIKE 'JR%%'")
j.update()
else:
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE)
j.sync()
except Exception as e:
print(TASK_NO, e)
def sync(t0, t1):
WHERE = "WHERE update_time BETWEEN '{t0}' AND '{t1}'".format(
t0=t0.strftime("%Y%m%d"), t1=t1.strftime("%Y%m%d"))
for TASK_NO in range(0, len(tasks)):
print("task_no: {no}; task: {src} --> {tgt}".format(
no=TASK_NO,
src=tasks[TASK_NO][0]._name,
tgt=tasks[TASK_NO][1]._name))
try:
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE)
j.sync()
except Exception as e:
print(TASK_NO, e)
def sync(t0, t1):
NOT_SYNC_PRODUCT = (
# 雷根
'JR006559',
'JR006560',
'JR006602',
'JR010118',
'JR013757',
'JR015022',
'JR018212',
'JR025431',
'JR027002',
'JR028121',
'JR031510',
'JR033010',
'JR034849',
'JR040097',
'JR058358',
'JR058516',
'JR065039',
'JR070143',
'JR078235',
'JR087376',
'JR100709',
'JR100710',
'JR100711',
'JR102591',
'JR103119',
'JR103556',
'JR103755',
'JR103939',
'JR105422',
'JR106452',
'JR109261',
'JR113887',
'JR114298',
'JR115117',
'JR122835',
'JR122945',
'JR123264',
'JR126054',
'JR126094',
'JR126880',
'JR131987',
'JR134330',
'JR134488',
'JR134505',
'JR134507',
'JR134515',
'JR135170',
'JR136217',
'JR137446',
'JR137524',
'JR137557',
'JR137577',
'JR138618',
'JR138619',
'JR139002',
'JR139099',
'JR139389',
'JR139390',
'JR139532',
'JR139769',
'JR141827',
'JR141896',
'JR142194',
'JR142961',
'JR143253',
'JR147568',
# # 洛书
# 'JR155148', 'JR155579', 'JR156934', 'JR147792', 'JR142156', 'JR142309', 'JR138694', 'JR087140',
# 'JR086911', 'JR086986', 'JR036144', 'JR063527', 'JR073645', 'JR085036', 'JR074126', 'JR046611',
# 'JR100769', 'JR109354', 'JR024256', 'JR071872', 'JR023887', 'JR027867', 'JR027868', 'JR027869',
# 'JR027870', 'JR049010', 'JR147330',
#上海一村
'JR026289',
'JR026290',
'JR058777',
'JR080930',
'JR080931',
'JR087316',
'JR105289',
'JR112182',
'JR113978',
'JR138881',
'JR147314',
'JR166291',
'JR176400',
'JR176668',
'JR183204',
'JR025917',
'JR024171',
'JR024172',
'JR003365',
'JR003496',
'JR015888',
'JR018800',
'JR019555',
'JR019802',
'JR023070',
)
WHERE = "WHERE update_time BETWEEN '{t0}' AND '{t1}'".format(
t0=t0.strftime("%Y%m%d%H%M%S"), t1=t1.strftime("%Y%m%d%H%M%S"))
WHERE_EXCEPT = " AND fund_id NOT IN {fid}".format(
fid=str(NOT_SYNC_PRODUCT))
for TASK_NO in range(0, len(tasks)):
print("task_no: {no}; task: {src} --> {tgt}".format(
no=TASK_NO,
src=tasks[TASK_NO][0]._name,
tgt=tasks[TASK_NO][1]._name))
try:
if tasks[TASK_NO][0]._name == "fund_type_mapping":
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE + " AND flag = 1" + WHERE_EXCEPT,
apply={
"typestandard_code": lambda x: {
601: 1,
603: 2,
604: 3,
602: 4,
600: 5,
605: 6
}.get(x)
})
j.sync()
elif tasks[TASK_NO][0]._name == "security_price":
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
apply={
"circulated_price": lambda x: x / 1e8,
"market_price": lambda x: x / 1e8
})
j.update()
elif tasks[TASK_NO][0]._name in ("fund_info",
"fund_info_aggregation",
"fund_nv_data_standard"):
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE + WHERE_EXCEPT,
where_del="WHERE fund_id LIKE 'JR%%'")
j.sync()
elif tasks[TASK_NO][0]._name in (
"fund_weekly_return",
"fund_weekly_risk",
"fund_subsidiary_weekly_index",
"fund_month_return",
"fund_month_risk",
"fund_subsidiary_month_index",
"fund_month_risk2",
"fund_weekly_risk2",
"fund_subsidiary_weekly_index2",
"fund_subsidiary_weekly_index3",
"fund_subsidiary_month_index2",
"fund_subsidiary_month_index3",
):
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE + WHERE_EXCEPT,
where_del="WHERE fund_id LIKE 'JR%%'")
j.update()
else:
j = splitter.Job(
tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
)
j.sync()
except Exception as e:
print(TASK_NO, e)
def main():
WHERE = "WHERE update_time BETWEEN '{t0}' AND '{t1}'".format(
t0=(dt.datetime.today() -
relativedelta(days=1, minutes=10)).strftime("%Y%m%d"),
t1=dt.datetime.today().strftime("%Y%m%d"))
for TASK_NO in range(0, len(tasks)):
print("task_no: {no}; task: {src} --> {tgt}".format(
no=TASK_NO,
src=tasks[TASK_NO][0]._name,
tgt=tasks[TASK_NO][1]._name))
try:
if tasks[TASK_NO][0]._name == "fund_type_mapping":
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where="WHERE flag = 1",
apply={
"typestandard_code": lambda x: {
601: 1,
603: 2,
604: 3,
602: 4,
600: 5,
605: 6
}.get(x)
})
j.sync()
elif tasks[TASK_NO][0]._name == "security_price":
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
apply={
"circulated_price": lambda x: x / 1e8,
"market_price": lambda x: x / 1e8
})
j.update()
elif tasks[TASK_NO][0]._name in ("fund_info",
"fund_info_aggregation",
"fund_nv_data_standard"):
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
where_del="WHERE fund_id LIKE 'JR%%'")
j.sync()
elif tasks[TASK_NO][0]._name in ("fund_weekly_return",
"fund_weekly_risk",
"fund_subsidiary_weekly_index",
"fund_month_return",
"fund_month_risk",
"fund_subsidiary_month_index"):
j = splitter.Job(tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
where_del="WHERE fund_id LIKE 'JR%%'")
j.update()
else:
j = splitter.Job(
tasks[TASK_NO][0],
tasks[TASK_NO][1],
pool_size={"update": 5},
where=WHERE,
)
j.sync()
except Exception as e:
print(TASK_NO, e)
