print(str(idn) + " " + sql + "\n")
cur.execute(sql) # , async=True)
cur.close()
conn.close()
if __name__ == "__main__":
the_conf = ConfigData(p_is_test=False)
if the_conf.is_test():
day_str = the_conf.test_date()
days = 190
else:
the_conf.m_project_id = StrTool.get_param_int(1, 1)
day_str = StrTool.get_param_str(2, "")
days = StrTool.get_param_int(3, 1)
if the_conf.m_project_id == 1:
return_code = subprocess.call("/app/code/posflow_loader/ftpcmd.sh",
shell=True)
print(return_code)
f_delta = the_conf.get_data("file_date_delta" + str(the_conf.m_project_id),
"0")
day_str = StrTool.get_the_date_str(day_str, -int(f_delta))
del_range = 30 # 删除旧数据的时间范围，天
keep_range = 7 # 保留最近旧数据的时间范围，天
client = Client(the_conf.hdfs_ip()) # "http://10.2.201.197:50070"
a = MyHdfsFile.get_child(client, "/data/posflow/allinpay_utf8_zc")
b = MyHdfsFile.get_child_file(client,"/data/posflow/allinpay_utf8_zc")
c = MyHdfsFile.get_child_dir(client, "/data/posflow/allinpay_utf8_zc")
# test
# MyHdfsFile.delete(client, "/data/posflow/allinpay_utf8_zc", "*agt_cpy*")
# test
if the_conf.is_test():
day_str = the_conf.test_date()
days = 9
else:
day_str = StrTool.get_param_str(1, "")
days = StrTool.get_param_int(2, 1)
day_str = StrTool.get_the_date_str(day_str)
date1 = StrTool.get_the_date(day_str)
for i in range(0, days):
delta = days - i - 1
date2 = date1 - datetime.timedelta(days=delta)
day_str2 = date2.strftime("%Y%m%d")
run_sftp_file(the_conf, day_str2)
run_unzip_file(the_conf, day_str2)
run_conv_file_local_to_hdfs(the_conf, day_str2)
run_hive(the_conf, the_date=day_str2)
run_remove_files(the_conf, day_str2, 0)
print("ok")
p_name = configData.get_file_name(f_date_str) # p_date+"*"
if MyLocalFile.check_file(a_file):
MyLocalFile.unzip_the_file(a_file, data_path, p_name=p_name)
if __name__ == "__main__":
m_is_test = False
# client = Client(the_conf.hdfs_ip()) # "http://10.2.201.197:50070"
if m_is_test:
m_project_id = 3
start_date_str = "20180901"
m_days = 9
m_project_id = StrTool.get_param_int(1, 3)
start_date_str = StrTool.get_the_date_str(StrTool.get_param_str(2, ""))
m_days = StrTool.get_param_int(3, 1)
else:
m_project_id = StrTool.get_param_int(1, 3)
start_date_str = StrTool.get_the_date_str(StrTool.get_param_str(2, ""))
m_days = StrTool.get_param_int(3, 1)
start_date = StrTool.get_the_date(start_date_str)
the_conf = ConfigData(m_project_id, StrTool.get_the_date_str_by_date(start_date, 0, 10), p_is_test=m_is_test)
for i in range(0, m_days):
delta = m_days - i - 1 # 不多加1天，20190108处理的是20190108文件夹
# delta = days - i - 1 + 1 # 多加1天，是因为20190108处理的是20190107文件夹
# 收银宝文件没有多 delta 1天