the_conf = ConfigData(p_is_test=False)
client = Client(the_conf.hdfs_ip()) # "http://10.2.201.197:50070"
a = MyHdfsFile.get_child(client, "/data/posflow/allinpay_utf8_zc")
b = MyHdfsFile.get_child_file(client,"/data/posflow/allinpay_utf8_zc")
c = MyHdfsFile.get_child_dir(client, "/data/posflow/allinpay_utf8_zc")
# test
# MyHdfsFile.delete(client, "/data/posflow/allinpay_utf8_zc", "*agt_cpy*")
# test
if the_conf.is_test():
day_str = the_conf.test_date()
days = 9
else:
day_str = StrTool.get_param_str(1, "")
days = StrTool.get_param_int(2, 1)
day_str = StrTool.get_the_date_str(day_str)
date1 = StrTool.get_the_date(day_str)
for i in range(0, days):
delta = days - i - 1
date2 = date1 - datetime.timedelta(days=delta)
day_str2 = date2.strftime("%Y%m%d")
run_sftp_file(the_conf, day_str2)
run_unzip_file(the_conf, day_str2)
run_conv_file_local_to_hdfs(the_conf, day_str2)
run_hive(the_conf, the_date=day_str2)
run_remove_files(the_conf, day_str2, 0)
if MyLocalFile.check_file(a_file):
MyLocalFile.unzip_the_file(a_file, data_path, p_name=p_name)
if __name__ == "__main__":
m_is_test = False
# client = Client(the_conf.hdfs_ip()) # "http://10.2.201.197:50070"
if m_is_test:
m_project_id = 3
start_date_str = "20180901"
m_days = 9
m_project_id = StrTool.get_param_int(1, 3)
start_date_str = StrTool.get_the_date_str(StrTool.get_param_str(2, ""))
m_days = StrTool.get_param_int(3, 1)
else:
m_project_id = StrTool.get_param_int(1, 3)
start_date_str = StrTool.get_the_date_str(StrTool.get_param_str(2, ""))
m_days = StrTool.get_param_int(3, 1)
start_date = StrTool.get_the_date(start_date_str)
the_conf = ConfigData(m_project_id, StrTool.get_the_date_str_by_date(start_date, 0, 10), p_is_test=m_is_test)
for i in range(0, m_days):
delta = m_days - i - 1 # 不多加1天，20190108处理的是20190108文件夹
# delta = days - i - 1 + 1 # 多加1天，是因为20190108处理的是20190107文件夹
# 收银宝文件没有多 delta 1天
# 1、20190110 191 2019-1-10 2018-7-4