the_conf = ConfigData(p_is_test=False) client = Client(the_conf.hdfs_ip()) # "http://10.2.201.197:50070" a = MyHdfsFile.get_child(client, "/data/posflow/allinpay_utf8_zc") b = MyHdfsFile.get_child_file(client,"/data/posflow/allinpay_utf8_zc") c = MyHdfsFile.get_child_dir(client, "/data/posflow/allinpay_utf8_zc") # test # MyHdfsFile.delete(client, "/data/posflow/allinpay_utf8_zc", "*agt_cpy*") # test if the_conf.is_test(): day_str = the_conf.test_date() days = 9 else: day_str = StrTool.get_param_str(1, "") days = StrTool.get_param_int(2, 1) day_str = StrTool.get_the_date_str(day_str) date1 = StrTool.get_the_date(day_str) for i in range(0, days): delta = days - i - 1 date2 = date1 - datetime.timedelta(days=delta) day_str2 = date2.strftime("%Y%m%d") run_sftp_file(the_conf, day_str2) run_unzip_file(the_conf, day_str2) run_conv_file_local_to_hdfs(the_conf, day_str2) run_hive(the_conf, the_date=day_str2) run_remove_files(the_conf, day_str2, 0)
if MyLocalFile.check_file(a_file): MyLocalFile.unzip_the_file(a_file, data_path, p_name=p_name) if __name__ == "__main__": m_is_test = False # client = Client(the_conf.hdfs_ip()) # "http://10.2.201.197:50070" if m_is_test: m_project_id = 3 start_date_str = "20180901" m_days = 9 m_project_id = StrTool.get_param_int(1, 3) start_date_str = StrTool.get_the_date_str(StrTool.get_param_str(2, "")) m_days = StrTool.get_param_int(3, 1) else: m_project_id = StrTool.get_param_int(1, 3) start_date_str = StrTool.get_the_date_str(StrTool.get_param_str(2, "")) m_days = StrTool.get_param_int(3, 1) start_date = StrTool.get_the_date(start_date_str) the_conf = ConfigData(m_project_id, StrTool.get_the_date_str_by_date(start_date, 0, 10), p_is_test=m_is_test) for i in range(0, m_days): delta = m_days - i - 1 # 不多加1天,20190108处理的是20190108文件夹 # delta = days - i - 1 + 1 # 多加1天,是因为20190108处理的是20190107文件夹 # 收银宝文件没有多 delta 1天 # 1、20190110 191 2019-1-10 2018-7-4