def run_unzip_file(configData: ConfigData):
f_date_str = configData.get_f_date() # "20181101"
zip_path = os.path.join(configData.get_zip_path(), f_date_str)
# root_path = configData.get_data("allinpay_data_zc")
data_path = os.path.join(configData.get_data_path(), f_date_str) # allinpay_data_zc
# ifile = '/home/testFolder/logflow/bl_shouyinbao/20181101/9999100000/t1_trxrecord_20181101_V2.csv'
# ofile = '/home/testFolder/logflow/bl_shouyinbao/UTF8/20181101/9999100000/t1_trxrecord_20181101_V2.csv'
print("Start\n")
# os.path.join(root_path, the_date) # real SYB folder don't have date folder
f_name = configData.get_zip_name(f_date_str) # 3= the_date+".zip" # 5 = the_date+"_agt.zip"
a_file = os.path.join(zip_path, f_name)
p_name = configData.get_file_name(f_date_str) # p_date+"*"
if MyLocalFile.check_file(a_file):
MyLocalFile.unzip_the_file(a_file, data_path, p_name=p_name)
def run_unzip_file(configData: ConfigData, folder_type=2):
f_date_str = configData.get_f_date() # "20181101"
if (type(f_date_str) is str) and len(f_date_str) == 8:
m_month = f_date_str[0:6]
m_day = f_date_str[6:8]
else:
return
zip_path = configData.get_zip_path()
data_path = configData.get_data_path()
f_name = configData.get_zip_name("") # "t1_trxrecord_" the_date # "_V2.csv"
print("Start\n")
# os.path.join(root_path, the_date) # real SYB folder don't have date folder
branches = MyLocalFile.get_child_dir(zip_path)
for aBranch in branches:
if MyLocalFile.check_branch(aBranch):
months = MyLocalFile.get_child_dir(aBranch)
for aMonth in months:
the_month = MyLocalFile.check_month(aMonth)
if the_month > 0 and "{:0>6d}".format(the_month) == m_month:
day_list = MyLocalFile.get_child_dir(aMonth)
for aDay in day_list:
the_day = MyLocalFile.check_day(aDay)
if the_day > 0 and "{:0>2d}".format(the_day) == m_day:
files = MyLocalFile.get_child_file(aDay)
for aFile in files:
if MyLocalFile.check_file(aFile, p_name=f_name):
short_name = os.path.basename(aBranch)
if folder_type == 1:
new_path = os.path.join(data_path, m_month, m_day, short_name)
# "{:0>6d}".format(month) "{:0>2d}".format(day)
else:
new_path = os.path.join(data_path, f_date_str, short_name)
# "{:0>6d}{:0>2d}".format(month, day)
p_name = configData.get_file_name(f_date_str)
MyLocalFile.unzip_the_file(aFile, new_path, p_name)
# 3、20180703 265 2018-7-2 2017-10-11
# 2、20171010 466 2017-10-9 2016-7-1
# 其他文件没有多 delta 1天
# 1、20190110 191 2019-1-10 2018-7-4 但实际看见的文件是到 2018-7-3 日
# 再给保理7、8号两个流水增加1天文件2018-7-2，运行 7 20180703 1
# 上面都是 205个文件
# 再风险 4、6、9， 运行 4 20180702 190
# 再流水 3、5、7，8 运行 3 20180702 190
date2 = start_date - datetime.timedelta(days=delta)
day_str2 = date2.strftime("%Y%m%d")
the_conf.find_row(date2.strftime("%Y-%m-%d"))
run_remove_files(the_conf)
run_remove_hive(the_conf)
run_sftp_file(the_conf)
g_zip_path = the_conf.get_zip_path()
if len(g_zip_path) > 0:
run_unzip_file(the_conf)
run_conv_file_local_to_hdfs(the_conf)
run_hive(the_conf)
run_remove_files(the_conf)
print("ok")