def copyFilesByDay(thedayStr=''):
thedayStr = getTheDateStr(thedayStr)
thedate = getTheDate(thedayStr)
logging.info('单日文件复制开始 ' + thedayStr)
print('单日文件复制开始 ' + thedayStr)
safeMakedir(dataClean.srcPath) # '/home/thjk01/thzc/'
safeMakedir(dataClean.aimPath) # '/home/thjk01/thzc/cleanedData/'
for fileName in dataClean.needFiles:
realFileName = fileName + thedayStr + '.csv'
if os.path.isfile(sftp_config.localDir + thedayStr + '/' +
realFileName):
isdownloaded = False
stinfo1 = os.stat(sftp_config.localDir + thedayStr + '/' +
realFileName)
if os.path.exists(dataClean.srcPath + realFileName):
stinfo2 = os.stat(dataClean.srcPath + realFileName)
if (stinfo1.st_size == stinfo2.st_size
and abs(stinfo2.st_mtime - stinfo1.st_mtime) <
10): # 本地文件时间是float
isdownloaded = True
if (isdownloaded == False):
shutil.copyfile(
sftp_config.localDir + thedayStr + '/' + realFileName,
dataClean.srcPath + realFileName)
# 修改访问和修改时间
os.chmod(
dataClean.srcPath + realFileName,
stat.S_IWOTH + stat.S_IROTH + stat.S_IWGRP + stat.S_IRGRP +
stat.S_IWUSR + stat.S_IRUSR)
os.utime(dataClean.srcPath + realFileName,
(stinfo1.st_atime, stinfo1.st_mtime))
logging.info('成功Copy文件 ' + realFileName)
print('成功Copy文件 ' + realFileName)
else:
logging.info('已经存在 ' + realFileName)
print('已经存在 ' + realFileName)
else:
logging.info('文件不存在 ' + realFileName)
print('文件不存在 ' + realFileName)
logging.info('文件清洗开始 ' + thedayStr + ' :clean、rename、append')
print(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) +
' 文件清洗开始:clean、rename、append ' + thedayStr)
dataClean.dataCleanTrustApply(thedayStr)
dataClean.renameFiles(thedayStr)
dataClean.appendData(thedate, thedayStr)
logging.info('文件清洗结束 ' + thedayStr)
print(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) +
' 文件清洗结束 ' + thedayStr)
def cleanFilesByDay(self, day_str: str = ''):
day_str = StrTool.get_the_date_str(day_str)
thedate = StrTool.get_the_date(day_str)
myLog.Log(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) +
' 文件清洗开始:clean、rename、append ' + day_str)
dataClean.dataCleanTrustApply(day_str)
dataClean.renameFiles(day_str)
dataClean.appendData(thedate, day_str)
myLog.Log(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) +
' 文件清洗结束 ' + day_str)
def scanTask(todayStr=''):
sftp = 'default'
try:
myLog.info('数据下载开始')
if len(todayStr) <= 0:
curYear = 2018
today = datetime.date.today()
today = today.replace(curYear, today.month, today.day)
todayStr = str(today).replace('-', '')
result = sftpUtil.getConnect(sftp_config.host, sftp_config.port,
sftp_config.username,
sftp_config.password)
if (result[0] == 1):
sftp = paramiko.SFTPClient.from_transport(result[2])
allFiles = sftp.listdir(sftp_config.remoteDir)
allFiles2 = sftp.listdir_attr(sftp_config.remoteDir)
if not os.path.exists(sftp_config.localDir):
os.makedirs(sftp_config.localDir)
if not os.path.exists(sftp_config.localDir + todayStr):
os.makedirs(sftp_config.localDir + todayStr)
if not os.path.exists(dataClean.srcPath): # '/home/thjk01/thzc/'
os.makedirs(dataClean.srcPath)
if not os.path.exists(
dataClean.aimPath): # '/home/thjk01/thzc/cleanedData/'
os.makedirs(dataClean.aimPath)
for fileName in dataClean.needFiles:
realFileName = fileName + todayStr + '.csv'
if (realFileName in allFiles):
# sftp.get(sftp_config.homeDir+realFileName, sftp_config.__m_localDir+realFileName)
sftp.get(
sftp_config.remoteDir + realFileName,
sftp_config.localDir + todayStr + '/' + realFileName)
shutil.copyfile(
sftp_config.localDir + todayStr + '/' + realFileName,
dataClean.srcPath + realFileName)
stinfo = sftp.stat(sftp_config.remoteDir + realFileName)
# 修改访问和修改时间
os.utime(dataClean.srcPath + realFileName,
(stinfo.st_atime, stinfo.st_mtime))
os.utime(
sftp_config.localDir + todayStr + '/' + realFileName,
(stinfo.st_atime, stinfo.st_mtime))
# os.utime(sftp_config.__m_localDir + realFileName, (stinfo.st_atime, stinfo.st_mtime))
myLog.info('成功下载 ' + realFileName)
print('成功下载 ' + realFileName)
else:
myLog.info('文件不存在 ' + realFileName)
print('文件不存在 ' + realFileName)
dataClean.dataCleanTrustApply(todayStr)
dataClean.renameFiles(todayStr)
dataClean.appendData(today, todayStr)
else:
myLog.info('sftp 连接失败')
except Exception as e:
myLog.error(str(e))
finally:
if (sftp != 'default'):
sftp.close()
def scanTask():
sftp = 'default'
try:
logging.info('数据下载开始')
nowYear = 2018
today = datetime.date.today()
#today=today.replace(nowYear,today.month,today.day)
today = today.replace(nowYear, today.month, today.day)
todayStr = str(today).replace('-', '')
deleteDay = str(today - relativedelta(days=+15)).replace('-', '')
result = sftpUtil.getConnect(sftp_config.host, sftp_config.port,
sftp_config.username,
sftp_config.password)
if (result[0] == 1):
sftp = paramiko.SFTPClient.from_transport(result[2])
allFiles = sftp.listdir(sftp_config.homeDir)
# needFiles=[
# 'DB2_POSL_BATCH_BOS_DEDUCT_',
# 'DB2_POSL_BATCH_BOS_DEDUCT_BALANCE_',
# 'DB2_POSL_BATCH_BOS_EARLY_BALANCE_',
# 'DB2_POSL_BATCH_BOS_GRANT_BALANCE_',
# 'DB2_POSL_BATCH_SPD_DEDUCT_',预下载文件
# 'DB2_POSL_BATCH_SPD_DEDUCT_BALANCE_',
# 'DB2_POSL_BATCH_SPD_EARLY_BALANCE_',
# 'DB2_POSL_BATCH_SPD_GRANT_BALANCE_',
# 'DB2_POSL_POSL_LOAN_APPLY_',
# 'DB2_POSL_POSL_TRUST_APPLY_',
# 'DB2_POSL_COMPENSATORY_',
# 'DB2_POSL_APMS_LOGINFO_',
# 'MYSQL_LOAN_T_LOAN_COMPERNSATORY_',
# 'MYSQL_LOAN_T_LOAN_CREDIT_REQ_',
# 'MYSQL_LOAN_T_LOAN_EARLYREPAY_REQ_',
# 'MYSQL_LOAN_T_LOAN_OUTSIDE_MCHT_RIG_',
# 'MYSQL_LOAN_T_LOAN_USE_REQ_',
# 'MYSQL_LOAN_T_REOCN_CREDIT_',
# 'MYSQL_LOAN_T_REOCN_DEDUCT_',
# 'MYSQL_LOAN_T_REOCN_DEDUCT_BALANCE_',
# 'MYSQL_LOAN_T_REOCN_EARLYREPAY_',
# 'MYSQL_LOAN_T_REOCN_LOANGRANT_',
# 'MYSQL_LOAN_T_REOCN_LOANINFO_'
# ]
#
# for fileName in needFiles:
# realFileName=fileName+todayStr+'.csv'
# if(realFileName in allFiles):
# sftp.get(sftp_config.homeDir+realFileName,sftp_config.__m_localDir+realFileName)
# logging.info('成功下载 '+realFileName)
# else:
# logging.info('文件不存在 '+realFileName)
# realFileName='dkyqqd_0002_'+todayStr+'.dat'
# if realFileName in allFiles:
# sftp.get(sftp_config.homeDir+realFileName,sftp_config.__m_localDir+realFileName)
# logging.info('成功下载 '+realFileName )
# else:
# logging.info('文件不存在 '+realFileName)
if (result[0] == 1):
sftp = paramiko.SFTPClient.from_transport(result[2])
allFiles = sftp.listdir(sftp_config.homeDir)
for fileName in allFiles:
if fileName.find(todayStr) >= 0:
logging.info('预下载 ' + fileName)
sftp.get(sftp_config.homeDir + fileName,
sftp_config.localDir + fileName)
logging.info('成功下载 ' + fileName)
if fileName.find(deleteDay) >= 0:
logging.info('预删除 ' + fileName)
sftp.remove(sftp_config.homeDir + fileName)
logging.info('成功删除 ' + fileName)
dataClean.dataCleanTrustApply(todayStr)
dataClean.renameFiles(todayStr)
dataClean.appendData(today, todayStr)
else:
logging.info('sftp 连接失败')
except Exception as e:
traceback.print_exc()
logging.error(str(e))
finally:
if (sftp != 'default'):
sftp.close()
def scanTask():
sftp = 'default'
try:
logging.info('数据下载开始')
curYear = 2018
today = datetime.date.today()
today = today.replace(curYear, today.month, today.day)
todayStr = str(today).replace('-', '')
result = sftpUtil.getConnect(sftp_config.host, sftp_config.port,
sftp_config.username,
sftp_config.password)
if (result[0] == 1):
sftp = paramiko.SFTPClient.from_transport(result[2])
allFiles = sftp.listdir(sftp_config.homeDir)
needFiles = [
'DB2_POSL_BATCH_BOS_DEDUCT_',
'DB2_POSL_BATCH_BOS_DEDUCT_BALANCE_',
'DB2_POSL_BATCH_BOS_EARLY_BALANCE_',
'DB2_POSL_BATCH_BOS_GRANT_BALANCE_',
'DB2_POSL_BATCH_SPD_DEDUCT_',
'DB2_POSL_BATCH_SPD_DEDUCT_BALANCE_',
'DB2_POSL_BATCH_SPD_EARLY_BALANCE_',
'DB2_POSL_BATCH_SPD_GRANT_BALANCE_',
'DB2_POSL_POSL_LOAN_APPLY_', 'DB2_POSL_POSL_TRUST_APPLY_',
'DB2_POSL_COMPENSATORY_', 'DB2_POSL_APMS_LOGINFO_',
'MYSQL_LOAN_T_LOAN_COMPERNSATORY_',
'MYSQL_LOAN_T_LOAN_CREDIT_REQ_',
'MYSQL_LOAN_T_LOAN_EARLYREPAY_REQ_',
'MYSQL_LOAN_T_LOAN_OUTSIDE_MCHT_RIG_',
'MYSQL_LOAN_T_LOAN_USE_REQ_', 'MYSQL_LOAN_T_REOCN_CREDIT_',
'MYSQL_LOAN_T_REOCN_DEDUCT_',
'MYSQL_LOAN_T_REOCN_DEDUCT_BALANCE_',
'MYSQL_LOAN_T_REOCN_EARLYREPAY_',
'MYSQL_LOAN_T_REOCN_LOANGRANT_', 'MYSQL_LOAN_T_REOCN_LOANINFO_'
]
for fileName in needFiles:
realFileName = fileName + todayStr + '.csv'
if (realFileName in allFiles):
sftp.get(sftp_config.homeDir + realFileName,
sftp_config.localDir + realFileName)
logging.info('成功下载 ' + realFileName)
else:
logging.info('文件不存在 ' + realFileName)
dataClean.dataCleanTrustApply(todayStr)
dataClean.renameFiles(todayStr)
dataClean.appendData(today, todayStr)
else:
logging.info('sftp 连接失败')
except Exception as e:
traceback.print_exc()
logging.error(str(e))
finally:
if (sftp != 'default'):
sftp.close()