def copyFilesByRange(self,
to_dir: str,
day_str: str = '',
days: int = 1,
file_names: list = [],
from_dir: str = ''):
# 检查默认值
if type(to_dir) is not str or len(to_dir) == 0:
return
if type(from_dir) is not str or len(from_dir) == 0:
from_dir = self.localDir
day_str = StrTool.get_the_date_str(day_str)
date1 = StrTool.get_the_date(day_str)
# 检查默认值
myLog.Log('批量复制文件开始')
for i in range(0, days):
date2 = date1 - datetime.timedelta(days=(days - i - 1))
thedayStr2 = date2.strftime("%Y%m%d")
self.copyFilesByDay(to_dir=to_dir,
day_str=thedayStr2,
file_names=file_names,
from_dir=from_dir)
myLog.Log('批量复制文件结束')
def downloadFilesByDay(self,
to_dir='',
day_str='',
file_names=None,
from_dir=''):
# 设置默认值
if self.__theSftp is None:
return
if type(from_dir) is not str or len(from_dir) == 0:
from_dir = self.remoteDir
if type(to_dir) is not str or len(to_dir) == 0:
to_dir = self.localDir
day_str = StrTool.get_the_date_str(day_str)
myLog.Log('单日文件下载开始' + day_str)
# 设置默认值
self.__theSftp.download_files(from_dir=from_dir,
to_dir=os.path.join(to_dir, day_str),
p_name="*" + day_str + "*",
sdate=day_str,
edate=day_str,
file_names=file_names,
and_op=False)
myLog.Log('单日文件下载结束 ' + day_str)
def downloadFilesByRange(self,
day_str: str = '',
days: int = 1,
from_dir: str = '',
to_dir: str = ''):
# 设置默认值
if self.__theSftp is None:
return
if type(from_dir) is not str or len(from_dir) == 0:
from_dir = self.remoteDir
if type(to_dir) is not str or len(to_dir) == 0:
to_dir = self.localDir
day_str = StrTool.get_the_date_str(day_str)
myLog.Log('批量下载文件开始')
date1 = StrTool.get_the_date(day_str)
# 设置默认值
for i in range(0, days):
date2 = date1 - datetime.timedelta(days=(days - i - 1))
day_str2 = date2.strftime("%Y%m%d")
self.__theSftp.download_files(from_dir=from_dir,
to_dir=os.path.join(
to_dir, day_str2),
p_name="*" + day_str2 + "*",
sdate=day_str2,
edate=day_str2,
file_names=[],
and_op=False)
myLog.Log('批量下载文件结束')
def closeSFTP(self):
### 需要调整!!!!!!!!!!!!!!!!!!!!!!!!!
if (self.__theSftp is not None) and isinstance(self.__theSftp,
paramiko.SFTPClient):
try:
self.__theSftp.close()
myLog.Log('sftp 关闭')
except Exception as e2:
myLog.Log("出错: " + str(e2), False)
self.__theSftp = None
def cleanFilesByRange(self, day_str: str = '', days: int = 1):
day_str = StrTool.get_the_date_str(day_str)
date1 = StrTool.get_the_date(day_str)
myLog.Log('批量文件清洗开始')
for i in range(0, days):
date2 = date1 - datetime.timedelta(days=(days - i - 1))
thedayStr2 = date2.strftime("%Y%m%d")
self.cleanFilesByDay(day_str=thedayStr2)
myLog.Log('批量文件清洗结束')
def cleanFilesByDay(self, day_str: str = ''):
day_str = StrTool.get_the_date_str(day_str)
thedate = StrTool.get_the_date(day_str)
myLog.Log(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) +
' 文件清洗开始:clean、rename、append ' + day_str)
dataClean.dataCleanTrustApply(day_str)
dataClean.renameFiles(day_str)
dataClean.appendData(thedate, day_str)
myLog.Log(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) +
' 文件清洗结束 ' + day_str)
def removeFiles(self, deleteDay: str):
if fileName.find(deleteDay) >= 0:
myLog.Log('预删除 ' + fileName)
sftp.remove(sftp_config.homeDir + fileName)
logging.info('成功删除 ' + fileName)
if self.__theSftp is None:
return
if type(from_dir) is not str or len(from_dir) == 0:
from_dir = self.remoteDir
if type(to_dir) is not str or len(to_dir) == 0:
to_dir = self.localDir
day_str = StrTool.get_the_date_str(day_str)
myLog.Log('批量下载文件开始')
date1 = StrTool.get_the_date(day_str)
def copyFilesByDay(self,
to_dir: str,
day_str: str = '',
file_names: list = [],
from_dir: str = ''):
# 检查默认值
if type(to_dir) is not str or len(to_dir) == 0:
return
if type(from_dir) is not str or len(from_dir) == 0:
from_dir = self.localDir
day_str = StrTool.get_the_date_str(day_str)
myLog.Log('单日文件复制开始 ' + day_str)
# 检查默认值
searchFileNames = []
if isinstance(file_names, list):
for fileName in file_names:
searchFileNames.append(fileName + day_str + '.csv')
self.__theSftp.copy_files(fromDir=os.path.join(from_dir, day_str),
toDir=to_dir,
p_name='',
sdate='',
edate='',
fileNames=searchFileNames,
and_op=False)
def openSFTP(self):
self.closeSFTP()
result = sftpUtil.getConnect(self.__m_host, self.__m_port,
self.__m_username, self.__m_password)
if result[0] == 1:
self.__theSftp = paramiko.SFTPClient.from_transport(result[2])
else:
self.__theSftp = None
myLog.Log('sftp 连接失败', False)
def copy_files(self,
fromDir='',
toDir='',
p_name: str = '',
sdate: str = '',
edate: str = '',
fileNames: list = [],
and_op: bool = True):
# 设置默认值
if type(fromDir) is not str or len(fromDir) <= 0:
fromDir = self.__m_localDir
if type(toDir) is not str or len(toDir) <= 0:
myLog.error('纯文件复制失败，没有复制目的地址')
return
myLog.Log('纯文件复制开始 from:' + fromDir + ' to: ' + toDir)
# 设置默认值
fileList = self.getLcFilesList(fromDir, p_name, sdate, edate,
fileNames, and_op) # 只处理列表中的文件
for fromFile in fileList:
if os.path.isfile(fromFile):
stinfo1 = os.stat(fromFile)
else:
myLog.Log('文件不存在 ' + fromFile)
continue
shortname = os.path.basename(fromFile)
toFile = os.path.join(toDir, shortname)
isdownloaded = False
if os.path.isfile(toFile):
stinfo2 = os.stat(toFile)
if (stinfo1.st_size == stinfo2.st_size
and abs(stinfo2.st_mtime - stinfo1.st_mtime) <
10): # 本地文件时间是float
isdownloaded = True
if not isdownloaded:
MyLocalFile.safe_make_dir(
toDir, stat.S_IRWXO + stat.S_IRWXG + stat.S_IRWXU)
try:
shutil.copyfile(fromFile, toFile)
# 修改访问和修改时间
os.chmod(
toFile, stat.S_IWOTH + stat.S_IROTH + stat.S_IWGRP +
stat.S_IRGRP + stat.S_IWUSR + stat.S_IRUSR)
os.utime(toFile, (stinfo1.st_atime, stinfo1.st_mtime))
myLog.Log('成功Copy文件 ' + toFile)
except Exception as e:
myLog.error(str(e))
myLog.Log('文件Copy失败：' + toFile)
else:
myLog.Log('已经存在 ' + toFile)
myLog.Log('纯文件复制结束 from:' + fromDir + ' to: ' + toDir)
def safe_make_dir(dir_str: str, mode: int = 0) -> object:
"""stat.S_IRWXO + stat.S_IRWXG + stat.S_IRWXU
:param dir_str:
:param mode:
:return:
"""
try:
if not os.path.exists(dir_str):
# os.makedirs(dir_str)
pathlib.Path(dir_str).mkdir(parents=True, exist_ok=True)
if mode > 0:
os.chmod(dir_str, mode)
except Exception as e:
myLog.Log("safeMakedir Error: " + str(e), False)
return
def download_files(self,
from_dir='',
to_dir='',
p_name: str = '',
sdate: str = '',
edate: str = '',
file_names: list = [],
and_op: bool = True):
# 设置默认值
if not isinstance(self.__theSftp, paramiko.SFTPClient):
myLog.error('文件下载失败，没有下载FTP类')
return
if type(from_dir) is not str or len(from_dir) <= 0:
from_dir = self.__m_remoteDir
if type(to_dir) is not str or len(to_dir) <= 0:
to_dir = self.__m_localDir
myLog.Log('文件下载开始 from:' + from_dir + ' to: ' + to_dir)
# 设置默认值
fileList = self.getRmFilesList(from_dir, p_name, sdate, edate,
file_names, and_op) # 只处理列表中的文件
for fromFile in fileList:
shortname = fromFile.filename # fromFile.filename # fromFile.st_atime # fromFile.st_mtime
toFile = os.path.join(to_dir, shortname)
isdownloaded = False
srcFile = os.path.join(from_dir, shortname)
# stinfo1 = theSftp.stat(srcFile)
# 可以用上面的 fromFile 里的信息代替
# - ``st_size``
# - ``st_uid``
# - ``st_gid``
# - ``st_mode``
# - ``st_atime``
# - ``st_mtime``
if os.path.isfile(toFile): # 文件已经存在，就比对下大小、时间
stinfo2 = os.stat(toFile)
if (fromFile.st_size == stinfo2.st_size
and abs(int(fromFile.st_mtime - stinfo2.st_mtime)) <
10): # 本地文件时间是float
isdownloaded = True
if not isdownloaded:
MyLocalFile.safe_make_dir(
to_dir, stat.S_IRWXO + stat.S_IRWXG + stat.S_IRWXU)
try:
srcFile = srcFile.replace('\\', '/')
self.__theSftp.get(srcFile, toFile)
# 修改访问和修改时间
os.chmod(
toFile, stat.S_IWOTH + stat.S_IROTH + stat.S_IWGRP +
stat.S_IRGRP + stat.S_IWUSR + stat.S_IRUSR)
os.utime(toFile, (fromFile.st_atime, fromFile.st_mtime))
myLog.Log('成功下载 ' + toFile)
except Exception as e:
myLog.error(str(e))
myLog.Log('文件下载失败：' + toFile)
else:
myLog.Log('已经存在 ' + toFile)
myLog.Log('文件下载结束 from:' + from_dir + ' to: ' + to_dir)
def getLcFilesList(self,
localdir: str,
p_name: str = '',
sdate: str = '',
edate: str = '',
fileNames: list = [],
and_op: bool = True):
"""### 需要调整!!!!!!!!!!!!!!!!!!!!!!!!!
# s_date, e_date = self.__getStartEndTime(sdate, edate)
# and 的时候，不给fileNames列表，__checkFileInList = True
# and 的时候，不给开始结束时间，__getStartEndTime = True
# and 的时候，不给开始结束字符，checkname = True
# checkname and __getStartEndTime and __checkFileInList
# or 的时候，不给fileNames列表，__checkFileInList = False
# or 的时候，不给开始结束时间，__getStartEndTime = False
# or 的时候，不给开始结束字符，checkname = False
# checkname or __getStartEndTime or __checkFileInList or
# ( not checkname and not __getStartEndTime or not __checkFileInList )
:param localdir:
:param p_name:
:param sdate:
:param edate:
:param fileNames:
:param and_op:
:return:
"""
retFiles = [] # type: list
if type(localdir) is not str or len(localdir) == 0:
localdir = self.__m_localDir
try:
listFiles = MyLocalFile.get_child(localdir)
if len(listFiles) > 0:
for f in listFiles:
st = os.stat(f)
if not stat.S_ISREG(st.st_mode):
continue
shortname = os.path.basename(f)
v1, d1 = self.__checkFileInList(shortname,
fileNames,
default=and_op)
v2, d2 = self.__checkStartEndTime(st.st_mtime,
sdate,
edate,
default=and_op)
v3, d3 = MyLocalFile.check_name(shortname,
p_name,
default=and_op)
if and_op:
if v1 and v2 and v3:
retFiles.append(f)
else:
if v1 or v2 or v3 or (d1 and d2 and d3):
retFiles.append(f)
except Exception as e:
myLog.Log('local 文件列表失败:' + str(e), False)
# traceback.print_exc()
return retFiles
def getRmFilesList(self,
rmdir: str,
p_name: str = '',
sdate: str = '',
edate: str = '',
fileNames: list = [],
and_op: bool = True):
"""### 需要调整!!!!!!!!!!!!!!!!!!!!!!!!!
# s_date, e_date = self.__getStartEndTime(sdate, edate)
# and 的时候，不给fileNames列表，__checkFileInList = True
# and 的时候，不给开始结束时间，__getStartEndTime = True
# and 的时候，不给开始结束字符，checkname = True
# checkname and __getStartEndTime and __checkFileInList
# or 的时候，不给fileNames列表，__checkFileInList = False
# or 的时候，不给开始结束时间，__getStartEndTime = False
# or 的时候，不给开始结束字符，checkname = False
# checkname or __getStartEndTime or __checkFileInList or
# ( not checkname and not __getStartEndTime or not __checkFileInList )
:param rmdir:
:param p_name:
:param sdate:
:param edate:
:param fileNames:
:param and_op:
:return:
"""
retFiles = [] # type: list[paramiko.SFTPAttributes]
if type(rmdir) is not str or len(rmdir) == 0:
rmdir = self.__m_remoteDir
if self.__theSftp is None or (not isinstance(self.__theSftp,
paramiko.SFTPClient)):
return retFiles
try:
rmdir = rmdir.replace('\\', '/')
listFiles = self.__theSftp.listdir_attr(rmdir)
if len(listFiles) > 0:
for f in listFiles:
if not stat.S_ISREG(f.st_mode):
continue
shortname = f.filename
v1, d1 = self.__checkFileInList(shortname,
fileNames,
default=and_op)
v2, d2 = self.__checkStartEndTime(f.st_mtime,
sdate,
edate,
default=and_op)
v3, d3 = MyLocalFile.check_name(shortname,
p_name,
default=and_op)
if and_op:
if v1 and v2 and v3:
retFiles.append(f)
else:
if v1 or v2 or v3 or (d1 and d2 and d3):
retFiles.append(f)
except Exception as e:
myLog.Log('sftp 文件列表失败:' + str(e), False)
# traceback.print_exc()
return retFiles
def openSFTP(self):
if isinstance(self.__theSftp, sftp_tool.Sftp_Tool):
self.__theSftp.openSFTP()
else:
myLog.Log('sftp 连接失败', False)