def delete_files(self):
# local path
credential = ftpinfo().ftp_Push()
directorypath = credential['ftp_pushed']
availablefiles = os.listdir(directorypath)
referencefiledatatime = int((datetime.today() - timedelta(days= int(config("daystostore")))).date().strftime('%Y%m%d'))
filestoremove = [x for x in [int(element.split("_")[-1].replace(".csv","")[:8]) for element in availablefiles] if x < referencefiledatatime]
for stringtomatch in filestoremove:
for filetoremove in availablefiles:
filepath = '{}{}'.format(directorypath, filetoremove) if str(
stringtomatch) in filetoremove else ""
if os.path.exists(filepath):
os.remove(filepath)
self.logger.info(
filetoremove+' - file is removed successfully')
else:
self.logger.warning(
'Could not delete file!,file does not exists')
def delete_logs(self):
# local path
directorypath = ftpinfo().ftp_logs()
availablefiles = os.listdir(directorypath)
print(directorypath)
referencefiledatatime = int(
(datetime.today() - timedelta(days=30)).date().strftime('%Y%m%d'))
filestoremove = [x for x in [int(element.split(
".")[0]) for element in availablefiles] if x <= referencefiledatatime]
print(filestoremove)
for filetoremove in filestoremove:
print("for")
filepath = '{}{}{}'.format(directorypath, filetoremove, '.log')
if os.path.exists(filepath):
os.remove(filepath)
self.logger.info(
filetoremove+' - file is removed successfully')
else:
self.logger.warning(
'Could not delete file!,file does not exists')
def data_pull(self):
try:
credentials = ftpinfo().sftp_Pull()
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
with pysftp.Connection(credentials['host'],
username=credentials['username'],
private_key=credentials['privatekey'],
port=22,
cnopts=cnopts) as connection:
connection.chdir(credentials['sftp_path'] + "/")
sftpfiles = connection.listdir(credentials['sftp_path'] + "/")
if connection.isdir(credentials['sftp_move']):
pass
else:
connection.mkdir(credentials['sftp_move'], mode=777)
for item in sftpfiles:
if os.path.isfile(credentials['sftp_pulled'] + item):
os.remove(credentials['sftp_pulled'] + item)
connection.get(credentials['sftp_path'] + "/" + item,
credentials['sftp_pulled'] + item)
connection.rename(
credentials['sftp_path'] + "/" + item,
credentials['sftp_move'] + item)
self.logger.info(item +
"successfully pulled to location" +
credentials['sftp_pulled'])
else:
connection.get(credentials['sftp_path'] + "/" + item,
credentials['sftp_pulled'] + item)
connection.rename(
credentials['sftp_path'] + "/" + item,
credentials['sftp_move'] + item)
self.logger.info(item +
"successfully pulled to location" +
credentials['sftp_pulled'])
# close conn
connection.close()
except:
self.logger.error(sys.exc_info())
def data_push(self):
try:
credentials = ftpinfo().ftp_Push()
# files list
local_files = sorted(list(os.listdir(credentials['ftp_push'])),
reverse=True)
filestopush = sorted([
x for x in [
element.split("_")[-1]
[:len(config("filenameendingformat"))]
for element in local_files
]
])
# check file count
if (len(local_files) > 0):
# connection
connection = ftplib.FTP_TLS(credentials['host'],
credentials['username'],
credentials['password'])
connection.prot_p()
ftpdir = set(connection.nlst())
if str(credentials['ftp_path']).split("/")[-1] in ftpdir:
ftpoperations().ftpdirexists(connection, credentials,
filestopush, local_files,
self.logger)
# close conn
connection.quit()
else:
connection.mkd(credentials['ftp_path'])
ftpoperations().ftpdirexists(connection, credentials,
filestopush, local_files,
self.logger)
# close conn
connection.quit()
else:
self.logger.warning('No files available to push')
except:
self.logger.error(sys.exc_info())
def data_pull(self):
try:
credentials = ftpinfo().ftp_Pull()
# connection
connection = ftplib.FTP_TLS(credentials['host'],
credentials['username'],
credentials['password'])
# files list
connection.cwd(credentials['ftp_path'] + "/")
connection.prot_p()
FTP_files = sorted(list(connection.nlst()), reverse=False)
self.logger.info('FTP directory files count ' +
str(len(FTP_files)))
# loop for filess
for item in FTP_files:
# check file is already available
if (os.path.exists(credentials['pulled'] + item)):
os.remove(credentials['pulled'] + item)
write_file = open(credentials['pulled'] + item, 'wb')
connection.retrbinary("RETR %s" % item, write_file.write)
self.logger.info(item + ' file successfully pulled from ' +
credentials['ftp_path'])
connection.rename(item,
credentials['ftp_move'] + "/" + item)
write_file.close()
else:
write_file = open(credentials['pulled'] + item, 'wb')
connection.retrbinary("RETR %s" % item, write_file.write)
self.logger.info(item + ' file successfully pulled from ' +
credentials['ftp_path'])
connection.rename(item,
credentials['ftp_move'] + "/" + item)
write_file.close()
# close conn
connection.quit()
except:
self.logger.error(sys.exc_info())
def data_push(self):
try:
credentials = ftpinfo().sftp_Push()
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
local_files = sorted(list(os.listdir(credentials['sftp_push'])),
reverse=False)
filestopush = sorted([
x for x in [
element.split("_")[-1]
[:len(config("filenameendingformat"))]
for element in local_files
]
])
# check file count
if (len(local_files) > 0):
with pysftp.Connection(credentials['host'],
username=credentials['username'],
private_key=credentials['privatekey'],
port=22,
cnopts=cnopts) as connection:
if connection.isdir(credentials['sftp_path']):
sftpoperations().direxists(connection, credentials,
local_files, filestopush,
self.logger)
else:
connection.mkdir(credentials['sftp_path'], mode=777)
sftpoperations().direxists(connection, credentials,
local_files, filestopush,
self.logger)
# close conn
connection.close()
else:
self.logger.warning('No files available to push')
except:
self.logger.error(sys.exc_info())
