def Push( FtpServer, Username, Password, uploadlist = FilesToPut, port = 21, passive = False, StartTls = False ):
print "Login to %s:%s using %s:%s (%s%s)"%(FtpServer, port, Username, 'xxx',
'passive' if passive else 'active',
'/tls' if StartTls else '')
if StartTls:
ftp = FTP_TLS()
else:
ftp = FTP()
#ftp.set_debuglevel(2)
ftp.connect( FtpServer, port )
ftp.login( Username, Password ) # user anonymous, passwd anonymous@
ftp.set_pasv( passive )
if StartTls:
ftp.prot_p()
for f in uploadlist:
print "uploading %s"%f
fp = open( f, 'rb')
ftp.storbinary('STOR %s'%os.path.basename(f), fp) # send the file
ftp.quit()
def connect_ftps():
#Connect to the server
print("Step 1")
if SECURE_CONN == '1':
ftps = FTP_TLS(SERVER)
print("Secure connection")
else:
ftps = FTP(SERVER)
print("Unsecure connection")
print("Step 2")
ftps.set_debuglevel(2)
print("Step 3")
ftps.set_pasv(False)
print("Step 4")
ftps.connect(port=int(PORT), timeout=160)
print("Step 5")
ftps.login(USER, PASS)
print("Step 6")
ftps.cwd(TARGET_DIR)
print("Step 7")
ftps.set_pasv(True)
print("Step 8")
#Set up the secure connection only in FTPS mode
if SECURE_CONN == '1':
ftps.prot_p()
else:
pass
#ftps.ccc()
return ftps
def connexionftp(adresseftp, nom, mdpasse, passif):
"""connexion au serveur ftp et ouverture de la session
- adresseftp: adresse du serveur ftp
- nom: nom de l'utilisateur enregistré ('anonymous' par défaut)
- mdpasse: mot de passe de l'utilisateur ('anonymous@' par défaut)
- passif: active ou désactive le mode passif (True par défaut)
retourne la variable 'ftplib.FTP' après connexion et ouverture de session
"""
try:
verbose('Attente connexion FTP .....')
if modeSSL:
ftp = FTP_TLS()
ftp.connect(adresseftp, 21)
ftp.login(nom, mdpasse)
ftp.prot_p()
ftp.set_pasv(passif)
else:
ftp = (ftplib.FTP(adresseftp, nom, mdpasse))
ftp.cwd(destination)
verbose ('Destination : '+destination)
verbose('Connexion FTP OK')
etat = ftp.getwelcome()
verbose("Etat : "+ etat)
return ftp
except:
verbose('Connexion FTP impossible', True)
suppressionDossierTemp(dossierTemporaireFTP)
sys.exit()
def ftpDownload(filename, system):
from ftplib import FTP_TLS
import os
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_filename = filename
with open(local_filename, 'wb') as f:
def callback(data):
f.write(data)
ftps.retrbinary('RETR %s' % filename, callback)
f.close()
file_extension = str(filename.split('.')[1])
if file_extension == '7z':
status = decompressit(local_filename, system)
if status:
print "file %s hash been downloaded." % local_filename
return True
def ftp_coon(self):
try:
self.ftp.close()
except Exception as e:
pass
ftp = FTP()
retry_time = 0
while retry_time < 3:
try:
ftp.connect(self.host, self.port)
ftp.login(self.username, self.password)
self.ftp = ftp
break
except Exception as e:
self.ftp = None
try:
ftps = FTP_TLS(self.host)
ftps.set_pasv(True)
ftps.login(self.username, self.password)
ftps.prot_p()
self.ftp = ftps
break
except Exception as e:
self.ftp = None
finally:
if self.ftp:
return self.ftp
retry_time += 1
time.sleep(5**retry_time)
def ftpUpload(filename):
from ftplib import FTP_TLS
import os
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_file = open(filename, 'rb')
ftps.storbinary('STOR ' + filename, local_file)
def ftpUpload(filename):
from ftplib import FTP_TLS
import os
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_file = open(filename, 'rb')
ftps.storbinary('STOR '+filename, local_file)
def upload_file():
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if filename is empty
if file.filename == '':
flash('No file selected for uploading')
return redirect(request.url)
# if file is available and have extension of mp4, avi, webm is saved in proect directory
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
# saving the video file
file.save(filename)
flash('File successfully uploaded')
secure = request.form['secure']
if secure == "FTP":
ftp = FTP()
ftp.connect('localhost', 21)
status = ftp.login('testuser1', 'testuser')
print("FTP Connected", status)
fp = open(filename, 'rb')
ftp.storbinary('STOR %s' % os.path.basename(filename), fp,
1024)
return render_template('success.html', name=filename)
elif secure == "FTP TLS":
ftps = FTP_TLS('localhost')
ftps.set_debuglevel(2)
ftps.login('testuser1', 'testuser')
ftps.set_pasv(False)
ftps.prot_p()
fp = open(filename, 'rb')
ftps.storbinary('STOR %s' % os.path.basename(filename), fp,
1024)
return render_template('success.html', name=filename)
else:
flash('Allowed file types are txt, pdf, png, jpg, jpeg, gif')
# return to the same page
return redirect(request.url)
def testFTPS(hostname, username, password, port, path, passive):
try:
ftp = FTP_TLS()
ftp.connect(hostname, port)
ftp.login(username, password)
ftp.prot_p()
ftp.set_pasv(passive)
ftp.cwd(path)
files = ftp.dir()
ftp.quit()
return True
except BaseException as error:
return error
def Push( FtpServer, Username, Password, uploadlist = FilesToPut, port = 21, passive = False, StartTls = False, Sftp = False ):
print "Login to %s:%s using %s:%s (%s%s)"%(FtpServer, port, Username, 'xxx',
'passive' if passive else 'active',
'/tls' if StartTls else '')
if Sftp:
paramiko.util.log_to_file('/tmp/paramiko.log')
transport = paramiko.Transport((FtpServer,int(port)))
transport.connect(username = Username, password = Password)
sftp = paramiko.SFTPClient.from_transport(transport)
print "Uploading file"
filepath = '../php/basedata.php'
localpath = 'basedata.php'
sftp.put(filepath, localpath)
sftp.close()
transport.close()
else:
if StartTls:
ftp = FTP_TLS()
else:
ftp = FTP()
ftp.connect( FtpServer, port )
ftp.login( Username, Password)
ftp.set_pasv( passive )
if StartTls:
ftp.prot_p()
for f in uploadlist:
print "uploading %s"%f
fp = open( f, 'rb')
ftp.storbinary('STOR %s'%os.path.basename(f), fp) # send the file
ftp.quit()
if __name__ == "__main__":
if len(sys.argv) < 5:
print >> sys.stderr, "usage %s <server> <port> <username> <password>"%sys.argv[0]
exit( 1 )
FtpServer = sys.argv[1]
Port = sys.argv[2]
Username = sys.argv[3]
Passwd = sys.argv[4]
Push( FtpServer, Username, Passwd, port = Port )
print >> sys.stderr, "Done"
def __SetupOneTest(self, user=None, passwd=''):
if self.use_tls:
ftp = FTP_TLS()
else:
ftp = FTP()
ftp.connect(self.host, int(self.port))
ftp.set_debuglevel(self.debug_level)
ftp.set_pasv(self.pasv_mode)
if user:
ftp.login(user, passwd)
if self.__IsTransferTlsEnabled():
ftp.prot_p()
return ftp
def ftpDownload(filename):
from ftplib import FTP_TLS
import os
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_filename = filename
with open(local_filename, 'wb') as f:
def callback(data):
f.write(data)
ftps.retrbinary('RETR %s' % filename, callback)
def FTPconnect(self):
#initial FTP object and login the HCA FTP server
try:
ftp = FTP_TLS()
ftp.set_pasv(True,self.host)
ftp.ssl_version = ssl.PROTOCOL_TLSv1
ftp.connect(host=self.host,port=21)
ftp.set_debuglevel(2)
ftp.auth()
ftp.login(self.user,self.password)
ftp.prot_p()
print (time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())+' FTP login successful')
return ftp
except Exception as e:
self.sns.informError()
print (e)
def ftpDownload(filename):
from ftplib import FTP_TLS
import os
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_filename = filename
with open(local_filename, 'wb') as f:
def callback(data):
f.write(data)
ftps.retrbinary('RETR %s' % filename, callback)
def login(self,debug=2,set_pasv=True):
_old_makepasv = FTP_TLS.makepasv
def _new_makepasv(self):
host, port = _old_makepasv(self)
host = self.sock.getpeername()[0]
return host, port
FTP_TLS.makepasv = _new_makepasv
ftps = FTP_TLS(self.host)
ftps.set_debuglevel(debug)
ftps.auth()
ftps.login(self.user,self.pwd)
ftps.makepasv()
ftps.sendcmd('pbsz 0')
ftps.set_pasv(set_pasv)
ftps.prot_p()
print("hello ")
ftps.getwelcome()
return ftps
def uploadFilesToFTP(ftpURL, ftpUser, ftpPassword, ftpPath, localPath,
filematch, historyBackupPath):
ftps = FTP_TLS(ftpURL)
ftps.set_debuglevel(1)
ftps.set_pasv(False)
ftps.connect(port=21, timeout=80)
ftps.login(ftpUser, ftpPassword)
ftps.prot_p()
ftps.ccc()
try:
ftps.cwd(ftpPath)
except Exception:
ftps.mkd(ftpPath)
for (localPathDir, _, files) in os.walk(localPath):
newdir = ftpPath
try:
ftps.cwd(newdir)
except Exception:
ftps.mkd(newdir)
LOGGER.info("filematch=" + filematch)
for f in fnmatch.filter(files, filematch):
fileLocalPath = os.path.join(localPathDir, f)
file = open(fileLocalPath, 'rb')
ftps.storbinary('STOR ' + f, file, blocksize=8192)
file.close()
LOGGER.info("Fichero transferido #:# " + fileLocalPath)
sleep(1)
now = datetime.datetime.now()
historyBackupPathYear = os.path.join(historyBackupPath,
str(now.year))
try:
os.stat(historyBackupPathYear)
except:
os.mkdir(historyBackupPathYear)
moveFilesUUID(fileLocalPath, historyBackupPathYear)
ftps.close()
def ftpUpload(filename, system):
from ftplib import FTP_TLS
import os
if os.path.isfile(filename):
zipFilename = compressit(filename, system)
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_file = open(zipFilename, 'rb')
ftps.storbinary('STOR '+zipFilename, local_file)
print "file %s has been uploaded." % zipFilename
return True
def ftpUpload(filename, system):
from ftplib import FTP_TLS
import os
if os.path.isfile(filename):
zipFilename = compressit(filename, system)
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_file = open(zipFilename, 'rb')
ftps.storbinary('STOR ' + zipFilename, local_file)
print "file %s has been uploaded." % zipFilename
return True
def acquire_stage_ftp(self, directory, package, local_dir, force, stage):
"""download one file from the ftp server"""
out = local_dir / package
if out.exists() and not force:
print(stage +
": not overwriting {file} since not forced to overwrite!".
format(**{"file": str(out)}))
return
ftp = FTP_TLS(self.remote_host)
print(
stage + ": " +
ftp.login(user="******", passwd="anonymous", acct="anonymous"))
print(stage + ": Downloading from " + directory)
print(stage + ": " + ftp.cwd(directory))
ftp.set_pasv(True)
with out.open(mode="wb") as filedes:
print(stage + ": downloading from " + directory + " to " +
str(out))
print(stage + ": " +
ftp.retrbinary("RETR " + package, filedes.write))
def upload_files(self):
if configBackup["FTP_BACKUP"]:
# Check to make server is up, if yes back up files! Test
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5)
try:
s.connect((configBackup["FTP_HOST"], 21))
listdir = os.listdir(configBackup["TMP_FOLDER"])
ftps = FTP_TLS(configBackup["FTP_HOST"])
ftps.login(configBackup["FTP_USER"],
configBackup["FTP_PASSWORD"])
ftps.set_pasv(True)
for item in listdir:
print('Upload: ' + item)
ftps.storbinary(
'STOR ' + item,
open(configBackup["TMP_FOLDER"] + item, 'rb'), 1024)
ftps.quit()
except socket.error as e:
print("Error on connect")
s.close()
if configBackup["SFTP_BACKUP"]:
try:
listdir = os.listdir(configBackup["TMP_FOLDER"])
with pysftp.Connection(
configBackup["SFTP_HOST"],
username=configBackup["SFTP_USER"],
password=configBackup["SFTP_PASSWORD"]) as sftp:
for item in listdir:
print('Upload: ' + item)
sftp.put(configBackup["TMP_FOLDER"] + item)
except socket.error as e:
print('Error on connect')
def ftp_orders_b2c_tecnofin_connector():
cfg = get_config_constant_file()
remote_host = cfg['ACCESS_B2C']['HOST']
remote_port = cfg['ACCESS_B2C']['PORT']
remote_username = cfg['ACCESS_B2C']['USERNAME']
remote_password = cfg['ACCESS_B2C']['PASSWORD']
remote_timeout = cfg['ACCESS_B2C']['TIME_OUT']
ftps = FTP_TLS(remote_host)
ftps.set_debuglevel(2)
ftps.set_pasv(True)
ftps.connect(port=remote_port, timeout=remote_timeout)
ftps.login(remote_username, remote_password)
logger.info('FTP Connected to: %s', remote_username+'@'+str(remote_host))
ftps.prot_p()
return ftps
def ftpDownload(filename, system):
from ftplib import FTP_TLS
import os
ftps = FTP_TLS()
ftps.connect('pwcrack.init6.me', '21')
ftps.auth()
ftps.login('DC214', 'passwordcrackingcontest')
ftps.prot_p()
ftps.set_pasv(True)
local_filename = filename
with open(local_filename, 'wb') as f:
def callback(data):
print "Downloading %s ..." % filename
f.write(data)
ftps.retrbinary('RETR %s' % filename, callback)
f.close()
file_extension = str(filename.rsplit('.')[2])
if file_extension == '7z':
status = decompressit(local_filename, system)
if status:
print "file %s has been downloaded." % local_filename
class Sftpimpl(Ftpimpl):
def __init__(self):
self.log = ProcessLogger()
def connectFTP(self,host='',port='',uname='',upass='',ssh_host_key='',acv_pcv=''):
if host=='' or port=='' or uname=='' or upass=='':
raise FTP_ERROR('Oops: Blank config parameters, Please dont leave any config parameter blank')
self.host = host
log_id = self.log.processlog(logid=0,f1=False,f2=False,hostnm=host,up_dw='login',typ='new',status='Pending',result='trying to connect...')
hostStr = '%s:%s' %(host,port)
usrHostStr = '%s@%s' %(uname,hostStr)
try:
self.ftp = FTP_TLS(hostStr)
if acv_pcv == 'Active':
self.ftp.set_pasv(False)
elif acv_pcv == 'Passive':
self.ftp.set_pasv(True)
self.ftp.login(uname,upass)
self.ftp.prot_p()
return_msg = self.ftp.getwelcome()
self.log.processlog(logid=log_id,f1=False,f2=False,hostnm=host,up_dw='login',typ='edit',status='Done',result=return_msg)
except Exception as e:
self.log.processlog(logid=log_id,f1=False,f2=False,hostnm=host,up_dw='login',typ='edit',status='Failed',result=str(e))
raise FTP_ERROR('Connection error: Reasons \n\n(Internet connection)\n(Remote server down)\n(Config settings) ')
class Ftps_client:
##初始化的时候会把登录参数赋值初始化
def __init__(self, host, user, pwd, port=21):
self.host = host
self.port = port
self.user = user
self.pwd = pwd
self.Ftp = None
#self._old_makepasv=FTP_TLS.makepasv
## ftp 登录项 含有闭包项
def login(self, debug=2, set_pasv=True):
_old_makepasv = FTP_TLS.makepasv
def _new_makepasv(self):
host, port = _old_makepasv(self)
host = self.sock.getpeername()[0]
return host, port
FTP_TLS.makepasv = _new_makepasv
self.Ftp = FTP_TLS(self.host)
self.Ftp.set_debuglevel(debug)
self.Ftp.auth()
self.Ftp.login(self.user, self.pwd)
self.Ftp.makepasv()
self.Ftp.sendcmd('pbsz 0')
self.Ftp.set_pasv(set_pasv)
self.Ftp.prot_p()
print("您好 您已经登录 ftp： %s 服务器" % self.host)
self.Ftp.getwelcome()
return self.Ftp
#显示 目录下的 文件列表
def ftplistDir(self, ftps, sever_path):
self.Ftp.cwd("/") #首先切换得到根目录下，否则会出现问题
self.Ftp.cwd(sever_path)
files = ftps.nlst()
for f in files:
print(f)
# 下载服务器文件
def ftpDownloadSeverFile(self,
sever_path,
sever_file,
new_localfile,
buffersize=1024):
self.Ftp.cwd("/")
self.Ftp.cwd(sever_path)
with open(new_localfile, 'wb') as download_file:
self.Ftp.retrbinary('RETR %s' % sever_file, download_file.write,
buffersize)
##上传文件 需要注意 上传文件的 new_severfile 只能是文件名，不能包含带目录 的 文件全路径
def ftpUploadLocalFile(self,
local_filepath,
sever_path,
new_severfile,
buffersize=1024):
self.Ftp.cwd("/")
self.Ftp.cwd(sever_path)
with open(local_filepath, 'rb') as upload_file:
self.Ftp.storbinary('STOR ' + new_severfile, upload_file,
buffersize)
call_shell_backupbdd()
#######################################################
# Send wordpress backup folder to ftp server #
#######################################################
lg.info('Send wordpress backup folder to FTP server')
try:
ftps = FTP_TLS(timeout=100)
ftps.connect(ftp_ip, ftp_port)
ftps.auth()
ftps.prot_p()
ftps.set_pasv(True)
ftps.login(ftp_login, ftp_password)
print('Connect to FTP ...')
print(ftps.getwelcome())
except ftplib.error_perm as e:
print('Ftp fail -> ', e)
ftps.nlst()
files_list = ftps.nlst()
lg.info("List of FTP remote folders:")
for filename in files_list:
lg.debug(f'{filename} ')
if folder in ftps.nlst(
): #check if 'wordpress-todayDate' exist inside '/home/wordpress'
def DownloadFile(self, dst_filename, local_filename=None):
res = ''
if local_filename is None:
local_filename = dst_filename
with open(local_filename, 'w+b') as f:
self.ptr = f.tell()
@setInterval(self.monitor_interval)
def monitor():
if not self.waiting:
i = f.tell()
if self.ptr < i:
logging.debug("%d - %0.1f Kb/s" %
(i, (i - self.ptr) /
(1024 * self.monitor_interval)))
self.ptr = i
os.system('clear')
print(
str(int((float(i) / float(dst_filesize)) * 100)) +
'%')
else:
ftp.close()
def connect():
ftp.connect(self.host, self.port)
ftp.login(self.login, self.passwd)
ftp.prot_p()
if self.directory != None:
ftp.cwd(self.directory)
# optimize socket params for download task
ftp.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
ftp.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
75)
ftp.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE,
60)
ftp = FTP_TLS()
ftp.set_pasv(True)
connect()
ftp.voidcmd('TYPE I')
dst_filesize = ftp.size(dst_filename)
mon = monitor()
while dst_filesize > f.tell():
try:
connect()
self.waiting = False
# retrieve file from position where we were disconnected
res = ftp.retrbinary('RETR %s' % dst_filename, f.write) if f.tell() == 0 else \
ftp.retrbinary('RETR %s' % dst_filename, f.write, rest=f.tell())
except:
self.max_attempts -= 1
if self.max_attempts == 0:
mon.set()
logging.exception('')
raise
self.waiting = True
logging.info('waiting 30 sec...')
time.sleep(30)
logging.info('reconnect')
mon.set() #stop monitor
ftp.close()
if not res.startswith('226 Transfer complete'):
logging.error(
'Downloaded file {0} is not full.'.format(dst_filename))
os.remove(local_filename)
return None
return 1
class FTPES:
def __init__(self, host, port=None, username=None, password=None, remote_path=None, absolute_zipfile_path=None):
"""
This is a helper class to manage the FTP commands from the ftplib.
@param host: The host for the connection.
@type host: String
@param port: The post for the connection.
@type port: Integer
@param username: The username for the connection. Leave blank to use "anonymous".
@type username: String
@param password: The password for the connection. Leave empty for none.
@type password: String
@param remote_path: The remote path of the server in which the zip file should be uploaded. If the path does not
exists, it will be created (recursive).
@type remote_path: String
@param absolute_zipfile_path: The absolute LOCAL filepath of the zip file.
@type absolute_zipfile_path: String
"""
self.ftps = None
self._host = host
self._port = port
self._username = username
self._password = password
self._remote_path = remote_path
self._absolute_zipfile_path = absolute_zipfile_path
self._bytesWritten = 0;
self._totalFileSize = os.path.getsize(self._absolute_zipfile_path)
self._uploadCallback = None
self._currentProgress = 0
# make the remote path relative if it isnt absolute or relative yet
if self._remote_path is not None and self._remote_path.startswith(
'.') is False and self._remote_path.startswith('/') is False:
self._remote_path = './' + self._remote_path
if self._username is None:
self._username = '******'
if self._port is None:
self._port = 22
def connect(self):
"""
Try to connect to the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps = FTP_TLS()
self.ftps.set_pasv(True)
self.ftps.connect(self._host, self._port)
except socket.gaierror:
raise
def login(self):
"""
Try to login in on the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps.login(self._username, self._password)
self.ftps.prot_p()
except ftplib.error_perm:
raise
def cwd(self):
"""
Try to switch the working directory on the FTP server (if set in the settings).
If the path does not exist, it will be created.
"""
if self._remote_path is not None:
try:
self.ftps.cwd(self._remote_path)
except ftplib.error_perm:
self.create_directory_tree(self._remote_path)
except IOError:
raise
def create_directory_tree(self, current_directory):
"""
Helper function to create the remote path.
@param current_directory: The current working directory.
@type current_directory: String
"""
if current_directory is not "":
try:
self.ftps.cwd(current_directory)
except ftplib.error_perm:
self.create_directory_tree("/".join(current_directory.split("/")[:-1]))
self.ftps.mkd(current_directory)
self.ftps.cwd(current_directory)
except IOError:
raise
def upload(self, callback=None):
"""
The upload function.
@param callback: The callback function for the upload progress.
@type callback: Function
"""
self._uploadCallback = callback
zipfile_to_upload = open(self._absolute_zipfile_path, 'rb')
zipfile_basename = os.path.basename(self._absolute_zipfile_path)
self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload, 1024, self.handle_upload_state)
zipfile_to_upload.close()
def handle_upload_state(self, block):
"""
The callback function for the upload progress.
@param block: The StringIO of the current upload state
@type block: StringIO
"""
self._bytesWritten += 1024
progress = math.floor((float(self._bytesWritten) / float(self._totalFileSize)) * 100)
do_update = False
if progress > self._currentProgress:
do_update = True
self._currentProgress = progress
if self._uploadCallback is not None:
self._uploadCallback(progress, self, do_update)
def quit(self):
"""
Try to quit everything and close the session.
"""
self.ftps.quit()
@property
def upload_path(self):
"""
Returns the upload path of the FTP server.
@return: The path of the uploaded file on the FTP server.
@rtype: String
"""
tmp_remote_path = ''
if self._remote_path is not None:
tmp_remote_path = self._remote_path
return os.path.join(tmp_remote_path, os.path.basename(self._absolute_zipfile_path))
class FTPES(object):
"""
The FTPES class manage all ftplib commands.
"""
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-arguments
def __init__(self,
host,
port=None,
username=None,
password=None,
remote_path=None,
absolute_zipfile_path=None):
"""
Initializer
@param host: The host for the connection.
@type host: String
@param port: The post for the connection.
@type port: Integer
@param username: The username for the connection. Leave blank to use "anonymous".
@type username: String
@param password: The password for the connection. Leave empty for none.
@type password: String
@param remote_path: The remote path of the server in which the zip file should be
uploaded. If the path does not
exists, it will be created (recursive).
@type remote_path: String
@param absolute_zipfile_path: The absolute LOCAL filepath of the zip file.
@type absolute_zipfile_path: String
"""
self.ftps = None
self._host = host
self._port = port
self._username = username
self._password = password
self._remote_path = remote_path
self._absolute_zipfile_path = absolute_zipfile_path
self._bytes_written = 0
self._total_file_size = os.path.getsize(self._absolute_zipfile_path)
self._upload_callback = None
self._current_progress = 0
# make the remote path relative if it isnt absolute or relative yet
if self._remote_path is not None and self._remote_path.startswith(
'.') is False and self._remote_path.startswith('/') is False:
self._remote_path = './' + self._remote_path
if self._username is None:
self._username = '******'
if self._port is None:
self._port = 22
# pylint: enable=too-many-instance-attributes
# pylint: enable=too-many-arguments
def connect(self):
"""
Try to connect to the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps = FTP_TLS()
self.ftps.set_pasv(True)
self.ftps.connect(self._host, self._port)
except socket.gaierror:
raise
def login(self):
"""
Try to login in on the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps.login(self._username, self._password)
self.ftps.prot_p()
except ftplib.error_perm:
raise
def cwd(self):
"""
Try to switch the working directory on the FTP server (if set in the settings).
If the path does not exist, it will be created.
"""
if self._remote_path is not None:
try:
self.ftps.cwd(self._remote_path)
except ftplib.error_perm:
self.create_directory_tree(self._remote_path)
except IOError:
raise
def create_directory_tree(self, current_directory):
"""
Helper function to create the remote path.
@param current_directory: The current working directory.
@type current_directory: String
"""
if current_directory is not "":
try:
self.ftps.cwd(current_directory)
except ftplib.error_perm:
self.create_directory_tree("/".join(
current_directory.split("/")[:-1]))
self.ftps.mkd(current_directory)
self.ftps.cwd(current_directory)
except IOError:
raise
def upload(self, callback=None):
"""
The upload function.
@param callback: The callback function for the upload progress.
@type callback: Function
"""
self._upload_callback = callback
zipfile_to_upload = open(self._absolute_zipfile_path, 'rb')
zipfile_basename = os.path.basename(self._absolute_zipfile_path)
self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload,
1024, self.handle_upload_state)
zipfile_to_upload.close()
# pylint: disable=unused-argument
def handle_upload_state(self, block):
"""
The callback function for the upload progress.
@param block: The StringIO of the current upload state
@type block: StringIO
"""
self._bytes_written += 1024
progress = math.floor(
(float(self._bytes_written) / float(self._total_file_size)) * 100)
do_update = False
if progress > self._current_progress:
do_update = True
self._current_progress = progress
if self._upload_callback is not None:
self._upload_callback(progress, self, do_update)
# pylint: enable=unused-argument
def quit(self):
"""
Try to quit everything and close the session.
"""
self.ftps.quit()
@property
def upload_path(self):
"""
Returns the upload path of the FTP server.
@return: The path of the uploaded file on the FTP server.
@rtype: String
"""
tmp_remote_path = ''
if self._remote_path is not None:
tmp_remote_path = self._remote_path
return os.path.join(tmp_remote_path,
os.path.basename(self._absolute_zipfile_path))
def run_module():
module_args = dict(
job_id=dict(type="str", required=False),
job_name=dict(type="str", required=False),
owner=dict(type="str", required=False),
ddname=dict(type="str", required=False),
)
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
job_id = module.params.get("job_id")
job_name = module.params.get("job_name")
owner = module.params.get("owner")
ddname = module.params.get("ddname")
if environ.get('FTP_SOCKS_PORT'):
import socks
import socket
socks.set_default_proxy(socks.SOCKS5, "127.0.0.1",
int(environ.get('FTP_SOCKS_PORT')))
socket.socket = socks.socksocket
try:
if environ.get('FTP_TLS_VERSION'):
from ftplib import FTP_TLS
import ssl
cert_file_path = environ.get('FTP_TLS_CERT_FILE')
if cert_file_path:
if not path.isfile(cert_file_path):
module.fail_json(
msg="The TLS cartificate file not found: {0}".format(
repr(cert_file_path)),
**result)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.load_verify_locations(cert_file_path)
context.check_hostname = False
ftp = FTP_TLS(context=context)
else:
ftp = FTP_TLS()
tls_version = environ.get('FTP_TLS_VERSION')
if tls_version == '1.2':
ftp.ssl_version = ssl.PROTOCOL_TLSv1_2
else:
ftp = FTP()
ftp.connect(environ.get('FTP_HOST'),
int(environ.get('FTP_PORT') or 21))
ftp.login(environ.get('FTP_USERID'), environ.get('FTP_PASSWORD'))
ftp.sendcmd("site filetype=jes")
ftp.set_pasv(True)
if environ.get('FTP_TLS_VERSION'):
ftp.prot_p()
except Exception as e:
module.fail_json(
msg="An unexpected error occurred during FTP login: {0}".format(
repr(e)),
**result)
if not job_id and not job_name and not owner:
module.fail_json(msg="Please provide a job_id or job_name or owner")
try:
result = {}
wait_time_s = 10
result["jobs"] = job_output(ftp, wait_time_s, job_id, owner, job_name,
ddname)
result["changed"] = False
except Exception as e:
module.fail_json(msg=repr(e))
module.exit_json(**result)
def run_module():
module_args = dict(src=dict(type="str", required=True),
wait=dict(type="bool", required=False),
location=dict(
type="str",
default="DATA_SET",
choices=["DATA_SET", "USS", "LOCAL"],
),
volume=dict(type="str", required=False),
return_output=dict(type="bool",
required=False,
default=True),
wait_time_s=dict(type="int", default=60),
max_rc=dict(type="int", required=False))
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
result = dict(changed=False, )
location = module.params["location"]
volume = module.params["volume"]
wait = module.params["wait"]
src = module.params["src"]
return_output = module.params["return_output"]
wait_time_s = module.params["wait_time_s"]
max_rc = module.params["max_rc"]
if wait_time_s <= 0:
module.fail_json(
msg=
"The option wait_time_s is not valid. It must be greater than 0.",
**result)
if environ.get('FTP_SOCKS_PORT'):
import socks
import socket
socks.set_default_proxy(socks.SOCKS5, "127.0.0.1",
int(environ.get('FTP_SOCKS_PORT')))
socket.socket = socks.socksocket
try:
if environ.get('FTP_TLS_VERSION'):
from ftplib import FTP_TLS
import ssl
cert_file_path = environ.get('FTP_TLS_CERT_FILE')
if cert_file_path:
if not path.isfile(cert_file_path):
module.fail_json(
msg="The TLS cartificate file not found: {0}".format(
repr(cert_file_path)),
**result)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.load_verify_locations(cert_file_path)
context.check_hostname = False
ftp = FTP_TLS(context=context)
else:
ftp = FTP_TLS()
tls_version = environ.get('FTP_TLS_VERSION')
if tls_version == '1.2':
ftp.ssl_version = ssl.PROTOCOL_TLSv1_2
else:
ftp = FTP()
ftp.connect(environ.get('FTP_HOST'),
int(environ.get('FTP_PORT') or 21))
ftp.login(environ.get('FTP_USERID'), environ.get('FTP_PASSWORD'))
ftp.sendcmd("site filetype=jes")
ftp.set_pasv(True)
if environ.get('FTP_TLS_VERSION'):
ftp.prot_p()
except Exception as e:
module.fail_json(
msg="An unexpected error occurred during FTP login: {0}".format(
repr(e)),
**result)
DSN_REGEX = r"^(?:(?:[A-Z$#@]{1}[A-Z0-9$#@-]{0,7})(?:[.]{1})){1,21}[A-Z$#@]{1}[A-Z0-9$#@-]{0,7}(?:\([A-Z$#@]{1}[A-Z0-9$#@]{0,7}\)){0,1}$"
try:
if location == "DATA_SET":
data_set_name_pattern = re.compile(DSN_REGEX, re.IGNORECASE)
if PY2:
check = data_set_name_pattern.match(src)
else:
check = data_set_name_pattern.fullmatch(src)
if check:
if volume is None or volume == "":
jobId = submit_pds_jcl(src, ftp, module)
else:
jobId = submit_jcl_in_volume(src, volume, ftp, module)
else:
ftp.quit()
module.fail_json(
msg=
"The parameter src for data set is not a valid name pattern. Please check the src input.",
**result)
else:
jobId = submit_ftp_jcl(src, ftp, module)
except SubmitJCLError as e:
module.fail_json(msg=repr(e), **result)
if jobId is None or jobId == "":
result["job_id"] = jobId
ftp.quit()
module.fail_json(
msg=
"JOB ID Returned is None. Please check whether the JCL is valid.",
**result)
result["job_id"] = jobId
if not wait:
wait_time_s = 10
# real time loop - will be used regardless of 'wait' to capture data
starttime = timer()
loopdone = False
foundissue = None
while not loopdone:
try:
job_output_txt = job_output(ftp, wait_time_s, job_id=jobId)
except IndexError:
pass
except Exception as e:
result["err_detail"] = "{1} {2}.\n".format(
"Error during job submission. The output is:", job_output_txt
or " ")
module.fail_json(msg=repr(e), **result)
if bool(job_output_txt):
jot_retcode = job_output_txt[0].get("ret_code")
if bool(jot_retcode):
job_msg = jot_retcode.get("msg")
if re.search(
"^(?:{0})".format("|".join(JOB_COMPLETION_MESSAGES)),
job_msg):
loopdone = True
# if the message doesn't have a CC, it is an improper completion (error/abend)
if re.search("^(?:CC)", job_msg) is None:
foundissue = job_msg
if not loopdone:
checktime = timer()
duration = round(checktime - starttime)
if duration >= wait_time_s:
loopdone = True
result["message"] = {
"stdout":
"Submit JCL operation succeeded but it is a long running job, exceeding the timeout of "
+ str(wait_time_s) + " seconds. JobID is " + str(jobId) +
"."
}
else:
sleep(1)
# End real time loop ^^^
if bool(job_output_txt):
result["jobs"] = job_output_txt
checktime = timer()
duration = round(checktime - starttime)
result["duration"] = duration
result["changed"] = True
if duration >= wait_time_s:
result["message"] = {
"stdout":
"Submit JCL operation succeeded but it is a long running job. Timeout is "
+ str(wait_time_s) + " seconds. JobID is " + str(jobId) + "."
}
else:
if foundissue is not None:
result["changed"] = False
result["message"] = {
"stderr": "Submit succeeded, but job failed: " + foundissue
}
result["failed"] = True
module.fail_json(msg=result["message"], **result)
elif (wait is True and return_output is True and max_rc is not None
and max_rc < result.get("jobs")[0].get("ret_code").get("code")):
result["message"] = {
"stderr":
"Submit succeeded, but the return code is more than max_rc: {0}"
.format(max_rc)
}
result["failed"] = True
module.fail_json(msg=result["message"], **result)
else:
result["message"] = {
"stdout":
"Submit JCL operation succeeded with id of " + str(jobId) + "."
}
module.exit_json(**result)
def run_module():
module_args = dict(commands=dict(type="raw",
required=True,
aliases=["command"]), )
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
result = dict(changed=False, )
commands = module.params['commands']
if environ.get('FTP_SOCKS_PORT'):
import socks
import socket
socks.set_default_proxy(socks.SOCKS5, "127.0.0.1",
int(environ.get('FTP_SOCKS_PORT')))
socket.socket = socks.socksocket
try:
if environ.get('FTP_TLS_VERSION'):
from ftplib import FTP_TLS
import ssl
cert_file_path = environ.get('FTP_TLS_CERT_FILE')
if cert_file_path:
if not path.isfile(cert_file_path):
module.fail_json(
msg="Certification file not found: {0}".format(
repr(cert_file_path)),
**result)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.load_verify_locations(cert_file_path)
context.check_hostname = False
ftp = FTP_TLS(context=context)
else:
ftp = FTP_TLS()
tls_version = environ.get('FTP_TLS_VERSION')
if tls_version == '1.2':
ftp.ssl_version = ssl.PROTOCOL_TLSv1_2
else:
ftp = FTP()
ftp.connect(environ.get('FTP_HOST'),
int(environ.get('FTP_PORT') or 21))
ftp.login(environ.get('FTP_USERID'), environ.get('FTP_PASSWORD'))
ftp.sendcmd("site filetype=jes")
ftp.set_pasv(True)
if environ.get('FTP_TLS_VERSION'):
ftp.prot_p()
except Exception as e:
module.fail_json(
msg="An unexpected error occurred during FTP login: {0}".format(
repr(e)),
**result)
try:
result = run_tso_command(ftp, commands, module)
ftp.quit()
for cmd in result.get("output"):
if cmd.get("rc") != 0:
module.fail_json(msg='The TSO command "' +
cmd.get("command", "") +
'" execution failed.',
**result)
result["changed"] = True
module.exit_json(**result)
except Exception as e:
ftp.quit()
module.fail_json(msg="An unexpected error occurred: {0}".format(
repr(e)),
**result)
def moveFTPFiles(serverName,
userName,
passWord,
fileToUpload,
remoteDirectoryPath,
useTLS=False):
lock.acquire(True)
"""Connect to an FTP server and bring down files to a local directory"""
if (serverName != '' and userName != '' and passWord != ''):
try:
ftp = None
if useTLS:
ftp = FTP_TLS(serverName, timeout=120)
if (ftp == None):
logger.info("LOG moveFTPFiles 3TLS ftp null")
else:
ftp = FTP(serverName)
ftp.login(userName, passWord)
if useTLS:
ftp.prot_p()
ftp.cwd(remoteDirectoryPath)
ftp.set_pasv(True)
filesMoved = 0
try:
logger.info("Connecting...")
# create a full local filepath
localFileName = path_leaf(fileToUpload)
if localFileName != "" and not localFileName.startswith("."):
#create a copy of the file before sending it
tempFile = create_temporary_copy(fileToUpload, workingDir,
localFileName)
# open a the local file
fileObj = open(tempFile, 'rb')
# Download the file a chunk at a time using RETR
ftp.storbinary('STOR ' + localFileName, fileObj)
# Close the file
fileObj.close()
filesMoved += 1
# remove the temp file
# to remove it I need to have the right permissions
os.chmod(tempFile, 0777)
os.remove(tempFile)
logger.info("Uploaded file: " + str(fileToUpload) + " on " +
timeStamp())
except Exception as inst:
logger.error(type(inst)) # the exception instance
logger.error(inst.args) # arguments stored in .args
logger.error(
inst) # __str__ allows args to be printed directly
logger.error("Connection Error - " + timeStamp())
ftp.quit() # Close FTP connection
ftp = None
except Exception as inst:
logger.error("Couldn't find server")
logger.error(type(inst)) # the exception instance
logger.error(inst.args) # arguments stored in .args
logger.error(inst) # __str__ allows args to be printed directly
else:
logger.error(
"Connection was not possible because one of the following var was not set in the configuration file: ftpServerName , ftpU or ftpP"
)
lock.release()
def lambda_handler(event, context):
exec_id = f"{time.time()}".split('.')[0]
### RETRIEVE FILE FROM TAW ###
# Set up connection to FTP server
ftps = FTP_TLS()
ftps.connect('ftp.tapww.com', 22, timeout=120)
ftps.set_debuglevel(1)
ftps.set_pasv(True)
ftps.login(os.environ['taw_user'], os.environ['taw_pass'])
ftps.prot_p()
# Create local file to store contents of Inventory file
f = open('/tmp/inv.txt', 'wb')
# Retrieve the Inventory file contents and store locally
ftps.retrbinary('RETR Inventory.txt', f.write)
# Close local file
f.close()
# Close the connection
ftps.quit()
### END RETRIEVE FILE FROM TAW ###
### READ INVENTORY FILE ###
item_list = []
with open('/tmp/inv.txt', newline='') as items:
reader = csv.reader(items, delimiter='\t')
item_list = [[item[2], item[12]] for item in reader
if item[2][:3] in LINES]
print(f"Items found: {len(item_list)}")
# Divide list into chunks for threading
chunks = []
for i in range(0, len(item_list), 200):
chunks.append(item_list[i:i + 200])
### END READ INVENTORY FILE ###
numChunks = len(chunks)
print(f"Number of chunks: {numChunks}")
### INVOKE LAMBDA ONCE FOR EACH CHUNK ###
i = 1
# only execute the first chunk while testing
for eachChunk in chunks:
print(f"Invoking lambda for chunk {i} of {numChunks}")
lambda_client.invoke(FunctionName='updateInventory',
InvocationType='Event',
Payload=json.dumps({
'id': exec_id,
'chunk': eachChunk,
'chunkNum': i
}))
i = i + 1
### END INVOKE LAMBDA ONCE FOR EACH CHUNK ###
return {
'statusCode':
202,
'headers': {
'Access-Control-Allow-Origin': '*'
},
'body':
json.dumps({
'id': exec_id,
'numItems': len(item_list),
'numChunks': numChunks
})
}
