def sendUpdateFilesFtp():
print "---- Send update files by FTP"
global serverFtp
global userFtp
global passFtp
from ftplib import FTP_TLS
ftps = FTP_TLS(serverFtp)
ftps.set_debuglevel(1)
ftps.login(userFtp, passFtp)
ftps.prot_p()
ftps.cwd('/files/updates')
ftps.storbinary('STOR ' + 'file_list.md5', open(quiterssFileRepoPath + '\\file_list.md5', 'rb'))
ftps.storbinary('STOR ' + 'VersionNo.h', open(quiterssSourcePath + '\\src\\VersionNo.h', 'rb'))
ftps.storbinary('STOR ' + 'HISTORY_EN', open(quiterssSourcePath + '\\HISTORY_EN', 'rb'))
ftps.storbinary('STOR ' + 'HISTORY_RU', open(quiterssSourcePath + '\\HISTORY_RU', 'rb'))
prepareFileList7z = []
for file in prepareFileList:
prepareFileList7z.append(file + '.7z')
for fileName in prepareFileList7z:
ftps.storbinary('STOR ' + 'windows' + fileName.replace('\\','/'), open(quiterssFileRepoPath + '\\windows' + fileName, 'rb'))
ftps.quit()
class ftpAccess():
def __init__(self, url):
self.ftp = FTP_TLS(url)
self.ftp.login(
"anonymous", "anonymous",
secure=False) # login anonymously before securing control channel
def GetFtpFile(self, InPath, filename, downloaddir):
outfile = downloaddir + filename
ensure_dir(downloaddir)
self.ftp.cwd(InPath)
self.ftp.retrbinary("RETR " + filename, open(outfile, 'wb').write)
def GetFileListFtp(self, pathname):
self.ftp.cwd(pathname)
ret = list()
out = self.ftp.retrlines('LIST',
addline) # list directory content securely
# search json files temp = entry.split(' ')
for filename in filenamelist:
try:
a = self.ftp.size(filename)
print("{} - {}".format(filename, a))
ret.append([filename, a])
except:
print("{} - xxx".format(filename))
pass
return ret
def conexionFTP():
print "conexionFTP"
global reIntento, rutaFtp,tiempoErr,hostFtp,passFtp,userFtp,portFtp,infoERR
#request = urllib.request.Request('http://'+hostFtp)
#response = urllib.request.urlopen(request)
estatusCftp=False
if response.getcode() == 200:
try:
ftp = FTP_TLS()
ftp.connect(hostFtp,portFtp)
ftp.login(userFtp, passFtp)
ftp.cwd(rutaFtp)
estatusCftp = True
except Exception as e:
if infoERR == True:logger.warning(formErro)
logger.warning('Error al conectar al servidor ftp '+str(e))
else:
logger.warning('Error al conectar al servidor Error de internet '+str(e))
for i in range(reIntento):
if conexionFTP()['ftp']:main()
else:logger.warning("reconecion internet intento "+str(i));slp(tiempoErr)#
return {'ftp':ftp,'estatusCftp':estatusCftp}
def main():
global ftp_client
scan_arguments()
ftp_client = FTP(host)
try:
ftp_client.login(username, password)
except ftplib.all_errors as e:
print "ERROR: cannot login with username '{0}' and relative password.\nMessage returned from server:".format(username)
print e
return
try:
ftp_client.cwd(remote_dir)
except ftplib.all_errors as e:
print "ERROR: emote directory '{0}' not existing.\nMessage returned from server:".format(remote_dir)
print e
return
else:
files = ftp_client.nlst()
print_directory_content(files)
setup_folder()
download_files(remote_dir, files)
if compress:
create_zip(local_dir)
try:
ftp_client.close()
print "!!!!! OPERATION COMPLETED SUCCESSFULLY !!!!!"
except ftplib.all_errors as e:
print "ERROR: cannot close the connection properly.\nMessage from server:"
print e
def DeleteTextFile():
filename = 'test.txt'
session = FTP_TLS('files.000webhost.com', 'vulcuns-webserver',
'AKPJJDT2lol')
session.cwd('public_html')
session.delete(filename)
session.quit()
def connect(self):
#初始化 FTP 链接
if self.ftp_ssl:
ftp = FTPS()
else:
ftp = FTP()
print('-'*20+self.ftp_name+'-'*20)
print('connect '+('ftps' if self.ftp_ssl else 'ftp')+'://'+self.ftp_host+':'+self.ftp_port)
try:
ftp.connect(self.ftp_host,self.ftp_port)
except Exception as e:
print (e)
print ('connect ftp server failed')
sys.exit()
try:
ftp.login(self.ftp_user,self.ftp_passwd)
print ('login ok')
except Exception as e:#可能服务器不支持ssl,或者用户名密码不正确
print (e)
print ('Username or password are not correct')
sys.exit()
if self.ftp_ssl:
try:
ftp.prot_p()
except Exception as e:
print (e)
print ('Make sure the SSL is on ;')
print(ftp.getwelcome())
ftp.cwd(self.ftp_webroot)
print('current path: '+ftp.pwd())
self.ftp=ftp
def connect(box_path):
ftp = FTP_TLS('ftp.box.com')
# ftp.debugging = 2
ftp.login('*****@*****.**', 'cJ19870915@#')
ftp.cwd(box_path[0])
print("Connected to Box FTP")
return ftp
def copy_ftp_file(project_id, client_name, file_name): #
"""
ftp file copy
"""
ftp = FTP_TLS(settings.FTP_LOCATION)
ftp.sendcmd("USER {}".format(settings.FTP_USER))
ftp.sendcmd("PASS {}".format(settings.FTP_PASS))
comp_logger.info(
'Initiating ftp file transfer for file {} for client {}'.format(
file_name, client_name))
ftp.cwd(client_name)
# create project input dir
project_dir = os.path.join(settings.PROJECT_DIR, str(project_id))
helper.create_dir(project_dir)
# copy remote ftp file to local project folder
file_format = file_name.split('.')[-1]
local_filename = os.path.join(settings.PROJECT_INPUT_FOLDER,
'{}.{}'.format(project_id, file_format))
if os.path.exists(local_filename):
os.remove(local_filename)
# project_input_dir = os.path.join(project_dir, settings.PROJECT_INPUT_FOLDER)
# helper.create_dir(project_input_dir)
# local_filename = os.path.join(project_input_dir, file_name)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + file_name, lf.write, 8 * 1024)
lf.close()
comp_logger.info('Completed Copying file {} for client {}'.format(
file_name, client_name))
def f_sending_order():
global l_selected
l = l_selected + [ent_cast.get(), txt_article.get(1.0, END), f_time_now()]
with open('client_app_order.pkl', 'wb') as f:
pickle.dump(l, f)
print('start sending......')
try:
ftps = FTP_TLS(helpic.place_ftp, helpic.user_ftp, helpic.password_ftp)
ftps.cwd('/www/{}'.format(helpic.directory_ftp))
except:
showinfo(
'Connection', 'You don\'t have internet conection\n\
or login/pasword were changed')
return None
sufix = random.randint(1, 1000000)
file_name = 'order{}.pkl'.format(sufix)
ftps.storbinary('STOR ' + file_name,
open('client_app_order.pkl',
'rb')) # загрузка файла НА сервер
print('finish sending')
ftps.quit()
f_get_items_table(
) # getting information from FTP-server and writing in file
def connect_ftps():
#Connect to the server
print("Step 1")
if SECURE_CONN == '1':
ftps = FTP_TLS(SERVER)
print("Secure connection")
else:
ftps = FTP(SERVER)
print("Unsecure connection")
print("Step 2")
ftps.set_debuglevel(2)
print("Step 3")
ftps.set_pasv(False)
print("Step 4")
ftps.connect(port=int(PORT), timeout=160)
print("Step 5")
ftps.login(USER, PASS)
print("Step 6")
ftps.cwd(TARGET_DIR)
print("Step 7")
ftps.set_pasv(True)
print("Step 8")
#Set up the secure connection only in FTPS mode
if SECURE_CONN == '1':
ftps.prot_p()
else:
pass
#ftps.ccc()
return ftps
def sendPackagesFtp():
print "---- Send packages by FTP"
global serverFtp
global userFtp
global passFtp
from ftplib import FTP_TLS
ftps = FTP_TLS(serverFtp)
ftps.set_debuglevel(1)
ftps.login(userFtp, passFtp)
ftps.prot_p()
try:
ftps.sendcmd('MKD ' + '/files/' + strProductVer)
except Exception:
print 'Directory already exists'
ftps.cwd('/files/' + strProductVer)
filesListFtp = ftps.nlst()
filesList = os.listdir(packagesPath)
newFilesList = [e for e in filesList if not(e in filesListFtp)]
for fileName in newFilesList:
ftps.storbinary('STOR ' + fileName, open(packagesPath + '\\' + fileName, 'rb'))
ftps.quit()
def download_all_in_one_path(targetdir,resultdir,check = True,num = 50):
if(os.path.exists(resultdir) == False):
os.makedirs(resultdir)
ftp = FTP('129.164.179.23')
ftp.login()
ftp.prot_p()
ftp.cwd(targetdir)
files = ftp.nlst()
target = 'https://heasarc.gsfc.nasa.gov/FTP' + targetdir
c = None
if(check):
c = []
data1 = []
ftp.voidcmd('TYPE I')
print('正在获取校验信息........')
for i in files:
#print(i)
data = os.path.join(target,i)
print(data)
data1.append(data)
if(check):
c.append(ftp.size(i))
ftp.quit()
if(check == False):
print('忽略数据大小校验。')
print('正在校验...............')
down(data1,resultdir,check=c,threadnum = num)
print('\n任务下载完成！！！')
def connexionftp(adresseftp, nom, mdpasse, passif):
"""connexion au serveur ftp et ouverture de la session
- adresseftp: adresse du serveur ftp
- nom: nom de l'utilisateur enregistré ('anonymous' par défaut)
- mdpasse: mot de passe de l'utilisateur ('anonymous@' par défaut)
- passif: active ou désactive le mode passif (True par défaut)
retourne la variable 'ftplib.FTP' après connexion et ouverture de session
"""
try:
verbose('Attente connexion FTP .....')
if modeSSL:
ftp = FTP_TLS()
ftp.connect(adresseftp, 21)
ftp.login(nom, mdpasse)
ftp.prot_p()
ftp.set_pasv(passif)
else:
ftp = (ftplib.FTP(adresseftp, nom, mdpasse))
ftp.cwd(destination)
verbose ('Destination : '+destination)
verbose('Connexion FTP OK')
etat = ftp.getwelcome()
verbose("Etat : "+ etat)
return ftp
except:
verbose('Connexion FTP impossible', True)
suppressionDossierTemp(dossierTemporaireFTP)
sys.exit()
def download_npc():
ftp_files = []
os_files = []
try:
ftps = FTP_TLS()
ftps.connect(CFG_FTPS_HOST, CFG_FTPS_PORT)
log_it(
'connected to ' + CFG_FTPS_HOST + ' welcome message: ' +
str(ftps.getwelcome()), 'info')
ftps.login(CFG_FTPS_USER, CFG_FTPS_PASS)
ftps.prot_p()
log_it('changing dir: ' + CFG_FTPS_DIR, 'info')
ftps.cwd(CFG_FTPS_DIR)
ftp_files = ftps.nlst()
for f in ftp_files:
if not os.path.isfile(CFG_ARCHIVE_DIR + f):
ftps.retrbinary('RETR ' + f,
open(CFG_ARCHIVE_DIR + f, 'wb').write)
log_it('downloading file ' + f, 'info')
else:
log_it(
'skipping ' + f + ' as it already exists in ' +
CFG_ARCHIVE_DIR, 'debug')
except ftplib.all_errors, e:
log_it('unable to connect to ' + CFG_FTPS_HOST + ' %s' % e, 'error')
def find_versions_above(
mirror: Dict[str, str], target_version: Version
) -> Generator[Mapping[str, Union[str, Version]], None, None]:
print("Using mirror:", mirror['host'])
tls = get_use_tls()
if tls:
ftp = FTP_TLS(mirror['host'])
else:
ftp = FTP(mirror['host'])
ftp.login(user=mirror.get('user', None),
passwd=mirror.get('passwd', ''),
acct=mirror.get('acct', ''))
ftp.cwd(mirror['path'])
for filename, attrs in sorted(ftp.mlsd(), key=lambda tup: tup[0]):
if not attrs["type"] == "file":
continue
if not filename.endswith(".tar.gz"):
continue
pkgver = get_package_version(filename)
if not pkgver:
continue
if pkgver > target_version:
fileinfo = mirror.copy()
fileinfo["filename"] = filename
fileinfo["version"] = pkgver
fileinfo["tls"] = tls
yield fileinfo
def _get_ftp(self):
try:
username, password = self._get_auth()
if self._tls:
ftp = FTP_TLS(self._uri, timeout=CONNECT_TIMEOUT)
else:
ftp = FTP(self._uri, timeout=CONNECT_TIMEOUT)
ftp.login(username, password)
if self._tls:
ftp.prot_d()
if not catch(ftp.cwd, self._path):
if self._path.startswith("/"):
ftp.cwd("/")
components = self._path.split("/")
for component in components:
if not catch(ftp.cwd, component):
ftp.mkd(component)
ftp.cwd(component)
return ftp
except Exception:
log.exception()
log.warning(
"[FTP] failed to establish server connection, disabled")
self._disabled = True
return None
def sync(self):
"""
downloads all needed_files from self.hostname (FTP)
of the downloaded files, extracts .gz files to same local_working_dir
-using self.extract function
parses the .txt downloaded needed_files
-using the self.parse function
"""
ftps = FTP_TLS(self.hostname) # connect to host, default port
ftps.login(self.username, self.password)
ftps.prot_p()
ftps.cwd(self.remote_dir) # change into "logs" directory
ftps.retrlines('LIST *.gz *.txt', self.ftp_list_callback) # list directory contents
for needed_file in self.needed_files:
if self.logging:
print "Writing {0} to {1}...".format(needed_file, self.local_working_dir)
ftps.retrbinary("RETR " + needed_file, open(os.path.join(self.local_working_dir, needed_file), 'wb').write)
if self.logging:
print "done syncing files"
for needed_file in self.needed_files:
if needed_file.endswith(".gz"):
self.extract(os.path.join(self.local_working_dir, needed_file))
txt_file_name = needed_file.replace('.gz','')#if already a .txt file, this is unnceccessary but works.
self.parse(txt_file_name)
if self.logging:
print "done extracting/parsing .gz files"
ftps.quit()
def _connect(server: str):
global _connection_cache
if server not in _connection_cache:
if server == 'in_server':
logger.debug(f'Connection to "in_server" requested')
conn_specs = config().ftp.in_server
elif server == 'out_server':
logger.debug(f'Connection to "out_server" requested')
conn_specs = config().ftp.out_server
else:
raise ValueError(f'Unknown server "{server}"')
logger.debug(f'conn specs = {conn_specs}')
if conn_specs.secure:
ftp = FTP_TLS(conn_specs.host)
ftp.auth()
else:
ftp = FTP(conn_specs.host)
# ftp.connect(conn_specs.host, conn_specs.port)
logger.debug(f'Connected to "{server}". Logging in...')
ftp.login(conn_specs.username, conn_specs.password)
logger.debug(f'Established connection to "{server}". Saving to cache')
_connection_cache[server] = ftp
return ftp
else:
ftp = _connection_cache[server]
ftp.cwd('/')
logger.debug(f'Recycling connection to "{server}"')
return ftp
def send_to_ftp(filepath):
ftps = FTP_TLS('ftpes.learn.inf.puc-rio.br') # Define Ftp server
ftps.login('learn', 'LRepoAdm18!!') # Login to ftp server
ftps.prot_p() # Enable data encryption
ftps.retrlines('LIST') # List Directory
ftps.cwd('/projeto/01_tarefas_de_geofisica/QualiSismo/CV_results') # Change Directory
ftps.storlines("STOR " + filepath, open(filepath,'rb'))
def UpdateLocalData(Force=False):
'''
This will download and convert any OMNI data which is missing from
the local archive.
'''
ftp = FTP_TLS(Globals.ftpbase)
ftp.login()
ftp.cwd(Globals.ftpdir)
#let's download and read the FTP index
status = _DownloadFTPIndex(ftp)
if not status:
print('Download failed; check for write permission to data folder')
ftp.close()
return
ftp.close()
FileNames,Addresses,UpdateDates,Res = _ParseFTP()
n = np.size(FileNames)
#check current data index
idx = _ReadDataIndex()
#now compare update dates
if Force:
update = np.ones(n,dtype='bool')
else:
update = _CompareUpdates(UpdateDates,FileNames,idx)
use = np.where(update)[0]
FileNames = FileNames[use]
Addresses = Addresses[use]
UpdateDates = UpdateDates[use]
Res = Res[use]
n = use.size
if n == 0:
print('No files to update.')
ftp.close()
return
for i in range(0,n):
print('Downloading file {0} of {1}'.format(i+1,n))
#download file
tmp = _DownloadFTPFile(FileNames[i])
print('Converting to binary')
#convert file
_ConvertFTPFile(tmp,FileNames[i],UpdateDates[i],Res[i])
#delete text file
_DeleteFTPFile(tmp)
ftp.close()
print('Done')
def ftp_to_ml(localfile, remote_file):
(user, acct, passwd) = netrc.netrc().authenticators('ftps.b2b.ml.com')
ftp = FTP_TLS('ftps.b2b.ml.com')
remote_dir = 'incoming/gmigefs2'
ftp.login(user, passwd, acct)
ftp.prot_p()
ftp.cwd(remote_dir)
local_fd = open(localfile, 'r')
ftp.storlines('STOR ' + remote_file, local_fd)
local_fd.close()
def send(self):
ftp = FTP(self.ftp_destination)
ftp.set_debuglevel(2)
ftp.ssl_version = ssl.PROTOCOL_TLSv1_2
ftp.login()
ftp.cwd("upload")
ftp.prot_p()
self.zip.close()
self.zip_contents.seek(0)
ftp.storbinary("STOR {}".format(self.zip_contents.name), self.zip_contents)
def fetch_data_via_ftp(ftp_config, local_directory):
""" Get benchmarking data from a remote ftp server.
:type ftp_config: config.FTPConfigurationRepresentation
:type local_directory: str
"""
if ftp_config.enabled:
# Create local directory tree if it does not exist
create_directory_tree(local_directory)
# Login to FTP server
if ftp_config.use_tls:
ftp = FTP_TLS(ftp_config.server)
ftp.login(ftp_config.username, ftp_config.password)
ftp.prot_p() # Request secure data connection for file retrieval
else:
ftp = FTP(ftp_config.server)
ftp.login(ftp_config.username, ftp_config.password)
if not ftp_config.files: # Auto-download all files in directory
fetch_data_via_ftp_recursive(ftp=ftp,
local_directory=local_directory,
remote_directory=ftp_config.directory)
else:
ftp.cwd(ftp_config.directory)
file_counter = 1
file_list_total = len(ftp_config.files)
for remote_filename in ftp_config.files:
local_filename = remote_filename
filepath = os.path.join(local_directory, local_filename)
if not os.path.exists(filepath):
with open(filepath, "wb") as local_file:
try:
ftp.retrbinary('RETR %s' % remote_filename,
local_file.write)
print("[Setup][FTP] ({}/{}) File downloaded: {}".
format(file_counter, file_list_total,
filepath))
except error_perm:
# Error downloading file. Display error message and delete local file
print(
"[Setup][FTP] ({}/{}) Error downloading file. Skipping: {}"
.format(file_counter, file_list_total,
filepath))
local_file.close()
os.remove(filepath)
else:
print(
"[Setup][FTP] ({}/{}) File already exists. Skipping: {}"
.format(file_counter, file_list_total, filepath))
file_counter = file_counter + 1
# Close FTP connection
ftp.close()
def connect_to_ftp(self, options):
host = options['ftp_server']
path = options['path']
user = options['user']
password = options['password']
ftp = FTP_TLS(host=host)
ftp.login(user=user, passwd=password)
ftp.cwd(path)
return ftp
def get_latest_file():
with open('filenames.json') as json_file:
files = json.load(json_file)
flag = 0
ftp = FTP_TLS(host='olftp.adesa.com',
user='******',
passwd='aU)kj7Qn8')
ftp.prot_p()
#ftp.retrlines('LIST')
ftp.cwd('outbound/')
file_list = []
ftp.retrlines('MLSD', file_list.append)
#ftp.dir(file_list.append)
max_value = 0
full_max = 0
filename = ''
full_file = ''
for i in file_list:
col = i.split(';')
col_max = int(re.search(r'\d+', col[0]).group())
if (col_max > max_value) & ('.txt' in col[-1]):
max_value = col_max
filename = col[-1].replace(' ', '')
if (col_max > full_max) & ('.txt' in col[-1]) & ('FULL' in col[-1]):
full_max = col_max
full_file = col[-1].replace(' ', '')
if (filename != files['inc_file']):
localfile = open(filename, 'wb')
ftp.retrbinary('RETR ' + filename, localfile.write, 1024)
localfile.close()
print("Inc file data tranfer complete")
flag = 1
else:
print("Inc already there")
if (full_file != files['full_file']):
localfile = open(full_file, 'wb')
ftp.retrbinary('RETR ' + full_file, localfile.write, 1024)
localfile.close()
print("Full file data tranfer complete")
flag = 1
else:
print("Full already there")
if flag == 1:
new_names = {'full_file': full_file, 'inc_file': filename}
with open('filenames.json', 'w') as outfile:
json.dump(new_names, outfile)
ftp.quit()
return filename, full_file, flag
def cliente_ftp_conexion(servidor, nombre_usuario, password):
#abrimos la conexion
ftps = FTP_TLS(servidor)
ftps.login(nombre_usuario, password)
#ftp = ftplib.FTP(servidor, nombre_usuario, password)
#listamos los archivos de /
ftps.cwd("/")
print "Archivos disponibles en %s:" % servidor
archivos = ftps.dir()
print archivos
ftps.quit()
def connect2box(self):
"""
Connects to the box sync server
"""
ftp = FTP_TLS('ftp.box.com')
print('logging in')
ftp.login(user=self.user, passwd=getpass.getpass())
# move to destination directory
ftp.cwd(self.boxFolder)
print('FTP: moving to folder ' + self.boxFolder + ' \n')
self.ftp = ftp
def testFTPS(hostname, username, password, port, path, passive):
try:
ftp = FTP_TLS()
ftp.connect(hostname, port)
ftp.login(username, password)
ftp.prot_p()
ftp.set_pasv(passive)
ftp.cwd(path)
files = ftp.dir()
ftp.quit()
return True
except BaseException as error:
return error
def fetch_temporary_gz_file(self, temporary_gz_file_path):
with open(temporary_gz_file_path, 'wb') as file_ftp:
file_name = ntpath.basename(temporary_gz_file_path)
try:
ftp = FTP_TLS(self.ftp_host, self.ftp_username,
self.ftp_password)
ftp.cwd(self.ftp_path)
ftp.retrbinary('RETR ' + file_name, file_ftp.write)
ftp.quit()
except:
Logger.file_parse(self.chain, file_name)
self.fetch_temporary_gz_file(temporary_gz_file_path)
def get_jobs_data():
ftp = FTP_TLS('ontario.files.com', timeout=10)
ftp.login(user=os.environ['211_username'],
passwd=os.environ['211_password'])
ftp.cwd('/211projects/BensTeam')
ftp.prot_p()
files = ftp.nlst()
for filename in files:
if not os.path.isfile('211_data/' + filename):
print(f"Getting file {filename}")
ftp.retrbinary("RETR " + filename,
open('211_data/' + filename, 'wb').write)
ftp.quit()
return 'Done'
def uploadToFtp(filename):
host = "88.122.218.102" # adresse du serveur FTP
user = "******" # votre identifiant
password = "******" # votre mot de passe
port = 49153
ftps = FTP_TLS()
ftps.connect(host, port)
ftps.login(user, password)
ftps.encoding = "utf-8"
ftps.cwd("Disque dur")
ftps.cwd("static")
with open('/static/'+filename, "rb") as file:
ftps.storbinary(f"STOR {filename}", file)
ftps.quit()
def _upload_qr_code_ftp(code_filename):
"""
TODO:
1. Remove DRY code around code_fullpath
"""
code_fullpath = _get_environ_var(
'GENERATED_CODES_DIRECTORY') + code_filename
with open(code_fullpath, 'rb') as fp:
ftps = FTP_TLS(_get_environ_var('FTP_HOST'))
ftps.login(_get_environ_var('FTP_USERNAME'),
_get_environ_var('FTP_PASSWORD'))
ftps.prot_p()
ftps.cwd(_get_environ_var('FTP_DIRECTORY'))
ftps.storbinary('STOR ' + code_filename, fp)
ftps.quit()
def linkFtp(folder=''):
"""
Handlser FTP per il caricamento dei dati su server remoto
:param folder: sotto cartella di salvataggio remota
:return: handler per la gestione
"""
# ftp = FTP_TLS()
ftp = FTP_TLS(host=rl['host'])
ftp.set_debuglevel(2)
status = ftp.login(user=rl['username'], passwd=rl['password'])
# comando per il cambio della directory di upload
ftp.cwd(backup_folder+'/'+folder)
# print the content of directory
print(ftp.dir())
return ftp, status
def upload_archive_file(self, local_filename, remote_filename, target_dir):
yield("Uploading {} to FTP in directory: {}, filename: {}".format(local_filename, target_dir, remote_filename))
local_filesize = os.stat(local_filename).st_size
self.upload_total = os.stat(local_filename).st_size
self.upload_current = 0
if self.settings.ftps['no_certificate_check']:
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False
else:
context = ssl.create_default_context()
ftps = FTP_TLS(
host=self.settings.ftps['address'],
user=self.settings.ftps['user'],
passwd=self.settings.ftps['passwd'],
context=context,
source_address=self.settings.ftps[
'source_address'],
timeout=self.settings.timeout_timer
)
ftps.cwd(target_dir)
ftps.encoding = 'utf8'
ftps.prot_p()
for line in ftps.mlsd(facts=["size"]):
if(line[0] == remote_filename
and local_filesize == int(line[1]["size"])):
yield("File exists and size is equal.")
ftps.close()
return
with open(local_filename, "rb") as file:
for retry_count in range(3):
try:
ftps.storbinary(
'STOR %s' % remote_filename,
file,
callback=lambda data, args=self.print_method: self.print_progress(data, args)
)
except (ConnectionResetError, socket.timeout, TimeoutError):
yield("Upload failed, retrying...")
else:
break
yield("\nFile uploaded.")
ftps.close()
def getfiles(server, port, user, password, db):
sqliteconnection = sqlite3.connect(db)
sqlitecursor = sqliteconnection.cursor()
sqlitecursor.execute('''CREATE TABLE IF NOT EXISTS latest (date int, CONSTRAINT 'id_UNIQUE' UNIQUE ('date'))''')
sqliteconnection.commit()
sqlitecursor.execute('''SELECT date FROM files WHERE date = (SELECT MAX(date) FROM files) LIMIT 1''')
latestfile = sqlitecursor.fetchone()
sqlitecursor.execute('''SELECT date FROM latest WHERE date = (SELECT MAX(date) FROM latest) LIMIT 1''')
latestfetch = sqlitecursor.fetchone()
if latestfetch is None:
latestfetch = 0
if latestfetch < latestfile:
ftpsconnection = FTP_TLS()
ftpsconnection.connect(server, port)
ftpsconnection.auth()
ftpsconnection.prot_p()
ftpsconnection.login(user, password)
ftpsconnection.prot_p()
sqlitecursor.execute('''SELECT name FROM files WHERE date > %d''' % latestfetch)
filestofetch = sqlitecursor.fetchall()
for currfile in filestofetch:
ftpsconnection.cwd(currfile[0])
filenames = ftpsconnection.nlst()
for filename in filenames:
print 'Now saving /mnt/folder' + currfile[0] + '/' + filename
localfile = open('/mnt/folder' + currfile + '/' + filename, 'wb')
ftpsconnection.retrbinary('RETR ' + filename, localfile.write)
localfile.close()
sqliteconnection.execute('''INSERT OR IGNORE INTO latest VALUES (%d)''' % time.time())
sqliteconnection.commit()
sqliteconnection.close()
ftpsconnection.quit()
ftpsconnection.close()
def connect(velkost_ftp,port):
ftp=FTP_TLS(server,meno2,ps,port)
ftp.prot_p()
ftp.cwd(my_list[2])
print "Posielam subor. Cakajte prosim."
obsah=open(file_to_send, 'rb')
obsah.close()
ftp.storbinary('STOR %s' % file_to_send, open(file_to_send, 'rb'))
obsah.close()
print "Subor odoslany [OK]"
print "Obsah adresara na serveri:"
ftp.retrlines("LIST")
size_ftp=ftp.nlst()
pocet=len(size_ftp)
velkost_ftp_subor=size_ftp[pocet-1] #berie posledne pridany subor zo zoznamu
ftp.sendcmd("TYPE i")
velkost_ftp=ftp.size(velkost_ftp_subor)
ftp.close()
return velkost_ftp
def download(downloaded, user, passwd, all_files=False, filename=None):
# Connect to the MAPS ftp server over FTPS
ftps = FTP_TLS('ftps.tsi.telecom-paristech.fr')
print 'Connected to MAPS FTP over TLS.'
try:
ftps.login(user=user, passwd=passwd)
ftps.cwd('maps')
except error_perm:
print "Incorrect username/password" ; quit
ftps.retrlines('LIST *.zip', get_file_list)
if filename is not None:
if not in_downloads(files, filename): print 'File not found' ; return
print 'Downloading', filename
res = ftps.retrbinary('RETR '+filename, open('./downloads/'+filename, 'wb').write)
ftps.close()
return [(filename, 0)]
if len(files) == len(downloaded):
print "All MAPS files downloaded. Continuing."
return
if all_files:
for f, s in files:
if not in_downloads(downloaded, f):
print "Downloading", f, "of size", s, "bytes"
res = ftps.retrbinary('RETR '+f, open('./downloads/'+f, 'wb').write)
elif filename is None:
f, s = random.choice(files)
while in_downloads(downloaded, f):
f, s = random.choice(files)
print "Downloading", f, "of size", s, "bytes"
res = ftps.retrbinary('RETR '+f, open('./downloads/'+f, 'wb').write)
ftps.close()
if all_files: return files
return [(f, s)]
class FTPES:
def __init__(self, host, port=None, username=None, password=None, remote_path=None, absolute_zipfile_path=None):
"""
This is a helper class to manage the FTP commands from the ftplib.
@param host: The host for the connection.
@type host: String
@param port: The post for the connection.
@type port: Integer
@param username: The username for the connection. Leave blank to use "anonymous".
@type username: String
@param password: The password for the connection. Leave empty for none.
@type password: String
@param remote_path: The remote path of the server in which the zip file should be uploaded. If the path does not
exists, it will be created (recursive).
@type remote_path: String
@param absolute_zipfile_path: The absolute LOCAL filepath of the zip file.
@type absolute_zipfile_path: String
"""
self.ftps = None
self._host = host
self._port = port
self._username = username
self._password = password
self._remote_path = remote_path
self._absolute_zipfile_path = absolute_zipfile_path
self._bytesWritten = 0;
self._totalFileSize = os.path.getsize(self._absolute_zipfile_path)
self._uploadCallback = None
self._currentProgress = 0
# make the remote path relative if it isnt absolute or relative yet
if self._remote_path is not None and self._remote_path.startswith(
'.') is False and self._remote_path.startswith('/') is False:
self._remote_path = './' + self._remote_path
if self._username is None:
self._username = '******'
if self._port is None:
self._port = 22
def connect(self):
"""
Try to connect to the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps = FTP_TLS()
self.ftps.set_pasv(True)
self.ftps.connect(self._host, self._port)
except socket.gaierror:
raise
def login(self):
"""
Try to login in on the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps.login(self._username, self._password)
self.ftps.prot_p()
except ftplib.error_perm:
raise
def cwd(self):
"""
Try to switch the working directory on the FTP server (if set in the settings).
If the path does not exist, it will be created.
"""
if self._remote_path is not None:
try:
self.ftps.cwd(self._remote_path)
except ftplib.error_perm:
self.create_directory_tree(self._remote_path)
except IOError:
raise
def create_directory_tree(self, current_directory):
"""
Helper function to create the remote path.
@param current_directory: The current working directory.
@type current_directory: String
"""
if current_directory is not "":
try:
self.ftps.cwd(current_directory)
except ftplib.error_perm:
self.create_directory_tree("/".join(current_directory.split("/")[:-1]))
self.ftps.mkd(current_directory)
self.ftps.cwd(current_directory)
except IOError:
raise
def upload(self, callback=None):
"""
The upload function.
@param callback: The callback function for the upload progress.
@type callback: Function
"""
self._uploadCallback = callback
zipfile_to_upload = open(self._absolute_zipfile_path, 'rb')
zipfile_basename = os.path.basename(self._absolute_zipfile_path)
self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload, 1024, self.handle_upload_state)
zipfile_to_upload.close()
def handle_upload_state(self, block):
"""
The callback function for the upload progress.
@param block: The StringIO of the current upload state
@type block: StringIO
"""
self._bytesWritten += 1024
progress = math.floor((float(self._bytesWritten) / float(self._totalFileSize)) * 100)
do_update = False
if progress > self._currentProgress:
do_update = True
self._currentProgress = progress
if self._uploadCallback is not None:
self._uploadCallback(progress, self, do_update)
def quit(self):
"""
Try to quit everything and close the session.
"""
self.ftps.quit()
@property
def upload_path(self):
"""
Returns the upload path of the FTP server.
@return: The path of the uploaded file on the FTP server.
@rtype: String
"""
tmp_remote_path = ''
if self._remote_path is not None:
tmp_remote_path = self._remote_path
return os.path.join(tmp_remote_path, os.path.basename(self._absolute_zipfile_path))
class FTPSession(object):
""" Attempt to create some robustness and performance to FTPing """
def __init__(self, server, username, password, tmpdir="/tmp", timeout=60):
"""Build a FTP session """
self.conn = None
self.server = server
self.username = username
self.password = password
self.tmpdir = tmpdir
self.timeout = timeout
def _connect(self):
"""Connect to FTP server """
if self.conn is not None:
return
logging.debug("Creating new connection to server %s", self.server)
not_connected = True
attempt = 1
while not_connected and attempt < 6:
try:
self.conn = FTP_TLS(self.server, timeout=self.timeout)
self.conn.login(self.username, self.password)
self.conn.prot_p()
not_connected = False
except Exception as exp:
logging.debug(exp)
time.sleep(5)
self.close()
attempt += 1
if not_connected is True:
raise Exception("Failed to make FTP connection after 5 tries!")
def _reconnect(self):
""" First attempt to shut down connection and then reconnect """
logging.debug("_reconnect() was called...")
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
self._connect()
def _put(self, path, localfn, remotefn):
""" """
self.chdir(path)
sz = os.path.getsize(localfn)
if sz > 14000000000:
# Step 1 Split this big file into 14GB chunks, each file will have
# suffix .aa then .ab then .ac etc
basefn = os.path.basename(localfn)
cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir, basefn)
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
files = glob.glob("%s/%s.??" % (self.tmpdir, basefn))
for filename in files:
suffix = filename.split(".")[-1]
self.conn.storbinary("STOR %s.%s" % (remotefn, suffix), open(filename))
os.unlink(filename)
else:
logging.debug("_put '%s' to '%s'", localfn, remotefn)
self.conn.storbinary("STOR %s" % (remotefn,), open(localfn))
return True
def close(self):
""" Good bye """
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
def chdir(self, path):
if self.pwd() == path.rstrip("/"):
return
self.conn.cwd("/")
for dirname in path.split("/"):
if dirname == "":
continue
bah = []
self.conn.retrlines("NLST", bah.append)
if dirname not in bah:
logging.debug("Creating directory '%s'", dirname)
self.conn.mkd(dirname)
logging.debug("Changing to directory '%s'", dirname)
self.conn.cwd(dirname)
def pwd(self):
""" Low friction function to get connectivity """
self._connect()
pwd = exponential_backoff(self.conn.pwd)
if pwd is None:
self._reconnect()
pwd = exponential_backoff(self.conn.pwd)
logging.debug("pwd() is currently '%s'", pwd)
return pwd
def put_file(self, path, localfn, remotefn):
""" Put the File """
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
self._reconnect()
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
logging.error("Double Failure to upload filename: '%s'", localfn)
def put_files(self, path, localfns, remotefns):
""" Put the File """
for localfn, remotefn in zip(localfns, remotefns):
self.put_file(path, localfn, remotefn)
class FTPClient(object):
"""Class FTPClient
"""
_mh = None
_client = None
_secured = None
_host = None
_port = None
_user = None
_passw = None
_path = None
_verbose = None
_is_connected = None
def __init__(self, secured=False, verbose=False):
"""Class constructor
Called when the object is initialized
Args:
secured (bool): secured FTP
verbose (bool): verbose mode
"""
self._mh = MasterHead.get_head()
self._secured = secured
if (not self._secured):
self._client = FTP()
else:
if (not(version_info[0] == 2 and version_info[1] == 6)):
self._client = FTP_TLS()
else:
raise NotImplementedError(
'Secured mode is not supported for Python 2.6')
self._verbose = verbose
if (self._verbose):
self._client.set_debuglevel(2)
@property
def client(self):
""" FTP client property getter """
return self._client
@property
def secured(self):
""" secured protocol mode property getter """
return self._secured
@property
def host(self):
""" server host property getter """
return self._host
@property
def port(self):
""" server port property getter """
return self._port
@property
def user(self):
""" username property getter """
return self._user
@property
def passw(self):
""" user password property getter """
return self._passw
@property
def path(self):
""" remote path property getter """
return self._path
@property
def verbose(self):
""" verbose mode property getter """
return self._verbose
@property
def is_connected(self):
""" is_connected property getter """
return self._is_connected
def connect(self, host, port=21, user=None, passw=None, path='/', timeout=10):
"""Method connects to server
Args:
host (str): server host
port (int): server port, default protocol port
user (str): username
passw (str): password
path (str): server path
timeout (int): timeout
Returns:
bool: result
Raises:
event: ftp_before_connect
event: ftp_after_connect
"""
try:
message = '{0}/{1}@{2}:{3}{4} timeout:{5}'.format(
user, passw, host, port, path, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connecting', message), self._mh.fromhere())
ev = event.Event(
'ftp_before_connect', host, port, user, passw, path, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
path = ev.argv(4)
timeout = ev.argv(5)
self._host = host
self._port = port
self._user = user
self._passw = passw
if (ev.will_run_default()):
self._client.connect(self._host, self._port, timeout=timeout)
if (self._user != None):
self._client.login(self._user, self._passw)
if (self._secured):
self._client.prot_p()
self._is_connected = True
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connected'), self._mh.fromhere())
if (path != None):
self.change_dir(path)
ev = event.Event('ftp_after_connect')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def disconnect(self):
"""Method disconnects from server
Args:
none
Returns:
bool: result
"""
try:
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
else:
self._client.quit()
self._is_connected = False
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_disconnected'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def list_dir(self):
"""Method lists remote working directory
Args:
none
Returns:
list: names
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_list_dir', self._path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
names = self._client.nlst()
if ('.' in names):
names.remove('.')
if ('..' in names):
names.remove('..')
return names
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return None
def change_dir(self, path):
"""Method changes remote working directory
Args:
path (str): new remote path
Returns:
bool: result
Raises:
event: ftp_before_change_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_change_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_change_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.cwd(path)
self._path = self._client.pwd()
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_cur_dir', self._path), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def download_file(self, remote_path, local_path=None):
"""Method downloads file from server
Args:
remote_path (str): remote path
local_path (str): local path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_download_file
event: ftp_after_download_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_downloading_file', remote_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event(
'ftp_before_download_file', remote_path, local_path)
if (self._mh.fire_event(ev) > 0):
remote_path = ev.argv(0)
local_path = ev.argv(1)
if (local_path != None and not path.exists(local_path)):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_dir', local_path), self._mh.fromhere())
return False
filename = remote_path.split('/')[-1]
lpath = filename if (local_path == None) else path.join(
local_path, filename)
if (ev.will_run_default()):
with open(lpath, 'wb') as f:
self._client.retrbinary('RETR ' + remote_path, f.write)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_downloaded'), self._mh.fromhere())
ev = event.Event('ftp_after_download_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
if (path.exists(lpath)):
remove(lpath)
return False
def upload_file(self, local_path, remote_path=None):
"""Method uploads file to server
Args:
local_path (str): local path
remote_path (str): remote path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_upload_file
event: ftp_after_upload_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_uploading_file', local_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_upload_file', local_path, remote_path)
if (self._mh.fire_event(ev) > 0):
local_path = ev.argv(0)
remote_path = ev.argv(1)
if (not(path.exists(local_path) or path.exists(path.relpath(local_path)))):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_file', local_path), self._mh.fromhere())
return False
filename = local_path.split('/')[-1]
rpath = filename if (remote_path == None) else path.join(
remote_path, filename)
if (ev.will_run_default()):
with open(local_path, 'rb') as f:
self._client.storbinary('STOR ' + rpath, f)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_uploaded'), self._mh.fromhere())
ev = event.Event('ftp_after_upload_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def delete_file(self, path):
"""Method deletes file from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_delete_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_deleting_file', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_delete_file', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.delete(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_deleted'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def make_dir(self, path):
"""Method makes directory on server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_make_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_making_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_make_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.mkd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_made'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def remove_dir(self, path):
"""Method removes directory from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_remove_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_removing_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_remove_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.rmd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_removed'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
data['bot_id'] = BOT_ID
def group_listing():
# Get list of groups user is a member of
r = requests.get('%s/groups?token=%s' % (GROUPME_API_URL, TOKEN))
resp = r.json()
print 'Groups:'
for item in resp['response']:
print item['name']
# Securely connect to FTP server
ftps = FTP_TLS(HOST, USER, PASSWD)
ftps.prot_p()
# Change working directory to directory containing images
ftps.cwd(IMAGE_DIRECTORY)
# Get list of items in current directory
directory_list = ftps.nlst()
# Get list of images
image_list = [item for item in directory_list if '.jpg' in item]
# Save oldest & newest images
images_to_upload = []
if image_list:
# Add first image
images_to_upload.append(image_list[0])
if len(image_list) > 1:
# Add last image (if more than 1 image)
images_to_upload.append(image_list[len(image_list)-1])
# Download oldest & newest image
for image in images_to_upload:
print 'Downloading %s...' % image
#See http://stackoverflow.com/questions/10207628/python-module-ftplib-ftp-tls-error-530
#Try TLS Lite or M2Crypto both are FTP/TLS client and server.
#ftps.login(user,secret)
ftps.sendcmd('USER ' + user)
ftps.sendcmd('PASS ' + secret)
print(ftps.getwelcome())
print('CURRENT WORKING DIRECTORY IS:',ftps.pwd())
#Enable data encryption
# TODO solve the encryption problem
#ftps.prot_p()
#define default DIR
d = 'feeds'
#Change to default DIR
ftps.cwd(d)
#Build list of files on servers
l = ftps.nlst()
l.sort()
for i in l:
print(i)
#Assign last element to var
litem = len(l)-1
print("MOST RECENT FILE ON SERVER IS; ",l[litem])
g = l[litem]
#Define local file
t = d + '/' + g
if os.path.exists(t):
print("FILE" ,g," EXISTS,WILL NOT DOWNLOAD FROM HOST:",host)
else:
class ServerWatcher(Watcher):
downloadProgress = Signal((int, int,))
uploadProgress = Signal((int, int,))
# Si added:
textStatus = Signal((str,))
fileEvent = Signal((str,))
fileEventCompleted = Signal()
loginCompleted = Signal((bool, str,))
badFilenameFound = Signal((str,))
LOCATION = 'server'
TEST_FILE = 'iqbox.test'
def __init__(self, host, ssl, parent=None):
"""
Initializes parent class and attributes. Decides
whether to use `FTP_TLS` or `FTP` based on the `ssl` param.
:param host: Location of the FTP server
:param ssl: Tells whether the FTP needs to support TLS or not
:param parent: Reference to a `QObject` instance a parent
"""
super(ServerWatcher, self).__init__(parent)
self.interval = 5000
self.localdir = ''
self.deleteQueue = []
self.downloadQueue = []
self.uploadQueue = []
self.warnedNames = []
self.ftp = None
self.useSSL = ssl
self.host = host
self.preemptiveCheck = False
self.preemptiveActions = []
self.testFile = 'iqbox.test'
@property
def currentdir(self):
"""Returns the current working directory at the server"""
return self.ftp.pwd()
def setLocalDir(self, localdir):
"""
Sets the local directory used to stored all
downloaded files. Creates the directory if needed.
:param localdir: Absolute path to local directory
"""
self.localdir = localdir
if not os.path.exists(self.localdir):
os.makedirs(self.localdir)
@pause_timer
@Slot()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... '+serverpath+'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
@Slot()
def onLogin(self, username, passwd):
ok = True
msg = ''
error_msg = 'Login failed.'
try:
if not self.ftp:
self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(self.host)
loginResponse = self.ftp.login(username, passwd)
except socket.gaierror:
self.ftp = None
ok = False
msg = 'Server address could not be found.'
except (error_perm, error_reply):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
ok = False
msg = error_msg
else:
if '230' in loginResponse:
ok = True
else:
ok = False
msg = error_msg
if ok:
# Logged in. Now let's do compability tests.
if not self.testPermissions():
# User doesn't have write permissions, don't bother doing next test.
ok = False
msg = 'It seems like you do not have write access to this server.'
else:
# Permissions test passed, now let's test MFMT for timestamp modification.
if not self.testMFMT():
ok = False
msg = 'This server does not support timestamp modification\n \
need by this application.'
self.loginCompleted.emit(ok, msg)
def getFiles(self, path):
"""
This method simply wraps the `nlst` method with an exception handler,
and returns an empty list in case an exception is caught.
:param path: Relative or absolute path on the server
"""
try:
nlst = self.ftp.nlst(path)
dirs = self.getDirs(path)
# Files are items in nlst that are not in dirs
files = [item for item in nlst if os.path.basename(item) not in dirs]
return files
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
def getDirs(self, path):
"""
Retrieves a list of the directories inside `path`,
uses `retrlines` and the LIST command to retrieve the items.
:param path: Relative or absolute path on the server
"""
dirs = list()
def handleLine(line):
"""
Recieves a line from the LIST command.
This function is meant to be used as callback for the `retrlines` method.
:params line: Line from the LIST command
"""
if line.startswith('d'):
# Only lines starting with 'd' are directories
# Parse the directory out of the line; lines look like:
# 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname'
dirname = line[55:].strip()
if dirname != '.' and dirname != '..':
# Ignoring '.' and '..' entries
dirs.append(dirname)
try:
self.ftp.retrlines('LIST %s' % path, handleLine)
return dirs
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
@upload_test
def testPermissions(self):
# For interface purposes. upload_test takes care of everything.
return True
@upload_test
def testMFMT(self):
# Absurd date to test whether the change really happened.
time = dt.utcfromtimestamp(100000000)
try:
self.setLastModified(self.testFile, time)
otherTime = self.lastModified(self.testFile)
diff = (time - otherTime).total_seconds()
if abs(diff) < 2:
# Let's give it a 2 seconds tolerance.
mdtm = True
else:
mdtm = False
except (ValueError, error_reply, error_perm):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
mdtm = False
return mdtm
@Slot(str)
def onDelete(self, filename):
self.deleteQueue.append(filename)
def deleteNext(self):
if len(self.deleteQueue) > 0:
next = self.deleteQueue.pop(0)
self.deleteFile(next)
def deleteAll(self):
for filename in self.deleteQueue:
self.deleteFile(filename)
self.deleteQueue = []
@Slot(str)
def deleteFile(self, filename):
"""
Deletes the file `filename` to the server
:param filename: Absolute or relative path to the file
"""
try:
print 'Deleting %s' % filename
self.ftp.delete(filename)
return True
except (error_reply, error_perm):
print 'Error deleting %s' % filename
return False
self.fileEventCompleted.emit()
@Slot(str)
def onDownload(self, filename):
self.downloadQueue.append(filename)
def downloadNext(self):
if len(self.downloadQueue) > 0:
next = self.downloadQueue.pop(0)
self.downloadFile(next)
def downloadAll(self):
for filename in self.downloadQueue:
self.downloadFile(filename)
self.downloadQueue = []
@Slot(str, str)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size, self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
@Slot(str)
def onUpload(self, filename):
self.uploadQueue.append(filename)
def uploadNext(self):
if len(self.uploadQueue) > 0:
next = self.uploadQueue.pop(0)
self.uploadFile(next)
def uploadAll(self):
for filename in self.uploadQueue:
self.uploadFile(filename)
self.uploadQueue = []
@Slot(str)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename,
open(localpath, 'rb'),
1024,
handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def lastModified(self, filename):
"""
Uses the MDTM FTP command to find the last modified timestamp
of the file `filename`.
Returns a `datetime.datetime` object in UTC representing the file's
last modified date and time.
:param filename: Relative or absolute path to the file
"""
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
if '213 ' not in timestamp:
# Second chance was found to be needed in some cases.
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
timestamp = timestamp.split(' ')[-1]
dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S'
try:
mtime = dt.strptime(timestamp, dateformat)
except ValueError:
mtime = dt.utcnow()
return mtime
def setLastModified(self, serverpath, newtime):
"""
Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the
file `serverpath` on the server.
:param serverpath: Relative or absolute path to the file
:param newtime: datedatime object holding the required time
"""
cmds = ['MFMT', 'MDTM']
for cmd in cmds:
try:
self.ftp.sendcmd(
'%s %s %s' % (cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath))
return
except (error_perm, error_reply) as e:
if cmd == cmds[len(cmds) - 1]:
# If is the last comand, re-raise the exception, else
# keep trying.
raise e
else:
continue
def mkpath(self, path):
"""
Creates the path `path` on the server by recursively
created folders, if needed.
:param path: Absolute path on the server to be created
"""
try:
self.ftp.cwd(path)
except error_perm:
# `cwd` call failed. Need to create some folders
make_dir = '/'
steps = path.split('/')
for step in steps:
if len(step) == 0:
continue
make_dir += '%s/' % step
try:
self.ftp.mkd(make_dir)
except error_perm:
# Probably already exists
continue
else:
# `cwd` call succeed. No need to create
# any folders
self.ftp.cwd('/')
return
@Slot(str, str)
def added(self, location, serverpath):
super(ServerWatcher, self).added(location, serverpath)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION)
return action
if self.preemptiveCheck:
if location == ServerWatcher.LOCATION:
localpath = self.localFromServer(serverpath)
if not os.path.exists(localpath):
action = FileAction(serverpath, FileAction.DOWNLOAD, ServerWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
elif location == LocalWatcher.LOCATION:
try:
self.ftp.sendcmd('SIZE %s' % serverpath)
except (error_reply, error_perm):
exists = False
else:
exists = True
if not exists:
action = FileAction(serverpath, FileAction.UPLOAD, LocalWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
@Slot(str, str)
def changed(self, location, serverpath):
super(ServerWatcher, self).changed(location, serverpath)
@Slot(str, str)
def deleted(self, location, serverpath):
super(ServerWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inserver = False
