def sendUpdateFilesFtp(): print "---- Send update files by FTP" global serverFtp global userFtp global passFtp from ftplib import FTP_TLS ftps = FTP_TLS(serverFtp) ftps.set_debuglevel(1) ftps.login(userFtp, passFtp) ftps.prot_p() ftps.cwd('/files/updates') ftps.storbinary('STOR ' + 'file_list.md5', open(quiterssFileRepoPath + '\\file_list.md5', 'rb')) ftps.storbinary('STOR ' + 'VersionNo.h', open(quiterssSourcePath + '\\src\\VersionNo.h', 'rb')) ftps.storbinary('STOR ' + 'HISTORY_EN', open(quiterssSourcePath + '\\HISTORY_EN', 'rb')) ftps.storbinary('STOR ' + 'HISTORY_RU', open(quiterssSourcePath + '\\HISTORY_RU', 'rb')) prepareFileList7z = [] for file in prepareFileList: prepareFileList7z.append(file + '.7z') for fileName in prepareFileList7z: ftps.storbinary('STOR ' + 'windows' + fileName.replace('\\','/'), open(quiterssFileRepoPath + '\\windows' + fileName, 'rb')) ftps.quit()
class ftpAccess(): def __init__(self, url): self.ftp = FTP_TLS(url) self.ftp.login( "anonymous", "anonymous", secure=False) # login anonymously before securing control channel def GetFtpFile(self, InPath, filename, downloaddir): outfile = downloaddir + filename ensure_dir(downloaddir) self.ftp.cwd(InPath) self.ftp.retrbinary("RETR " + filename, open(outfile, 'wb').write) def GetFileListFtp(self, pathname): self.ftp.cwd(pathname) ret = list() out = self.ftp.retrlines('LIST', addline) # list directory content securely # search json files temp = entry.split(' ') for filename in filenamelist: try: a = self.ftp.size(filename) print("{} - {}".format(filename, a)) ret.append([filename, a]) except: print("{} - xxx".format(filename)) pass return ret
def conexionFTP(): print "conexionFTP" global reIntento, rutaFtp,tiempoErr,hostFtp,passFtp,userFtp,portFtp,infoERR #request = urllib.request.Request('http://'+hostFtp) #response = urllib.request.urlopen(request) estatusCftp=False if response.getcode() == 200: try: ftp = FTP_TLS() ftp.connect(hostFtp,portFtp) ftp.login(userFtp, passFtp) ftp.cwd(rutaFtp) estatusCftp = True except Exception as e: if infoERR == True:logger.warning(formErro) logger.warning('Error al conectar al servidor ftp '+str(e)) else: logger.warning('Error al conectar al servidor Error de internet '+str(e)) for i in range(reIntento): if conexionFTP()['ftp']:main() else:logger.warning("reconecion internet intento "+str(i));slp(tiempoErr)# return {'ftp':ftp,'estatusCftp':estatusCftp}
def main(): global ftp_client scan_arguments() ftp_client = FTP(host) try: ftp_client.login(username, password) except ftplib.all_errors as e: print "ERROR: cannot login with username '{0}' and relative password.\nMessage returned from server:".format(username) print e return try: ftp_client.cwd(remote_dir) except ftplib.all_errors as e: print "ERROR: emote directory '{0}' not existing.\nMessage returned from server:".format(remote_dir) print e return else: files = ftp_client.nlst() print_directory_content(files) setup_folder() download_files(remote_dir, files) if compress: create_zip(local_dir) try: ftp_client.close() print "!!!!! OPERATION COMPLETED SUCCESSFULLY !!!!!" except ftplib.all_errors as e: print "ERROR: cannot close the connection properly.\nMessage from server:" print e
def DeleteTextFile(): filename = 'test.txt' session = FTP_TLS('files.000webhost.com', 'vulcuns-webserver', 'AKPJJDT2lol') session.cwd('public_html') session.delete(filename) session.quit()
def connect(self): #初始化 FTP 链接 if self.ftp_ssl: ftp = FTPS() else: ftp = FTP() print('-'*20+self.ftp_name+'-'*20) print('connect '+('ftps' if self.ftp_ssl else 'ftp')+'://'+self.ftp_host+':'+self.ftp_port) try: ftp.connect(self.ftp_host,self.ftp_port) except Exception as e: print (e) print ('connect ftp server failed') sys.exit() try: ftp.login(self.ftp_user,self.ftp_passwd) print ('login ok') except Exception as e:#可能服务器不支持ssl,或者用户名密码不正确 print (e) print ('Username or password are not correct') sys.exit() if self.ftp_ssl: try: ftp.prot_p() except Exception as e: print (e) print ('Make sure the SSL is on ;') print(ftp.getwelcome()) ftp.cwd(self.ftp_webroot) print('current path: '+ftp.pwd()) self.ftp=ftp
def connect(box_path): ftp = FTP_TLS('ftp.box.com') # ftp.debugging = 2 ftp.login('*****@*****.**', 'cJ19870915@#') ftp.cwd(box_path[0]) print("Connected to Box FTP") return ftp
def copy_ftp_file(project_id, client_name, file_name): # """ ftp file copy """ ftp = FTP_TLS(settings.FTP_LOCATION) ftp.sendcmd("USER {}".format(settings.FTP_USER)) ftp.sendcmd("PASS {}".format(settings.FTP_PASS)) comp_logger.info( 'Initiating ftp file transfer for file {} for client {}'.format( file_name, client_name)) ftp.cwd(client_name) # create project input dir project_dir = os.path.join(settings.PROJECT_DIR, str(project_id)) helper.create_dir(project_dir) # copy remote ftp file to local project folder file_format = file_name.split('.')[-1] local_filename = os.path.join(settings.PROJECT_INPUT_FOLDER, '{}.{}'.format(project_id, file_format)) if os.path.exists(local_filename): os.remove(local_filename) # project_input_dir = os.path.join(project_dir, settings.PROJECT_INPUT_FOLDER) # helper.create_dir(project_input_dir) # local_filename = os.path.join(project_input_dir, file_name) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + file_name, lf.write, 8 * 1024) lf.close() comp_logger.info('Completed Copying file {} for client {}'.format( file_name, client_name))
def f_sending_order(): global l_selected l = l_selected + [ent_cast.get(), txt_article.get(1.0, END), f_time_now()] with open('client_app_order.pkl', 'wb') as f: pickle.dump(l, f) print('start sending......') try: ftps = FTP_TLS(helpic.place_ftp, helpic.user_ftp, helpic.password_ftp) ftps.cwd('/www/{}'.format(helpic.directory_ftp)) except: showinfo( 'Connection', 'You don\'t have internet conection\n\ or login/pasword were changed') return None sufix = random.randint(1, 1000000) file_name = 'order{}.pkl'.format(sufix) ftps.storbinary('STOR ' + file_name, open('client_app_order.pkl', 'rb')) # загрузка файла НА сервер print('finish sending') ftps.quit() f_get_items_table( ) # getting information from FTP-server and writing in file
def connect_ftps(): #Connect to the server print("Step 1") if SECURE_CONN == '1': ftps = FTP_TLS(SERVER) print("Secure connection") else: ftps = FTP(SERVER) print("Unsecure connection") print("Step 2") ftps.set_debuglevel(2) print("Step 3") ftps.set_pasv(False) print("Step 4") ftps.connect(port=int(PORT), timeout=160) print("Step 5") ftps.login(USER, PASS) print("Step 6") ftps.cwd(TARGET_DIR) print("Step 7") ftps.set_pasv(True) print("Step 8") #Set up the secure connection only in FTPS mode if SECURE_CONN == '1': ftps.prot_p() else: pass #ftps.ccc() return ftps
def sendPackagesFtp(): print "---- Send packages by FTP" global serverFtp global userFtp global passFtp from ftplib import FTP_TLS ftps = FTP_TLS(serverFtp) ftps.set_debuglevel(1) ftps.login(userFtp, passFtp) ftps.prot_p() try: ftps.sendcmd('MKD ' + '/files/' + strProductVer) except Exception: print 'Directory already exists' ftps.cwd('/files/' + strProductVer) filesListFtp = ftps.nlst() filesList = os.listdir(packagesPath) newFilesList = [e for e in filesList if not(e in filesListFtp)] for fileName in newFilesList: ftps.storbinary('STOR ' + fileName, open(packagesPath + '\\' + fileName, 'rb')) ftps.quit()
def download_all_in_one_path(targetdir,resultdir,check = True,num = 50): if(os.path.exists(resultdir) == False): os.makedirs(resultdir) ftp = FTP('129.164.179.23') ftp.login() ftp.prot_p() ftp.cwd(targetdir) files = ftp.nlst() target = 'https://heasarc.gsfc.nasa.gov/FTP' + targetdir c = None if(check): c = [] data1 = [] ftp.voidcmd('TYPE I') print('正在获取校验信息........') for i in files: #print(i) data = os.path.join(target,i) print(data) data1.append(data) if(check): c.append(ftp.size(i)) ftp.quit() if(check == False): print('忽略数据大小校验。') print('正在校验...............') down(data1,resultdir,check=c,threadnum = num) print('\n任务下载完成!!!')
def connexionftp(adresseftp, nom, mdpasse, passif): """connexion au serveur ftp et ouverture de la session - adresseftp: adresse du serveur ftp - nom: nom de l'utilisateur enregistré ('anonymous' par défaut) - mdpasse: mot de passe de l'utilisateur ('anonymous@' par défaut) - passif: active ou désactive le mode passif (True par défaut) retourne la variable 'ftplib.FTP' après connexion et ouverture de session """ try: verbose('Attente connexion FTP .....') if modeSSL: ftp = FTP_TLS() ftp.connect(adresseftp, 21) ftp.login(nom, mdpasse) ftp.prot_p() ftp.set_pasv(passif) else: ftp = (ftplib.FTP(adresseftp, nom, mdpasse)) ftp.cwd(destination) verbose ('Destination : '+destination) verbose('Connexion FTP OK') etat = ftp.getwelcome() verbose("Etat : "+ etat) return ftp except: verbose('Connexion FTP impossible', True) suppressionDossierTemp(dossierTemporaireFTP) sys.exit()
def download_npc(): ftp_files = [] os_files = [] try: ftps = FTP_TLS() ftps.connect(CFG_FTPS_HOST, CFG_FTPS_PORT) log_it( 'connected to ' + CFG_FTPS_HOST + ' welcome message: ' + str(ftps.getwelcome()), 'info') ftps.login(CFG_FTPS_USER, CFG_FTPS_PASS) ftps.prot_p() log_it('changing dir: ' + CFG_FTPS_DIR, 'info') ftps.cwd(CFG_FTPS_DIR) ftp_files = ftps.nlst() for f in ftp_files: if not os.path.isfile(CFG_ARCHIVE_DIR + f): ftps.retrbinary('RETR ' + f, open(CFG_ARCHIVE_DIR + f, 'wb').write) log_it('downloading file ' + f, 'info') else: log_it( 'skipping ' + f + ' as it already exists in ' + CFG_ARCHIVE_DIR, 'debug') except ftplib.all_errors, e: log_it('unable to connect to ' + CFG_FTPS_HOST + ' %s' % e, 'error')
def find_versions_above( mirror: Dict[str, str], target_version: Version ) -> Generator[Mapping[str, Union[str, Version]], None, None]: print("Using mirror:", mirror['host']) tls = get_use_tls() if tls: ftp = FTP_TLS(mirror['host']) else: ftp = FTP(mirror['host']) ftp.login(user=mirror.get('user', None), passwd=mirror.get('passwd', ''), acct=mirror.get('acct', '')) ftp.cwd(mirror['path']) for filename, attrs in sorted(ftp.mlsd(), key=lambda tup: tup[0]): if not attrs["type"] == "file": continue if not filename.endswith(".tar.gz"): continue pkgver = get_package_version(filename) if not pkgver: continue if pkgver > target_version: fileinfo = mirror.copy() fileinfo["filename"] = filename fileinfo["version"] = pkgver fileinfo["tls"] = tls yield fileinfo
def _get_ftp(self): try: username, password = self._get_auth() if self._tls: ftp = FTP_TLS(self._uri, timeout=CONNECT_TIMEOUT) else: ftp = FTP(self._uri, timeout=CONNECT_TIMEOUT) ftp.login(username, password) if self._tls: ftp.prot_d() if not catch(ftp.cwd, self._path): if self._path.startswith("/"): ftp.cwd("/") components = self._path.split("/") for component in components: if not catch(ftp.cwd, component): ftp.mkd(component) ftp.cwd(component) return ftp except Exception: log.exception() log.warning( "[FTP] failed to establish server connection, disabled") self._disabled = True return None
def sync(self): """ downloads all needed_files from self.hostname (FTP) of the downloaded files, extracts .gz files to same local_working_dir -using self.extract function parses the .txt downloaded needed_files -using the self.parse function """ ftps = FTP_TLS(self.hostname) # connect to host, default port ftps.login(self.username, self.password) ftps.prot_p() ftps.cwd(self.remote_dir) # change into "logs" directory ftps.retrlines('LIST *.gz *.txt', self.ftp_list_callback) # list directory contents for needed_file in self.needed_files: if self.logging: print "Writing {0} to {1}...".format(needed_file, self.local_working_dir) ftps.retrbinary("RETR " + needed_file, open(os.path.join(self.local_working_dir, needed_file), 'wb').write) if self.logging: print "done syncing files" for needed_file in self.needed_files: if needed_file.endswith(".gz"): self.extract(os.path.join(self.local_working_dir, needed_file)) txt_file_name = needed_file.replace('.gz','')#if already a .txt file, this is unnceccessary but works. self.parse(txt_file_name) if self.logging: print "done extracting/parsing .gz files" ftps.quit()
def _connect(server: str): global _connection_cache if server not in _connection_cache: if server == 'in_server': logger.debug(f'Connection to "in_server" requested') conn_specs = config().ftp.in_server elif server == 'out_server': logger.debug(f'Connection to "out_server" requested') conn_specs = config().ftp.out_server else: raise ValueError(f'Unknown server "{server}"') logger.debug(f'conn specs = {conn_specs}') if conn_specs.secure: ftp = FTP_TLS(conn_specs.host) ftp.auth() else: ftp = FTP(conn_specs.host) # ftp.connect(conn_specs.host, conn_specs.port) logger.debug(f'Connected to "{server}". Logging in...') ftp.login(conn_specs.username, conn_specs.password) logger.debug(f'Established connection to "{server}". Saving to cache') _connection_cache[server] = ftp return ftp else: ftp = _connection_cache[server] ftp.cwd('/') logger.debug(f'Recycling connection to "{server}"') return ftp
def send_to_ftp(filepath): ftps = FTP_TLS('ftpes.learn.inf.puc-rio.br') # Define Ftp server ftps.login('learn', 'LRepoAdm18!!') # Login to ftp server ftps.prot_p() # Enable data encryption ftps.retrlines('LIST') # List Directory ftps.cwd('/projeto/01_tarefas_de_geofisica/QualiSismo/CV_results') # Change Directory ftps.storlines("STOR " + filepath, open(filepath,'rb'))
def UpdateLocalData(Force=False): ''' This will download and convert any OMNI data which is missing from the local archive. ''' ftp = FTP_TLS(Globals.ftpbase) ftp.login() ftp.cwd(Globals.ftpdir) #let's download and read the FTP index status = _DownloadFTPIndex(ftp) if not status: print('Download failed; check for write permission to data folder') ftp.close() return ftp.close() FileNames,Addresses,UpdateDates,Res = _ParseFTP() n = np.size(FileNames) #check current data index idx = _ReadDataIndex() #now compare update dates if Force: update = np.ones(n,dtype='bool') else: update = _CompareUpdates(UpdateDates,FileNames,idx) use = np.where(update)[0] FileNames = FileNames[use] Addresses = Addresses[use] UpdateDates = UpdateDates[use] Res = Res[use] n = use.size if n == 0: print('No files to update.') ftp.close() return for i in range(0,n): print('Downloading file {0} of {1}'.format(i+1,n)) #download file tmp = _DownloadFTPFile(FileNames[i]) print('Converting to binary') #convert file _ConvertFTPFile(tmp,FileNames[i],UpdateDates[i],Res[i]) #delete text file _DeleteFTPFile(tmp) ftp.close() print('Done')
def ftp_to_ml(localfile, remote_file): (user, acct, passwd) = netrc.netrc().authenticators('ftps.b2b.ml.com') ftp = FTP_TLS('ftps.b2b.ml.com') remote_dir = 'incoming/gmigefs2' ftp.login(user, passwd, acct) ftp.prot_p() ftp.cwd(remote_dir) local_fd = open(localfile, 'r') ftp.storlines('STOR ' + remote_file, local_fd) local_fd.close()
def send(self): ftp = FTP(self.ftp_destination) ftp.set_debuglevel(2) ftp.ssl_version = ssl.PROTOCOL_TLSv1_2 ftp.login() ftp.cwd("upload") ftp.prot_p() self.zip.close() self.zip_contents.seek(0) ftp.storbinary("STOR {}".format(self.zip_contents.name), self.zip_contents)
def fetch_data_via_ftp(ftp_config, local_directory): """ Get benchmarking data from a remote ftp server. :type ftp_config: config.FTPConfigurationRepresentation :type local_directory: str """ if ftp_config.enabled: # Create local directory tree if it does not exist create_directory_tree(local_directory) # Login to FTP server if ftp_config.use_tls: ftp = FTP_TLS(ftp_config.server) ftp.login(ftp_config.username, ftp_config.password) ftp.prot_p() # Request secure data connection for file retrieval else: ftp = FTP(ftp_config.server) ftp.login(ftp_config.username, ftp_config.password) if not ftp_config.files: # Auto-download all files in directory fetch_data_via_ftp_recursive(ftp=ftp, local_directory=local_directory, remote_directory=ftp_config.directory) else: ftp.cwd(ftp_config.directory) file_counter = 1 file_list_total = len(ftp_config.files) for remote_filename in ftp_config.files: local_filename = remote_filename filepath = os.path.join(local_directory, local_filename) if not os.path.exists(filepath): with open(filepath, "wb") as local_file: try: ftp.retrbinary('RETR %s' % remote_filename, local_file.write) print("[Setup][FTP] ({}/{}) File downloaded: {}". format(file_counter, file_list_total, filepath)) except error_perm: # Error downloading file. Display error message and delete local file print( "[Setup][FTP] ({}/{}) Error downloading file. Skipping: {}" .format(file_counter, file_list_total, filepath)) local_file.close() os.remove(filepath) else: print( "[Setup][FTP] ({}/{}) File already exists. Skipping: {}" .format(file_counter, file_list_total, filepath)) file_counter = file_counter + 1 # Close FTP connection ftp.close()
def connect_to_ftp(self, options): host = options['ftp_server'] path = options['path'] user = options['user'] password = options['password'] ftp = FTP_TLS(host=host) ftp.login(user=user, passwd=password) ftp.cwd(path) return ftp
def get_latest_file(): with open('filenames.json') as json_file: files = json.load(json_file) flag = 0 ftp = FTP_TLS(host='olftp.adesa.com', user='******', passwd='aU)kj7Qn8') ftp.prot_p() #ftp.retrlines('LIST') ftp.cwd('outbound/') file_list = [] ftp.retrlines('MLSD', file_list.append) #ftp.dir(file_list.append) max_value = 0 full_max = 0 filename = '' full_file = '' for i in file_list: col = i.split(';') col_max = int(re.search(r'\d+', col[0]).group()) if (col_max > max_value) & ('.txt' in col[-1]): max_value = col_max filename = col[-1].replace(' ', '') if (col_max > full_max) & ('.txt' in col[-1]) & ('FULL' in col[-1]): full_max = col_max full_file = col[-1].replace(' ', '') if (filename != files['inc_file']): localfile = open(filename, 'wb') ftp.retrbinary('RETR ' + filename, localfile.write, 1024) localfile.close() print("Inc file data tranfer complete") flag = 1 else: print("Inc already there") if (full_file != files['full_file']): localfile = open(full_file, 'wb') ftp.retrbinary('RETR ' + full_file, localfile.write, 1024) localfile.close() print("Full file data tranfer complete") flag = 1 else: print("Full already there") if flag == 1: new_names = {'full_file': full_file, 'inc_file': filename} with open('filenames.json', 'w') as outfile: json.dump(new_names, outfile) ftp.quit() return filename, full_file, flag
def cliente_ftp_conexion(servidor, nombre_usuario, password): #abrimos la conexion ftps = FTP_TLS(servidor) ftps.login(nombre_usuario, password) #ftp = ftplib.FTP(servidor, nombre_usuario, password) #listamos los archivos de / ftps.cwd("/") print "Archivos disponibles en %s:" % servidor archivos = ftps.dir() print archivos ftps.quit()
def connect2box(self): """ Connects to the box sync server """ ftp = FTP_TLS('ftp.box.com') print('logging in') ftp.login(user=self.user, passwd=getpass.getpass()) # move to destination directory ftp.cwd(self.boxFolder) print('FTP: moving to folder ' + self.boxFolder + ' \n') self.ftp = ftp
def testFTPS(hostname, username, password, port, path, passive): try: ftp = FTP_TLS() ftp.connect(hostname, port) ftp.login(username, password) ftp.prot_p() ftp.set_pasv(passive) ftp.cwd(path) files = ftp.dir() ftp.quit() return True except BaseException as error: return error
def fetch_temporary_gz_file(self, temporary_gz_file_path): with open(temporary_gz_file_path, 'wb') as file_ftp: file_name = ntpath.basename(temporary_gz_file_path) try: ftp = FTP_TLS(self.ftp_host, self.ftp_username, self.ftp_password) ftp.cwd(self.ftp_path) ftp.retrbinary('RETR ' + file_name, file_ftp.write) ftp.quit() except: Logger.file_parse(self.chain, file_name) self.fetch_temporary_gz_file(temporary_gz_file_path)
def get_jobs_data(): ftp = FTP_TLS('ontario.files.com', timeout=10) ftp.login(user=os.environ['211_username'], passwd=os.environ['211_password']) ftp.cwd('/211projects/BensTeam') ftp.prot_p() files = ftp.nlst() for filename in files: if not os.path.isfile('211_data/' + filename): print(f"Getting file {filename}") ftp.retrbinary("RETR " + filename, open('211_data/' + filename, 'wb').write) ftp.quit() return 'Done'
def uploadToFtp(filename): host = "88.122.218.102" # adresse du serveur FTP user = "******" # votre identifiant password = "******" # votre mot de passe port = 49153 ftps = FTP_TLS() ftps.connect(host, port) ftps.login(user, password) ftps.encoding = "utf-8" ftps.cwd("Disque dur") ftps.cwd("static") with open('/static/'+filename, "rb") as file: ftps.storbinary(f"STOR {filename}", file) ftps.quit()
def _upload_qr_code_ftp(code_filename): """ TODO: 1. Remove DRY code around code_fullpath """ code_fullpath = _get_environ_var( 'GENERATED_CODES_DIRECTORY') + code_filename with open(code_fullpath, 'rb') as fp: ftps = FTP_TLS(_get_environ_var('FTP_HOST')) ftps.login(_get_environ_var('FTP_USERNAME'), _get_environ_var('FTP_PASSWORD')) ftps.prot_p() ftps.cwd(_get_environ_var('FTP_DIRECTORY')) ftps.storbinary('STOR ' + code_filename, fp) ftps.quit()
def linkFtp(folder=''): """ Handlser FTP per il caricamento dei dati su server remoto :param folder: sotto cartella di salvataggio remota :return: handler per la gestione """ # ftp = FTP_TLS() ftp = FTP_TLS(host=rl['host']) ftp.set_debuglevel(2) status = ftp.login(user=rl['username'], passwd=rl['password']) # comando per il cambio della directory di upload ftp.cwd(backup_folder+'/'+folder) # print the content of directory print(ftp.dir()) return ftp, status
def upload_archive_file(self, local_filename, remote_filename, target_dir): yield("Uploading {} to FTP in directory: {}, filename: {}".format(local_filename, target_dir, remote_filename)) local_filesize = os.stat(local_filename).st_size self.upload_total = os.stat(local_filename).st_size self.upload_current = 0 if self.settings.ftps['no_certificate_check']: context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) context.verify_mode = ssl.CERT_NONE context.check_hostname = False else: context = ssl.create_default_context() ftps = FTP_TLS( host=self.settings.ftps['address'], user=self.settings.ftps['user'], passwd=self.settings.ftps['passwd'], context=context, source_address=self.settings.ftps[ 'source_address'], timeout=self.settings.timeout_timer ) ftps.cwd(target_dir) ftps.encoding = 'utf8' ftps.prot_p() for line in ftps.mlsd(facts=["size"]): if(line[0] == remote_filename and local_filesize == int(line[1]["size"])): yield("File exists and size is equal.") ftps.close() return with open(local_filename, "rb") as file: for retry_count in range(3): try: ftps.storbinary( 'STOR %s' % remote_filename, file, callback=lambda data, args=self.print_method: self.print_progress(data, args) ) except (ConnectionResetError, socket.timeout, TimeoutError): yield("Upload failed, retrying...") else: break yield("\nFile uploaded.") ftps.close()
def getfiles(server, port, user, password, db): sqliteconnection = sqlite3.connect(db) sqlitecursor = sqliteconnection.cursor() sqlitecursor.execute('''CREATE TABLE IF NOT EXISTS latest (date int, CONSTRAINT 'id_UNIQUE' UNIQUE ('date'))''') sqliteconnection.commit() sqlitecursor.execute('''SELECT date FROM files WHERE date = (SELECT MAX(date) FROM files) LIMIT 1''') latestfile = sqlitecursor.fetchone() sqlitecursor.execute('''SELECT date FROM latest WHERE date = (SELECT MAX(date) FROM latest) LIMIT 1''') latestfetch = sqlitecursor.fetchone() if latestfetch is None: latestfetch = 0 if latestfetch < latestfile: ftpsconnection = FTP_TLS() ftpsconnection.connect(server, port) ftpsconnection.auth() ftpsconnection.prot_p() ftpsconnection.login(user, password) ftpsconnection.prot_p() sqlitecursor.execute('''SELECT name FROM files WHERE date > %d''' % latestfetch) filestofetch = sqlitecursor.fetchall() for currfile in filestofetch: ftpsconnection.cwd(currfile[0]) filenames = ftpsconnection.nlst() for filename in filenames: print 'Now saving /mnt/folder' + currfile[0] + '/' + filename localfile = open('/mnt/folder' + currfile + '/' + filename, 'wb') ftpsconnection.retrbinary('RETR ' + filename, localfile.write) localfile.close() sqliteconnection.execute('''INSERT OR IGNORE INTO latest VALUES (%d)''' % time.time()) sqliteconnection.commit() sqliteconnection.close() ftpsconnection.quit() ftpsconnection.close()
def connect(velkost_ftp,port): ftp=FTP_TLS(server,meno2,ps,port) ftp.prot_p() ftp.cwd(my_list[2]) print "Posielam subor. Cakajte prosim." obsah=open(file_to_send, 'rb') obsah.close() ftp.storbinary('STOR %s' % file_to_send, open(file_to_send, 'rb')) obsah.close() print "Subor odoslany [OK]" print "Obsah adresara na serveri:" ftp.retrlines("LIST") size_ftp=ftp.nlst() pocet=len(size_ftp) velkost_ftp_subor=size_ftp[pocet-1] #berie posledne pridany subor zo zoznamu ftp.sendcmd("TYPE i") velkost_ftp=ftp.size(velkost_ftp_subor) ftp.close() return velkost_ftp
def download(downloaded, user, passwd, all_files=False, filename=None): # Connect to the MAPS ftp server over FTPS ftps = FTP_TLS('ftps.tsi.telecom-paristech.fr') print 'Connected to MAPS FTP over TLS.' try: ftps.login(user=user, passwd=passwd) ftps.cwd('maps') except error_perm: print "Incorrect username/password" ; quit ftps.retrlines('LIST *.zip', get_file_list) if filename is not None: if not in_downloads(files, filename): print 'File not found' ; return print 'Downloading', filename res = ftps.retrbinary('RETR '+filename, open('./downloads/'+filename, 'wb').write) ftps.close() return [(filename, 0)] if len(files) == len(downloaded): print "All MAPS files downloaded. Continuing." return if all_files: for f, s in files: if not in_downloads(downloaded, f): print "Downloading", f, "of size", s, "bytes" res = ftps.retrbinary('RETR '+f, open('./downloads/'+f, 'wb').write) elif filename is None: f, s = random.choice(files) while in_downloads(downloaded, f): f, s = random.choice(files) print "Downloading", f, "of size", s, "bytes" res = ftps.retrbinary('RETR '+f, open('./downloads/'+f, 'wb').write) ftps.close() if all_files: return files return [(f, s)]
class FTPES: def __init__(self, host, port=None, username=None, password=None, remote_path=None, absolute_zipfile_path=None): """ This is a helper class to manage the FTP commands from the ftplib. @param host: The host for the connection. @type host: String @param port: The post for the connection. @type port: Integer @param username: The username for the connection. Leave blank to use "anonymous". @type username: String @param password: The password for the connection. Leave empty for none. @type password: String @param remote_path: The remote path of the server in which the zip file should be uploaded. If the path does not exists, it will be created (recursive). @type remote_path: String @param absolute_zipfile_path: The absolute LOCAL filepath of the zip file. @type absolute_zipfile_path: String """ self.ftps = None self._host = host self._port = port self._username = username self._password = password self._remote_path = remote_path self._absolute_zipfile_path = absolute_zipfile_path self._bytesWritten = 0; self._totalFileSize = os.path.getsize(self._absolute_zipfile_path) self._uploadCallback = None self._currentProgress = 0 # make the remote path relative if it isnt absolute or relative yet if self._remote_path is not None and self._remote_path.startswith( '.') is False and self._remote_path.startswith('/') is False: self._remote_path = './' + self._remote_path if self._username is None: self._username = '******' if self._port is None: self._port = 22 def connect(self): """ Try to connect to the FTP server. Raise an error if something went wrong. """ try: self.ftps = FTP_TLS() self.ftps.set_pasv(True) self.ftps.connect(self._host, self._port) except socket.gaierror: raise def login(self): """ Try to login in on the FTP server. Raise an error if something went wrong. """ try: self.ftps.login(self._username, self._password) self.ftps.prot_p() except ftplib.error_perm: raise def cwd(self): """ Try to switch the working directory on the FTP server (if set in the settings). If the path does not exist, it will be created. """ if self._remote_path is not None: try: self.ftps.cwd(self._remote_path) except ftplib.error_perm: self.create_directory_tree(self._remote_path) except IOError: raise def create_directory_tree(self, current_directory): """ Helper function to create the remote path. @param current_directory: The current working directory. @type current_directory: String """ if current_directory is not "": try: self.ftps.cwd(current_directory) except ftplib.error_perm: self.create_directory_tree("/".join(current_directory.split("/")[:-1])) self.ftps.mkd(current_directory) self.ftps.cwd(current_directory) except IOError: raise def upload(self, callback=None): """ The upload function. @param callback: The callback function for the upload progress. @type callback: Function """ self._uploadCallback = callback zipfile_to_upload = open(self._absolute_zipfile_path, 'rb') zipfile_basename = os.path.basename(self._absolute_zipfile_path) self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload, 1024, self.handle_upload_state) zipfile_to_upload.close() def handle_upload_state(self, block): """ The callback function for the upload progress. @param block: The StringIO of the current upload state @type block: StringIO """ self._bytesWritten += 1024 progress = math.floor((float(self._bytesWritten) / float(self._totalFileSize)) * 100) do_update = False if progress > self._currentProgress: do_update = True self._currentProgress = progress if self._uploadCallback is not None: self._uploadCallback(progress, self, do_update) def quit(self): """ Try to quit everything and close the session. """ self.ftps.quit() @property def upload_path(self): """ Returns the upload path of the FTP server. @return: The path of the uploaded file on the FTP server. @rtype: String """ tmp_remote_path = '' if self._remote_path is not None: tmp_remote_path = self._remote_path return os.path.join(tmp_remote_path, os.path.basename(self._absolute_zipfile_path))
class FTPSession(object): """ Attempt to create some robustness and performance to FTPing """ def __init__(self, server, username, password, tmpdir="/tmp", timeout=60): """Build a FTP session """ self.conn = None self.server = server self.username = username self.password = password self.tmpdir = tmpdir self.timeout = timeout def _connect(self): """Connect to FTP server """ if self.conn is not None: return logging.debug("Creating new connection to server %s", self.server) not_connected = True attempt = 1 while not_connected and attempt < 6: try: self.conn = FTP_TLS(self.server, timeout=self.timeout) self.conn.login(self.username, self.password) self.conn.prot_p() not_connected = False except Exception as exp: logging.debug(exp) time.sleep(5) self.close() attempt += 1 if not_connected is True: raise Exception("Failed to make FTP connection after 5 tries!") def _reconnect(self): """ First attempt to shut down connection and then reconnect """ logging.debug("_reconnect() was called...") try: self.conn.quit() self.conn.close() except: pass finally: self.conn = None self._connect() def _put(self, path, localfn, remotefn): """ """ self.chdir(path) sz = os.path.getsize(localfn) if sz > 14000000000: # Step 1 Split this big file into 14GB chunks, each file will have # suffix .aa then .ab then .ac etc basefn = os.path.basename(localfn) cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir, basefn) subprocess.call(cmd, shell=True, stderr=subprocess.PIPE) files = glob.glob("%s/%s.??" % (self.tmpdir, basefn)) for filename in files: suffix = filename.split(".")[-1] self.conn.storbinary("STOR %s.%s" % (remotefn, suffix), open(filename)) os.unlink(filename) else: logging.debug("_put '%s' to '%s'", localfn, remotefn) self.conn.storbinary("STOR %s" % (remotefn,), open(localfn)) return True def close(self): """ Good bye """ try: self.conn.quit() self.conn.close() except: pass finally: self.conn = None def chdir(self, path): if self.pwd() == path.rstrip("/"): return self.conn.cwd("/") for dirname in path.split("/"): if dirname == "": continue bah = [] self.conn.retrlines("NLST", bah.append) if dirname not in bah: logging.debug("Creating directory '%s'", dirname) self.conn.mkd(dirname) logging.debug("Changing to directory '%s'", dirname) self.conn.cwd(dirname) def pwd(self): """ Low friction function to get connectivity """ self._connect() pwd = exponential_backoff(self.conn.pwd) if pwd is None: self._reconnect() pwd = exponential_backoff(self.conn.pwd) logging.debug("pwd() is currently '%s'", pwd) return pwd def put_file(self, path, localfn, remotefn): """ Put the File """ res = exponential_backoff(self._put, path, localfn, remotefn) if not res: self._reconnect() res = exponential_backoff(self._put, path, localfn, remotefn) if not res: logging.error("Double Failure to upload filename: '%s'", localfn) def put_files(self, path, localfns, remotefns): """ Put the File """ for localfn, remotefn in zip(localfns, remotefns): self.put_file(path, localfn, remotefn)
class FTPClient(object): """Class FTPClient """ _mh = None _client = None _secured = None _host = None _port = None _user = None _passw = None _path = None _verbose = None _is_connected = None def __init__(self, secured=False, verbose=False): """Class constructor Called when the object is initialized Args: secured (bool): secured FTP verbose (bool): verbose mode """ self._mh = MasterHead.get_head() self._secured = secured if (not self._secured): self._client = FTP() else: if (not(version_info[0] == 2 and version_info[1] == 6)): self._client = FTP_TLS() else: raise NotImplementedError( 'Secured mode is not supported for Python 2.6') self._verbose = verbose if (self._verbose): self._client.set_debuglevel(2) @property def client(self): """ FTP client property getter """ return self._client @property def secured(self): """ secured protocol mode property getter """ return self._secured @property def host(self): """ server host property getter """ return self._host @property def port(self): """ server port property getter """ return self._port @property def user(self): """ username property getter """ return self._user @property def passw(self): """ user password property getter """ return self._passw @property def path(self): """ remote path property getter """ return self._path @property def verbose(self): """ verbose mode property getter """ return self._verbose @property def is_connected(self): """ is_connected property getter """ return self._is_connected def connect(self, host, port=21, user=None, passw=None, path='/', timeout=10): """Method connects to server Args: host (str): server host port (int): server port, default protocol port user (str): username passw (str): password path (str): server path timeout (int): timeout Returns: bool: result Raises: event: ftp_before_connect event: ftp_after_connect """ try: message = '{0}/{1}@{2}:{3}{4} timeout:{5}'.format( user, passw, host, port, path, timeout) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_connecting', message), self._mh.fromhere()) ev = event.Event( 'ftp_before_connect', host, port, user, passw, path, timeout) if (self._mh.fire_event(ev) > 0): host = ev.argv(0) port = ev.argv(1) user = ev.argv(2) passw = ev.argv(3) path = ev.argv(4) timeout = ev.argv(5) self._host = host self._port = port self._user = user self._passw = passw if (ev.will_run_default()): self._client.connect(self._host, self._port, timeout=timeout) if (self._user != None): self._client.login(self._user, self._passw) if (self._secured): self._client.prot_p() self._is_connected = True self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_connected'), self._mh.fromhere()) if (path != None): self.change_dir(path) ev = event.Event('ftp_after_connect') self._mh.fire_event(ev) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def disconnect(self): """Method disconnects from server Args: none Returns: bool: result """ try: if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False else: self._client.quit() self._is_connected = False self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_disconnected'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def list_dir(self): """Method lists remote working directory Args: none Returns: list: names """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_list_dir', self._path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False names = self._client.nlst() if ('.' in names): names.remove('.') if ('..' in names): names.remove('..') return names except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return None def change_dir(self, path): """Method changes remote working directory Args: path (str): new remote path Returns: bool: result Raises: event: ftp_before_change_dir """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_change_dir', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_change_dir', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.cwd(path) self._path = self._client.pwd() self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_cur_dir', self._path), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def download_file(self, remote_path, local_path=None): """Method downloads file from server Args: remote_path (str): remote path local_path (str): local path, default ./filename Returns: bool: result Raises: event: ftp_before_download_file event: ftp_after_download_file """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_downloading_file', remote_path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event( 'ftp_before_download_file', remote_path, local_path) if (self._mh.fire_event(ev) > 0): remote_path = ev.argv(0) local_path = ev.argv(1) if (local_path != None and not path.exists(local_path)): self._mh.demsg('htk_on_error', self._mh._trn.msg( 'htk_ftp_unknown_dir', local_path), self._mh.fromhere()) return False filename = remote_path.split('/')[-1] lpath = filename if (local_path == None) else path.join( local_path, filename) if (ev.will_run_default()): with open(lpath, 'wb') as f: self._client.retrbinary('RETR ' + remote_path, f.write) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_file_downloaded'), self._mh.fromhere()) ev = event.Event('ftp_after_download_file') self._mh.fire_event(ev) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) if (path.exists(lpath)): remove(lpath) return False def upload_file(self, local_path, remote_path=None): """Method uploads file to server Args: local_path (str): local path remote_path (str): remote path, default ./filename Returns: bool: result Raises: event: ftp_before_upload_file event: ftp_after_upload_file """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_uploading_file', local_path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_upload_file', local_path, remote_path) if (self._mh.fire_event(ev) > 0): local_path = ev.argv(0) remote_path = ev.argv(1) if (not(path.exists(local_path) or path.exists(path.relpath(local_path)))): self._mh.demsg('htk_on_error', self._mh._trn.msg( 'htk_ftp_unknown_file', local_path), self._mh.fromhere()) return False filename = local_path.split('/')[-1] rpath = filename if (remote_path == None) else path.join( remote_path, filename) if (ev.will_run_default()): with open(local_path, 'rb') as f: self._client.storbinary('STOR ' + rpath, f) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_file_uploaded'), self._mh.fromhere()) ev = event.Event('ftp_after_upload_file') self._mh.fire_event(ev) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def delete_file(self, path): """Method deletes file from server Args: path (str): remote path Returns: bool: result Raises: event: ftp_before_delete_file """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_deleting_file', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_delete_file', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.delete(path) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_file_deleted'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def make_dir(self, path): """Method makes directory on server Args: path (str): remote path Returns: bool: result Raises: event: ftp_before_make_dir """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_making_dir', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_make_dir', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.mkd(path) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_dir_made'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def remove_dir(self, path): """Method removes directory from server Args: path (str): remote path Returns: bool: result Raises: event: ftp_before_remove_dir """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_removing_dir', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_remove_dir', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.rmd(path) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_dir_removed'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False
data['bot_id'] = BOT_ID def group_listing(): # Get list of groups user is a member of r = requests.get('%s/groups?token=%s' % (GROUPME_API_URL, TOKEN)) resp = r.json() print 'Groups:' for item in resp['response']: print item['name'] # Securely connect to FTP server ftps = FTP_TLS(HOST, USER, PASSWD) ftps.prot_p() # Change working directory to directory containing images ftps.cwd(IMAGE_DIRECTORY) # Get list of items in current directory directory_list = ftps.nlst() # Get list of images image_list = [item for item in directory_list if '.jpg' in item] # Save oldest & newest images images_to_upload = [] if image_list: # Add first image images_to_upload.append(image_list[0]) if len(image_list) > 1: # Add last image (if more than 1 image) images_to_upload.append(image_list[len(image_list)-1]) # Download oldest & newest image for image in images_to_upload: print 'Downloading %s...' % image
#See http://stackoverflow.com/questions/10207628/python-module-ftplib-ftp-tls-error-530 #Try TLS Lite or M2Crypto both are FTP/TLS client and server. #ftps.login(user,secret) ftps.sendcmd('USER ' + user) ftps.sendcmd('PASS ' + secret) print(ftps.getwelcome()) print('CURRENT WORKING DIRECTORY IS:',ftps.pwd()) #Enable data encryption # TODO solve the encryption problem #ftps.prot_p() #define default DIR d = 'feeds' #Change to default DIR ftps.cwd(d) #Build list of files on servers l = ftps.nlst() l.sort() for i in l: print(i) #Assign last element to var litem = len(l)-1 print("MOST RECENT FILE ON SERVER IS; ",l[litem]) g = l[litem] #Define local file t = d + '/' + g if os.path.exists(t): print("FILE" ,g," EXISTS,WILL NOT DOWNLOAD FROM HOST:",host) else:
class ServerWatcher(Watcher): downloadProgress = Signal((int, int,)) uploadProgress = Signal((int, int,)) # Si added: textStatus = Signal((str,)) fileEvent = Signal((str,)) fileEventCompleted = Signal() loginCompleted = Signal((bool, str,)) badFilenameFound = Signal((str,)) LOCATION = 'server' TEST_FILE = 'iqbox.test' def __init__(self, host, ssl, parent=None): """ Initializes parent class and attributes. Decides whether to use `FTP_TLS` or `FTP` based on the `ssl` param. :param host: Location of the FTP server :param ssl: Tells whether the FTP needs to support TLS or not :param parent: Reference to a `QObject` instance a parent """ super(ServerWatcher, self).__init__(parent) self.interval = 5000 self.localdir = '' self.deleteQueue = [] self.downloadQueue = [] self.uploadQueue = [] self.warnedNames = [] self.ftp = None self.useSSL = ssl self.host = host self.preemptiveCheck = False self.preemptiveActions = [] self.testFile = 'iqbox.test' @property def currentdir(self): """Returns the current working directory at the server""" return self.ftp.pwd() def setLocalDir(self, localdir): """ Sets the local directory used to stored all downloaded files. Creates the directory if needed. :param localdir: Absolute path to local directory """ self.localdir = localdir if not os.path.exists(self.localdir): os.makedirs(self.localdir) @pause_timer @Slot() def checkout(self): """ Recursively checks out all files on the server. Returns a dictionary of files on the server with their last modified date. :param download: Indicates whether or not the files should be downloaded """ # Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues. # These tasks are done in queues to make sure all FTP commands # are done sequentially, in the same thread. self.deleteAll() self.uploadAll() self.downloadAll() # Handy list to keep track of the checkout process. # This list contain absolute paths only. checked_dirs = list() # Sets '/' as initial directory and initializes `downloading_dir` self.ftp.cwd('/') downloading_dir = self.currentdir check_date = dt.utcnow() sidirlist = list() root_cached = False fileC = 0 while True: # Gets the list of sub directories and files inside the # current directory `downloading_dir`. self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...') if root_cached and downloading_dir == '/': dir_subdirs = saved_root_dirs dirfiles = saved_root_files else: dir_subdirs = self.getDirs(downloading_dir) if downloading_dir == '/': saved_root_dirs = dir_subdirs # sidirlist.extend(dir_subdirs) self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...') dirfiles = self.getFiles(downloading_dir) if downloading_dir == '/': saved_root_files = dirfiles root_cached = True # Leading '/' in `downloading_dir` breaks the `os.path.join` call localdir = os.path.join(self.localdir, downloading_dir[1:]) if not os.path.exists(localdir): # Creates the directory if it doesn't already exists. os.makedirs(localdir) for file_ in dirfiles: # `serverpath` is the absolute path of the file on the server, # download it only if it hasn't been already downloaded serverpath = os.path.join(downloading_dir, file_) serverpath = QDir.fromNativeSeparators(serverpath) server_file = File.fromPath(serverpath) self.textStatus.emit('Scanning remote file... '+serverpath+'...') # How do we know if we should check this server file? # We see if the date last checked is the check start time. if server_file.last_checked_server != check_date: # Do this process only once per file # Added by Simon # Give feedback on scanning of files. fileC += 1 if fileC % 1 == 2: self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.') # STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB? just_added = not server_file.inserver # STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT # Example: If it's a temporary file, or a Unix file with a name we don't support. if just_added: filename = os.path.basename(serverpath) if platform.system() == 'Windows': badName = False for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']: if chr in filename: badName = True break if badName: if filename not in self.warnedNames: self.warnedNames.append(filename) self.badFilenameFound.emit(filename) continue # STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE lastmdate = server_file.servermdate # STEP: SAVE THE MOD DATE TO A VARIABLE # Now we get the last mod time. # We expect this to work fine since this file # was found on the server servermdate = self.lastModified(serverpath) # STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS. server_file.inserver = True # STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME server_file.last_checked_server = check_date # STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT server_file.servermdate = servermdate # STEP: SAVE THIS CHANGE TO THE DATABASE server_file.session.commit() delta = 0 if server_file.inlocal: delta = server_file.timeDiff() # Emit the signals after the attributes has been set and committed if just_added is True: self.fileAdded.emit(ServerWatcher.LOCATION, serverpath) elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE: self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False) #END FOR self.textStatus.emit('Remote scan- Finding next folder...') dir_ready = True for dir_ in dir_subdirs: # `dirpath` is the absolute path of the subdirectory on the server, dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_)) # `downloading_dir` is ready only when all its subdirectory are on the # `checked_dirs` list. if dirpath not in checked_dirs: # Found one subdirectory that is not on `checked_dirs`, # will process it in the next iteration. downloading_dir = dirpath dir_ready = False break if dir_ready is True: # All subdirectories of `downloading_dir` are already in `checked_dirs` if downloading_dir == '/': # All directories ready and at '/', means checkout is complete # So, exit the main While loop!! break else: # Not at '/'. Current directory is ready so is appended to `checked_dirs` # Back one directory to find directories that are not in `checked_dirs` checked_dirs.append(downloading_dir) downloading_dir = os.path.dirname(downloading_dir) self.textStatus.emit('Remote scan- Found Folder...') ##### END OF WHILE ################ ################################################################### # Deleted files are the ones whose `last_checked_server` attribute # didn't get updated in the recursive run. session = Session() deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True) for file_ in deleted: self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path) # Wraps up the checkout process, commits to the database. session.commit() @Slot() def onLogin(self, username, passwd): ok = True msg = '' error_msg = 'Login failed.' try: if not self.ftp: self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(self.host) loginResponse = self.ftp.login(username, passwd) except socket.gaierror: self.ftp = None ok = False msg = 'Server address could not be found.' except (error_perm, error_reply): info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) ok = False msg = error_msg else: if '230' in loginResponse: ok = True else: ok = False msg = error_msg if ok: # Logged in. Now let's do compability tests. if not self.testPermissions(): # User doesn't have write permissions, don't bother doing next test. ok = False msg = 'It seems like you do not have write access to this server.' else: # Permissions test passed, now let's test MFMT for timestamp modification. if not self.testMFMT(): ok = False msg = 'This server does not support timestamp modification\n \ need by this application.' self.loginCompleted.emit(ok, msg) def getFiles(self, path): """ This method simply wraps the `nlst` method with an exception handler, and returns an empty list in case an exception is caught. :param path: Relative or absolute path on the server """ try: nlst = self.ftp.nlst(path) dirs = self.getDirs(path) # Files are items in nlst that are not in dirs files = [item for item in nlst if os.path.basename(item) not in dirs] return files except: print 'Exception in ServerWatcher.getDirs' info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) return [] def getDirs(self, path): """ Retrieves a list of the directories inside `path`, uses `retrlines` and the LIST command to retrieve the items. :param path: Relative or absolute path on the server """ dirs = list() def handleLine(line): """ Recieves a line from the LIST command. This function is meant to be used as callback for the `retrlines` method. :params line: Line from the LIST command """ if line.startswith('d'): # Only lines starting with 'd' are directories # Parse the directory out of the line; lines look like: # 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname' dirname = line[55:].strip() if dirname != '.' and dirname != '..': # Ignoring '.' and '..' entries dirs.append(dirname) try: self.ftp.retrlines('LIST %s' % path, handleLine) return dirs except: print 'Exception in ServerWatcher.getDirs' info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) return [] @upload_test def testPermissions(self): # For interface purposes. upload_test takes care of everything. return True @upload_test def testMFMT(self): # Absurd date to test whether the change really happened. time = dt.utcfromtimestamp(100000000) try: self.setLastModified(self.testFile, time) otherTime = self.lastModified(self.testFile) diff = (time - otherTime).total_seconds() if abs(diff) < 2: # Let's give it a 2 seconds tolerance. mdtm = True else: mdtm = False except (ValueError, error_reply, error_perm): info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) mdtm = False return mdtm @Slot(str) def onDelete(self, filename): self.deleteQueue.append(filename) def deleteNext(self): if len(self.deleteQueue) > 0: next = self.deleteQueue.pop(0) self.deleteFile(next) def deleteAll(self): for filename in self.deleteQueue: self.deleteFile(filename) self.deleteQueue = [] @Slot(str) def deleteFile(self, filename): """ Deletes the file `filename` to the server :param filename: Absolute or relative path to the file """ try: print 'Deleting %s' % filename self.ftp.delete(filename) return True except (error_reply, error_perm): print 'Error deleting %s' % filename return False self.fileEventCompleted.emit() @Slot(str) def onDownload(self, filename): self.downloadQueue.append(filename) def downloadNext(self): if len(self.downloadQueue) > 0: next = self.downloadQueue.pop(0) self.downloadFile(next) def downloadAll(self): for filename in self.downloadQueue: self.downloadFile(filename) self.downloadQueue = [] @Slot(str, str) def downloadFile(self, filename, localpath=None): """ Performs a binary download to the file `filename` located on the server. `filename` parameter can be either absolute or relative, though it can fail for relative paths if the current directory is not appropiate. :param filename: Relative or absolute path to the file :param localpath: Absolute local path where the file will be saved """ def handleChunk(chunk): """ Receives chuncks of data downloaded from the server. This function is meant to be used as callback for the `retrbinary` method. :params chunk: Chunk of downloaded bytes to be written into the file """ # Simply writes the received data into the file `self.downloading` self.downloading.write(chunk) self.download_progress += len(chunk) self.downloadProgress.emit(self.download_size, self.download_progress) if localpath is None: localpath = self.localFromServer(filename) localdir = os.path.dirname(localpath) if not os.path.exists(localdir): # Creates the directory if it doesn't already exists. os.makedirs(localdir) print 'Downloading: %s to %s' % (filename, localpath) try: with open(localpath, 'wb') as f: # Opens the file at `localname` which will hold the downloaded file. # Object attributes regarding download status are updated accordingly. self.fileEvent.emit(filename) self.downloading = f self.download_progress = 0 self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1]) self.ftp.retrbinary('RETR %s' % filename, handleChunk) print 'Download finished' # Let's set the same modified time to that on the server. with File.fromPath(filename) as downloadedfile: mdate = LocalWatcher.lastModified(localpath) downloadedfile.localmdate = mdate downloadedfile.servermdate = mdate self.setLastModified(filename, mdate) downloaded = True except (IOError, OSError): downloaded = False self.ioError.emit(localpath) except (error_reply, error_perm) as ftperr: print 'Error downloading %s, %s' % (filename, ftperr) downloaded = False # TODO: Sometimes the file doesn't complete properly. # in that case we maybe shouldn't call this? self.fileEventCompleted.emit() return downloaded @Slot(str) def onUpload(self, filename): self.uploadQueue.append(filename) def uploadNext(self): if len(self.uploadQueue) > 0: next = self.uploadQueue.pop(0) self.uploadFile(next) def uploadAll(self): for filename in self.uploadQueue: self.uploadFile(filename) self.uploadQueue = [] @Slot(str) def uploadFile(self, filename): """ Uploads the file `filename` to the server, creating the needed directories. :param filename: Absolute or relative path to the file """ def handle(buf): """This function is meant to be used as callback for the `storbinary` method.""" self.upload_progress += 1024 self.uploadProgress.emit(self.upload_size, self.upload_progress) # Creates the directory where the file will be uploaded to self.mkpath(os.path.dirname(filename)) localpath = self.localFromServer(filename) print 'Uploading %s to %s' % (localpath, filename) try: # Uploads file and updates its modified date in the server # to match the date in the local filesystem. self.upload_progress = 0 self.upload_size = os.path.getsize(localpath) self.fileEvent.emit(localpath) self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'), 1024, handle) print 'Upload finished' with File.fromPath(filename) as uploaded: modified = uploaded.localmdate uploaded.servermdate = modified self.setLastModified(filename, modified) uploaded = True except (IOError, OSError): uploaded = False self.ioError.emit(localpath) except (error_reply, error_perm, OSError) as err: print 'Error uploading %s, %s' % (filename, err) uploaded = False # TODO: Sometimes the file doesn't complete properly. # in that case we maybe shouldn't call this? self.fileEventCompleted.emit() return uploaded def lastModified(self, filename): """ Uses the MDTM FTP command to find the last modified timestamp of the file `filename`. Returns a `datetime.datetime` object in UTC representing the file's last modified date and time. :param filename: Relative or absolute path to the file """ timestamp = self.ftp.sendcmd('MDTM %s' % filename) if '213 ' not in timestamp: # Second chance was found to be needed in some cases. timestamp = self.ftp.sendcmd('MDTM %s' % filename) timestamp = timestamp.split(' ')[-1] dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S' try: mtime = dt.strptime(timestamp, dateformat) except ValueError: mtime = dt.utcnow() return mtime def setLastModified(self, serverpath, newtime): """ Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the file `serverpath` on the server. :param serverpath: Relative or absolute path to the file :param newtime: datedatime object holding the required time """ cmds = ['MFMT', 'MDTM'] for cmd in cmds: try: self.ftp.sendcmd( '%s %s %s' % (cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath)) return except (error_perm, error_reply) as e: if cmd == cmds[len(cmds) - 1]: # If is the last comand, re-raise the exception, else # keep trying. raise e else: continue def mkpath(self, path): """ Creates the path `path` on the server by recursively created folders, if needed. :param path: Absolute path on the server to be created """ try: self.ftp.cwd(path) except error_perm: # `cwd` call failed. Need to create some folders make_dir = '/' steps = path.split('/') for step in steps: if len(step) == 0: continue make_dir += '%s/' % step try: self.ftp.mkd(make_dir) except error_perm: # Probably already exists continue else: # `cwd` call succeed. No need to create # any folders self.ftp.cwd('/') return @Slot(str, str) def added(self, location, serverpath): super(ServerWatcher, self).added(location, serverpath) def actionFromPath(serverpath): f = File() fileExistsOnServer = True try: f.servermdate = self.lastModified(serverpath) except error_perm: fileExistsOnServer = False f.servermdate = 0 f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath)) diff = f.timeDiff() action = None if abs(diff) > Watcher.TOLERANCE: if not fileExistsOnServer or diff > 0: action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION) else: action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION) return action if self.preemptiveCheck: if location == ServerWatcher.LOCATION: localpath = self.localFromServer(serverpath) if not os.path.exists(localpath): action = FileAction(serverpath, FileAction.DOWNLOAD, ServerWatcher.LOCATION) self.preemptiveActions.append(action) else: action = actionFromPath(serverpath) if action is not None: self.preemptiveActions.append(action) elif location == LocalWatcher.LOCATION: try: self.ftp.sendcmd('SIZE %s' % serverpath) except (error_reply, error_perm): exists = False else: exists = True if not exists: action = FileAction(serverpath, FileAction.UPLOAD, LocalWatcher.LOCATION) self.preemptiveActions.append(action) else: action = actionFromPath(serverpath) if action is not None: self.preemptiveActions.append(action) @Slot(str, str) def changed(self, location, serverpath): super(ServerWatcher, self).changed(location, serverpath) @Slot(str, str) def deleted(self, location, serverpath): super(ServerWatcher, self).deleted(location, serverpath) with File.fromPath(serverpath) as deleted: deleted.inserver = False