def send_chunks(self): if self.final_chunks is None: return ERR if self.tls_flag: if self.auth_flag: ftp_obj = FTP_TLS(host=self.server, user=self.creds[0], passwd=self.creds[1]) else: ftp_obj = FTP_TLS(host=self.server) else: if self.auth_flag: ftp_obj = FTP(host=self.server, user=self.creds[0], passwd=self.creds[1]) else: ftp_obj = FTP(host=self.server) try: ftp_obj.login() sys.stdout.write("\t[+]\tConnected to server %s.\n" % self.server) except: sys.stderr.write("\t[-]\tCould not login to the server.\n") return ERR for chunk in self.final_chunks: ftp_obj.mkd(chunk) time.sleep(SLEEP) ftp_obj.quit() sys.stdout.write("\t[+]\tWrote %s(+1) folders.\n" % (len(self.final_chunks) - 1)) return OKAY
def _get_ftp(self): try: username, password = self._get_auth() if self._tls: ftp = FTP_TLS(self._uri, timeout=CONNECT_TIMEOUT) else: ftp = FTP(self._uri, timeout=CONNECT_TIMEOUT) ftp.login(username, password) if self._tls: ftp.prot_d() if not catch(ftp.cwd, self._path): if self._path.startswith("/"): ftp.cwd("/") components = self._path.split("/") for component in components: if not catch(ftp.cwd, component): ftp.mkd(component) ftp.cwd(component) return ftp except Exception: log.exception() log.warning( "[FTP] failed to establish server connection, disabled") self._disabled = True return None
def upload_to_ftp(root_dir, upload_path): ftp = FTP_TLS() ftp.connect(host, port) ftp.set_debuglevel(2) ftp.login("account_id", "account_password") files, directories = get_list_local(upload_path) directory_name = os.path.dirname(upload_path) directory_name = os.path.basename(directory_name) root_dir = root_dir + directory_name + "/" directories.insert(0, "") for directory in directories: path = directory.replace(upload_path, "").replace("\\", "/") path = root_dir + path # for fName in ftp.nlst(): # print(fName) # if folderName in ftp.nlst(): # continue try: ftp.mkd(path) except: continue print("Make directory is " + path) for file in files: path = file.replace(upload_path, "").replace("\\", "/") with open(file, "rb") as localfile: path = root_dir + path ftp.storbinary('STOR ' + path, localfile) print("Upload file is " + path) print("done upload")
def send_chunks(self): if self.final_chunks is None: return ERR if self.tls_flag: if self.auth_flag: ftp_obj = FTP_TLS(host=self.server, user=self.creds[0], passwd=self.creds[1]) else: ftp_obj = FTP_TLS(host=self.server) else: if self.auth_flag: ftp_obj = FTP(host=self.server, user=self.creds[0], passwd=self.creds[1]) else: ftp_obj = FTP(host=self.server) try: ftp_obj.login() sys.stdout.write("\t[+]\tConnected to server %s.\n" % self.server) except: sys.stderr.write("\t[-]\tCould not login to the server.\n") return ERR for chunk in self.final_chunks: ftp_obj.mkd(chunk) time.sleep(SLEEP) ftp_obj.quit() sys.stdout.write("\t[+]\tWrote %s(+1) folders.\n" % (len(self.final_chunks)-1)) return OKAY
def upload(self,path,type,name=None): try: fname = name if name else path.split('\\')[-1] ftps = FTP_TLS(self.host) ftps.login(self.user,self.password) ftps.prot_p() # force encoding to utf-8, this will let us to work with unicode file names ftps.encoding = "utf-8" ftps.cwd(self.user_dir) ftps.cwd(type) if type != 'files': # if type of requested file is screenshot or keylog # upload it to special folder on ftp today = date.today().strftime("%b-%d-%Y") if today not in ftps.nlst(): ftps.mkd(today) ftps.cwd(today) ftps.storbinary('STOR %s' % fname, open(path, 'rb')) return 'Upload Started!' if fname in ftps.nlst(): return 'File Already on FTP' ftps.storbinary('STOR %s' %fname, open(path,'rb')) ftps.close() return 'Upload Started!' except Exception as e: if e.args[0] == 0: return 'Uploaded to FTP!' return 'Upload Failed!'
def uploadFilesToFTP(ftpURL, ftpUser, ftpPassword, ftpPath, localPath, filematch, historyBackupPath): ftps = FTP_TLS(ftpURL) ftps.set_debuglevel(1) ftps.set_pasv(False) ftps.connect(port=21, timeout=80) ftps.login(ftpUser, ftpPassword) ftps.prot_p() ftps.ccc() try: ftps.cwd(ftpPath) except Exception: ftps.mkd(ftpPath) for (localPathDir, _, files) in os.walk(localPath): newdir = ftpPath try: ftps.cwd(newdir) except Exception: ftps.mkd(newdir) LOGGER.info("filematch=" + filematch) for f in fnmatch.filter(files, filematch): fileLocalPath = os.path.join(localPathDir, f) file = open(fileLocalPath, 'rb') ftps.storbinary('STOR ' + f, file, blocksize=8192) file.close() LOGGER.info("Fichero transferido #:# " + fileLocalPath) sleep(1) now = datetime.datetime.now() historyBackupPathYear = os.path.join(historyBackupPath, str(now.year)) try: os.stat(historyBackupPathYear) except: os.mkdir(historyBackupPathYear) moveFilesUUID(fileLocalPath, historyBackupPathYear) ftps.close()
def ftp_login(): global ftp # Use ftps if config.get_value("ftp", "protocol") == "ftps": try: ftp = FTP_TLS(config.get_value("ftp", "host")) except: return ("host_error") try: ftp.login(config.get_value("ftp", "username"), config.get_value("ftp", "password")) ftp.prot_p() try: ftp.cwd(config.get_value("ftp", "path")) except: ftp.mkd(config.get_value("ftp", "path")) ftp.cwd(config.get_value("ftp", "path")) except: return ("login_error") # Use ftp if config.get_value("ftp", "protocol") == "ftp": try: ftp = FTP(config.get_value("ftp", "host")) except: return ("host_error") try: ftp.login(config.get_value("ftp", "username"), config.get_value("ftp", "password")) try: ftp.cwd(config.get_value("ftp", "path")) except: ftp.mkd(config.get_value("ftp", "path")) ftp.cwd(config.get_value("ftp", "path")) except: return ("login_error")
CSV3 = '_Inverter3.csv' CSV4 = '_Inverter4.csv' CSV = [CSV1, CSV2, CSV3, CSV4] foldername = str(datetime.date.today()) PATH_TO_SAVE_CVSDIRECTORIES = '/root/regs/' client1 = mqtt.Client() try: ftps = FTP_TLS('host') ftps.login('user', 'password') # login anonymously before securing control channel ftps.prot_p() # switch to secure data connection ftps.cwd('dir') # execute a cd command to access to directory if (not (foldername in ftps.nlst())): ftps.mkd(foldername) #create a new folder with current date ftps.cwd(str(datetime.date.today())) #access to current date folder for i in range(0, len(CSV)): filename = str(PATH_TO_SAVE_CVSDIRECTORIES + str(datetime.date.today()) + '/' + str(datetime.date.today()) + CSV[i]) file = open(filename, 'rb') filename2 = str(datetime.date.today()) + CSV[i] ftps.storbinary('STOR ' + filename2, file) file.close() ftps.quit() print("End try") except: print("except")
ftp.mkd(name) except error_perm as e: if not e.args[0].startswith('550'): raise print("CWD", name) ftp.cwd(name) upload_files(ftp, localpath) print("CWD", "..") ftp.cwd("..") ftp_username = os.environ['FTP_USERNAME'] ftp_password = os.environ['FTP_PASSWORD'] with open('version') as file: version = file.read().rstrip() working_directory = os.getcwd() ftp = FTP_TLS("ftp.keycdn.com") ftp.login(user=ftp_username, passwd=ftp_password) ftp.cwd('alted') ftp.mkd(version) ftp.cwd(version) upload_files(ftp, '{}/{}'.format(working_directory, 'static')) ftp.quit()
def genTiles(task): time_start = datetime.now() try: cpu_count = 1 if thread_count is None: cpu_count = multiprocessing.cpu_count() else: cpu_count = thread_count s_file = None if source_file.endswith(".zip"): src = source_file.replace("\\", "/") fileinfo = QFileInfo(src) filename = fileinfo.completeBaseName() s_file = [src, filename] else: return [0, time_start] #cpu_count = min(6, cpu_count) #if cpu_count % 2 == 0: # cpu_count = int(cpu_count / 2) # converted_source = os.path.join(source_folder, "converted").replace("\\","/") QgsMessageLog.logMessage( 'Started creating dataset from {name}'.format(name=s_file[1]), CATEGORY, Qgis.Info) source_folder = os.path.dirname(source_file) org_vrt = os.path.join(source_folder, "OrgSource.vrt").replace("\\", "/") if os.path.isfile(org_vrt): os.remove(org_vrt) org_files = [] c_file = '/vsizip/{archive}/{file}'.format(archive=s_file[0], file=source_file_name) QgsMessageLog.logMessage(c_file, CATEGORY, Qgis.Info) org_files.append(c_file) ds = gdal.BuildVRT(org_vrt, org_files, resolution="highest", resampleAlg="bilinear") ds.FlushCache() ds = None sleep(0.05) QgsMessageLog.logMessage('Created original vrt', CATEGORY, Qgis.Info) vrt = os.path.join(source_folder, "Source.vrt").replace("\\", "/") if os.path.isfile(vrt): os.remove(vrt) pds = gdal.Open(org_vrt) pds = gdal.Warp(vrt, pds, dstSRS="EPSG:3857") pds = None QgsMessageLog.logMessage('Created reporojected vrt', CATEGORY, Qgis.Info) terrarium_tile = os.path.join(source_folder, "TerrariumSource.vrt").replace( "\\", "/") if os.path.isfile(terrarium_tile): os.remove(terrarium_tile) statistics = [] task.setProgress(0) ress = calculateStat(c_file) sleep(0.05) if ress is not None: statistics.append(ress) task.setProgress(12) else: return [0, time_start] ress = None org_files = None # Find minV and maxV from statistics minV = None maxV = None for stat in statistics: if stat.minV is not None and stat.maxV is not None: if minV is None: minV = stat.minV elif stat.minV < minV: minV = stat.minV if maxV is None: maxV = stat.maxV elif stat.maxV > maxV: maxV = stat.maxV if minV is None or maxV is None: QgsMessageLog.logMessage( 'Error: Minimum and maximum height are None', CATEGORY, Qgis.Info) return None else: QgsMessageLog.logMessage( 'Minimum and maximum height are {minv} and {maxv}'.format( minv=minV, maxv=maxV), CATEGORY, Qgis.Info) statistics = None color_ramp_file = os.path.join(source_folder, "TerrariumSource.txt").replace( "\\", "/") if os.path.isfile(color_ramp_file): os.remove(color_ramp_file) color_ramp = [] altitudes = [] # Create color ramp for meters in range(minV, maxV): for fraction in range(0, 256): altitudes.append(meters + (fraction / 256)) exx = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) ff = exx.map(createRamp, repeat(convert_feet_to_meters), altitudes) for res in ff: if res is not None: color_ramp.append(res) QgsMessageLog.logMessage('Creating color ramp file', CATEGORY, Qgis.Info) #sorted_color_ramp = sorted(color_ramp, key=itemgetter(0)) color_ramp.sort(key=lambda x: x.altitude) f = open(color_ramp_file, "w") rt = 0 cf = len(color_ramp) for ramp in color_ramp: if manual_nodata_value is not None: if ramp.altitude == manual_nodata_value: f.write('{altitude}\t0\t0\t0\t0\n'.format( altitude=manual_nodata_value)) continue f.write('{altitude}\t{red}\t{green}\t{blue}\n'.format( altitude=ramp.altitude, red=ramp.red, green=ramp.green, blue=ramp.blue)) rt += 1 task.setProgress(max(0, min(int(((rt * 8) / cf) + 12), 100))) f.write('nv\t0\t0\t0\t0') f.close() sleep(0.05) QgsMessageLog.logMessage('Created color ramp file', CATEGORY, Qgis.Info) QgsMessageLog.logMessage( 'Rendering vrt to terrarium format. This might take a while', CATEGORY, Qgis.Info) dst_ds = gdal.Open(vrt) ds = gdal.DEMProcessing(terrarium_tile, dst_ds, 'color-relief', colorFilename=color_ramp_file, format="VRT", addAlpha=True) sleep(0.05) dst_ds = None QgsMessageLog.logMessage('Created vrt in terrarium format', CATEGORY, Qgis.Info) input_file = terrarium_tile QgsMessageLog.logMessage('Input file is {inp}'.format(inp=input_file), CATEGORY, Qgis.Info) dst_ds = gdal.Open(input_file, gdal.GA_ReadOnly) QgsMessageLog.logMessage('Calculating bounds', CATEGORY, Qgis.Info) diff = (20037508.3427892439067364 + 20037508.3427892439067364) / 2**zoom x_diff = diff y_diff = diff ulx, xres, xskew, uly, yskew, yres = dst_ds.GetGeoTransform() #lrx = ulx + (dst_ds.RasterXSize * xres) #lry = uly + (dst_ds.RasterYSize * yres) info = gdal.Info(dst_ds, format='json') ulx, uly = info['cornerCoordinates']['upperLeft'][0:2] lrx, lry = info['cornerCoordinates']['lowerRight'][0:2] wgs_minx, wgs_minz = info['wgs84Extent']['coordinates'][0][1][0:2] wgs_maxx, wgs_maxz = info['wgs84Extent']['coordinates'][0][3][0:2] QgsMessageLog.logMessage( 'The dataset bounds are (in WGS84 [EPSG:4326]), minX: {minX}, minZ: {minZ}, maxX: {maxX}, maxZ: {maxZ}' .format(minX=str(wgs_minx), minZ=str(wgs_minz), maxX=str(wgs_maxx), maxZ=str(wgs_maxz)), CATEGORY, Qgis.Info) QgsMessageLog.logMessage( 'Use this info for following step F in Part two: Generating/using your dataset', CATEGORY, Qgis.Info) base_x = 0 base_y = 0 start_tile_x = base_x start_tile_y = base_y min_x = -20037508.3427892439067364 max_y = 20037508.3427892439067364 x_tiles = 0 y_tiles = 0 # Find min lot lon = min_x while lon <= 20037508.3427892439067364: if lon >= ulx: break start_tile_x += 1 min_x = lon lon += x_diff # Find max lat lat = max_y while lat >= -20037508.3427892550826073: if lat <= uly: break start_tile_y += 1 max_y = lat lat -= y_diff # Find how many lon tiles to make lon = min_x while lon < lrx: x_tiles += 1 lon += x_diff # Find how many lat tiles to make lat = max_y while lat >= lry: y_tiles += 1 lat -= y_diff if start_tile_x > 0: start_tile_x -= 1 if start_tile_y > 0: start_tile_y -= 1 QgsMessageLog.logMessage( 'Start tile: {tx} ({mlon}), {ty} ({mxlat}), Tiles to generate: {xt} (Width: {xtx} Height: {xty})' .format(tx=start_tile_x, mlon=min_x, ty=start_tile_y, mxlat=max_y, xt=((x_tiles + 1) * (y_tiles + 1)), xtx=x_tiles, xty=y_tiles), CATEGORY, Qgis.Info) QgsMessageLog.logMessage('Creating output folders', CATEGORY, Qgis.Info) mx = start_tile_x + x_tiles zoom_folder = '' temp_folder = os.path.join(source_folder, 'TEMP') if not ftp_upload: zoom_folder = os.path.join(output_directory, str(zoom)).replace("\\", "/") else: if os.path.isdir(temp_folder): shutil.rmtree(temp_folder) os.mkdir(temp_folder) zoom_folder = os.path.join(temp_folder, str(zoom)).replace("\\", "/") if not os.path.isdir(zoom_folder): os.mkdir(zoom_folder) for x in range(start_tile_x, mx): folderx = os.path.join(zoom_folder, str(x)).replace("\\", "/") if not os.path.isdir(folderx): os.mkdir(folderx) zip_file = os.path.join(zoom_folder, 'RenderedDataset.zip').replace("\\", "/") rd_file = None if ftp_upload: if ftp_one_file: rd_file = zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED) else: QgsMessageLog.logMessage( 'Creating output folders on target ftp server', CATEGORY, Qgis.Info) ftp = None if ftp_s: ftp = FTP_TLS() else: ftp = FTP() ftp.connect(ftp_upload_url, ftp_upload_port) if ftp_user is None or ftp_password is None: ftp.login() else: ftp.login(user=ftp_user, passwd=ftp_password) if ftp_upload_folder is not None: ftp.cwd(ftp_upload_folder) if not isFTPDir(ftp, str(zoom)): ftp.mkd(str(zoom)) ftp.cwd(str(zoom)) for x in range(start_tile_x, mx): ftp.mkd(str(x)) ftp.quit() sleep(0.01) QgsMessageLog.logMessage('Created {ct} folders'.format(ct=x_tiles), CATEGORY, Qgis.Info) tiled = [] # Tile dataset sub_min_x = min_x sub_max_x = min_x + x_diff for x in range(start_tile_x, start_tile_x + x_tiles): sub_min_y = max_y - y_diff sub_max_y = max_y for y in range(start_tile_y, start_tile_y + y_tiles): tiled.append( Tile(dst_ds, str(x), str(y), sub_min_x, sub_max_y, sub_max_x, sub_min_y, ulx, xres, uly, yres, querysize=1024)) sub_min_y -= y_diff sub_max_y -= y_diff sub_min_x += x_diff sub_max_x += x_diff job = Job(zoom_folder, str(zoom), input_file, getBands(dst_ds), resampling_algorithm, ftp_upload, ftp_one_file, ftp_s, ftp_upload_url, ftp_upload_port, ftp_upload_folder, ftp_user, ftp_password) dst_ds = None # Tile the dataset realtiles = 0 if cpu_count == 1: QgsMessageLog.logMessage( 'Started tilling vrt in singlethread mode', CATEGORY, Qgis.Info) rt = 0 cf = len(tiled) for tile in tiled: res = tileVrt(job, tile) if res is not None: if job.ftpUpload and job.ftpOnefile: addToZIP(job, res, rd_file) realtiles += 1 rt += 1 if job.ftpUpload and job.ftpOnefile: task.setProgress( max(0, min(int(((rt * 40) / cf) + 20), 100))) else: task.setProgress( max(0, min(int(((rt * 80) / cf) + 20), 100))) if getattr(localThread, 'ds', None): del localThread.ds else: QgsMessageLog.logMessage( 'Started tilling vrt in multithread mode with {count} threads'. format(count=cpu_count), CATEGORY, Qgis.Info) gdal_cache_max = gdal.GetCacheMax() gdal_cache_max_per_process = max( 1024 * 1024, math.floor(gdal_cache_max / cpu_count)) setCacheMax(gdal_cache_max_per_process) tpe = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) tm = tpe.map(tileVrt, repeat(job), tiled) rt = 0 cf = len(tiled) realtiles = 0 for res in tm: if res is not None: if job.ftpUpload and job.ftpOnefile: addToZIP(job, res, rd_file) realtiles += 1 rt += 1 if job.ftpUpload and job.ftpOnefile: task.setProgress( max(0, min(int(((rt * 40) / cf) + 20), 100))) else: task.setProgress( max(0, min(int(((rt * 80) / cf) + 20), 100))) setCacheMax(gdal_cache_max) if job.ftpUpload and job.ftpOnefile: rd_file.close() totalSize = os.path.getsize(zip_file) QgsMessageLog.logMessage( 'Starting uploading renderned archive ({size} GB) out of {count} tiles' .format(count=realtiles, size=str(round(totalSize * 9.3132257461548E-10, 4))), CATEGORY, Qgis.Info) ftp = None if job.ftpS: ftp = FTP_TLS() else: ftp = FTP() ftp.connect(job.ftpUrl, job.ftpPort) if job.ftpUser is None or job.ftpPassword is None: ftp.login() else: ftp.login(user=job.ftpUser, passwd=job.ftpPassword) if job.ftpFolder is not None: ftp.cwd(job.ftpFolder) ftapr = FTPArchiveProgress(int(totalSize), task) with open(zip_file, 'rb') as rdf_file: ftp.storbinary('STOR RenderedDataset.zip', rdf_file, blocksize=1024, callback=ftapr.updateProgress) ftp.quit() QgsMessageLog.logMessage( 'Uploaded rendered dataset archive RenderedDataset.zip with {count} tiles. You can now unzip it.' .format(count=realtiles), CATEGORY, Qgis.Info) elif job.ftpUpload: QgsMessageLog.logMessage( 'Tiled and uploaded dataset with {count} tiles to ftp server'. format(count=realtiles), CATEGORY, Qgis.Info) else: QgsMessageLog.logMessage( 'Tiled dataset with {count} tiles'.format(count=realtiles), CATEGORY, Qgis.Info) # Clean up if cleanup: os.remove(org_vrt) os.remove(vrt) os.remove(color_ramp_file) os.remove(terrarium_tile) if job.ftpUpload: shutil.rmtree(temp_folder) QgsMessageLog.logMessage('Cleaned up temp files', CATEGORY, Qgis.Info) return [realtiles, time_start] except Exception as e: QgsMessageLog.logMessage('Error: ' + str(e), CATEGORY, Qgis.Info) return None
class FTPClient(object): """Class FTPClient """ _mh = None _client = None _secured = None _host = None _port = None _user = None _passw = None _path = None _verbose = None _is_connected = None def __init__(self, secured=False, verbose=False): """Class constructor Called when the object is initialized Args: secured (bool): secured FTP verbose (bool): verbose mode """ self._mh = MasterHead.get_head() self._secured = secured if (not self._secured): self._client = FTP() else: if (not(version_info[0] == 2 and version_info[1] == 6)): self._client = FTP_TLS() else: raise NotImplementedError( 'Secured mode is not supported for Python 2.6') self._verbose = verbose if (self._verbose): self._client.set_debuglevel(2) @property def client(self): """ FTP client property getter """ return self._client @property def secured(self): """ secured protocol mode property getter """ return self._secured @property def host(self): """ server host property getter """ return self._host @property def port(self): """ server port property getter """ return self._port @property def user(self): """ username property getter """ return self._user @property def passw(self): """ user password property getter """ return self._passw @property def path(self): """ remote path property getter """ return self._path @property def verbose(self): """ verbose mode property getter """ return self._verbose @property def is_connected(self): """ is_connected property getter """ return self._is_connected def connect(self, host, port=21, user=None, passw=None, path='/', timeout=10): """Method connects to server Args: host (str): server host port (int): server port, default protocol port user (str): username passw (str): password path (str): server path timeout (int): timeout Returns: bool: result Raises: event: ftp_before_connect event: ftp_after_connect """ try: message = '{0}/{1}@{2}:{3}{4} timeout:{5}'.format( user, passw, host, port, path, timeout) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_connecting', message), self._mh.fromhere()) ev = event.Event( 'ftp_before_connect', host, port, user, passw, path, timeout) if (self._mh.fire_event(ev) > 0): host = ev.argv(0) port = ev.argv(1) user = ev.argv(2) passw = ev.argv(3) path = ev.argv(4) timeout = ev.argv(5) self._host = host self._port = port self._user = user self._passw = passw if (ev.will_run_default()): self._client.connect(self._host, self._port, timeout=timeout) if (self._user != None): self._client.login(self._user, self._passw) if (self._secured): self._client.prot_p() self._is_connected = True self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_connected'), self._mh.fromhere()) if (path != None): self.change_dir(path) ev = event.Event('ftp_after_connect') self._mh.fire_event(ev) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def disconnect(self): """Method disconnects from server Args: none Returns: bool: result """ try: if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False else: self._client.quit() self._is_connected = False self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_disconnected'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def list_dir(self): """Method lists remote working directory Args: none Returns: list: names """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_list_dir', self._path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False names = self._client.nlst() if ('.' in names): names.remove('.') if ('..' in names): names.remove('..') return names except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return None def change_dir(self, path): """Method changes remote working directory Args: path (str): new remote path Returns: bool: result Raises: event: ftp_before_change_dir """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_change_dir', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_change_dir', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.cwd(path) self._path = self._client.pwd() self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_cur_dir', self._path), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def download_file(self, remote_path, local_path=None): """Method downloads file from server Args: remote_path (str): remote path local_path (str): local path, default ./filename Returns: bool: result Raises: event: ftp_before_download_file event: ftp_after_download_file """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_downloading_file', remote_path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event( 'ftp_before_download_file', remote_path, local_path) if (self._mh.fire_event(ev) > 0): remote_path = ev.argv(0) local_path = ev.argv(1) if (local_path != None and not path.exists(local_path)): self._mh.demsg('htk_on_error', self._mh._trn.msg( 'htk_ftp_unknown_dir', local_path), self._mh.fromhere()) return False filename = remote_path.split('/')[-1] lpath = filename if (local_path == None) else path.join( local_path, filename) if (ev.will_run_default()): with open(lpath, 'wb') as f: self._client.retrbinary('RETR ' + remote_path, f.write) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_file_downloaded'), self._mh.fromhere()) ev = event.Event('ftp_after_download_file') self._mh.fire_event(ev) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) if (path.exists(lpath)): remove(lpath) return False def upload_file(self, local_path, remote_path=None): """Method uploads file to server Args: local_path (str): local path remote_path (str): remote path, default ./filename Returns: bool: result Raises: event: ftp_before_upload_file event: ftp_after_upload_file """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_uploading_file', local_path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_upload_file', local_path, remote_path) if (self._mh.fire_event(ev) > 0): local_path = ev.argv(0) remote_path = ev.argv(1) if (not(path.exists(local_path) or path.exists(path.relpath(local_path)))): self._mh.demsg('htk_on_error', self._mh._trn.msg( 'htk_ftp_unknown_file', local_path), self._mh.fromhere()) return False filename = local_path.split('/')[-1] rpath = filename if (remote_path == None) else path.join( remote_path, filename) if (ev.will_run_default()): with open(local_path, 'rb') as f: self._client.storbinary('STOR ' + rpath, f) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_file_uploaded'), self._mh.fromhere()) ev = event.Event('ftp_after_upload_file') self._mh.fire_event(ev) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def delete_file(self, path): """Method deletes file from server Args: path (str): remote path Returns: bool: result Raises: event: ftp_before_delete_file """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_deleting_file', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_delete_file', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.delete(path) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_file_deleted'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def make_dir(self, path): """Method makes directory on server Args: path (str): remote path Returns: bool: result Raises: event: ftp_before_make_dir """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_making_dir', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_make_dir', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.mkd(path) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_dir_made'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False def remove_dir(self, path): """Method removes directory from server Args: path (str): remote path Returns: bool: result Raises: event: ftp_before_remove_dir """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_removing_dir', path), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_ftp_not_connected'), self._mh.fromhere()) return False ev = event.Event('ftp_before_remove_dir', path) if (self._mh.fire_event(ev) > 0): path = ev.argv(0) if (ev.will_run_default()): self._client.rmd(path) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_dir_removed'), self._mh.fromhere()) return True except all_errors as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False
class FTPSession(object): """ Attempt to create some robustness and performance to FTPing """ def __init__(self, server, username, password, tmpdir='/tmp', timeout=60): """Build a FTP session """ self.conn = None self.server = server self.username = username self.password = password self.tmpdir = tmpdir self.timeout = timeout def _connect(self): """Connect to FTP server """ if self.conn is not None: return logging.debug('Creating new connection to server %s', self.server) not_connected = True attempt = 1 while not_connected and attempt < 6: try: self.conn = FTP_TLS(self.server, timeout=self.timeout) self.conn.login(self.username, self.password) self.conn.prot_p() not_connected = False except Exception as exp: logging.debug(exp) time.sleep(5) self.close() attempt += 1 if not_connected is True: raise Exception("Failed to make FTP connection after 5 tries!") def _reconnect(self): """ First attempt to shut down connection and then reconnect """ logging.debug('_reconnect() was called...') try: self.conn.quit() self.conn.close() except: pass finally: self.conn = None self._connect() def _put(self, path, localfn, remotefn): """ """ self.chdir(path) sz = os.path.getsize(localfn) if sz > 14000000000: # Step 1 Split this big file into 14GB chunks, each file will have # suffix .aa then .ab then .ac etc basefn = os.path.basename(localfn) cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir, basefn) subprocess.call(cmd, shell=True, stderr=subprocess.PIPE) files = glob.glob("%s/%s.??" % (self.tmpdir, basefn)) for filename in files: suffix = filename.split(".")[-1] self.conn.storbinary('STOR %s.%s' % (remotefn, suffix), open(filename)) os.unlink(filename) else: logging.debug("_put '%s' to '%s'", localfn, remotefn) self.conn.storbinary('STOR %s' % (remotefn, ), open(localfn)) return True def close(self): """ Good bye """ try: self.conn.quit() self.conn.close() except: pass finally: self.conn = None def chdir(self, path): if self.pwd() == path.rstrip("/"): return self.conn.cwd("/") for dirname in path.split("/"): if dirname == '': continue bah = [] self.conn.retrlines('NLST', bah.append) if dirname not in bah: logging.debug("Creating directory '%s'", dirname) self.conn.mkd(dirname) logging.debug("Changing to directory '%s'", dirname) self.conn.cwd(dirname) def pwd(self): """ Low friction function to get connectivity """ self._connect() pwd = exponential_backoff(self.conn.pwd) if pwd is None: self._reconnect() pwd = exponential_backoff(self.conn.pwd) logging.debug("pwd() is currently '%s'", pwd) return pwd def put_file(self, path, localfn, remotefn): """ Put the File """ res = exponential_backoff(self._put, path, localfn, remotefn) if not res: self._reconnect() res = exponential_backoff(self._put, path, localfn, remotefn) if not res: logging.error("Double Failure to upload filename: '%s'", localfn) return False return True def put_files(self, path, localfns, remotefns): """ Put the File """ res = [] for localfn, remotefn in zip(localfns, remotefns): res.append(self.put_file(path, localfn, remotefn)) return res
#picamera.resolution = (1024, 768) # こちらは554KBで済む picamera.resolution = (800, 600) # こちらは●●●KBで済む #picamera.start_preview() # Camera warm-up time、Whiteバランスをとるための猶予時間。これがないと色が青白くて使い物にならない time.sleep(2) file_name = file_name+'.jpg' picamera.capture(dir_path+'/'+file_name) print "Now start Upload." _ftps = FTP_TLS(userID) _ftps.set_debuglevel(1) # デバッグログをリアルタイムで確認 _ftps.login(archive_server,pw) _file = open(dir_path+'/'+file_name, 'rb') #target file. 次のステップでアップロード成功したら削除した方がよいのではないか? #SD Memoryがパンクする恐れがあるので、次のステップでアップロードが成功したらファイルは削除するように、改良しましょう。 _ftps.cwd(put_directory) try: _ftps.cwd(dir_name) #target_dir 撮影データは日付付きのフォルダ内に納められる。 except Exception as e: #ディレクトリがなければ '550 Failed to change directory.' が帰ってくる。 #もし、"550"があれば...まだその日のフォルダはないので、新規作成 ## if str(e).count('550') : 文字列が含まれるかどうかの判定はこの関数を使ってもよいが、次の書き方が分かりやすい。 if '550' in str(e): _ftps.mkd(dir_name) _ftps.cwd(dir_name) _ftps.storbinary('STOR ' + file_name, _file) _file.close() _ftps.quit()
class ServerWatcher(Watcher): downloadProgress = Signal((int, int,)) uploadProgress = Signal((int, int,)) # Si added: textStatus = Signal((str,)) fileEvent = Signal((str,)) fileEventCompleted = Signal() loginCompleted = Signal((bool, str,)) badFilenameFound = Signal((str,)) LOCATION = 'server' TEST_FILE = 'iqbox.test' def __init__(self, host, ssl, parent=None): """ Initializes parent class and attributes. Decides whether to use `FTP_TLS` or `FTP` based on the `ssl` param. :param host: Location of the FTP server :param ssl: Tells whether the FTP needs to support TLS or not :param parent: Reference to a `QObject` instance a parent """ super(ServerWatcher, self).__init__(parent) self.interval = 5000 self.localdir = '' self.deleteQueue = [] self.downloadQueue = [] self.uploadQueue = [] self.warnedNames = [] self.ftp = None self.useSSL = ssl self.host = host self.preemptiveCheck = False self.preemptiveActions = [] self.testFile = 'iqbox.test' @property def currentdir(self): """Returns the current working directory at the server""" return self.ftp.pwd() def setLocalDir(self, localdir): """ Sets the local directory used to stored all downloaded files. Creates the directory if needed. :param localdir: Absolute path to local directory """ self.localdir = localdir if not os.path.exists(self.localdir): os.makedirs(self.localdir) @pause_timer @Slot() def checkout(self): """ Recursively checks out all files on the server. Returns a dictionary of files on the server with their last modified date. :param download: Indicates whether or not the files should be downloaded """ # Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues. # These tasks are done in queues to make sure all FTP commands # are done sequentially, in the same thread. self.deleteAll() self.uploadAll() self.downloadAll() # Handy list to keep track of the checkout process. # This list contain absolute paths only. checked_dirs = list() # Sets '/' as initial directory and initializes `downloading_dir` self.ftp.cwd('/') downloading_dir = self.currentdir check_date = dt.utcnow() sidirlist = list() root_cached = False fileC = 0 while True: # Gets the list of sub directories and files inside the # current directory `downloading_dir`. self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...') if root_cached and downloading_dir == '/': dir_subdirs = saved_root_dirs dirfiles = saved_root_files else: dir_subdirs = self.getDirs(downloading_dir) if downloading_dir == '/': saved_root_dirs = dir_subdirs # sidirlist.extend(dir_subdirs) self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...') dirfiles = self.getFiles(downloading_dir) if downloading_dir == '/': saved_root_files = dirfiles root_cached = True # Leading '/' in `downloading_dir` breaks the `os.path.join` call localdir = os.path.join(self.localdir, downloading_dir[1:]) if not os.path.exists(localdir): # Creates the directory if it doesn't already exists. os.makedirs(localdir) for file_ in dirfiles: # `serverpath` is the absolute path of the file on the server, # download it only if it hasn't been already downloaded serverpath = os.path.join(downloading_dir, file_) serverpath = QDir.fromNativeSeparators(serverpath) server_file = File.fromPath(serverpath) self.textStatus.emit('Scanning remote file... '+serverpath+'...') # How do we know if we should check this server file? # We see if the date last checked is the check start time. if server_file.last_checked_server != check_date: # Do this process only once per file # Added by Simon # Give feedback on scanning of files. fileC += 1 if fileC % 1 == 2: self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.') # STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB? just_added = not server_file.inserver # STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT # Example: If it's a temporary file, or a Unix file with a name we don't support. if just_added: filename = os.path.basename(serverpath) if platform.system() == 'Windows': badName = False for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']: if chr in filename: badName = True break if badName: if filename not in self.warnedNames: self.warnedNames.append(filename) self.badFilenameFound.emit(filename) continue # STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE lastmdate = server_file.servermdate # STEP: SAVE THE MOD DATE TO A VARIABLE # Now we get the last mod time. # We expect this to work fine since this file # was found on the server servermdate = self.lastModified(serverpath) # STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS. server_file.inserver = True # STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME server_file.last_checked_server = check_date # STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT server_file.servermdate = servermdate # STEP: SAVE THIS CHANGE TO THE DATABASE server_file.session.commit() delta = 0 if server_file.inlocal: delta = server_file.timeDiff() # Emit the signals after the attributes has been set and committed if just_added is True: self.fileAdded.emit(ServerWatcher.LOCATION, serverpath) elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE: self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False) #END FOR self.textStatus.emit('Remote scan- Finding next folder...') dir_ready = True for dir_ in dir_subdirs: # `dirpath` is the absolute path of the subdirectory on the server, dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_)) # `downloading_dir` is ready only when all its subdirectory are on the # `checked_dirs` list. if dirpath not in checked_dirs: # Found one subdirectory that is not on `checked_dirs`, # will process it in the next iteration. downloading_dir = dirpath dir_ready = False break if dir_ready is True: # All subdirectories of `downloading_dir` are already in `checked_dirs` if downloading_dir == '/': # All directories ready and at '/', means checkout is complete # So, exit the main While loop!! break else: # Not at '/'. Current directory is ready so is appended to `checked_dirs` # Back one directory to find directories that are not in `checked_dirs` checked_dirs.append(downloading_dir) downloading_dir = os.path.dirname(downloading_dir) self.textStatus.emit('Remote scan- Found Folder...') ##### END OF WHILE ################ ################################################################### # Deleted files are the ones whose `last_checked_server` attribute # didn't get updated in the recursive run. session = Session() deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True) for file_ in deleted: self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path) # Wraps up the checkout process, commits to the database. session.commit() @Slot() def onLogin(self, username, passwd): ok = True msg = '' error_msg = 'Login failed.' try: if not self.ftp: self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(self.host) loginResponse = self.ftp.login(username, passwd) except socket.gaierror: self.ftp = None ok = False msg = 'Server address could not be found.' except (error_perm, error_reply): info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) ok = False msg = error_msg else: if '230' in loginResponse: ok = True else: ok = False msg = error_msg if ok: # Logged in. Now let's do compability tests. if not self.testPermissions(): # User doesn't have write permissions, don't bother doing next test. ok = False msg = 'It seems like you do not have write access to this server.' else: # Permissions test passed, now let's test MFMT for timestamp modification. if not self.testMFMT(): ok = False msg = 'This server does not support timestamp modification\n \ need by this application.' self.loginCompleted.emit(ok, msg) def getFiles(self, path): """ This method simply wraps the `nlst` method with an exception handler, and returns an empty list in case an exception is caught. :param path: Relative or absolute path on the server """ try: nlst = self.ftp.nlst(path) dirs = self.getDirs(path) # Files are items in nlst that are not in dirs files = [item for item in nlst if os.path.basename(item) not in dirs] return files except: print 'Exception in ServerWatcher.getDirs' info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) return [] def getDirs(self, path): """ Retrieves a list of the directories inside `path`, uses `retrlines` and the LIST command to retrieve the items. :param path: Relative or absolute path on the server """ dirs = list() def handleLine(line): """ Recieves a line from the LIST command. This function is meant to be used as callback for the `retrlines` method. :params line: Line from the LIST command """ if line.startswith('d'): # Only lines starting with 'd' are directories # Parse the directory out of the line; lines look like: # 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname' dirname = line[55:].strip() if dirname != '.' and dirname != '..': # Ignoring '.' and '..' entries dirs.append(dirname) try: self.ftp.retrlines('LIST %s' % path, handleLine) return dirs except: print 'Exception in ServerWatcher.getDirs' info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) return [] @upload_test def testPermissions(self): # For interface purposes. upload_test takes care of everything. return True @upload_test def testMFMT(self): # Absurd date to test whether the change really happened. time = dt.utcfromtimestamp(100000000) try: self.setLastModified(self.testFile, time) otherTime = self.lastModified(self.testFile) diff = (time - otherTime).total_seconds() if abs(diff) < 2: # Let's give it a 2 seconds tolerance. mdtm = True else: mdtm = False except (ValueError, error_reply, error_perm): info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) mdtm = False return mdtm @Slot(str) def onDelete(self, filename): self.deleteQueue.append(filename) def deleteNext(self): if len(self.deleteQueue) > 0: next = self.deleteQueue.pop(0) self.deleteFile(next) def deleteAll(self): for filename in self.deleteQueue: self.deleteFile(filename) self.deleteQueue = [] @Slot(str) def deleteFile(self, filename): """ Deletes the file `filename` to the server :param filename: Absolute or relative path to the file """ try: print 'Deleting %s' % filename self.ftp.delete(filename) return True except (error_reply, error_perm): print 'Error deleting %s' % filename return False self.fileEventCompleted.emit() @Slot(str) def onDownload(self, filename): self.downloadQueue.append(filename) def downloadNext(self): if len(self.downloadQueue) > 0: next = self.downloadQueue.pop(0) self.downloadFile(next) def downloadAll(self): for filename in self.downloadQueue: self.downloadFile(filename) self.downloadQueue = [] @Slot(str, str) def downloadFile(self, filename, localpath=None): """ Performs a binary download to the file `filename` located on the server. `filename` parameter can be either absolute or relative, though it can fail for relative paths if the current directory is not appropiate. :param filename: Relative or absolute path to the file :param localpath: Absolute local path where the file will be saved """ def handleChunk(chunk): """ Receives chuncks of data downloaded from the server. This function is meant to be used as callback for the `retrbinary` method. :params chunk: Chunk of downloaded bytes to be written into the file """ # Simply writes the received data into the file `self.downloading` self.downloading.write(chunk) self.download_progress += len(chunk) self.downloadProgress.emit(self.download_size, self.download_progress) if localpath is None: localpath = self.localFromServer(filename) localdir = os.path.dirname(localpath) if not os.path.exists(localdir): # Creates the directory if it doesn't already exists. os.makedirs(localdir) print 'Downloading: %s to %s' % (filename, localpath) try: with open(localpath, 'wb') as f: # Opens the file at `localname` which will hold the downloaded file. # Object attributes regarding download status are updated accordingly. self.fileEvent.emit(filename) self.downloading = f self.download_progress = 0 self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1]) self.ftp.retrbinary('RETR %s' % filename, handleChunk) print 'Download finished' # Let's set the same modified time to that on the server. with File.fromPath(filename) as downloadedfile: mdate = LocalWatcher.lastModified(localpath) downloadedfile.localmdate = mdate downloadedfile.servermdate = mdate self.setLastModified(filename, mdate) downloaded = True except (IOError, OSError): downloaded = False self.ioError.emit(localpath) except (error_reply, error_perm) as ftperr: print 'Error downloading %s, %s' % (filename, ftperr) downloaded = False # TODO: Sometimes the file doesn't complete properly. # in that case we maybe shouldn't call this? self.fileEventCompleted.emit() return downloaded @Slot(str) def onUpload(self, filename): self.uploadQueue.append(filename) def uploadNext(self): if len(self.uploadQueue) > 0: next = self.uploadQueue.pop(0) self.uploadFile(next) def uploadAll(self): for filename in self.uploadQueue: self.uploadFile(filename) self.uploadQueue = [] @Slot(str) def uploadFile(self, filename): """ Uploads the file `filename` to the server, creating the needed directories. :param filename: Absolute or relative path to the file """ def handle(buf): """This function is meant to be used as callback for the `storbinary` method.""" self.upload_progress += 1024 self.uploadProgress.emit(self.upload_size, self.upload_progress) # Creates the directory where the file will be uploaded to self.mkpath(os.path.dirname(filename)) localpath = self.localFromServer(filename) print 'Uploading %s to %s' % (localpath, filename) try: # Uploads file and updates its modified date in the server # to match the date in the local filesystem. self.upload_progress = 0 self.upload_size = os.path.getsize(localpath) self.fileEvent.emit(localpath) self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'), 1024, handle) print 'Upload finished' with File.fromPath(filename) as uploaded: modified = uploaded.localmdate uploaded.servermdate = modified self.setLastModified(filename, modified) uploaded = True except (IOError, OSError): uploaded = False self.ioError.emit(localpath) except (error_reply, error_perm, OSError) as err: print 'Error uploading %s, %s' % (filename, err) uploaded = False # TODO: Sometimes the file doesn't complete properly. # in that case we maybe shouldn't call this? self.fileEventCompleted.emit() return uploaded def lastModified(self, filename): """ Uses the MDTM FTP command to find the last modified timestamp of the file `filename`. Returns a `datetime.datetime` object in UTC representing the file's last modified date and time. :param filename: Relative or absolute path to the file """ timestamp = self.ftp.sendcmd('MDTM %s' % filename) if '213 ' not in timestamp: # Second chance was found to be needed in some cases. timestamp = self.ftp.sendcmd('MDTM %s' % filename) timestamp = timestamp.split(' ')[-1] dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S' try: mtime = dt.strptime(timestamp, dateformat) except ValueError: mtime = dt.utcnow() return mtime def setLastModified(self, serverpath, newtime): """ Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the file `serverpath` on the server. :param serverpath: Relative or absolute path to the file :param newtime: datedatime object holding the required time """ cmds = ['MFMT', 'MDTM'] for cmd in cmds: try: self.ftp.sendcmd( '%s %s %s' % (cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath)) return except (error_perm, error_reply) as e: if cmd == cmds[len(cmds) - 1]: # If is the last comand, re-raise the exception, else # keep trying. raise e else: continue def mkpath(self, path): """ Creates the path `path` on the server by recursively created folders, if needed. :param path: Absolute path on the server to be created """ try: self.ftp.cwd(path) except error_perm: # `cwd` call failed. Need to create some folders make_dir = '/' steps = path.split('/') for step in steps: if len(step) == 0: continue make_dir += '%s/' % step try: self.ftp.mkd(make_dir) except error_perm: # Probably already exists continue else: # `cwd` call succeed. No need to create # any folders self.ftp.cwd('/') return @Slot(str, str) def added(self, location, serverpath): super(ServerWatcher, self).added(location, serverpath) def actionFromPath(serverpath): f = File() fileExistsOnServer = True try: f.servermdate = self.lastModified(serverpath) except error_perm: fileExistsOnServer = False f.servermdate = 0 f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath)) diff = f.timeDiff() action = None if abs(diff) > Watcher.TOLERANCE: if not fileExistsOnServer or diff > 0: action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION) else: action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION) return action if self.preemptiveCheck: if location == ServerWatcher.LOCATION: localpath = self.localFromServer(serverpath) if not os.path.exists(localpath): action = FileAction(serverpath, FileAction.DOWNLOAD, ServerWatcher.LOCATION) self.preemptiveActions.append(action) else: action = actionFromPath(serverpath) if action is not None: self.preemptiveActions.append(action) elif location == LocalWatcher.LOCATION: try: self.ftp.sendcmd('SIZE %s' % serverpath) except (error_reply, error_perm): exists = False else: exists = True if not exists: action = FileAction(serverpath, FileAction.UPLOAD, LocalWatcher.LOCATION) self.preemptiveActions.append(action) else: action = actionFromPath(serverpath) if action is not None: self.preemptiveActions.append(action) @Slot(str, str) def changed(self, location, serverpath): super(ServerWatcher, self).changed(location, serverpath) @Slot(str, str) def deleted(self, location, serverpath): super(ServerWatcher, self).deleted(location, serverpath) with File.fromPath(serverpath) as deleted: deleted.inserver = False
import sys,json server = json.loads(sys.argv[1]) try: if server['secure'] == True: from ftplib import FTP_TLS ftp = FTP_TLS() else: from ftplib import FTP ftp = FTP() ftp.connect(server['host'], server['port']) ftp.login(server['user'], server['pass']) try: ftp.storbinary('STOR %s' % sys.argv[2], open(sys.argv[3], 'rb')) except: strsplt = sys.argv[2][1:].split('/') strsplt = strsplt[:-1] wdir = "/"+"/".join(strsplt) ftp.mkd(wdir) ftp.storbinary('STOR %s' % sys.argv[2], open(sys.argv[3], 'rb')) ftp.quit() except: sys.exit(1)
class ServerWatcher(Watcher): downloadProgress = Signal(( int, int, )) uploadProgress = Signal(( int, int, )) # Si added: textStatus = Signal((str, )) fileEvent = Signal((str, )) fileEventCompleted = Signal() loginCompleted = Signal(( bool, str, )) badFilenameFound = Signal((str, )) LOCATION = 'server' TEST_FILE = 'iqbox.test' def __init__(self, host, ssl, parent=None): """ Initializes parent class and attributes. Decides whether to use `FTP_TLS` or `FTP` based on the `ssl` param. :param host: Location of the FTP server :param ssl: Tells whether the FTP needs to support TLS or not :param parent: Reference to a `QObject` instance a parent """ super(ServerWatcher, self).__init__(parent) self.interval = 5000 self.localdir = '' self.deleteQueue = [] self.downloadQueue = [] self.uploadQueue = [] self.warnedNames = [] self.ftp = None self.useSSL = ssl self.host = host self.preemptiveCheck = False self.preemptiveActions = [] self.testFile = 'iqbox.test' @property def currentdir(self): """Returns the current working directory at the server""" return self.ftp.pwd() def setLocalDir(self, localdir): """ Sets the local directory used to stored all downloaded files. Creates the directory if needed. :param localdir: Absolute path to local directory """ self.localdir = localdir if not os.path.exists(self.localdir): os.makedirs(self.localdir) @pause_timer @Slot() def checkout(self): """ Recursively checks out all files on the server. Returns a dictionary of files on the server with their last modified date. :param download: Indicates whether or not the files should be downloaded """ # Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues. # These tasks are done in queues to make sure all FTP commands # are done sequentially, in the same thread. self.deleteAll() self.uploadAll() self.downloadAll() # Handy list to keep track of the checkout process. # This list contain absolute paths only. checked_dirs = list() # Sets '/' as initial directory and initializes `downloading_dir` self.ftp.cwd('/') downloading_dir = self.currentdir check_date = dt.utcnow() sidirlist = list() root_cached = False fileC = 0 while True: # Gets the list of sub directories and files inside the # current directory `downloading_dir`. self.textStatus.emit('Remote scan- Downloading folder list of ' + downloading_dir + '...') if root_cached and downloading_dir == '/': dir_subdirs = saved_root_dirs dirfiles = saved_root_files else: dir_subdirs = self.getDirs(downloading_dir) if downloading_dir == '/': saved_root_dirs = dir_subdirs # sidirlist.extend(dir_subdirs) self.textStatus.emit( 'Remote scan- Downloading files list of ' + downloading_dir + '...') dirfiles = self.getFiles(downloading_dir) if downloading_dir == '/': saved_root_files = dirfiles root_cached = True # Leading '/' in `downloading_dir` breaks the `os.path.join` call localdir = os.path.join(self.localdir, downloading_dir[1:]) if not os.path.exists(localdir): # Creates the directory if it doesn't already exists. os.makedirs(localdir) for file_ in dirfiles: # `serverpath` is the absolute path of the file on the server, # download it only if it hasn't been already downloaded serverpath = os.path.join(downloading_dir, file_) serverpath = QDir.fromNativeSeparators(serverpath) server_file = File.fromPath(serverpath) self.textStatus.emit('Scanning remote file... ' + serverpath + '...') # How do we know if we should check this server file? # We see if the date last checked is the check start time. if server_file.last_checked_server != check_date: # Do this process only once per file # Added by Simon # Give feedback on scanning of files. fileC += 1 if fileC % 1 == 2: self.textStatus.emit( 'Scanning remote files for changes, ' + str(fileC) + ' files scanned.') # STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB? just_added = not server_file.inserver # STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT # Example: If it's a temporary file, or a Unix file with a name we don't support. if just_added: filename = os.path.basename(serverpath) if platform.system() == 'Windows': badName = False for chr in [ '\\', '/', ':', '?', '"', '<', '>', '|' ]: if chr in filename: badName = True break if badName: if filename not in self.warnedNames: self.warnedNames.append(filename) self.badFilenameFound.emit(filename) continue # STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE lastmdate = server_file.servermdate # STEP: SAVE THE MOD DATE TO A VARIABLE # Now we get the last mod time. # We expect this to work fine since this file # was found on the server servermdate = self.lastModified(serverpath) # STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS. server_file.inserver = True # STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME server_file.last_checked_server = check_date # STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT server_file.servermdate = servermdate # STEP: SAVE THIS CHANGE TO THE DATABASE server_file.session.commit() delta = 0 if server_file.inlocal: delta = server_file.timeDiff() # Emit the signals after the attributes has been set and committed if just_added is True: self.fileAdded.emit(ServerWatcher.LOCATION, serverpath) elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE: self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False) #END FOR self.textStatus.emit('Remote scan- Finding next folder...') dir_ready = True for dir_ in dir_subdirs: # `dirpath` is the absolute path of the subdirectory on the server, dirpath = QDir.fromNativeSeparators( os.path.join(downloading_dir, dir_)) # `downloading_dir` is ready only when all its subdirectory are on the # `checked_dirs` list. if dirpath not in checked_dirs: # Found one subdirectory that is not on `checked_dirs`, # will process it in the next iteration. downloading_dir = dirpath dir_ready = False break if dir_ready is True: # All subdirectories of `downloading_dir` are already in `checked_dirs` if downloading_dir == '/': # All directories ready and at '/', means checkout is complete # So, exit the main While loop!! break else: # Not at '/'. Current directory is ready so is appended to `checked_dirs` # Back one directory to find directories that are not in `checked_dirs` checked_dirs.append(downloading_dir) downloading_dir = os.path.dirname(downloading_dir) self.textStatus.emit('Remote scan- Found Folder...') ##### END OF WHILE ################ ################################################################### # Deleted files are the ones whose `last_checked_server` attribute # didn't get updated in the recursive run. session = Session() deleted = session.query(File).filter( File.last_checked_server < check_date).filter( File.inserver == True) for file_ in deleted: self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path) # Wraps up the checkout process, commits to the database. session.commit() @Slot() def onLogin(self, username, passwd): ok = True msg = '' error_msg = 'Login failed.' try: if not self.ftp: self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP( self.host) loginResponse = self.ftp.login(username, passwd) except socket.gaierror: self.ftp = None ok = False msg = 'Server address could not be found.' except (error_perm, error_reply): info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) ok = False msg = error_msg else: if '230' in loginResponse: ok = True else: ok = False msg = error_msg if ok: # Logged in. Now let's do compability tests. if not self.testPermissions(): # User doesn't have write permissions, don't bother doing next test. ok = False msg = 'It seems like you do not have write access to this server.' else: # Permissions test passed, now let's test MFMT for timestamp modification. if not self.testMFMT(): ok = False msg = 'This server does not support timestamp modification\n \ need by this application.' self.loginCompleted.emit(ok, msg) def getFiles(self, path): """ This method simply wraps the `nlst` method with an exception handler, and returns an empty list in case an exception is caught. :param path: Relative or absolute path on the server """ try: nlst = self.ftp.nlst(path) dirs = self.getDirs(path) # Files are items in nlst that are not in dirs files = [ item for item in nlst if os.path.basename(item) not in dirs ] return files except: print 'Exception in ServerWatcher.getDirs' info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) return [] def getDirs(self, path): """ Retrieves a list of the directories inside `path`, uses `retrlines` and the LIST command to retrieve the items. :param path: Relative or absolute path on the server """ dirs = list() def handleLine(line): """ Recieves a line from the LIST command. This function is meant to be used as callback for the `retrlines` method. :params line: Line from the LIST command """ if line.startswith('d'): # Only lines starting with 'd' are directories # Parse the directory out of the line; lines look like: # 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname' dirname = line[55:].strip() if dirname != '.' and dirname != '..': # Ignoring '.' and '..' entries dirs.append(dirname) try: self.ftp.retrlines('LIST %s' % path, handleLine) return dirs except: print 'Exception in ServerWatcher.getDirs' info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) return [] @upload_test def testPermissions(self): # For interface purposes. upload_test takes care of everything. return True @upload_test def testMFMT(self): # Absurd date to test whether the change really happened. time = dt.utcfromtimestamp(100000000) try: self.setLastModified(self.testFile, time) otherTime = self.lastModified(self.testFile) diff = (time - otherTime).total_seconds() if abs(diff) < 2: # Let's give it a 2 seconds tolerance. mdtm = True else: mdtm = False except (ValueError, error_reply, error_perm): info = traceback.format_exception(*sys.exc_info()) for i in info: sys.stderr.write(i) mdtm = False return mdtm @Slot(str) def onDelete(self, filename): self.deleteQueue.append(filename) def deleteNext(self): if len(self.deleteQueue) > 0: next = self.deleteQueue.pop(0) self.deleteFile(next) def deleteAll(self): for filename in self.deleteQueue: self.deleteFile(filename) self.deleteQueue = [] @Slot(str) def deleteFile(self, filename): """ Deletes the file `filename` to the server :param filename: Absolute or relative path to the file """ try: print 'Deleting %s' % filename self.ftp.delete(filename) return True except (error_reply, error_perm): print 'Error deleting %s' % filename return False self.fileEventCompleted.emit() @Slot(str) def onDownload(self, filename): self.downloadQueue.append(filename) def downloadNext(self): if len(self.downloadQueue) > 0: next = self.downloadQueue.pop(0) self.downloadFile(next) def downloadAll(self): for filename in self.downloadQueue: self.downloadFile(filename) self.downloadQueue = [] @Slot(str, str) def downloadFile(self, filename, localpath=None): """ Performs a binary download to the file `filename` located on the server. `filename` parameter can be either absolute or relative, though it can fail for relative paths if the current directory is not appropiate. :param filename: Relative or absolute path to the file :param localpath: Absolute local path where the file will be saved """ def handleChunk(chunk): """ Receives chuncks of data downloaded from the server. This function is meant to be used as callback for the `retrbinary` method. :params chunk: Chunk of downloaded bytes to be written into the file """ # Simply writes the received data into the file `self.downloading` self.downloading.write(chunk) self.download_progress += len(chunk) self.downloadProgress.emit(self.download_size, self.download_progress) if localpath is None: localpath = self.localFromServer(filename) localdir = os.path.dirname(localpath) if not os.path.exists(localdir): # Creates the directory if it doesn't already exists. os.makedirs(localdir) print 'Downloading: %s to %s' % (filename, localpath) try: with open(localpath, 'wb') as f: # Opens the file at `localname` which will hold the downloaded file. # Object attributes regarding download status are updated accordingly. self.fileEvent.emit(filename) self.downloading = f self.download_progress = 0 self.download_size = int( self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1]) self.ftp.retrbinary('RETR %s' % filename, handleChunk) print 'Download finished' # Let's set the same modified time to that on the server. with File.fromPath(filename) as downloadedfile: mdate = LocalWatcher.lastModified(localpath) downloadedfile.localmdate = mdate downloadedfile.servermdate = mdate self.setLastModified(filename, mdate) downloaded = True except (IOError, OSError): downloaded = False self.ioError.emit(localpath) except (error_reply, error_perm) as ftperr: print 'Error downloading %s, %s' % (filename, ftperr) downloaded = False # TODO: Sometimes the file doesn't complete properly. # in that case we maybe shouldn't call this? self.fileEventCompleted.emit() return downloaded @Slot(str) def onUpload(self, filename): self.uploadQueue.append(filename) def uploadNext(self): if len(self.uploadQueue) > 0: next = self.uploadQueue.pop(0) self.uploadFile(next) def uploadAll(self): for filename in self.uploadQueue: self.uploadFile(filename) self.uploadQueue = [] @Slot(str) def uploadFile(self, filename): """ Uploads the file `filename` to the server, creating the needed directories. :param filename: Absolute or relative path to the file """ def handle(buf): """This function is meant to be used as callback for the `storbinary` method.""" self.upload_progress += 1024 self.uploadProgress.emit(self.upload_size, self.upload_progress) # Creates the directory where the file will be uploaded to self.mkpath(os.path.dirname(filename)) localpath = self.localFromServer(filename) print 'Uploading %s to %s' % (localpath, filename) try: # Uploads file and updates its modified date in the server # to match the date in the local filesystem. self.upload_progress = 0 self.upload_size = os.path.getsize(localpath) self.fileEvent.emit(localpath) self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'), 1024, handle) print 'Upload finished' with File.fromPath(filename) as uploaded: modified = uploaded.localmdate uploaded.servermdate = modified self.setLastModified(filename, modified) uploaded = True except (IOError, OSError): uploaded = False self.ioError.emit(localpath) except (error_reply, error_perm, OSError) as err: print 'Error uploading %s, %s' % (filename, err) uploaded = False # TODO: Sometimes the file doesn't complete properly. # in that case we maybe shouldn't call this? self.fileEventCompleted.emit() return uploaded def lastModified(self, filename): """ Uses the MDTM FTP command to find the last modified timestamp of the file `filename`. Returns a `datetime.datetime` object in UTC representing the file's last modified date and time. :param filename: Relative or absolute path to the file """ timestamp = self.ftp.sendcmd('MDTM %s' % filename) if '213 ' not in timestamp: # Second chance was found to be needed in some cases. timestamp = self.ftp.sendcmd('MDTM %s' % filename) timestamp = timestamp.split(' ')[-1] dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S' try: mtime = dt.strptime(timestamp, dateformat) except ValueError: mtime = dt.utcnow() return mtime def setLastModified(self, serverpath, newtime): """ Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the file `serverpath` on the server. :param serverpath: Relative or absolute path to the file :param newtime: datedatime object holding the required time """ cmds = ['MFMT', 'MDTM'] for cmd in cmds: try: self.ftp.sendcmd( '%s %s %s' % (cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath)) return except (error_perm, error_reply) as e: if cmd == cmds[len(cmds) - 1]: # If is the last comand, re-raise the exception, else # keep trying. raise e else: continue def mkpath(self, path): """ Creates the path `path` on the server by recursively created folders, if needed. :param path: Absolute path on the server to be created """ try: self.ftp.cwd(path) except error_perm: # `cwd` call failed. Need to create some folders make_dir = '/' steps = path.split('/') for step in steps: if len(step) == 0: continue make_dir += '%s/' % step try: self.ftp.mkd(make_dir) except error_perm: # Probably already exists continue else: # `cwd` call succeed. No need to create # any folders self.ftp.cwd('/') return @Slot(str, str) def added(self, location, serverpath): super(ServerWatcher, self).added(location, serverpath) def actionFromPath(serverpath): f = File() fileExistsOnServer = True try: f.servermdate = self.lastModified(serverpath) except error_perm: fileExistsOnServer = False f.servermdate = 0 f.localmdate = LocalWatcher.lastModified( self.localFromServer(serverpath)) diff = f.timeDiff() action = None if abs(diff) > Watcher.TOLERANCE: if not fileExistsOnServer or diff > 0: action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION) else: action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION) return action if self.preemptiveCheck: if location == ServerWatcher.LOCATION: localpath = self.localFromServer(serverpath) if not os.path.exists(localpath): action = FileAction(serverpath, FileAction.DOWNLOAD, ServerWatcher.LOCATION) self.preemptiveActions.append(action) else: action = actionFromPath(serverpath) if action is not None: self.preemptiveActions.append(action) elif location == LocalWatcher.LOCATION: try: self.ftp.sendcmd('SIZE %s' % serverpath) except (error_reply, error_perm): exists = False else: exists = True if not exists: action = FileAction(serverpath, FileAction.UPLOAD, LocalWatcher.LOCATION) self.preemptiveActions.append(action) else: action = actionFromPath(serverpath) if action is not None: self.preemptiveActions.append(action) @Slot(str, str) def changed(self, location, serverpath): super(ServerWatcher, self).changed(location, serverpath) @Slot(str, str) def deleted(self, location, serverpath): super(ServerWatcher, self).deleted(location, serverpath) with File.fromPath(serverpath) as deleted: deleted.inserver = False
# ftps.retrlines('LIST') # list directory content securely # Open the upload directory ftps.cwd(FtpUploadDir) # Open or create 'year' folder try: # Open 'year' folder ftps.cwd(year) print("Opened '" + year + "' folder") except: print("Failed to open '" + year + "' folder.\nCreating new '" + year + "' folder...") try: # Create new 'year' folder ftps.mkd(year) # Open 'year' folder ftps.cwd(year) print("Created '" + year + "' folder!") except: print("Failed to create '" + year + "' folder.") sys.exit() # Open or create 'month' folder try: # Open 'month' folder ftps.cwd(month) print("Opened '" + month + "' folder") except: print("Failed to open '" + month + "' folder.\nCreating new '" + month + "' folder...")
except ftplib.error_perm as e: print('Ftp fail -> ', e) ftps.nlst() files_list = ftps.nlst() lg.info("List of FTP remote folders:") for filename in files_list: lg.debug(f'{filename} ') if folder in ftps.nlst( ): #check if 'wordpress-todayDate' exist inside '/home/wordpress' ftps.cwd(folder) #change into "wordpress-todayDate" directory lg.debug(f'{folder} ') else: ftps.mkd( folder ) #Create a new directory called "wordpress-todayDate" on the server. lg.debug(f'{folder} 2') ftps.cwd(folder) def placeFiles(): for file in os.listdir(wp_backup_path): localpath = os.path.join(wp_backup_path, file) if os.path.isfile(localpath): filesize = os.path.getsize(localpath) #print("STOR", file, localpath) with tqdm(unit='blocks', unit_scale=True, leave=False, miniters=1,
class FTPSession(object): """ Attempt to create some robustness and performance to FTPing """ def __init__(self, server, username, password, tmpdir="/tmp", timeout=60): """Build a FTP session """ self.conn = None self.server = server self.username = username self.password = password self.tmpdir = tmpdir self.timeout = timeout def _connect(self): """Connect to FTP server """ if self.conn is not None: return logging.debug("Creating new connection to server %s", self.server) not_connected = True attempt = 1 while not_connected and attempt < 6: try: self.conn = FTP_TLS(self.server, timeout=self.timeout) self.conn.login(self.username, self.password) self.conn.prot_p() not_connected = False except Exception as exp: logging.debug(exp) time.sleep(5) self.close() attempt += 1 if not_connected is True: raise Exception("Failed to make FTP connection after 5 tries!") def _reconnect(self): """ First attempt to shut down connection and then reconnect """ logging.debug("_reconnect() was called...") try: self.conn.quit() self.conn.close() except: pass finally: self.conn = None self._connect() def _put(self, path, localfn, remotefn): """ """ self.chdir(path) sz = os.path.getsize(localfn) if sz > 14000000000: # Step 1 Split this big file into 14GB chunks, each file will have # suffix .aa then .ab then .ac etc basefn = os.path.basename(localfn) cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir, basefn) subprocess.call(cmd, shell=True, stderr=subprocess.PIPE) files = glob.glob("%s/%s.??" % (self.tmpdir, basefn)) for filename in files: suffix = filename.split(".")[-1] self.conn.storbinary("STOR %s.%s" % (remotefn, suffix), open(filename)) os.unlink(filename) else: logging.debug("_put '%s' to '%s'", localfn, remotefn) self.conn.storbinary("STOR %s" % (remotefn,), open(localfn)) return True def close(self): """ Good bye """ try: self.conn.quit() self.conn.close() except: pass finally: self.conn = None def chdir(self, path): if self.pwd() == path.rstrip("/"): return self.conn.cwd("/") for dirname in path.split("/"): if dirname == "": continue bah = [] self.conn.retrlines("NLST", bah.append) if dirname not in bah: logging.debug("Creating directory '%s'", dirname) self.conn.mkd(dirname) logging.debug("Changing to directory '%s'", dirname) self.conn.cwd(dirname) def pwd(self): """ Low friction function to get connectivity """ self._connect() pwd = exponential_backoff(self.conn.pwd) if pwd is None: self._reconnect() pwd = exponential_backoff(self.conn.pwd) logging.debug("pwd() is currently '%s'", pwd) return pwd def put_file(self, path, localfn, remotefn): """ Put the File """ res = exponential_backoff(self._put, path, localfn, remotefn) if not res: self._reconnect() res = exponential_backoff(self._put, path, localfn, remotefn) if not res: logging.error("Double Failure to upload filename: '%s'", localfn) def put_files(self, path, localfns, remotefns): """ Put the File """ for localfn, remotefn in zip(localfns, remotefns): self.put_file(path, localfn, remotefn)
images_to_upload = [] if image_list: # Add first image images_to_upload.append(image_list[0]) if len(image_list) > 1: # Add last image (if more than 1 image) images_to_upload.append(image_list[len(image_list)-1]) # Download oldest & newest image for image in images_to_upload: print 'Downloading %s...' % image ftps.retrbinary('RETR %s' % image, open(image, 'wb').write) # Check if directory for old images exists, if not create it if 'old' not in directory_list: print 'Creating dir "old"...' ftps.mkd('old') # Move checked images to old for image in image_list: #date_str = image.split('_')[1].split('.')[0] #img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S') print 'Moving %s to "old"...' % image ftps.rename(image, 'old/%s' % image) # Disconnect from FTP server ftps.quit() def upload_image(filename): image_file = {'file': open(filename, 'rb')} print 'Uploading %s to GroupMe image service...' % filename r = requests.post('%s?access_token=%s' % (GROUPME_IMAGE_URL, TOKEN), files=image_file)
class FTPES: def __init__(self, host, port=None, username=None, password=None, remote_path=None, absolute_zipfile_path=None): """ This is a helper class to manage the FTP commands from the ftplib. @param host: The host for the connection. @type host: String @param port: The post for the connection. @type port: Integer @param username: The username for the connection. Leave blank to use "anonymous". @type username: String @param password: The password for the connection. Leave empty for none. @type password: String @param remote_path: The remote path of the server in which the zip file should be uploaded. If the path does not exists, it will be created (recursive). @type remote_path: String @param absolute_zipfile_path: The absolute LOCAL filepath of the zip file. @type absolute_zipfile_path: String """ self.ftps = None self._host = host self._port = port self._username = username self._password = password self._remote_path = remote_path self._absolute_zipfile_path = absolute_zipfile_path self._bytesWritten = 0; self._totalFileSize = os.path.getsize(self._absolute_zipfile_path) self._uploadCallback = None self._currentProgress = 0 # make the remote path relative if it isnt absolute or relative yet if self._remote_path is not None and self._remote_path.startswith( '.') is False and self._remote_path.startswith('/') is False: self._remote_path = './' + self._remote_path if self._username is None: self._username = '******' if self._port is None: self._port = 22 def connect(self): """ Try to connect to the FTP server. Raise an error if something went wrong. """ try: self.ftps = FTP_TLS() self.ftps.set_pasv(True) self.ftps.connect(self._host, self._port) except socket.gaierror: raise def login(self): """ Try to login in on the FTP server. Raise an error if something went wrong. """ try: self.ftps.login(self._username, self._password) self.ftps.prot_p() except ftplib.error_perm: raise def cwd(self): """ Try to switch the working directory on the FTP server (if set in the settings). If the path does not exist, it will be created. """ if self._remote_path is not None: try: self.ftps.cwd(self._remote_path) except ftplib.error_perm: self.create_directory_tree(self._remote_path) except IOError: raise def create_directory_tree(self, current_directory): """ Helper function to create the remote path. @param current_directory: The current working directory. @type current_directory: String """ if current_directory is not "": try: self.ftps.cwd(current_directory) except ftplib.error_perm: self.create_directory_tree("/".join(current_directory.split("/")[:-1])) self.ftps.mkd(current_directory) self.ftps.cwd(current_directory) except IOError: raise def upload(self, callback=None): """ The upload function. @param callback: The callback function for the upload progress. @type callback: Function """ self._uploadCallback = callback zipfile_to_upload = open(self._absolute_zipfile_path, 'rb') zipfile_basename = os.path.basename(self._absolute_zipfile_path) self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload, 1024, self.handle_upload_state) zipfile_to_upload.close() def handle_upload_state(self, block): """ The callback function for the upload progress. @param block: The StringIO of the current upload state @type block: StringIO """ self._bytesWritten += 1024 progress = math.floor((float(self._bytesWritten) / float(self._totalFileSize)) * 100) do_update = False if progress > self._currentProgress: do_update = True self._currentProgress = progress if self._uploadCallback is not None: self._uploadCallback(progress, self, do_update) def quit(self): """ Try to quit everything and close the session. """ self.ftps.quit() @property def upload_path(self): """ Returns the upload path of the FTP server. @return: The path of the uploaded file on the FTP server. @rtype: String """ tmp_remote_path = '' if self._remote_path is not None: tmp_remote_path = self._remote_path return os.path.join(tmp_remote_path, os.path.basename(self._absolute_zipfile_path))
bus.write_byte_data(I2C_ADDRESS, 0x09, relay_status) # GPIO.output(DENYLED, GPIO.LOW) # FTP if lpr_filepath != '': try: ftp = FTP_TLS('192.168.0.200') ftp.sendcmd('USER vlada') ftp.sendcmd('PASS raspberry2018') timestamp = "%04d%02d%02d" % (datetime.datetime.now().year, datetime.datetime.now().month, datetime.datetime.now().day) try: ftp.cwd('/PARKING_08_iz/' + timestamp) except Exception as ex: ftp.mkd('/PARKING_08_iz/' + timestamp) ftp.cwd('/PARKING_08_iz/' + timestamp) lpr_file = open(DIR_PATH + lpr_filepath, 'rb') print("To ftp: " + timestamp + '/' + ftp_file_case) ftp.storbinary('STOR ' + ftp_file_case, lpr_file) ftp.quit() ftp.close() except Exception as exc: displayMsg("FTP Error", "ftp_error.jpg") print("FTP error") lpr_filepath = '' continue lpr_filepath = ''
class FTPES(object): """ The FTPES class manage all ftplib commands. """ # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments def __init__(self, host, port=None, username=None, password=None, remote_path=None, absolute_zipfile_path=None): """ Initializer @param host: The host for the connection. @type host: String @param port: The post for the connection. @type port: Integer @param username: The username for the connection. Leave blank to use "anonymous". @type username: String @param password: The password for the connection. Leave empty for none. @type password: String @param remote_path: The remote path of the server in which the zip file should be uploaded. If the path does not exists, it will be created (recursive). @type remote_path: String @param absolute_zipfile_path: The absolute LOCAL filepath of the zip file. @type absolute_zipfile_path: String """ self.ftps = None self._host = host self._port = port self._username = username self._password = password self._remote_path = remote_path self._absolute_zipfile_path = absolute_zipfile_path self._bytes_written = 0 self._total_file_size = os.path.getsize(self._absolute_zipfile_path) self._upload_callback = None self._current_progress = 0 # make the remote path relative if it isnt absolute or relative yet if self._remote_path is not None and self._remote_path.startswith( '.') is False and self._remote_path.startswith('/') is False: self._remote_path = './' + self._remote_path if self._username is None: self._username = '******' if self._port is None: self._port = 22 # pylint: enable=too-many-instance-attributes # pylint: enable=too-many-arguments def connect(self): """ Try to connect to the FTP server. Raise an error if something went wrong. """ try: self.ftps = FTP_TLS() self.ftps.set_pasv(True) self.ftps.connect(self._host, self._port) except socket.gaierror: raise def login(self): """ Try to login in on the FTP server. Raise an error if something went wrong. """ try: self.ftps.login(self._username, self._password) self.ftps.prot_p() except ftplib.error_perm: raise def cwd(self): """ Try to switch the working directory on the FTP server (if set in the settings). If the path does not exist, it will be created. """ if self._remote_path is not None: try: self.ftps.cwd(self._remote_path) except ftplib.error_perm: self.create_directory_tree(self._remote_path) except IOError: raise def create_directory_tree(self, current_directory): """ Helper function to create the remote path. @param current_directory: The current working directory. @type current_directory: String """ if current_directory is not "": try: self.ftps.cwd(current_directory) except ftplib.error_perm: self.create_directory_tree("/".join( current_directory.split("/")[:-1])) self.ftps.mkd(current_directory) self.ftps.cwd(current_directory) except IOError: raise def upload(self, callback=None): """ The upload function. @param callback: The callback function for the upload progress. @type callback: Function """ self._upload_callback = callback zipfile_to_upload = open(self._absolute_zipfile_path, 'rb') zipfile_basename = os.path.basename(self._absolute_zipfile_path) self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload, 1024, self.handle_upload_state) zipfile_to_upload.close() # pylint: disable=unused-argument def handle_upload_state(self, block): """ The callback function for the upload progress. @param block: The StringIO of the current upload state @type block: StringIO """ self._bytes_written += 1024 progress = math.floor( (float(self._bytes_written) / float(self._total_file_size)) * 100) do_update = False if progress > self._current_progress: do_update = True self._current_progress = progress if self._upload_callback is not None: self._upload_callback(progress, self, do_update) # pylint: enable=unused-argument def quit(self): """ Try to quit everything and close the session. """ self.ftps.quit() @property def upload_path(self): """ Returns the upload path of the FTP server. @return: The path of the uploaded file on the FTP server. @rtype: String """ tmp_remote_path = '' if self._remote_path is not None: tmp_remote_path = self._remote_path return os.path.join(tmp_remote_path, os.path.basename(self._absolute_zipfile_path))
images_to_upload = [] if image_list: # Add first image images_to_upload.append(image_list[0]) if len(image_list) > 1: # Add last image (if more than 1 image) images_to_upload.append(image_list[len(image_list) - 1]) # Download oldest & newest image for image in images_to_upload: print 'Downloading %s...' % image ftps.retrbinary('RETR %s' % image, open(image, 'wb').write) # Check if directory for old images exists, if not create it if 'old' not in directory_list: print 'Creating dir "old"...' ftps.mkd('old') # Move checked images to old for image in image_list: #date_str = image.split('_')[1].split('.')[0] #img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S') print 'Moving %s to "old"...' % image ftps.rename(image, 'old/%s' % image) # Disconnect from FTP server ftps.quit() def upload_image(filename): image_file = {'file': open(filename, 'rb')} print 'Uploading %s to GroupMe image service...' % filename