async def download_csv(): global state error = None state['status'] = "Files downloading" try: async with aioftp.ClientSession(host=host, user=ftp_user, password=ftp_password) as client: for path, info in (await client.list()): try: await client.download( path, destination=f"{pwd}/r-driveproject/src/data/" + path.name, write_into=True) except Exception as e: state['status'] = 'FAILED' state['error'] = e error = 'CSV download failed' except Exception as exc: state['status'] = 'FAILED' state['error'] = exc error = 'Something went wrong with connection to FTP' return error
async def prove(self): anonymous = False flag = 3 usernamedic = self.read_file( self.parameter['U']) if 'U' in self.parameter.keys( ) else self.read_file('dict/ftp_usernames.txt') passworddic = self.read_file( self.parameter['P']) if 'P' in self.parameter.keys( ) else self.read_file('dict/ftp_passwords.txt') async for (username, password) in self.generate_dict(usernamedic, passworddic): try: if username == 'anonymous': if anonymous: continue else: anonymous = True async with aioftp.ClientSession(self.target_host, self.target_port, username, password) as client: self.flag = 1 self.req.append({ "username": username, "password": password }) self.res.append({ "info": username + "/" + password, "key": "ftp burst" }) except Exception as e: if "timed out" in str(e): if flag == 0: return flag -= 1
async def on_chat(self, nick, message): if message.find('!map ') == 0: mapname = message.split('!map ')[1:][0] if os.path.exists(get_map_download_path(mapname)): self.say("{C}9Map {C}A" + mapname + "{C}9 is already on the server, changing") self.new_map(mapname) else: async with aioftp.ClientSession("ic3y.de", 21) as client: if await client.exists(get_map_server_path(mapname)): self.say("{C}9Downloading {C}A" + mapname + "{C}9 from ic3y.de...") print(get_map_server_path(mapname)) await client.download(get_map_server_path(mapname), get_map_download_path(mapname), write_into=True) self.say( "{C}A" + mapname + "{C}9 has been downloaded successfully, changing map..." ) self.new_map(mapname) else: self.say("{C}9 Map {C}A" + mapname + "{C}9 is not available at ic3y.de") self.new_map(mapname)
async def get_file(host, path, out_dir=CURRENT_DIR): with (await SEMA): async with aioftp.ClientSession(host, socket_timeout=30) as client: # get download file stat if await client.exists(path): stat = await client.stat(path) size = int(stat["size"]) else: print('{fi} not exists!'.format(fi=path)) file_name = pathlib.PurePath(path).name outfile = out_dir / file_name if pathlib.Path(outfile).exists(): outfile_stat = pathlib.Path(outfile).stat() outfile_size = outfile_stat.st_size if outfile_size != size: print('Downloading {fi}'.format(fi=file_name)) file_out = gzip.open(outfile, 'ab') async with client.download_stream( path, offset=outfile_size) as stream: file_out.write(stream) file_out.close() else: pass else: print('Downloading {fi}'.format(fi=file_name)) await client.download(path, outfile, write_into=True) print('Download {fi} finished.'.format(fi=file_name))
async def fetch_mapcycle(self): log.info("Fetching mapcycle for gameserver %s", self.name) async with aioftp.ClientSession(self.host, self.port, self.user, self.password) as client: await client.download("/tf/addons/sourcemod/data/mapcycle.txt", f"data/mapcycle/{self.name}.txt", write_into=True)
async def download_ftp(urlFTP, pathFTP, listFileName): async with aioftp.ClientSession(urlFTP) as client: for strFileName in listFileName: if await client.is_file(pathFTP + '/' + strFileName): print("Download: " + pathFTP + '/' + strFileName) await client.download(pathFTP + '/' + strFileName) print("End download: " + pathFTP + '/' + strFileName)
async def __async_upload(self, file_path): try: file_dest = file_path.replace(self.path_sync, "").replace( os.environ["HOMEDRIVE"] + os.sep, "").replace("\\", '/') async with aioftp.ClientSession("host", 21, "user", "pass") as client: if not await client.exists(file_dest.split('/')[1]): try: await client.make_directory(file_dest.split('/')[1]) except: pass if not await client.exists(file_dest): await client.upload(file_path, file_dest, write_into=True) self.files_list.remove(file_path) self.files.emit(self.files_list) else: self.files_list.remove(file_path) self.files.emit(self.files_list) # verificar se o arquivo é mais recente e então fazer o upload pass except Exception as e: self.files_error.append(file_path) print(e)
async def ls(request): """ftp LS command Optional query params: path: directory to list (defaults to "/") recursive: recurse down folders (defaults to "false") """ host, port, login, password = parse_headers(request) root_path = request.query.get('path', '/') recursive = request.query.get('recursive', 'false') == 'true' extension = request.query.get('extension') files = [] try: async with aioftp.ClientSession(host, port, login, password, socket_timeout=FTP_TIMEOUT, path_timeout=FTP_TIMEOUT) as client: async for path, info in client.list(root_path, recursive=recursive): if extension is None or path.suffix == extension: files.append(str(path)) except (OSError, asyncio.TimeoutError, TimeoutError): raise ServerUnreachable except aioftp.errors.StatusCodeError as ftp_error: raise AioftpError(ftp_error) return web.json_response(files)
async def download(request): host, port, login, password = parse_headers(request) path = request.query.get('path') if not path: raise MissingMandatoryQueryParameter('path') try: async with aioftp.ClientSession(host, port, login, password, socket_timeout=FTP_TIMEOUT, path_timeout=FTP_TIMEOUT) as client: ftp_stream = await client.download_stream(path) response = web.StreamResponse() response.content_type = 'application/octet-stream' await response.prepare(request) async for chunk in ftp_stream.iter_by_block(): await response.write(chunk) return response except (OSError, asyncio.TimeoutError, TimeoutError): raise ServerUnreachable except aioftp.errors.StatusCodeError as ftp_error: raise AioftpError(ftp_error)
async def get_inf(host, species, version, info_dict): with (await SEMA): async with aioftp.ClientSession(host) as client: path = '/pub/plants/release-{ver}/fasta/{sp}/cds/'.format( ver=version, sp=species) for path, info in (await client.list(path)): if info['type'] == 'file' and path.suffix == '.gz': pattern = re.compile('{pref}.(\S+).cds.all.fa.gz'.format( pref=species.capitalize())) g_version = pattern.match(path.name).groups()[0] info_dict[species] = g_version
async def download_meta(self, prefix_path): async with aioftp.ClientSession(host=self.server, user=self.username, password=self.password) as client: await client.change_directory(prefix_path) list = await client.list(recursive=True) return [ to_file(item, prefix_path) for item in list if item[1]['type'] == 'file' ]
async def connection(rule): cs = aioftp.ClientSession( host=rule.host, port=rule.port, user=rule.username, password=rule.password, ) async with cs as client: if rule.remote_path: await client.change_directory(rule.remote_path) yield client
async def ftp_download(cls, method: str, info: Dict, path: str): url = furl(info['url']) fileName = url.path.segments[-1] filePath = os.path.join(path, fileName) if cls.use_existing is True and os.path.exists(filePath): return filePath cls.logger.debug(f"Start to download file: {info}") async with aioftp.ClientSession(url.host) as session: await session.change_directory('/'.join(url.path.segments[:-1])) await session.download(fileName, path) # , write_info=True cls.logger.debug(f"File has been saved in: {filePath}") return filePath
async def get_ftp_listing(self, url): """Returns list of files at FTP **url**""" logger.debug("FTP: listing %s", url) if self.cache and url in self.cache["ftp_list"]: return self.cache["ftp_list"][url] parsed = urlparse(url) async with aioftp.ClientSession(parsed.netloc, password=self.USER_AGENT+"@") as client: res = [str(path) for path, _info in await client.list(parsed.path)] if self.cache: self.cache["ftp_list"][url] = res return res
async def get_checksum_from_ftp(self, url, _desc=None): """Compute sha256 checksum of content at ftp **url** Does not show progress monitor at this time (would need to get file size first) """ parsed = urlparse(url) checksum = sha256() async with aioftp.ClientSession(parsed.netloc, password=self.USER_AGENT+"@") as client: async with client.download_stream(parsed.path) as stream: async for block in stream.iter_by_block(): checksum.update(block) return checksum.hexdigest()
async def _fetch(self): log.info("Fetching CDN maplist and writing to disk") async with aioftp.ClientSession(self.host, self.port, self.user, self.password) as client: for path, info in (await client.list('/maps')): if info['type'] == 'file': if path.name.endswith('.bsp.bz2'): self.maps[path.name[:-8]] = int(info['size']) elif path.name.endswith('.bsp'): self.maps[path.name[:-4]] = int(info['size']) with open('data/maplist/cdn.txt', 'w') as f: f.writelines( sorted( [f'{name}:{size}\n' for name, size in self.maps.items()]))
async def config_sync(self, server, username, password): """Configuration syncronization""" Continue = True while Continue: async with aioftp.ClientSession(server, user=username, password=password) as client: print("Syncronizing config file") try: await client.upload("user.ini") except: print("An error occured during upload.") finally: print("Done!") await asyncio.sleep(300)
async def _read(self, server: dict, path): ip = server['ip'] pt = int(server['pt']) un = server['un'] pw = server['pw'] with io.BytesIO() as file_out: # Initialize connection log.info(f'Downloading {path} from {ip}') async with aioftp.ClientSession(ip, pt, un, pw) as client: async with client.download_stream(path) as stream: async for block in stream.iter_by_block(block_size): file_out.write(block) file_out.seek(0) return file_out.read()
async def _ftp_fetch(sensor, outDir, startTime=None, endTime=None, overwrite=False): start = datetime.datetime.strptime(startTime, "%Y-%m-%d") end = datetime.datetime.strptime(endTime, "%Y-%m-%d") outDir = Path(outDir).resolve() flist = [] async with aioftp.ClientSession('ftp-access.aviso.altimetry.fr',user='******',password='******') as client: await client.change_directory(path=f'geophysical-data-record/{sensor}/gdr_t') for path, info in (await client.list(recursive=True)): if info["type"] == "file": fdate = datetime.datetime.strptime( str(path).split('/')[-1].split('_')[4], '%Y%m%d') if start <= fdate <= end: outPath = outDir / path.name if outPath.exists() is False or overwrite is True: await client.download(path, destination=outDir) # print(f'{path} -> {outPath}') flist.append(outPath) return flist
async def run(gutenberg): files = [] rows = gutenberg.iterate_csv_file() for row in rows: file_location = gutenberg.obtain_directory_location(row["FileNumber"]) files.append(file_location) async with aioftp.ClientSession(gutenberg.ftp_uri) as client: tasks = [] for file_to_download in files: for path, info in (await client.list(file_to_download, raw_command='LIST')): text_file = gutenberg.find_text_file(str(path), row) if text_file: print(file_to_download) file_name = file_to_download[file_to_download.rfind('/')+1:] task = asyncio.ensure_future(download_file(gutenberg, text_file, file_name)) tasks.append(task) results = await asyncio.gather(*tasks) return results
async def get_file(host, path, out_dir=CURRENT_DIR, retry_lim=2, logger=DEFAULT_LOGGER): with (await SEMA): retry_times = 0 while retry_times <= retry_lim: try: async with aioftp.ClientSession( host, socket_timeout=30) as client: # get download file stat if await client.exists(path): stat = await client.stat(path) size = int(stat["size"]) else: logger.error(f'{file_name} not exists in server!') file_name = pathlib.PurePath(path).name outfile = out_dir / file_name if pathlib.Path(outfile).exists(): outfile_stat = pathlib.Path(outfile).stat() outfile_size = outfile_stat.st_size else: outfile_size = 0 if outfile_size != size: logger.info(f'|Downloading...| {file_name}') file_out = open(outfile, 'ab') async with client.download_stream( path, offset=outfile_size) as stream: async for block in stream.iter_by_block(): file_out.write(block) stream.close() file_out.close() else: pass logger.info(f'|Downloaded| {file_name}') return True except TimeoutError: retry_times += 1 except concurrent.futures._base.TimeoutError: retry_times += 1 except aioftp.errors.StatusCodeError: retry_times += 1 except ConnectionResetError: retry_times += 1 logger.error(f'|Failed!| {file_name}') return False
async def ping(request): """test FTP connection by sending a minimal LS command returns "pong" on success """ host, port, login, password = parse_headers(request) try: async with aioftp.ClientSession(host, port, login, password, socket_timeout=FTP_TIMEOUT, path_timeout=FTP_TIMEOUT) as client: async for _ in client.list('/'): # noqa # Iterate once if any result, only list command matters not the results break return web.json_response({'success': True}) except (OSError, asyncio.TimeoutError, TimeoutError): raise ServerUnreachable except aioftp.errors.StatusCodeError as ftp_error: raise AioftpError(ftp_error)
async def get_inf(host, name=None, retry_lim=1, logger=DEFAULT_LOGGER): file_list = [] retry_num = 0 out_name = name # name is None means to retrieve all files of blastdb if name is None: out_name = 'NCBI Blast Database' with (await SEMA): while retry_num <= retry_lim: try: async with aioftp.ClientSession( host, socket_timeout=10) as client: path = '/blast/db/' logger.info( f'Retriving blastdb files. Try {retry_num + 1}.') for path, info in (await client.list(path)): if info['type'] == 'file' and (path.suffix == '.gz' or path.suffix == '.md5'): if name is None: name = '.*' pattern = re.compile( '{pref}\..*tar.gz.*'.format( pref=name )) if pattern.match(path.name): file_list.append(path) file_number = len(file_list) logger.info( f'Total {file_number} files for [{out_name}] database.') return file_list except concurrent.futures._base.TimeoutError: retry_num += 1 except aioftp.errors.StatusCodeError: retry_num += 1 except ConnectionResetError: retry_num += 1 logger.error('Failed to connect to the FTP host.') logger.error(f'Please check host IP [{host}] and try again!') return None
async def _write(self, server: dict, name, isize, file_in, path='/', overwrite=False): ip = server['ip'] pt = int(server['pt']) un = server['un'] pw = server['pw'] log.info(f'Uploading {name} to {ip}') # Initialize connection async with aioftp.ClientSession(ip, pt, un, pw) as client: await client.change_directory(path) # Check if name already exists if await client.exists(name): if not overwrite: log.warning(f'{path}{name} already exists') return 2 log.info(f'Overwriting {path}{name}') await client.remove_file(name) # Send file file_in.seek(0) async with client.upload_stream(name) as stream: while True: data = file_in.read(block_size) if not data: break await stream.write(data) # Size verification stat = await client.stat(name) dsize = int(stat['size']) log.info(f'Origin: {isize} Destination: {dsize}') return isize != dsize
async def _ftp_fetch(sensor, outDir, startTime=None, endTime=None, overwrite=False): start = datetime.datetime.strptime(startTime, "%Y-%m-%d") end = datetime.datetime.strptime(endTime, "%Y-%m-%d") outDir = Path(outDir).resolve() flist = [] async with aioftp.ClientSession('ftp.nodc.noaa.gov') as client: await client.change_directory( path=f'pub/data.nodc/{sensor}/gdr/gdr/') for path, info in (await client.list(recursive=True)): if info["type"] == "file": fdate = datetime.datetime.strptime( str(path).split('_')[4], '%Y%m%d') if start <= fdate <= end: outPath = outDir / path.name if outPath.exists() is False or overwrite is True: await client.download(path, destination=outDir) # print(f'{path} -> {outPath}') flist.append(outPath) return flist
async def fetch_async(sem, pid, posixPath): async with sem: #ensure that we abide by Semaphore limit. I.e. do not DOS attack the FTP server start = time.time() print('Fetch async process {0} started at {1}'.format(pid, start)) toCdDir = posixPath.parent #[1:-1]) #the directory inside the ftp server to change_directory (cd) into pathName = posixPath.name #the folder/filename which we wish to get async with aioftp.ClientSession(siteURL, 21, user=usr, password=pwd) as client: await client.change_directory( toCdDir) #change ftp directory to the folder we want await smart_recursive_dwn(client, pathName, pathlib.Path(dataDir, satName, toCdDir)) #for path, info in (await client.list(recursive=True)): #print(path,info) #if info["type"] == "file" and path.suffix == ".HDR": # await client.download(path) print('Process {}, took: {:.2f} seconds'.format( pid, time.time() - start))
async def async_download(site, cdpath, usr, pwd): start = time.time() #Get list of filenames of the data files we want in a particular ftp folder async with aioftp.ClientSession(site, 21, user=usr, password=pwd) as client: filenames = [ filename for filename, info in (await client.list(cdpath)) ] #List of pathlib.PurePosixPaths to the files we want print("{0} files inside {1}".format(len(filenames), cdpath)) #Create tasks and run them concurrently via asyncio sem = asyncio.Semaphore( MAX_CLIENTS) #Pass in Semaphore FTP connection limit tasks = [ asyncio.ensure_future(fetch_async(sem, i, filename)) for i, filename in enumerate(filenames, start=1) ] #Create series of coroutines tasks (download tasks) to execute await asyncio.wait( tasks) #Execute those coroutines concurrently and wait for them to fin print("Asynchronous process took: {:.2f} seconds".format(time.time() - start))
async def download(self, prefix_path, file_path): async with aioftp.ClientSession(host=self.server, user=self.username, password=self.password) as client: await client.change_directory(prefix_path) path_for_download = pathlib.PurePosixPath( file_path.replace(prefix_path, '')) _, _, filename = file_path.rpartition('/') path_for_destination = DOWNLOAD_FOLDER + file_path.replace( filename, '') try: if not os.path.isfile(file_path): path = Path(path_for_destination) path.mkdir(parents=True, exist_ok=True) await client.download(source=path_for_download, destination=path_for_destination, write_into=False) except Exception as e: print('Exception 2 {0} on path {1}'.format(e, file_path))
async def test_client_session_context_manager(pair_factory): async with pair_factory(connected=False) as pair: async with aioftp.ClientSession(*pair.server.address) as client: await client.list()
async def get_list(urlFTP, pathFTP, filenames): async with aioftp.ClientSession(urlFTP) as client: await client.change_directory(pathFTP) for path, info in (await client.list(recursive=False)): if info["type"] == "file": filenames.append(path.name)