async def download_csv():
global state
error = None
state['status'] = "Files downloading"
try:
async with aioftp.ClientSession(host=host,
user=ftp_user,
password=ftp_password) as client:
for path, info in (await client.list()):
try:
await client.download(
path,
destination=f"{pwd}/r-driveproject/src/data/" +
path.name,
write_into=True)
except Exception as e:
state['status'] = 'FAILED'
state['error'] = e
error = 'CSV download failed'
except Exception as exc:
state['status'] = 'FAILED'
state['error'] = exc
error = 'Something went wrong with connection to FTP'
return error
async def prove(self):
anonymous = False
flag = 3
usernamedic = self.read_file(
self.parameter['U']) if 'U' in self.parameter.keys(
) else self.read_file('dict/ftp_usernames.txt')
passworddic = self.read_file(
self.parameter['P']) if 'P' in self.parameter.keys(
) else self.read_file('dict/ftp_passwords.txt')
async for (username,
password) in self.generate_dict(usernamedic, passworddic):
try:
if username == 'anonymous':
if anonymous:
continue
else:
anonymous = True
async with aioftp.ClientSession(self.target_host,
self.target_port, username,
password) as client:
self.flag = 1
self.req.append({
"username": username,
"password": password
})
self.res.append({
"info": username + "/" + password,
"key": "ftp burst"
})
except Exception as e:
if "timed out" in str(e):
if flag == 0:
return
flag -= 1
async def on_chat(self, nick, message):
if message.find('!map ') == 0:
mapname = message.split('!map ')[1:][0]
if os.path.exists(get_map_download_path(mapname)):
self.say("{C}9Map {C}A" + mapname +
"{C}9 is already on the server, changing")
self.new_map(mapname)
else:
async with aioftp.ClientSession("ic3y.de", 21) as client:
if await client.exists(get_map_server_path(mapname)):
self.say("{C}9Downloading {C}A" + mapname +
"{C}9 from ic3y.de...")
print(get_map_server_path(mapname))
await client.download(get_map_server_path(mapname),
get_map_download_path(mapname),
write_into=True)
self.say(
"{C}A" + mapname +
"{C}9 has been downloaded successfully, changing map..."
)
self.new_map(mapname)
else:
self.say("{C}9 Map {C}A" + mapname +
"{C}9 is not available at ic3y.de")
self.new_map(mapname)
async def get_file(host, path, out_dir=CURRENT_DIR):
with (await SEMA):
async with aioftp.ClientSession(host, socket_timeout=30) as client:
# get download file stat
if await client.exists(path):
stat = await client.stat(path)
size = int(stat["size"])
else:
print('{fi} not exists!'.format(fi=path))
file_name = pathlib.PurePath(path).name
outfile = out_dir / file_name
if pathlib.Path(outfile).exists():
outfile_stat = pathlib.Path(outfile).stat()
outfile_size = outfile_stat.st_size
if outfile_size != size:
print('Downloading {fi}'.format(fi=file_name))
file_out = gzip.open(outfile, 'ab')
async with client.download_stream(
path, offset=outfile_size) as stream:
file_out.write(stream)
file_out.close()
else:
pass
else:
print('Downloading {fi}'.format(fi=file_name))
await client.download(path, outfile, write_into=True)
print('Download {fi} finished.'.format(fi=file_name))
async def fetch_mapcycle(self):
log.info("Fetching mapcycle for gameserver %s", self.name)
async with aioftp.ClientSession(self.host, self.port, self.user,
self.password) as client:
await client.download("/tf/addons/sourcemod/data/mapcycle.txt",
f"data/mapcycle/{self.name}.txt",
write_into=True)
async def download_ftp(urlFTP, pathFTP, listFileName):
async with aioftp.ClientSession(urlFTP) as client:
for strFileName in listFileName:
if await client.is_file(pathFTP + '/' + strFileName):
print("Download: " + pathFTP + '/' + strFileName)
await client.download(pathFTP + '/' + strFileName)
print("End download: " + pathFTP + '/' + strFileName)
async def __async_upload(self, file_path):
try:
file_dest = file_path.replace(self.path_sync, "").replace(
os.environ["HOMEDRIVE"] + os.sep, "").replace("\\", '/')
async with aioftp.ClientSession("host", 21, "user",
"pass") as client:
if not await client.exists(file_dest.split('/')[1]):
try:
await client.make_directory(file_dest.split('/')[1])
except:
pass
if not await client.exists(file_dest):
await client.upload(file_path, file_dest, write_into=True)
self.files_list.remove(file_path)
self.files.emit(self.files_list)
else:
self.files_list.remove(file_path)
self.files.emit(self.files_list)
# verificar se o arquivo é mais recente e então fazer o upload
pass
except Exception as e:
self.files_error.append(file_path)
print(e)
async def ls(request):
"""ftp LS command
Optional query params:
path: directory to list (defaults to "/")
recursive: recurse down folders (defaults to "false")
"""
host, port, login, password = parse_headers(request)
root_path = request.query.get('path', '/')
recursive = request.query.get('recursive', 'false') == 'true'
extension = request.query.get('extension')
files = []
try:
async with aioftp.ClientSession(host,
port,
login,
password,
socket_timeout=FTP_TIMEOUT,
path_timeout=FTP_TIMEOUT) as client:
async for path, info in client.list(root_path,
recursive=recursive):
if extension is None or path.suffix == extension:
files.append(str(path))
except (OSError, asyncio.TimeoutError, TimeoutError):
raise ServerUnreachable
except aioftp.errors.StatusCodeError as ftp_error:
raise AioftpError(ftp_error)
return web.json_response(files)
async def download(request):
host, port, login, password = parse_headers(request)
path = request.query.get('path')
if not path:
raise MissingMandatoryQueryParameter('path')
try:
async with aioftp.ClientSession(host,
port,
login,
password,
socket_timeout=FTP_TIMEOUT,
path_timeout=FTP_TIMEOUT) as client:
ftp_stream = await client.download_stream(path)
response = web.StreamResponse()
response.content_type = 'application/octet-stream'
await response.prepare(request)
async for chunk in ftp_stream.iter_by_block():
await response.write(chunk)
return response
except (OSError, asyncio.TimeoutError, TimeoutError):
raise ServerUnreachable
except aioftp.errors.StatusCodeError as ftp_error:
raise AioftpError(ftp_error)
async def get_inf(host, species, version, info_dict):
with (await SEMA):
async with aioftp.ClientSession(host) as client:
path = '/pub/plants/release-{ver}/fasta/{sp}/cds/'.format(
ver=version, sp=species)
for path, info in (await client.list(path)):
if info['type'] == 'file' and path.suffix == '.gz':
pattern = re.compile('{pref}.(\S+).cds.all.fa.gz'.format(
pref=species.capitalize()))
g_version = pattern.match(path.name).groups()[0]
info_dict[species] = g_version
async def download_meta(self, prefix_path):
async with aioftp.ClientSession(host=self.server,
user=self.username,
password=self.password) as client:
await client.change_directory(prefix_path)
list = await client.list(recursive=True)
return [
to_file(item, prefix_path) for item in list
if item[1]['type'] == 'file'
]
async def connection(rule):
cs = aioftp.ClientSession(
host=rule.host,
port=rule.port,
user=rule.username,
password=rule.password,
)
async with cs as client:
if rule.remote_path:
await client.change_directory(rule.remote_path)
yield client
async def ftp_download(cls, method: str, info: Dict, path: str):
url = furl(info['url'])
fileName = url.path.segments[-1]
filePath = os.path.join(path, fileName)
if cls.use_existing is True and os.path.exists(filePath):
return filePath
cls.logger.debug(f"Start to download file: {info}")
async with aioftp.ClientSession(url.host) as session:
await session.change_directory('/'.join(url.path.segments[:-1]))
await session.download(fileName, path) # , write_info=True
cls.logger.debug(f"File has been saved in: {filePath}")
return filePath
async def get_ftp_listing(self, url):
"""Returns list of files at FTP **url**"""
logger.debug("FTP: listing %s", url)
if self.cache and url in self.cache["ftp_list"]:
return self.cache["ftp_list"][url]
parsed = urlparse(url)
async with aioftp.ClientSession(parsed.netloc,
password=self.USER_AGENT+"@") as client:
res = [str(path) for path, _info in await client.list(parsed.path)]
if self.cache:
self.cache["ftp_list"][url] = res
return res
async def get_checksum_from_ftp(self, url, _desc=None):
"""Compute sha256 checksum of content at ftp **url**
Does not show progress monitor at this time (would need to
get file size first)
"""
parsed = urlparse(url)
checksum = sha256()
async with aioftp.ClientSession(parsed.netloc,
password=self.USER_AGENT+"@") as client:
async with client.download_stream(parsed.path) as stream:
async for block in stream.iter_by_block():
checksum.update(block)
return checksum.hexdigest()
async def _fetch(self):
log.info("Fetching CDN maplist and writing to disk")
async with aioftp.ClientSession(self.host, self.port, self.user,
self.password) as client:
for path, info in (await client.list('/maps')):
if info['type'] == 'file':
if path.name.endswith('.bsp.bz2'):
self.maps[path.name[:-8]] = int(info['size'])
elif path.name.endswith('.bsp'):
self.maps[path.name[:-4]] = int(info['size'])
with open('data/maplist/cdn.txt', 'w') as f:
f.writelines(
sorted(
[f'{name}:{size}\n' for name, size in self.maps.items()]))
async def config_sync(self, server, username, password):
"""Configuration syncronization"""
Continue = True
while Continue:
async with aioftp.ClientSession(server,
user=username,
password=password) as client:
print("Syncronizing config file")
try:
await client.upload("user.ini")
except:
print("An error occured during upload.")
finally:
print("Done!")
await asyncio.sleep(300)
async def _read(self, server: dict, path):
ip = server['ip']
pt = int(server['pt'])
un = server['un']
pw = server['pw']
with io.BytesIO() as file_out:
# Initialize connection
log.info(f'Downloading {path} from {ip}')
async with aioftp.ClientSession(ip, pt, un, pw) as client:
async with client.download_stream(path) as stream:
async for block in stream.iter_by_block(block_size):
file_out.write(block)
file_out.seek(0)
return file_out.read()
async def _ftp_fetch(sensor, outDir, startTime=None, endTime=None, overwrite=False):
start = datetime.datetime.strptime(startTime, "%Y-%m-%d")
end = datetime.datetime.strptime(endTime, "%Y-%m-%d")
outDir = Path(outDir).resolve()
flist = []
async with aioftp.ClientSession('ftp-access.aviso.altimetry.fr',user='******',password='******') as client:
await client.change_directory(path=f'geophysical-data-record/{sensor}/gdr_t')
for path, info in (await client.list(recursive=True)):
if info["type"] == "file":
fdate = datetime.datetime.strptime(
str(path).split('/')[-1].split('_')[4], '%Y%m%d')
if start <= fdate <= end:
outPath = outDir / path.name
if outPath.exists() is False or overwrite is True:
await client.download(path, destination=outDir)
# print(f'{path} -> {outPath}')
flist.append(outPath)
return flist
async def run(gutenberg):
files = []
rows = gutenberg.iterate_csv_file()
for row in rows:
file_location = gutenberg.obtain_directory_location(row["FileNumber"])
files.append(file_location)
async with aioftp.ClientSession(gutenberg.ftp_uri) as client:
tasks = []
for file_to_download in files:
for path, info in (await client.list(file_to_download, raw_command='LIST')):
text_file = gutenberg.find_text_file(str(path), row)
if text_file:
print(file_to_download)
file_name = file_to_download[file_to_download.rfind('/')+1:]
task = asyncio.ensure_future(download_file(gutenberg, text_file, file_name))
tasks.append(task)
results = await asyncio.gather(*tasks)
return results
async def get_file(host, path, out_dir=CURRENT_DIR,
retry_lim=2, logger=DEFAULT_LOGGER):
with (await SEMA):
retry_times = 0
while retry_times <= retry_lim:
try:
async with aioftp.ClientSession(
host, socket_timeout=30) as client:
# get download file stat
if await client.exists(path):
stat = await client.stat(path)
size = int(stat["size"])
else:
logger.error(f'{file_name} not exists in server!')
file_name = pathlib.PurePath(path).name
outfile = out_dir / file_name
if pathlib.Path(outfile).exists():
outfile_stat = pathlib.Path(outfile).stat()
outfile_size = outfile_stat.st_size
else:
outfile_size = 0
if outfile_size != size:
logger.info(f'|Downloading...| {file_name}')
file_out = open(outfile, 'ab')
async with client.download_stream(
path, offset=outfile_size) as stream:
async for block in stream.iter_by_block():
file_out.write(block)
stream.close()
file_out.close()
else:
pass
logger.info(f'|Downloaded| {file_name}')
return True
except TimeoutError:
retry_times += 1
except concurrent.futures._base.TimeoutError:
retry_times += 1
except aioftp.errors.StatusCodeError:
retry_times += 1
except ConnectionResetError:
retry_times += 1
logger.error(f'|Failed!| {file_name}')
return False
async def ping(request):
"""test FTP connection by sending a minimal LS command
returns "pong" on success
"""
host, port, login, password = parse_headers(request)
try:
async with aioftp.ClientSession(host,
port,
login,
password,
socket_timeout=FTP_TIMEOUT,
path_timeout=FTP_TIMEOUT) as client:
async for _ in client.list('/'): # noqa
# Iterate once if any result, only list command matters not the results
break
return web.json_response({'success': True})
except (OSError, asyncio.TimeoutError, TimeoutError):
raise ServerUnreachable
except aioftp.errors.StatusCodeError as ftp_error:
raise AioftpError(ftp_error)
async def get_inf(host, name=None,
retry_lim=1,
logger=DEFAULT_LOGGER):
file_list = []
retry_num = 0
out_name = name
# name is None means to retrieve all files of blastdb
if name is None:
out_name = 'NCBI Blast Database'
with (await SEMA):
while retry_num <= retry_lim:
try:
async with aioftp.ClientSession(
host, socket_timeout=10) as client:
path = '/blast/db/'
logger.info(
f'Retriving blastdb files. Try {retry_num + 1}.')
for path, info in (await client.list(path)):
if info['type'] == 'file' and (path.suffix == '.gz' or
path.suffix == '.md5'):
if name is None:
name = '.*'
pattern = re.compile(
'{pref}\..*tar.gz.*'.format(
pref=name
))
if pattern.match(path.name):
file_list.append(path)
file_number = len(file_list)
logger.info(
f'Total {file_number} files for [{out_name}] database.')
return file_list
except concurrent.futures._base.TimeoutError:
retry_num += 1
except aioftp.errors.StatusCodeError:
retry_num += 1
except ConnectionResetError:
retry_num += 1
logger.error('Failed to connect to the FTP host.')
logger.error(f'Please check host IP [{host}] and try again!')
return None
async def _write(self,
server: dict,
name,
isize,
file_in,
path='/',
overwrite=False):
ip = server['ip']
pt = int(server['pt'])
un = server['un']
pw = server['pw']
log.info(f'Uploading {name} to {ip}')
# Initialize connection
async with aioftp.ClientSession(ip, pt, un, pw) as client:
await client.change_directory(path)
# Check if name already exists
if await client.exists(name):
if not overwrite:
log.warning(f'{path}{name} already exists')
return 2
log.info(f'Overwriting {path}{name}')
await client.remove_file(name)
# Send file
file_in.seek(0)
async with client.upload_stream(name) as stream:
while True:
data = file_in.read(block_size)
if not data:
break
await stream.write(data)
# Size verification
stat = await client.stat(name)
dsize = int(stat['size'])
log.info(f'Origin: {isize} Destination: {dsize}')
return isize != dsize
async def _ftp_fetch(sensor,
outDir,
startTime=None,
endTime=None,
overwrite=False):
start = datetime.datetime.strptime(startTime, "%Y-%m-%d")
end = datetime.datetime.strptime(endTime, "%Y-%m-%d")
outDir = Path(outDir).resolve()
flist = []
async with aioftp.ClientSession('ftp.nodc.noaa.gov') as client:
await client.change_directory(
path=f'pub/data.nodc/{sensor}/gdr/gdr/')
for path, info in (await client.list(recursive=True)):
if info["type"] == "file":
fdate = datetime.datetime.strptime(
str(path).split('_')[4], '%Y%m%d')
if start <= fdate <= end:
outPath = outDir / path.name
if outPath.exists() is False or overwrite is True:
await client.download(path, destination=outDir)
# print(f'{path} -> {outPath}')
flist.append(outPath)
return flist
async def fetch_async(sem, pid, posixPath):
async with sem: #ensure that we abide by Semaphore limit. I.e. do not DOS attack the FTP server
start = time.time()
print('Fetch async process {0} started at {1}'.format(pid, start))
toCdDir = posixPath.parent #[1:-1]) #the directory inside the ftp server to change_directory (cd) into
pathName = posixPath.name #the folder/filename which we wish to get
async with aioftp.ClientSession(siteURL, 21, user=usr,
password=pwd) as client:
await client.change_directory(
toCdDir) #change ftp directory to the folder we want
await smart_recursive_dwn(client, pathName,
pathlib.Path(dataDir, satName, toCdDir))
#for path, info in (await client.list(recursive=True)):
#print(path,info)
#if info["type"] == "file" and path.suffix == ".HDR":
# await client.download(path)
print('Process {}, took: {:.2f} seconds'.format(
pid,
time.time() - start))
async def async_download(site, cdpath, usr, pwd):
start = time.time()
#Get list of filenames of the data files we want in a particular ftp folder
async with aioftp.ClientSession(site, 21, user=usr,
password=pwd) as client:
filenames = [
filename for filename, info in (await client.list(cdpath))
] #List of pathlib.PurePosixPaths to the files we want
print("{0} files inside {1}".format(len(filenames), cdpath))
#Create tasks and run them concurrently via asyncio
sem = asyncio.Semaphore(
MAX_CLIENTS) #Pass in Semaphore FTP connection limit
tasks = [
asyncio.ensure_future(fetch_async(sem, i, filename))
for i, filename in enumerate(filenames, start=1)
] #Create series of coroutines tasks (download tasks) to execute
await asyncio.wait(
tasks) #Execute those coroutines concurrently and wait for them to fin
print("Asynchronous process took: {:.2f} seconds".format(time.time() -
start))
async def download(self, prefix_path, file_path):
async with aioftp.ClientSession(host=self.server,
user=self.username,
password=self.password) as client:
await client.change_directory(prefix_path)
path_for_download = pathlib.PurePosixPath(
file_path.replace(prefix_path, ''))
_, _, filename = file_path.rpartition('/')
path_for_destination = DOWNLOAD_FOLDER + file_path.replace(
filename, '')
try:
if not os.path.isfile(file_path):
path = Path(path_for_destination)
path.mkdir(parents=True, exist_ok=True)
await client.download(source=path_for_download,
destination=path_for_destination,
write_into=False)
except Exception as e:
print('Exception 2 {0} on path {1}'.format(e, file_path))
async def test_client_session_context_manager(pair_factory):
async with pair_factory(connected=False) as pair:
async with aioftp.ClientSession(*pair.server.address) as client:
await client.list()
async def get_list(urlFTP, pathFTP, filenames):
async with aioftp.ClientSession(urlFTP) as client:
await client.change_directory(pathFTP)
for path, info in (await client.list(recursive=False)):
if info["type"] == "file":
filenames.append(path.name)
