def get_ftp():
ftp_path = getattr(settings, 'FTP_PATH', 'ftp://*****:*****@192.168.60.70/soft/测试文件')
parsed = urlparse(ftp_path)
ftp = FTPHost(parsed.hostname, parsed.username, parsed.password)
ftp.chdir(parsed.path)
return ftp
class RFTP:
def __init__(self):
ftp_path = getattr(settings, 'FTP_PATH', 'ftp://*****:*****@192.168.60.70/soft/测试文件')
parsed = urlparse(ftp_path)
self.hostname = parsed.hostname
self.username = parsed.username
self.password = parsed.password
self.path = parsed.path
self.ftp = None
def get_ftp(self):
if self.ftp is None:
self.ftp = FTPHost(self.hostname, self.username, self.password)
self.ftp.chdir(self.path)
else:
try:
self.ftp.chdir(self.path)
except Exception as e:
self.ftp = FTPHost(self.hostname, self.username, self.password)
self.ftp.chdir(self.path)
return self.ftp
def remove_file(self, name):
try:
self.ftp.remove(name)
except Exception as e:
self.ftp.rmdir(name)
def fopen(top):
"""Open file.
Returns file handle specified by path
"""
(hostname,path,user,password) = parseuri(top)
host = FTPHost(hostname,user,password)
return host.open(path)
def get_ftp(self):
if self.ftp is None:
self.ftp = FTPHost(self.hostname, self.username, self.password)
self.ftp.chdir(self.path)
else:
try:
self.ftp.chdir(self.path)
except Exception as e:
self.ftp = FTPHost(self.hostname, self.username, self.password)
self.ftp.chdir(self.path)
return self.ftp
def __init__(self, *args, **kwargs):
kw = {}
if "expire" in kwargs:
kw["expire"] = kwargs["expire"]
del kwargs["expire"]
if "size" in kwargs:
kw["size"] = kwargs["size"]
del kwargs["size"]
self.cache = Cache(**kw)
FTPHost.__init__(self, *args, **kwargs)
self.CWD = None
self.setcwd()
def autoupdateftp():
ftp_host, ftp_name, ftp_pwd, ftp_path, keep_update_dirs, exclude_dirs, root_dir = read_conf(
)
from ftputil import FTPHost
from ftputil.error import FTPError
from ftputil.error import TemporaryError
try:
ftp_s = FTPHost(ftp_host, ftp_name, ftp_pwd)
except FTPError as e:
print('cannot connect ftp sever ---- ' + str(e))
exit(0)
if ftp_path != '/':
ftp_s.chdir(ftp_path)
try:
ftp_s.keep_alive()
except TemporaryError as e:
print('Program execute exception ---- ' + str(e))
exit(0)
import os
if keep_update_dirs == '/':
if exclude_dirs != '':
download_file(ftp_s, keep_update_dirs, root_dir, exclude_dirs)
else:
download_file(ftp_s, keep_update_dirs, root_dir)
else:
sdirs = keep_update_dirs.split(',')
for dir in sdirs:
download_file(ftp_s, dir.strip(), root_dir + os.sep + dir.strip())
ftp_s.close()
def get_host():
c = current_app.config
session_factory = FTP_TLS if c['FTP_USE_TLS'] else FTP
return FTPHost(c['FTP_ADDRESS'],
c['FTP_USERNAME'],
c['FTP_PASSWORD'],
session_factory=session_factory)
def stat(top):
"""Get file information (stats)
For the file given by path returns a 10-tuple
mode, ino, dev, nlink, uid, gid, *size, atime, *mtime, *ctime
(Only attributes preceeded with * are currently required,
all others may be set 0 if not determinable)
"""
(hostname,path,user,password) = parseuri(top)
host = FTPHost(hostname,user,password)
return host.stat(path)
def listDirectory(self, dir: str, host: ftputil.FTPHost):
exc: ftputil.error.PermanentError
try:
return host.listdir(dir)
except ftputil.error.PermanentError as exc: # noqa: F841
raise BackendError(str(exc))
def run(self):
try:
with FTPHost(self.hostname, self.username, self.password) as ftp:
for folder in ftp.listdir('.'):
log.info('Found remote folder: {}'.format(folder))
except Exception as e:
print(e)
def move_nas(local_dir, remote_dir):
with FTPHost('10.100.4.102', 'root', 'originp123') as ftp_host:
for root, dirs, files in os.walk(local_dir, topdown=False):
for name in files:
local_file = os.path.join(root, name)
remote_file = remote_dir + '/' + name
ftp_host.upload(local_file, remote_file)
os.remove(local_file)
def install_server_configuration(host, username, password):
with FTPHost(host, username, password) as remote:
if remote.path.isfile('config_inc.php.bak'):
remote.remove('config_inc.php.bak')
if remote.path.isfile('config_inc.php'):
remote.rename('config_inc.php', 'config_inc.php.bak')
remote.upload(
os.path.join(os.path.dirname(__file__),
"recources/config_inc.php"), 'config_inc.php')
def walk(top):
"""Directory tree generator.
For each directory in the directory tree rooted at top (including top
itself, but excluding '.' and '..'), yields a 3-tuple
dirpath, dirnames, filenames
dirpath is a string, the path to the directory. dirnames is a list of
the names of the subdirectories in dirpath (excluding '.' and '..').
filenames is a list of the names of the non-directory files in dirpath.
Note that the names in the lists are just names, with no path components.
To get a full path (which begins with top) to a file or directory in
dirpath, do os.path.join(dirpath, name).
"""
(hostname,path,user,password) = parseuri(top)
host = FTPHost(hostname,user,password)
for dir in host.walk(path):
yield dir
def _dir(self, path):
path = self.path.normcase(path)
try:
lines = self.cache[path]
except KeyError:
self.logger.debug("cache miss: %s" % path)
lines = FTPHost._dir(self, path)
self.cache[path] = lines
else:
self.logger.debug("cache hit: %s" % path)
return lines
def recursively_download_ftp(host, username, password, ftp_root_path,
local_dest_dir):
# Data preparation
ftp_root_path = sub('\\\\', '/', ftp_root_path)
ftp_root_path = sub('^[/]*', '/', ftp_root_path)
ftp_root_path = sub('/$', '', ftp_root_path)
print('Recursively downloading from ftp://{0}:{1}@{2}{3} to {4}'.format(
username, password, host, ftp_root_path, local_dest_dir))
host = FTPHost(host, username, password)
recursive_file_walk = host.walk(ftp_root_path, topdown=True, onerror=None)
for folder_path, _, folder_files in recursive_file_walk:
print('REMOTE DIR\t', folder_path)
short_folder_path = sub('^' + ftp_root_path, '', folder_path)
local_folder_path = local_dest_dir
if short_folder_path:
local_folder_path = path.join(local_dest_dir, sub('^/', '',
short_folder_path))
if not path.exists(local_folder_path):
makedirs(local_folder_path)
print('LOCAL DIR\t', local_folder_path)
if len(folder_files) > 0:
for folder_file in folder_files:
remote_path = folder_path + '/' + folder_file
print('REMOTE FILE\t', remote_path)
local_file_path = path.join(local_folder_path, folder_file)
local_file_path = path.abspath(local_file_path)
print('LOCAL FILE\t', local_file_path)
host.download_if_newer(remote_path, local_file_path)
else:
print('NO FILES')
print('')
host.close
def move_nas(local_dir, remote_dir):
with FTPHost('10.100.4.102', 'root', 'originp123') as ftp_host:
for root, dirs, files in os.walk(local_dir, topdown=False):
for name in files:
file_year = root.split('/')[-1]
remote_fdir = os.path.join(remote_dir, file_year)
if not ftp_host.path.exists(remote_fdir):
ftp_host.mkdir(remote_fdir)
local_file = os.path.join(root, name)
ftp_host.upload(local_file, os.path.join(remote_fdir, name))
os.remove(local_file)
for name in dirs:
os.rmdir(os.path.join(root, name))
def get_ftputil_client(self):
if self.protocol == "sftp":
raise TypeError(
"ftputil does not manage sftp protocol, please use another ftp client"
)
if self.host == "":
raise ValueError(
"ftp account is not configured, can't initialize ftputil client"
)
password = self.get_password()
session_factory = ftputil.session.session_factory(port=self.port)
return FTPHost(self.host,
self.login,
password,
session_factory=session_factory)
def upload_to_ena(self):
# Connect #
print "Connecting..."
ftp = FTPHost(ftp_server, ftp_login, str(ftp_password))
# Gzip if not there yet #
if not self.raw_gz.exists:
self.raw.fwd.gzip_to(self.p.raw_forward_gz)
self.raw.rev.gzip_to(self.p.raw_reverse_gz)
# Make directory #
directory = '/ILLUMITAG/run%03d/pool%02d/sample%02d/'
directory = directory % (self.pool.run_num, self.pool.num, self.num)
print "Making directories..."
ftp.makedirs(directory)
# Upload #
base_path = directory + 'run%03d_pool%02d_sample%02d_{}_reads.fastq.gz'
base_path = base_path % (self.pool.run_num, self.pool.num, self.num)
print "Uploading forward..."
ftp.upload(self.p.raw_forward_gz, base_path.format("forward"))
print "Uploading reverse..."
ftp.upload(self.p.raw_reverse_gz, base_path.format("reverse"))
# Return #
ftp.close()
def upload_to_sra(self, verbose=True):
"""They have an FTP site where you should drop the files first"""
# Print #
if verbose: print self.s.short_name + ' (' + self.s.name + ')'
# Connect #
if verbose: print "Connecting..."
ftp = FTPHost(ftp_server, ftp_login, str(ftp_password))
# Gzip if not there yet #
if not self.s.raw_gz.exists:
self.s.raw.fwd.gzip_to(self.s.p.raw_forward_gz)
self.s.raw.rev.gzip_to(self.s.p.raw_reverse_gz)
# Make directory #
if verbose: print "Making directories..."
ftp.makedirs(self.directory)
# Upload #
base_path = self.directory + self.base_name
if verbose: print "Uploading forward..."
ftp.upload(self.s.p.raw_forward_gz, base_path.format("forward"))
if verbose: print "Uploading reverse..."
ftp.upload(self.s.p.raw_reverse_gz, base_path.format("reverse"))
# Return #
ftp.close()
def upload_to_sra(self):
# Print #
print self.s.short_name
# Connect #
print "Connecting..."
ftp = FTPHost(sra.ftp_server, sra.ftp_login, str(sra.ftp_password))
# Make directory #
print "Making directories..."
ftp.makedirs(self.directory)
# Upload #
dest_path = self.directory + self.name
print "Uploading to '%s' (%s)..." % (dest_path, self.s.p.raw_sff.size)
ftp.upload(self.s.p.raw_sff, dest_path)
# Return #
ftp.close()
def install_blast(base_dir):
"""Deprecated, look into 'home_linux/setup/bioinfo_tools'
On Ubuntu: sudo apt-get install ncbi-blast+
"""
# Default location #
if base_dir is None: base_dir = os.environ.get('HOME', '/') + '/programs/blast/'
# Download from FTP #
ftp_url = "ftp.ncbi.nlm.nih.gov"
ftp_dir = "/blast/executables/blast+/LATEST/"
pattern = 'ncbi-blast-*+-src.zip'
ftp = FTPHost(ftp_url, "anonymous")
ftp.chdir(ftp_dir)
files = ftp.listdir(self.ftp.curdir)
ftp.download(source, dest)
def move_nas(local_dir, remote_dir):
fs_reports = search_fs_reports(local_dir)
db_reports = search_db_reports(fs_reports)
src_files = pick_files(fs_reports, db_reports)
with FTPHost('10.100.4.102', 'root', 'originp123') as ftp_host:
for local_file in src_files:
*_, file_year, filename = local_file.split(os.sep)
remote_fdir = os.path.join(remote_dir, file_year)
if not ftp_host.path.exists(remote_fdir):
ftp_host.mkdir(remote_fdir)
ftp_host.upload(local_file, os.path.join(remote_fdir, filename))
os.remove(local_file)
# 删除空目录
for it in local_dir:
dir_path = os.path.join(local_dir, it)
try:
os.rmdir(dir_path)
except OSError:
pass
def _append_todo(start_date, sidecar_dir, ftp_site, ftp_dir, listing,
original_dirs_list):
try:
with FTPHost(ftp_site, 'anonymous', '@anonymous') as ftp_host:
dirs = ftp_host.listdir(ftp_dir)
for entry in dirs:
entry_fqn = '{}/{}'.format(ftp_dir, entry)
entry_stats = ftp_host.stat(entry_fqn)
if entry_stats.st_mtime >= start_date:
if stat.S_ISDIR(entry_stats.st_mode):
# True - it's a directory, follow it down later
if entry_fqn not in original_dirs_list:
listing[entry_fqn] = [True, entry_stats.st_mtime]
elif entry.endswith('.fits') or entry.endswith('.fits.gz'):
# False - it's a file, just leave it in the list
listing[entry_fqn] = [False, entry_stats.st_mtime]
logging.debug(f'Adding entry {entry_fqn}')
ftp_host.close()
temp_listing = {}
for entry, value in listing.items():
if value[0] and entry not in original_dirs_list: # is a directory
temp_listing.update(
_append_todo(start_date, sidecar_dir, ftp_site, entry,
temp_listing, original_dirs_list))
_sidecar(entry, value, sidecar_dir)
original_dirs_list[entry] = value[1]
logging.info(f'Added results for {entry}')
_cache(temp_listing, sidecar_dir)
listing.update(temp_listing)
except Exception as e:
logging.error(e)
logging.debug(traceback.format_exc())
raise mc.CadcException(f'Could not list {ftp_dir} on {ftp_site}')
return listing
def run(self):
if form.stop_sync is not True:
for folder in self.list_of_folders:
log.info(
'Attempting to retrieve file list from {}'.format(folder))
try:
with FTPHost(self.hostname, self.username,
self.password) as ftp:
for root, dirs, files in ftp.walk(folder):
for file in files:
form.list_transfer.addItem(root + '/' + file)
except Exception as e:
log.exception(e)
for i in range(0, form.list_transfer.count()):
if form.stop_sync is not True:
remote_file = Path(form.list_transfer.takeItem(0).text())
local_file = Path(self.download_path + '/' + str(remote_file))
# create folder for file
try:
os.chdir(self.download_path)
os.makedirs(str(local_file.parent))
log.info('Created local folder: {}'.format(
str(local_file.parent)))
except FileExistsError as e:
log.warning('{} already exists, skipping..'.format(
str(local_file.parent)))
# enter the local folder
try:
os.chdir(str(local_file.parent))
except Exception as e:
log.exception(e)
# enter the remote folder
log.info('Attempting to download: {}'.format(
str(local_file.parts[-1])))
try:
with ftp2(self.hostname, self.username,
self.password) as ftp:
for i in remote_file.parts[0:-1]:
ftp.cwd(str(i))
fh = open(str(local_file), 'wb')
ftp.retrbinary(
'RETR {}'.format(str(remote_file.parts[-1])),
fh.write)
fh.close()
log.info('Download Complete: {}'.format(
str(local_file)))
form.list_finished_transfer.addItem(str(local_file))
except Exception as e:
log.exception(e)
else:
form.list_transfer.clear()
form.action_add_folder.setEnabled(True)
form.action_download_path.setEnabled(True)
form.action_load_settings.setEnabled(True)
form.action_save_settings.setEnabled(True)
form.action_find_remote_folders.setEnabled(True)
form.action_sync.setEnabled(True)
form.action_stop_sync.setEnabled(False)
form.input_hostname.setEnabled(True)
form.input_username.setEnabled(True)
form.input_password.setEnabled(True)
def mkdir(self, path, mode=None):
path = self.path.abspath(path)
FTPHost.mkdir(self, path, mode)
self._invalidate_dir(path)
if __name__ == "__main__":
ftp_host = None
try:
while True:
server = input("Set server address: ")
if server == "":
server = "192.168.0.81"
user = input("User: "******"":
user = "******"
password = input("Password: "******"":
password = "******"
try:
ftp_host = FTPHost(server, user, password)
break
except error.PermanentError as e:
print(e)
actions(ftp_host)
except KeyboardInterrupt:
print("Force exit...")
if ftp_host:
ftp_host.close()
# names = ftp_host.listdir(ftp_host.curdir)
# for name in names:
# if ftp_host.path.isfile(name):
# ftp_host.download(name, name) # remote, local
# # Make a new directory and copy a remote file into it.
def rmdir(self, path):
FTPHost.rmdir(self, path)
self.cache.invalidate(self.path.normcase(
self.path.abspath(path)))
self._invalidate_dir(path)
def getStats(self, path: str, host: ftputil.FTPHost) -> Any:
return host.stat(path)
external_deps = ['ftputil', 'easysnmp', 'dateutil']
cisco_miburl = urlparse("ftp://ftp.cisco.com/pub/mibs/v2/v2.tar.gz")
mibs_dest = '/usr/share/snmp/mibs'
try:
for dep in external_deps:
imp.find_module(dep)
except ImportError:
deplist = ", ".join(external_deps)
print "Unmet dependencies, the following external dependencies are required: {}".format(
deplist)
from ftputil import FTPHost # noqa
print "Downloading required mibs..."
with FTPHost(cisco_miburl.netloc, 'anonymous', 'anonymous') as host:
if host.path.isfile(cisco_miburl.path):
tempfile = mkstemp(suffix='.tar.gz')[1]
host.download(cisco_miburl.path, tempfile)
print "Unpacking tarball..."
tar = tarfile.open(tempfile)
tar.extractall()
tar.close()
os.unlink(tempfile)
print "Installing MIBs..."
for mib in os.listdir("./auto/mibs/v2"):
if not mib.endswith(".my"):
continue
srcmib = "auto/mibs/v2/{}".format(mib)
destmib = "{}/{}".format(mibs_dest, mib)
if os.path.exists(destmib):
def ftp(self):
"""If the data is to be obtained by FTP here is the ftputil object."""
ftp = FTPHost(self.ftp_url, "anonymous")
ftp.chdir(self.ftp_dir)
return ftp
def rename(self, source, target):
FTPHost.rename(self, source, target)
self._invalidate_dir(source)
self._invalidate_dir(target)
def file(self, path, mode='r'):
path = self.path.abspath(path)
ret = FTPHost.file(self, path, mode)
if 'w' in mode:
self._invalidate_dir(path)
return ret
def restore_server_configuration(host, username, password):
with FTPHost(host, username, password) as remote:
if remote.path.isfile('config_inc.php.bak'):
if remote.path.isfile('config_inc.php'):
remote.remove('config_inc.php')
remote.rename('config_inc.php.bak', 'config_inc.php')
def chdir(self, path):
FTPHost.chdir(self, path)
self.setcwd()
def setcwd(self):
"""
Update cached working directory.
"""
self.CWD = self.path.normpath(FTPHost.getcwd(self))
self.logger.info("New cwd: %s" % self.CWD)
rmtdir = '/modelos/io/produtos/MERGE' # diretorio remoto
for d in days:
year = d[:4]
binfile = 'prec_{0}.bin'.format(d)
ctlfile = 'prec_{0}.ctl'.format(d)
locbinfile = '{0}/{1}/{2}'.format(locdir, year, binfile)
locctlfile = '{0}/{1}/{2}'.format(locdir, year, ctlfile)
if not os.path.exists(locbinfile):
msg = 'File not found: {0}'.format(locbinfile)
print(msg)
rmtbinfile = '{0}/{1}/{2}'.format(rmtdir, year, binfile)
rmtctlfile = '{0}/{1}/{2}'.format(rmtdir, year, ctlfile)
host = FTPHost(hostnm, usernm, passwd)
host.download(rmtbinfile, locbinfile) # remoto, local
host.download(rmtctlfile, locctlfile)
else:
msg = 'File already exists: {0}'.format(locbinfile)
print(msg)
def remove(self, path):
FTPHost.remove(self, path)
self._invalidate_dir(path)
def ftp(self):
"""If the data is to be obtained by FTP, here is the ftputil object."""
from ftputil import FTPHost
ftp = FTPHost(self.ftp_url, "anonymous")
ftp.chdir(self.ftp_dir)
return ftp