def store_ftp(self, zip_loc, name, job_id):
"optional fall back if camlistore doesn't pan out"
conf = util.conf()['ftp']
f = ftplib.FTP_TLS(
conf['ftp_host']) #We don't actually want to upload here
f.login() #Anonymous login
f.prot_p() #secure the line
f.cwd(os.path.join(conf['remotepath'], name))
f.storbinary("STOR %s" % os.path.split(zip_loc)[1], open(zip_loc))
def get_conn(self):
"""
Returns a FTPS connection object.
"""
if self.conn is None:
params = self.get_connection(self.ftp_conn_id)
self.conn = ftplib.FTP_TLS(params.host, params.login,
params.password)
return self.conn
def __init__(self, one=None, globus_client=None):
super().__init__(one=one)
self.ftp = ftplib.FTP_TLS(host=FTP_HOST,
user=one._par.FTP_DATA_SERVER_LOGIN,
passwd=one._par.FTP_DATA_SERVER_PWD)
# self.ftp.ssl_version = ssl.PROTOCOL_TLSv1
# self.ftp.auth()
self.ftp.prot_p()
self.ftp.login(one._par.FTP_DATA_SERVER_LOGIN, one._par.FTP_DATA_SERVER_PWD)
def test_ftp_dir(mocker):
"""nlst now returns the parent dir. We should retrieve only the filenames"""
client_mock = mocker.patch("peakina.io.ftp.ftp_utils.client")
client_mock.return_value.__enter__.return_value = (ftplib.FTP_TLS(),
"path")
mocker.patch("ftplib.FTP.nlst").return_value = [
"/somepath/file1.csv", "/somepath/file2.csv"
]
assert ftp_listdir("ftps://somepath") == ["file1.csv", "file2.csv"]
def connect(self):
self.logger.debug("sr_ftp connect %s" % self.parent.destination)
self.connected = False
self.destination = self.parent.destination
if not self.credentials(): return False
# timeout alarm 100 secs to connect
alarm_set(self.iotime)
try:
expire = -999
if self.parent.timeout: expire = self.parent.timeout
if self.port == '' or self.port == None: self.port = 21
if not self.tls:
ftp = ftplib.FTP()
ftp.connect(self.host, self.port, timeout=expire)
ftp.login(self.user, self.password)
else:
# ftplib supports FTPS with TLS
ftp = ftplib.FTP_TLS(self.host,
self.user,
self.password,
timeout=expire)
if self.prot_p: ftp.prot_p()
# needed only if prot_p then set back to prot_c
#else : ftp.prot_c()
ftp.set_pasv(self.passive)
self.originalDir = '.'
try:
self.originalDir = ftp.pwd()
except:
(stype, svalue, tb) = sys.exc_info()
self.logger.warning("Unable to ftp.pwd (Type: %s, Value: %s)" %
(stype, svalue))
self.pwd = self.originalDir
self.connected = True
self.ftp = ftp
alarm_cancel()
return True
except:
(stype, svalue, tb) = sys.exc_info()
self.logger.error(
"Unable to connect to %s (user:%s). Type: %s, Value: %s" %
(self.host, self.user, stype, svalue))
alarm_cancel()
return False
def _ftp_connect(self):
if self.tls:
self.conn = ftplib.FTP_TLS()
else:
self.conn = ftplib.FTP()
self.conn.connect(self.host, self.port, timeout=self.timeout)
self.conn.login(self.username, self.password)
if self.tls:
self.conn.prot_p()
def __makeFtpClient(self,
hostName,
userName="******",
pw=None,
keyFilePath=None,
certFilePath=None,
certFilePw=None):
"""
Make an FTP client connected to the specified host using provided authentication parameters
(username and password for standard FTP connection, and/or certificate and private key files for FTP_TLS connection).
Arguments:
hostName (str): hostname of FTP server (e.g., 'ftp.rcsb.org')
userName (str): username to login with
pw (str): password to login with (if required)
keyFilePath (str): path to a file containing the private SSL key (for TLS connections)
certFilePath (str): "path to a single file in PEM format containing the certificate as
well as any number of CA certificates needed to establish the
certificate’s authenticity" (for TLS connections)
(https://docs.python.org/3/library/ssl.html)
certFilePw (str): password to access the certificate file (if required)
Returns:
ftplib.FTP or ftplib.FTP_TLS: ftplib FTP connection object, as either a standard FTP connection or with FTP_TLS
(depending if an SSL certificate was provided for the connection or not).
"""
ftp = None
self.__context = None
try:
if certFilePath is not None:
# Create SSLContext object
self.__context = ftplib.ssl.SSLContext()
# Load the certificate and key (if provided) into the context object
self.__context.load_cert_chain(certfile=certFilePath,
keyfile=keyFilePath,
password=certFilePw)
# Connect via FTP_TLS using supplied method of authentication.
# For TLS with certificate, host and user may need to be set to empty string '' or None
ftp = ftplib.FTP_TLS(host=hostName,
user=userName,
passwd=pw,
context=self.__context)
else:
# Connect via FTP using username and password authentication
ftp = ftplib.FTP(host=hostName, user=userName, passwd=pw)
return ftp
except Exception as e:
logger.error("Error occurred creating FTP client: %s: %s",
e.__class__, str(e))
if ftp is not None:
ftp.close()
if self.__raiseExceptions:
raise e
def connect(self):
self.logger.debug("sr_ftp connect %s" % self.parent.destination)
self.connected = False
self.destination = self.parent.destination
if not self.credentials() : return False
# timeout alarm 100 secs to connect
alarm_set(self.iotime)
try:
expire = -999
if self.parent.timeout : expire = self.parent.timeout
if self.port == '' or self.port == None : self.port = 21
if not self.tls :
ftp = ftplib.FTP()
ftp.connect(self.host,self.port,timeout=expire)
ftp.login(self.user, self.password)
else :
# ftplib supports FTPS with TLS
ftp = ftplib.FTP_TLS(self.host,self.user,self.password,timeout=expire)
if self.prot_p : ftp.prot_p()
# needed only if prot_p then set back to prot_c
#else : ftp.prot_c()
ftp.set_pasv(self.passive)
self.originalDir = '.'
try:
self.originalDir = ftp.pwd()
except:
self.logger.warning("Unable to ftp.pwd")
self.logger.debug('Exception details: ', exc_info=True)
self.pwd = self.originalDir
self.connected = True
self.ftp = ftp
self.file_index_cache = self.user_cache_dir + os.sep + '.dest_file_index'
if os.path.isfile(self.file_index_cache): self.load_file_index()
else: self.init_file_index()
#alarm_cancel()
return True
except:
self.logger.error("Unable to connect to %s (user:%s)" % ( self.host, self.user ) )
self.logger.debug('Exception details: ', exc_info=True)
alarm_cancel()
return False
def ftp_connect(self, host, user='******', password='******', port=21, timeout=30, connId='default', tls=False, mode='passive'):
"""
Constructs FTP object, opens a connection and login. TLS support is optional.
Call this function before any other (otherwise raises exception).
Returns server output.
Parameters:
- host - server host address
- user(optional) - FTP user name. If not given, 'anonymous' is used.
- password(optional) - FTP password. If not given, 'anonymous@' is used.
- port(optional) - TCP port. By default 21.
- timeout(optional) - timeout in seconds. By default 30.
- connId(optional) - connection identifier. By default equals 'default'
- tls(optional) - TLS connections flag. By default False
- mode(optional) - set the transfer mode to 'active' or 'passive'. By default 'passive'
Examples:
| ftp connect | 192.168.1.10 | mylogin | mypassword | | |
| ftp connect | 192.168.1.10 | | | | |
| ftp connect | 192.168.1.10 | mylogin | mypassword | connId=secondConn | |
| ftp connect | 192.168.1.10 | mylogin | mypassword | 29 | 20 |
| ftp connect | 192.168.1.10 | mylogin | mypassword | 29 | |
| ftp connect | 192.168.1.10 | mylogin | mypassword | timeout=20 | |
| ftp connect | 192.168.1.10 | port=29 | timeout=20 | | |
| ftp connect | 192.168.1.10 | port=29 | timeout=20 | mode=active | |
"""
if connId in self.ftpList:
errMsg = "Connection with ID %s already exist. It should be deleted before this step." % connId
raise FtpLibraryError(errMsg)
else:
newFtp = None
outputMsg = ""
try:
timeout = int(timeout)
port = int(port)
newFtp = None
if tls:
newFtp = ftplib.FTP_TLS()
else:
newFtp = ftplib.FTP()
outputMsg += newFtp.connect(host, port, timeout)
self.__addNewConnection(newFtp, connId)
outputMsg += newFtp.login(user, password)
# set mode depending of "mode" value. if it is not "active" or "passive" default to passive
newFtp.set_pasv({'passive': True, 'active': False}.get(mode, True))
except socket.error as se:
raise FtpLibraryError('Socket error exception occured.')
except ftplib.all_errors as e:
if connId in self.ftpList:
self.ftp_close(connId)
raise FtpLibraryError(str(e))
except Exception as e:
raise FtpLibraryError(str(e))
if self.printOutput:
logger.info(outputMsg)
def __init__(self, download_dir, method='ftp', *args, **kwargs):
super().__init__(download_dir, *args, **kwargs)
self.method = method
if method == 'ftp':
logger.info("connecting to server")
self.server = ftplib.FTP_TLS("spdf.gsfc.nasa.gov")
self.server.login()
self._download_file = self._download_ftp_file
elif method == 'http':
self._download_file = self._download_http_file
def upload(host, dirname, path, user, passwd, isProtected):
ret = False
try:
if isProtected:
if user and passwd:
ftp = ftplib.FTP_TLS(host, user, passwd)
else:
ftp = ftplib.FTP_TLS(host)
else:
if user and passwd:
ftp = ftplib.FTP(host, user, passwd)
else:
ftp = ftplib.FTP(host)
except Exception, e:
print 'Failed to connect host "%s" with' % host, user, 'and', passwd, ":", e
return ret
def _connect(self):
""" Log in to ftp """
if self.tls:
self.ftpcon = ftplib.FTP_TLS()
else:
self.ftpcon = ftplib.FTP()
self.ftpcon.connect(self.host, self.port)
self.ftpcon.login(self.username, self.password)
if self.tls:
self.ftpcon.prot_p()
def ftpupload():
try:
session = ftplib.FTP_TLS('ftp.russellvalefootball.com', 'user', 'pass')
file = open('test.json', 'rb')
session.prot_p()
session.storbinary('STOR /public_html/test.json', file)
file.close()
session.quit()
except:
print('ftp error')
def __init__(self, config, generic_config, name):
self.config = config
self.generic_config = generic_config
self.name = name
self.isClosed = False
if self.config['tls'] is True:
self.connection = ftplib.FTP_TLS()
else:
self.connection = ftplib.FTP()
def __init__(self, one=None, globus_client=None):
super().__init__(one=one)
self.ftp = ftplib.FTP_TLS(host=FTP_HOST,
user=one._par.FTP_DATA_SERVER_LOGIN,
passwd=one._par.FTP_DATA_SERVER_PWD)
# self.ftp.ssl_version = ssl.PROTOCOL_TLSv1
# self.ftp.auth()
self.ftp.prot_p()
self.ftp.login(one._par.FTP_DATA_SERVER_LOGIN, one._par.FTP_DATA_SERVER_PWD)
# pre-fetch the repositories so as not to query them for every file registered
self.repositories = self.one.alyx.rest("data-repository", "list")
def ftpget(url, directory, filename):
"""Return contents of a file on an ftp-ssl site"""
contents = []
ftps = ftplib.FTP_TLS(url)
# login and encrypt connection
ftps.login()
ftps.prot_p()
ftps.cwd(directory)
ftps.retrlines("RETR {:s}".format(filename), contents.append)
return contents
def __init__(self, ftp_target, ftp_port, ssl=False, verbosity=False):
self.ftp_target = ftp_target
self.ftp_port = ftp_port
self.verbosity = verbosity
self.peer = "{}:{}".format(self.ftp_target, ftp_port)
if ssl:
self.ftp_client = ftplib.FTP_TLS()
else:
self.ftp_client = ftplib.FTP()
def task(ftps_host,
ftps_user,
ftps_pwd,
import_host="localhost",
import_port="15000",
specific_dates=None):
print("Running import task at", datetime.datetime.now())
ds = [datetime.date.today()] if specific_dates is None else specific_dates
ftps = ftplib.FTP_TLS(ftps_host, user=ftps_user, passwd=ftps_pwd)
ftps.prot_p()
ftps.cwd("dwd")
cap = capnp.TwoPartyClient(import_host + ":" +
import_port).bootstrap().cast_as(
dwd_service_capnp.DWLABImport)
dates = [f"{d:%Y%m%d}" for d in ds]
files = defaultdict(lambda: defaultdict(dict))
for entry in filter(lambda e: e[1]["type"] == "file",
ftps.mlsd(facts=["type"])):
date = entry[0][9:17]
type = entry[0][4:8]
time = int(entry[0][18:22])
if date in dates and type in ["DWLA", "DWLB"]:
files[date][type][time] = entry[0]
dates = list(files.keys())
dates.sort()
for date in dates:
type_to_times = files[date]
def retrieve(type):
times = list(type_to_times[type].keys())
if len(times) > 0:
times.sort()
time = times[-1]
with io.BytesIO() as f:
ftps.retrbinary("RETR " + type_to_times[type][time],
f.write)
return f.getvalue().decode("cp1252")
dwla = retrieve("DWLA")
dwla_comp = zlib.compress(dwla.encode("cp1252"))
#print("DWLA:\n", dwla)
dwlb = retrieve("DWLB")
dwlb_comp = zlib.compress(dwlb.encode("cp1252"))
#print("DWLB:\n", dwlb)
d = datetime.datetime.strptime(date, "%Y%m%d")
#print("len(dwla)=",len(dwla), " len(dwlb)=",len(dwlb))
#print("len(dwla_comp)=",len(dwla_comp), " len(dwlb_comp)=",len(dwlb_comp))
success = cap.importData(f"{d:%Y-%m-%d}", dwla_comp, dwlb_comp).wait()
print("Import succeeded?", success)
def ftp_retrieve(host, from_path, to_path, port=None, secure=False):
if secure:
ftp = ftplib.FTP_TLS(context=ssl.create_default_context())
else:
ftp = ftplib.FTP()
host = (host, port) if port else (host, )
ftp.connect(*host)
ftp.login()
with open(to_path, 'wb') as f:
ftp.retrbinary('RETR ' + from_path, f.write, 1024)
ftp.close()
def ftps_brute_force(host, ports, usernames, passwords, timeout):
ftp_connection = ftplib.FTP_TLS(timeout=int(timeout))
ftp_connection.connect(host, int(ports))
ftp_connection.login(usernames, passwords)
ftp_connection.close()
return {
"host": host,
"username": usernames,
"password": passwords,
"port": ports
}
def __init__(self, ip_, port_=0, usn_='anonymous', pwd_='', secure_=False):
self.ip = ip_
self.port = port_
self.usn = usn_
self.passwd = pwd_
self.secure_ = secure_
if secure_:
self.con = ftplib.FTP_TLS()
else:
self.con = ftplib.FTP()
self.buffer_ = []
def _setup(self,
tls_control_required=False,
tls_data_required=False,
ssl_protocol=ssl.PROTOCOL_SSLv23,
):
self.server = FTPSServer()
self.server.handler.tls_control_required = tls_control_required
self.server.handler.tls_data_required = tls_data_required
self.server.handler.ssl_protocol = ssl_protocol
self.server.start()
self.client = ftplib.FTP_TLS(timeout=TIMEOUT)
self.client.connect(self.server.host, self.server.port)
def ftpUpload():
session = ftplib.FTP_TLS('sgp63.siteground.asia',
'*****@*****.**',
'ty%o7E&^k7-^') #FTP Connexion
session.cwd('/public_html/app/scinternational/scifiles/assets/'
) # Change difrectory
for photo in os.listdir(
"photos"): # Loop through the file in /photos/ folder
file = open('photos/' + photo, 'rb') # open file to send
session.storlines('STOR ' + photo, file) # send the file
file.close() # close file
print('All files uploaded')
def ftp_init():
CONFIG = Config()
if CONFIG["FTPSSL"]:
ftp = ftplib.FTP_TLS(CONFIG["FTPHost"])
ftp.prot_p()
else:
ftp = ftplib.FTP(CONFIG["FTPHost"])
ftp.login(CONFIG["FTPUser"], CONFIG["FTPPass"])
ftp.cwd(CONFIG["FTPDir"])
return ftp
def get_ftp_class(options):
if options.ftp.ssl:
if hasattr(ftplib, 'FTP_TLS'): # SSL new in 2.7+
ftp = ftplib.FTP_TLS(options.ftp.hostname, options.ftp.username, options.ftp.password)
ftp.prot_p()
logging.info("Using SSL")
else:
raise FtpSslNotSupported(
"Python is too old for FTP SSL. Try using Python 2.7 or later.")
else:
ftp = ftplib.FTP(options.ftp.hostname, options.ftp.username, options.ftp.password)
return ftp
def copyFTPFolderToTemp(host, user, password, ftpPath, packageName):
dirPath = tempfile.mkdtemp()
tempPackageDir = os.path.join(dirPath, packageName)
ftps = ftplib.FTP_TLS(host)
ftps.login(user,
password) # login anonymously before securing control channel
ftps.prot_p(
) # switch to secure data connection.. IMPORTANT! Otherwise, only the user and password is encrypted and not all the file data.
ftps.cwd(r"{0}/".format(ftpPath))
ftp_utility.download_ftp_tree(ftps, packageName, dirPath, overwrite=True)
return tempPackageDir
def log(self, fileOrgignal, fileCrash, crashReport = None, fileDesc = None, maxPerIssue=25):
self.workDone = 0
ftp = ftplib.FTP_TLS(self.host)
ftp.FtpsLoging = self
#ftp.login(self.username, self.password)
ftp.sendcmd("USER " + self.username)
ftp.sendcmd("PASS " + self.password)
tmpDir = self.dir
self.newDir(ftp, tmpDir)
if crashReport != None:
if crashReport.nearNull:
tmpDir = "nearNull"
else:
tmpDir = "notNearNull"
self.newDir(ftp, tmpDir)
tmpDir = crashReport.type
self.newDir(ftp, tmpDir)
tmpDir = crashReport.location.replace(":",".")
self.newDir(ftp, tmpDir)
count = 0
try:
flist = []
ftp.retrlines('MLSD', flist.append)
count = len(flist)/3
except:
count = (len(ftp.nlst("")))/3
if count >= maxPerIssue:
ftp.close()
return
ftp.storbinary('STOR ' + ("%04d_crash" % count) + os.path.splitext(fileCrash)[1], open(fileCrash, 'rb'), callback = self.workDoneFunc)
ftp.storbinary('STOR ' + ("%04d_original" % count) + os.path.splitext(fileOrgignal)[1], open(fileOrgignal, 'rb'), callback = self.workDoneFunc)
fout = tempfile.TemporaryFile()
if fileDesc != None:
fout.write(fileDesc + "\n\n")
if crashReport != None:
fout.write(crashReport.getInfo())
fout.seek(0)
ftp.storbinary('STOR ' + ("%04d_description.txt" % count), fout, callback = self.workDoneFunc)
while self.workDone < 3:
time.sleep(1)
fout.close()
ftp.close()
count = 0
def main():
Git.git_binary = 'git' # Windows doesn't like env
repo, options, args = parse_args()
if repo.is_dirty() and not options.commit:
logging.warning("Working copy is dirty; uncommitted changes will NOT be uploaded")
base = options.ftp.remotepath
logging.info("Base directory is %s", base)
try:
branch = (h for h in repo.heads if h.name == options.branch).next()
except StopIteration:
raise BranchNotFound
commit = branch.commit
if options.commit:
commit = repo.commit(options.commit)
tree = commit.tree
if options.ftp.ssl:
if hasattr(ftplib, 'FTP_TLS'): # SSL new in 2.7+
ftp = ftplib.FTP_TLS(options.ftp.hostname, options.ftp.username, options.ftp.password)
ftp.prot_p()
logging.info("Using SSL")
else:
raise FtpSslNotSupported("Python is too old for FTP SSL. Try using Python 2.7 or later.")
else:
ftp = ftplib.FTP(options.ftp.hostname, options.ftp.username, options.ftp.password)
ftp.cwd(base)
# Check revision
hash = options.revision
if not options.force and not hash:
hashFile = cStringIO.StringIO()
try:
ftp.retrbinary('RETR git-rev.txt', hashFile.write)
hash = hashFile.getvalue()
except ftplib.error_perm:
pass
if not hash:
# Diffing against an empty tree will cause a full upload.
oldtree = get_empty_tree(repo)
else:
oldtree = repo.commit(hash).tree
if oldtree.hexsha == tree.hexsha:
logging.info('Nothing to do!')
else:
upload_diff(repo, oldtree, tree, ftp, base)
ftp.storbinary('STOR git-rev.txt', cStringIO.StringIO(commit.hexsha))
ftp.quit()
def on_task_input(self, task, config):
connection_config = config['config']
if connection_config['use-ssl']:
ftp = ftplib.FTP_TLS()
else:
ftp = ftplib.FTP()
# ftp.set_debuglevel(2)
log.debug('Trying connecting to: %s', (connection_config['host']))
try:
ftp.connect(connection_config['host'], connection_config['port'])
ftp.login(connection_config['username'], connection_config['password'])
except ftplib.all_errors as e:
raise plugin.PluginError(e)
log.debug('Connected.')
encoding = connection_config['encoding']
files_only = connection_config['files-only']
recursive = connection_config['recursive']
get_size = connection_config['get-size']
mlst_supported = False
feat_response = ftp.sendcmd('FEAT').splitlines()
supported_extensions = [feat_item.strip().upper() for feat_item in feat_response[1:len(feat_response) - 1]]
if encoding.lower() == 'auto' and 'UTF8' in supported_extensions:
encoding = 'utf8'
else:
encoding = 'ascii'
for supported_extension in supported_extensions:
if supported_extension.startswith('MLST'):
mlst_supported = True
break
if not mlst_supported:
log.warning('MLST Command is not supported by the FTP server %s@%s:%s', connection_config['username'],
connection_config['host'], connection_config['port'])
ftp.sendcmd('TYPE I')
ftp.set_pasv(True)
entries = []
for path in config['dirs']:
baseurl = "ftp://%s:%s@%s:%s/" % (connection_config['username'], connection_config['password'],
connection_config['host'], connection_config['port'])
self._handle_path(entries, ftp, baseurl, path, mlst_supported, files_only, recursive, get_size, encoding)
return entries
def ftp_connect(self, config, ftp_url, current_path):
if config['use-ssl']:
ftp = ftplib.FTP_TLS()
else:
ftp = ftplib.FTP()
log.debug("Connecting to " + ftp_url.hostname)
ftp.connect(ftp_url.hostname, ftp_url.port)
ftp.login(ftp_url.username, ftp_url.password)
ftp.sendcmd('TYPE I')
ftp.set_pasv(True)
ftp.cwd(current_path)
return ftp
