def downloadSarZIP(value, ftpParams):
ftps = FTP_TLS()
#ftps.set_debuglevel(2)
ftps.connect(ftpParams.host)
ftps.sendcmd('USER ' + ftpParams.user)
ftps.sendcmd('PASS ' + ftpParams.password)
list_of_files = []
checkFile = []
ftps.retrlines("NLST", list_of_files.append)
# ...or use the existing helper
# list_of_files = ftps.nlst()
dest_dir = "./"
# download files from ftp
for name in list_of_files:
if fnmatch.fnmatch(name, value + "_*"):
checkFile.append(name)
with open(os.path.join(dest_dir, name), "wb") as f:
ftps.retrbinary("RETR {}".format(name), f.write)
#delete files from ftp
for name in list_of_files:
if fnmatch.fnmatch(name, value + "_*"):
ftps.delete(name)
ftps.quit()
return checkFile
def sync(self):
"""
downloads all needed_files from self.hostname (FTP)
of the downloaded files, extracts .gz files to same local_working_dir
-using self.extract function
parses the .txt downloaded needed_files
-using the self.parse function
"""
ftps = FTP_TLS(self.hostname) # connect to host, default port
ftps.login(self.username, self.password)
ftps.prot_p()
ftps.cwd(self.remote_dir) # change into "logs" directory
ftps.retrlines('LIST *.gz *.txt', self.ftp_list_callback) # list directory contents
for needed_file in self.needed_files:
if self.logging:
print "Writing {0} to {1}...".format(needed_file, self.local_working_dir)
ftps.retrbinary("RETR " + needed_file, open(os.path.join(self.local_working_dir, needed_file), 'wb').write)
if self.logging:
print "done syncing files"
for needed_file in self.needed_files:
if needed_file.endswith(".gz"):
self.extract(os.path.join(self.local_working_dir, needed_file))
txt_file_name = needed_file.replace('.gz','')#if already a .txt file, this is unnceccessary but works.
self.parse(txt_file_name)
if self.logging:
print "done extracting/parsing .gz files"
ftps.quit()
def send_to_ftp(filepath):
ftps = FTP_TLS('ftpes.learn.inf.puc-rio.br') # Define Ftp server
ftps.login('learn', 'LRepoAdm18!!') # Login to ftp server
ftps.prot_p() # Enable data encryption
ftps.retrlines('LIST') # List Directory
ftps.cwd('/projeto/01_tarefas_de_geofisica/QualiSismo/CV_results') # Change Directory
ftps.storlines("STOR " + filepath, open(filepath,'rb'))
def connect(user, password, ip_address='pensieve.usc.edu', port=41590):
ftp = FTP_TLS()
ftp.ssl_version = ssl.PROTOCOL_SSLv23
ftp.debugging = 2
ftp.connect(ip_address, port)
ftp.login(user, password)
ftp.prot_p()
ftp.retrlines('LIST home')
return(ftp)
def connectFTPS(hst, usr, pwd, prt):
try:
ftps = FTP_TLS(hst, usr, pwd)
retr = ftps.retrlines('SYST')
retr.join(ftps.retrlines('PWD'))
retr.join(ftps.retrlines('LIST'))
ftps.quit()
return retr
except Exception as e:
return "%s" % (e)
def start(self):
# create connection
for ftpserver, users in self.db.iteritems():
for s_user in users:
self.log.log("Connecting to %s: user: %s pass: %s" % (ftpserver, s_user.user, s_user.passwd))
ftp = FTP_TLS(ftpserver) # connect to host, default port
ftp.login(user=s_user.user, passwd=s_user.passwd)
ftp.prot_p() # switch to secure data connection
ftp.retrlines('LIST', self.parseFiles)
self.log.log("Done! now quit...")
ftp.quit()
def get_latest_file():
with open('filenames.json') as json_file:
files = json.load(json_file)
flag = 0
ftp = FTP_TLS(host='olftp.adesa.com',
user='******',
passwd='aU)kj7Qn8')
ftp.prot_p()
#ftp.retrlines('LIST')
ftp.cwd('outbound/')
file_list = []
ftp.retrlines('MLSD', file_list.append)
#ftp.dir(file_list.append)
max_value = 0
full_max = 0
filename = ''
full_file = ''
for i in file_list:
col = i.split(';')
col_max = int(re.search(r'\d+', col[0]).group())
if (col_max > max_value) & ('.txt' in col[-1]):
max_value = col_max
filename = col[-1].replace(' ', '')
if (col_max > full_max) & ('.txt' in col[-1]) & ('FULL' in col[-1]):
full_max = col_max
full_file = col[-1].replace(' ', '')
if (filename != files['inc_file']):
localfile = open(filename, 'wb')
ftp.retrbinary('RETR ' + filename, localfile.write, 1024)
localfile.close()
print("Inc file data tranfer complete")
flag = 1
else:
print("Inc already there")
if (full_file != files['full_file']):
localfile = open(full_file, 'wb')
ftp.retrbinary('RETR ' + full_file, localfile.write, 1024)
localfile.close()
print("Full file data tranfer complete")
flag = 1
else:
print("Full already there")
if flag == 1:
new_names = {'full_file': full_file, 'inc_file': filename}
with open('filenames.json', 'w') as outfile:
json.dump(new_names, outfile)
ftp.quit()
return filename, full_file, flag
def TimerCallback():
data = getConfig()
# loop processes list and kill each one
processes = data["processes"]
for line in processes.splitlines():
Log("Killing Process %s " % line[line.rindex('\\')+1:])
os.system('taskkill /f /im %s' % line[line.rindex('\\')+1:])
# Connect to SFTP
ftps = FTP_TLS('fuge.it')
ftps.login('testuser', 'testpass') # login anonymously before securing control channel
ftps.prot_p() # switch to secure data connection.. IMPORTANT! Otherwise, only the user and password is encrypted and not all the file data.
ftps.retrlines('LIST')
# Loop directories/files and sftp each one
directories = data["directories"]
for line in directories.splitlines():
# If nothing after last slash then this is a directory we need to loop for files
if line[line.rindex('\\')+1:] == "":
for fn in os.listdir(line):
if os.path.isfile(fn):
# upload file to public/ on remote
myfile = open(fn, 'r')
ftps.storlines('STOR ' + fn, myfile)
myfile.close()
else: # otherwise it's a single file
if os.path.isfile(line):
# upload file to public/ on remote
localpath = line
myfile = open(line, 'r')
ftps.storlines('STOR ' + filename, myfile)
myfile.close()
ftps.close()
# reset daemon for tomorrow's run
try: win32api.WinExec('daemon.exe %d %d %d %d' % (day, hr, min, sec)) # Works seamlessly
except: pass
# loop processes list and kill each one
processes = data["processes"]
for line in processes.splitlines():
Log("Restarting Process %s " % line)
try: win32api.WinExec(line) # Works seamlessly
except: pass
def corrReportDL(exp_id, vgos_tag):
year = '20' + str(vgos_tag[0:2])
tag = str(vgos_tag.rstrip())
exp_id = str(exp_id)
vgos_exists = []
if os.path.isfile(dirname + "/corr_files/" + exp_id + '.corr'):
print("Corr report already exists for experiment " + exp_id +
", skipping re-download.")
return
else:
ftps = FTP_TLS(host='gdc.cddis.eosdis.nasa.gov')
ftps.login()
ftps.prot_p()
try:
ftps.retrlines(
"LIST /pub/vlbi/ivsdata/vgosdb/" + year + "/" + tag + ".tgz",
vgos_exists.append)
if len(vgos_exists) > 0:
local_filename = os.path.join(dirname, tag + ".tgz")
ftps.sendcmd('TYPE I')
lf = open(local_filename, "wb")
ftps.retrbinary(
"RETR /pub/vlbi/ivsdata/vgosdb/" + year + "/" + tag +
".tgz", lf.write)
lf.close()
tar = tarfile.open(dirname + '/' + tag + ".tgz")
if tag + '/History/' + tag + '_V000_kMk4.hist' in tar.getnames(
):
member = tar.getmember(tag + '/History/' + tag +
'_V000_kMk4.hist')
member.name = dirname + '/corr_files/' + exp_id + '.corr'
tar.extract(member)
tar.close()
else:
file_list = tar.getnames()
regex = re.compile('.*V...\.hist')
for file in file_list:
if re.match(regex, file):
member = tar.getmember(file)
member.name = dirname + '/corr_files/' + exp_id + '.corr'
tar.extract(member)
tar.close()
break
os.remove(dirname + '/' + tag + ".tgz")
print("Corr report download complete for experiment " +
exp_id + ".")
return
except Exception:
print("Corr report not available for experiment " + exp_id + ".")
return
class ftpAccess():
def __init__(self, url):
self.ftp = FTP_TLS(url)
self.ftp.login(
"anonymous", "anonymous",
secure=False) # login anonymously before securing control channel
def GetFtpFile(self, InPath, filename, downloaddir):
outfile = downloaddir + filename
ensure_dir(downloaddir)
self.ftp.cwd(InPath)
self.ftp.retrbinary("RETR " + filename, open(outfile, 'wb').write)
def GetFileListFtp(self, pathname):
self.ftp.cwd(pathname)
ret = list()
out = self.ftp.retrlines('LIST',
addline) # list directory content securely
# search json files temp = entry.split(' ')
for filename in filenamelist:
try:
a = self.ftp.size(filename)
print("{} - {}".format(filename, a))
ret.append([filename, a])
except:
print("{} - xxx".format(filename))
pass
return ret
def ftp_backup(dmpdir,dbname):
try:
#
ftp =FTP_TLS()
ftp.connect(ftphost,ftpport)
ftp.login(ftpuser,ftppass)
ftp.prot_p()
print "Welcome:",ftp.getwelcome()
print ftp.retrlines('LIST')
# ftp =FTP()
# ftp.connect(ftphost,ftpport)
# ftp.login(ftpuser,ftppass)
# print "Welcome:",ftp.getwelcome()
# print ftp.retrlines('LIST')
except Exception,e:
print e
return
def connect(velkost_ftp,port):
ftp=FTP_TLS(server,meno2,ps,port)
ftp.prot_p()
ftp.cwd(my_list[2])
print "Posielam subor. Cakajte prosim."
obsah=open(file_to_send, 'rb')
obsah.close()
ftp.storbinary('STOR %s' % file_to_send, open(file_to_send, 'rb'))
obsah.close()
print "Subor odoslany [OK]"
print "Obsah adresara na serveri:"
ftp.retrlines("LIST")
size_ftp=ftp.nlst()
pocet=len(size_ftp)
velkost_ftp_subor=size_ftp[pocet-1] #berie posledne pridany subor zo zoznamu
ftp.sendcmd("TYPE i")
velkost_ftp=ftp.size(velkost_ftp_subor)
ftp.close()
return velkost_ftp
def download(downloaded, user, passwd, all_files=False, filename=None):
# Connect to the MAPS ftp server over FTPS
ftps = FTP_TLS('ftps.tsi.telecom-paristech.fr')
print 'Connected to MAPS FTP over TLS.'
try:
ftps.login(user=user, passwd=passwd)
ftps.cwd('maps')
except error_perm:
print "Incorrect username/password" ; quit
ftps.retrlines('LIST *.zip', get_file_list)
if filename is not None:
if not in_downloads(files, filename): print 'File not found' ; return
print 'Downloading', filename
res = ftps.retrbinary('RETR '+filename, open('./downloads/'+filename, 'wb').write)
ftps.close()
return [(filename, 0)]
if len(files) == len(downloaded):
print "All MAPS files downloaded. Continuing."
return
if all_files:
for f, s in files:
if not in_downloads(downloaded, f):
print "Downloading", f, "of size", s, "bytes"
res = ftps.retrbinary('RETR '+f, open('./downloads/'+f, 'wb').write)
elif filename is None:
f, s = random.choice(files)
while in_downloads(downloaded, f):
f, s = random.choice(files)
print "Downloading", f, "of size", s, "bytes"
res = ftps.retrbinary('RETR '+f, open('./downloads/'+f, 'wb').write)
ftps.close()
if all_files: return files
return [(f, s)]
class Client:
def __init__(self):
self._url = None
self._handler = None
self._usr = '******'
self._pwd = 'guest'
def set_url(self, url):
parsed_result = urlparse(url)
protocol = parsed_result.scheme
domain = parsed_result.netloc
dirs = parsed_result.path.split('/')
file = dirs[-1]
dirs = dirs[:-2]
if not protocol.lower().startswith('ftp://'):
raise AssertionError(f'Not a valid ftp resource, url: {url}')
self._url = url
self._domain = domain
self._dirs = dirs
self._file = file
return self
def login(self):
self._handler = FTP(self._url)
self._handler.login(self._usr, self._pwd)
return self
def login_tls(self):
self._handler = FTP_TLS(self._url)
self._handler.login(self._usr, self._pwd)
return self
def available(self):
self._handler.cwd('pub1')
a = self._handler.retrlines('LIST')
return a
class FtpProxy(object):
def __init__(self):
self.connectionType = ''
self.connection = None
self.username = ''
self.password = ''
self.serverName = ''
self.serverIp = ''
self.isConnected = False
self.logger = None
@classmethod
def Init(cls, ConnectionType, ServerName, ServerIp, UserName, Password,
Logger):
ftpProxy = cls()
## initializing logger
ftpProxy.logger = Logger
## initializing...
ftpProxy.connectionType = str(ConnectionType)
ftpProxy.username = str(UserName)
if ftpProxy.username == '':
ftpProxy.username = '******'
ftpProxy.password = str(Password)
ftpProxy.serverName = str(ServerName)
ftpProxy.serverIp = str(ServerIp)
ftpProxy.isConnected = False
return ftpProxy
def connectByServerName(self):
try:
if (self.connectionType == 'ftp'):
self.connection = FTP(self.serverName)
elif (self.connectionType == 'ftps'):
self.connection = FTP_TLS(self.serverName)
else:
raise Exception('Unknown connection type...')
self.connection.login(user=self.username, passwd=self.password)
if (self.connectionType == 'ftps'):
self.connection.prot_p()
self.isConnected = True
return self.isConnected
except Exception as e:
self.logger.LogMessage('error', str(e))
return False
def connectByServerIp(self):
try:
if (self.connectionType == 'ftp'):
self.connection = FTP(self.serverIp)
elif (self.connectionType == 'ftps'):
self.connection = FTP_TLS(self.serverIp)
else:
raise Exception('Unknown connection type...')
self.connection.login(user=self.username, passwd=self.password)
if (self.connectionType == 'ftps'):
self.connection.prot_p()
self.isConnected = True
return self.isConnected
except Exception as e:
self.logger.LogMessage('error', str(e))
return False
def Connect(self):
if self.connectByServerName():
return True
if self.connectByServerIp():
return True
return False
def parseLocationEntry(self, locationPath, entry, entries):
ftpEntry = None
try:
ftpEntry = FtpEntry.Init(locationPath, entry)
except Exception as e:
self.logger.LogMessage(
'error', 'error parsing entry {0} in {1} {2}'.format(
entry, locationPath, str(e)))
if ftpEntry:
entries.append(ftpEntry)
def ListLocation(self, LocationPath):
if not self.isConnected:
return None
if len(LocationPath) == 0:
LocationPath = '/'
if LocationPath[0:1] != '/':
LocationPath = '/{0}'.format(LocationPath)
LocationPath = re.sub('[\/]+', '/', LocationPath)
try:
entries = []
self.connection.retrlines(
'LIST {0}'.format(LocationPath), lambda entry: self.
parseLocationEntry(LocationPath, entry, entries))
return entries
except Exception as e:
self.logger.LogMessage(
'error', 'error listing {0} {1}'.format(LocationPath, str(e)))
return None
def RetrieveFile(self, FullFileName):
if not self.isConnected:
return None
try:
fileStream = io.BytesIO()
self.connection.retrbinary('RETR {0}'.format(FullFileName),
fileStream.write,
blocksize=1024)
return fileStream
except Exception as e:
self.logger.LogMessage(
'info',
'could not retrieve file {0} {1}'.format(FullFileName, str(e)))
return None
def shell(host, port, user, passwd, use_ftps):
""" Shell Mode function. Provides a CLI to interact with FTP server """
""" Args: host address, port number to connect, user[name] and passw[or]d to login, """
""" use_ftps - if true use FTPS instead of FTP """
#preparation: connect to the server
try:
if (use_ftps == "y"):
if (user != ""): #if username defined ---> use it
if (port == "def" or port == ""): #use default port 21
ftps = FTP_TLS(host=host, user=user, passwd=passwd)
else:
ftps = FTP_TLS(host=host, user=user, passwd=passwd, acct=port)
else: #anon login
if (port == "def" or port == ""): #use default port 21
ftps = FTP_TLS(host=host)
ftps.login()
else:
ftps = FTP_TLS(host=host, acct=port, user=user, passwd=str(passwd))
ftps.login()
print("Connecting to host '{0}' through port '{1}'... ".format(host, port))
os.system("echo Entering Shell Mode in 2 seconds")
os.system("sleep 2")
os.system("clear")
print("Server response:\n" + ftp.getwelcome())
if (use_ftps == "n"):
if (user != ""): #if username defined ---> use it
if (port == "def" or port == ""): #use default port 21
ftp = FTP(host=host, user=user, passwd=passwd)
else:
ftp = FTP(host=host, user=user, passwd=passwd, acct=port)
else: #anon login
if (port == "def" or port == ""): #use default port 21
ftp = FTP(host=host)
ftp.login()
else:
ftp = FTP(host=host, acct=port, user=user, passwd=str(passwd))
ftp.login()
print("Connecting to host '{0}' through port '{1}'... ".format(host, port))
os.system("echo Entering Shell Mode in 2 seconds")
os.system("sleep 2")
os.system("clear")
print("Server response:\n" + ftp.getwelcome())
except ftplib.all_errors as ex:
print(color.red + "An error occured while login: {0}".format(ex) + color.std)
return
#enter shell mode
while(1):
try:
if (use_ftps == "n"):
cmd = input("ftp@{0}~#".format(host))
cmd, arg = parse(cmd)
if (cmd == "cd"):
ftp.cwd(arg)
elif (cmd == "exit"):
ftp.quit()
return
elif (cmd == "ls"): #unix's LS without flags support
print(ftp.dir())
elif (cmd == "cmd"): #send a command to the server
ftp.sendcmd(arg)
elif (cmd == "getbin"): #download file in binary mode
ftp.retrbinary(arg)
elif (cmd == "gettxt"): #download file on text mode
ftp.retrlines(arg)
elif (cmd == "unknown"):
print("Unknown command.")
else: #FTPS
cmd = input("ftps@{0}~#".format(host))
cmd, arg = parse(cmd)
if (cmd == "cd"):
ftps.cwd(arg)
elif (cmd == "exit"):
ftps.quit()
return
elif (cmd == "ls"): #unix's LS without flags support
print(ftps.dir())
elif (cmd == "cmd"): #send a command to the server
ftps.sendcmd(arg)
elif (cmd == "getbin"): #download file in binary mode
ftps.retrbinary(arg)
elif (cmd == "gettxt"): #download file on text mode
ftps.retrlines(arg)
elif (cmd == "unknown"):
print("Unknown command.")
except ftplib.all_errors as ex:
print(color.red + "Shell mode error: {0}".format(ex) + color.std)
class BrowseWindowRemote(QMainWindow):
'''This class creates a new windows for browsing'''
def __init__(self, parent=None, initdir='/',filter=[''],search='file',credentials=['','',''],outputline=''):
super(BrowseWindowRemote, self).__init__(parent)
self.setGeometry(50, 50, 400, 300)
self.pathdisplay = QLineEdit()
self.pathdisplay.setEnabled(False)
splitter0 = QSplitter(Qt.Vertical)
self.qtype = self.parent().qtype
self.qcommand = self.parent().qcommand
self.topleft = QListWidget()
self.topleft.itemDoubleClicked.connect(self.repopulate_folder_list)
self.topright = QListWidget()
if search == 'file': self.topright.itemDoubleClicked.connect(self.select_file)
splitter1 = QSplitter(Qt.Horizontal)
splitter1.addWidget(self.topleft)
splitter1.addWidget(self.topright)
splitter1.setSizes([200,200])
bottom_layout = QHBoxLayout()
ok_button = QPushButton(text='OK')
ok_button.clicked.connect(self.select_item)
cancel_button = QPushButton(text='Cancel')
cancel_button.clicked.connect(self.cancel)
bottom = QWidget()
bottom_layout.addWidget(ok_button)
bottom_layout.addWidget(cancel_button)
bottom_layout.addStretch(1)
bottom.setLayout(bottom_layout)
splitter0.addWidget(self.pathdisplay)
splitter0.addWidget(splitter1)
splitter0.addWidget(bottom)
self.success = False
self.ftps=None
self.initdir = initdir
self.folderpath = initdir
self.filters = filter
self.search = search
self.outputline = outputline
try:
self.servername = str(credentials[0])
self.username = str(credentials[1])
self.password = str(credentials[2])
except:
pass
#self.servername,self.username,self.password = '******','emuser','#99@3584cg'
try:
self.connect_ftp_server(self.servername, self.username, self.password)
self.setCentralWidget(splitter0)
self.success = True
self.add_folders()
self.show()
except:
QMessageBox().critical(self, "Credentials not valid.",
"Please provide a valid combination of servername, username, and password",
QMessageBox.Ok)
#self.destroy(True,True)
self.close()
def select_item(self):
if self.outputline: self.outputline.setText(self.folderpath)
self.close()
def select_file(self):
self.folderpath = os.path.join(self.folderpath, self.topright.currentItem().text())
self.outputline.setText(self.folderpath)
self.close()
def cancel(self):
#self.pathselect = self.initdir
self.close()
def connect_ftp_server(self,servername, username, password):
self.ftps = FTP_TLS(servername,username,password)
self.ftps.prot_p()
def append(self,line):
words = line.split(None, 3)
datestamp = words[0]
timestamp = words[1]
folder = words[2]
filename = words[3]
size = None
if folder == '<DIR>':
folder = True
else:
size = int(folder)
folder = False
if folder:
self.subdirs.append(filename)
else:
self.matchingfiles.append(filename)
def add_folders(self, folders=[]):
self.data()
self.topleft.clear()
self.topright.clear()
self.topleft.insertItems(0,self.subdirs)
self.topright.insertItems(0,self.matchingfiles)
#if self.search == 'file': self.topleft.itemDoubleClicked.connect(self.repopulate_folder_list)
def data(self):
self.subdirs = [os.pardir]
self.matchingfiles = []
self.ftps.cwd(self.folderpath)
self.ftps.retrlines('LIST', self.append)
matchingfiles = []
for pat in self.filters:
matchingfiles += [line for line in self.matchingfiles if line.endswith(pat.split('.')[-1])]
self.matchingfiles = matchingfiles
def repopulate_folder_list(self):
extra = self.topleft.currentItem().text()
if extra != '..':
self.folderpath = os.path.join(self.folderpath, extra)
elif len(self.folderpath) >1:
self.folderpath = os.path.dirname(self.folderpath)
self.pathdisplay.setText(self.folderpath)
self.add_folders()
def main(master_schedule, db_name):
schedule = str(master_schedule)
ftps = FTP_TLS(host='gdc.cddis.eosdis.nasa.gov')
ftps.login()
ftps.prot_p()
master_sched_filename = os.path.join(dirname, schedule)
mf = open(master_sched_filename, "wb")
ftps.sendcmd('TYPE I')
ftps.retrbinary('RETR /pub/vlbi/ivscontrol/' + schedule, mf.write)
mf.close()
valid_experiment = validExpFinder(os.path.join(dirname, schedule))
existing_experiments = checkExistingData(str(db_name))
if existing_experiments == None:
experiments_to_download = valid_experiment
else:
experiments_to_download = [
x for x in valid_experiment if x not in existing_experiments
]
year = '20' + schedule[6:8]
for exp in experiments_to_download:
if os.path.isfile(dirname + '/analysis_reports/' + exp.lower() +
'_report.txt'):
print("Analysis report already exists for " + exp.lower() +
", skipping file downloads.")
continue
else:
#ftp = FTP('cddis.gsfc.nasa.gov')
exp = exp.lower()
print('Beginning file downloads for experiment ' + exp + ".")
ftps = FTP_TLS(host='gdc.cddis.eosdis.nasa.gov')
ftps.login()
ftps.prot_p()
# Download SKED file
try:
filename_skd = []
ftps.retrlines(
'LIST /pub/vlbi/ivsdata/aux/' + str(year) + '/' + exp +
'/' + exp + '.skd', filename_skd.append)
if len(filename_skd) > 0:
local_filename_skd = os.path.join(
dirname, 'skd_files/' + exp + '.skd')
ftps.sendcmd('TYPE I')
lf3 = open(local_filename_skd, "wb")
ftps.retrbinary(
'RETR /pub/vlbi/ivsdata/aux/' + str(year) + '/' + exp +
'/' + exp + ".skd", lf3.write)
lf3.close()
except Exception:
print('No SKED file found for ' + exp)
# Spelling options need to be here because analysis report names are unfortunately not standardised - sometimes they are even different within the same experiment (e.g. 'ivs' and 'IVS')
# Now time to download analysis report
options = ['ivs', 'IVS', 'usno', 'USNO', 'NASA']
for spelling in options:
filename_report = []
try:
ftps.retrlines(
'LIST /pub/vlbi/ivsdata/aux/' + str(year) + '/' + exp +
'/' + exp + '-' + spelling + '-analysis-report*',
filename_report.append)
if len(filename_report) > 0:
local_filename_report = os.path.join(
dirname, 'analysis_reports/' + exp + '_report.txt')
ftps.sendcmd('TYPE I')
lf1 = open(local_filename_report, "wb")
ftps.retrbinary(
'RETR /pub/vlbi/ivsdata/aux/' + str(year) + '/' +
exp + '/' + filename_report[len(filename_report) -
1].split()[8],
lf1.write)
lf1.close()
print('Analysis report downloaded for experiment ' +
exp + ".")
break
except Exception:
pass
# Download spool file
for spelling in options:
filename_spool = []
try:
ftps.retrlines(
'LIST /pub/vlbi/ivsdata/aux/' + str(year) + '/' + exp +
'/' + exp + '-' + spelling + '-analysis-spoolfile*',
filename_spool.append)
if len(filename_spool) > 0:
local_filename_spool = os.path.join(
dirname,
'analysis_reports/' + exp + '_spoolfile.txt')
ftps.sendcmd('TYPE I')
lf2 = open(local_filename_spool, "wb")
ftps.retrbinary(
'RETR /pub/vlbi/ivsdata/aux/' + str(year) + '/' +
exp + '/' + filename_spool[len(filename_report) -
1].split()[8],
lf2.write)
lf2.close()
print('Spoolfile downloaded for experiment ' + exp +
".")
break
except Exception:
pass
# Download old style analysis report if it exists.
try:
filename_report_old = []
ftps.retrlines(
'LIST /pub/vlbi/ivsdata/aux/' + str(year) + '/' + exp +
'/' + exp + '-analyst.txt', filename_report_old.append)
if len(filename_report_old) > 0:
local_filename_report = os.path.join(
dirname, 'analysis_reports/' + exp + '_report.txt')
ftps.sendcmd('TYPE I')
lf1 = open(local_filename_report, "wb")
ftps.retrbinary(
'RETR /pub/vlbi/ivsdata/aux/' + str(year) + '/' + exp +
'/' + exp + "-analyst.txt", lf1.write)
lf1.close()
except Exception:
pass
from ftplib import FTP_TLS
ftps = FTP_TLS('202.157.143.104:12126')
ftps.login()
ftps.prot_p()
ftps.retrlines('LIST')
class ServerWatcher(Watcher):
downloadProgress = Signal((
int,
int,
))
uploadProgress = Signal((
int,
int,
))
# Si added:
textStatus = Signal((str, ))
fileEvent = Signal((str, ))
fileEventCompleted = Signal()
loginCompleted = Signal((
bool,
str,
))
badFilenameFound = Signal((str, ))
LOCATION = 'server'
TEST_FILE = 'iqbox.test'
def __init__(self, host, ssl, parent=None):
"""
Initializes parent class and attributes. Decides
whether to use `FTP_TLS` or `FTP` based on the `ssl` param.
:param host: Location of the FTP server
:param ssl: Tells whether the FTP needs to support TLS or not
:param parent: Reference to a `QObject` instance a parent
"""
super(ServerWatcher, self).__init__(parent)
self.interval = 5000
self.localdir = ''
self.deleteQueue = []
self.downloadQueue = []
self.uploadQueue = []
self.warnedNames = []
self.ftp = None
self.useSSL = ssl
self.host = host
self.preemptiveCheck = False
self.preemptiveActions = []
self.testFile = 'iqbox.test'
@property
def currentdir(self):
"""Returns the current working directory at the server"""
return self.ftp.pwd()
def setLocalDir(self, localdir):
"""
Sets the local directory used to stored all
downloaded files. Creates the directory if needed.
:param localdir: Absolute path to local directory
"""
self.localdir = localdir
if not os.path.exists(self.localdir):
os.makedirs(self.localdir)
@pause_timer
@Slot()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of ' +
downloading_dir + '...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit(
'Remote scan- Downloading files list of ' +
downloading_dir + '...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... ' + serverpath +
'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit(
'Scanning remote files for changes, ' +
str(fileC) + ' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in [
'\\', '/', ':', '?', '"', '<', '>', '|'
]:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION,
serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(
os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(
File.last_checked_server < check_date).filter(
File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
@Slot()
def onLogin(self, username, passwd):
ok = True
msg = ''
error_msg = 'Login failed.'
try:
if not self.ftp:
self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(
self.host)
loginResponse = self.ftp.login(username, passwd)
except socket.gaierror:
self.ftp = None
ok = False
msg = 'Server address could not be found.'
except (error_perm, error_reply):
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
ok = False
msg = error_msg
else:
if '230' in loginResponse:
ok = True
else:
ok = False
msg = error_msg
if ok:
# Logged in. Now let's do compability tests.
if not self.testPermissions():
# User doesn't have write permissions, don't bother doing next test.
ok = False
msg = 'It seems like you do not have write access to this server.'
else:
# Permissions test passed, now let's test MFMT for timestamp modification.
if not self.testMFMT():
ok = False
msg = 'This server does not support timestamp modification\n \
need by this application.'
self.loginCompleted.emit(ok, msg)
def getFiles(self, path):
"""
This method simply wraps the `nlst` method with an exception handler,
and returns an empty list in case an exception is caught.
:param path: Relative or absolute path on the server
"""
try:
nlst = self.ftp.nlst(path)
dirs = self.getDirs(path)
# Files are items in nlst that are not in dirs
files = [
item for item in nlst if os.path.basename(item) not in dirs
]
return files
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
return []
def getDirs(self, path):
"""
Retrieves a list of the directories inside `path`,
uses `retrlines` and the LIST command to retrieve the items.
:param path: Relative or absolute path on the server
"""
dirs = list()
def handleLine(line):
"""
Recieves a line from the LIST command.
This function is meant to be used as callback for the `retrlines` method.
:params line: Line from the LIST command
"""
if line.startswith('d'):
# Only lines starting with 'd' are directories
# Parse the directory out of the line; lines look like:
# 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname'
dirname = line[55:].strip()
if dirname != '.' and dirname != '..':
# Ignoring '.' and '..' entries
dirs.append(dirname)
try:
self.ftp.retrlines('LIST %s' % path, handleLine)
return dirs
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
return []
@upload_test
def testPermissions(self):
# For interface purposes. upload_test takes care of everything.
return True
@upload_test
def testMFMT(self):
# Absurd date to test whether the change really happened.
time = dt.utcfromtimestamp(100000000)
try:
self.setLastModified(self.testFile, time)
otherTime = self.lastModified(self.testFile)
diff = (time - otherTime).total_seconds()
if abs(diff) < 2:
# Let's give it a 2 seconds tolerance.
mdtm = True
else:
mdtm = False
except (ValueError, error_reply, error_perm):
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
mdtm = False
return mdtm
@Slot(str)
def onDelete(self, filename):
self.deleteQueue.append(filename)
def deleteNext(self):
if len(self.deleteQueue) > 0:
next = self.deleteQueue.pop(0)
self.deleteFile(next)
def deleteAll(self):
for filename in self.deleteQueue:
self.deleteFile(filename)
self.deleteQueue = []
@Slot(str)
def deleteFile(self, filename):
"""
Deletes the file `filename` to the server
:param filename: Absolute or relative path to the file
"""
try:
print 'Deleting %s' % filename
self.ftp.delete(filename)
return True
except (error_reply, error_perm):
print 'Error deleting %s' % filename
return False
self.fileEventCompleted.emit()
@Slot(str)
def onDownload(self, filename):
self.downloadQueue.append(filename)
def downloadNext(self):
if len(self.downloadQueue) > 0:
next = self.downloadQueue.pop(0)
self.downloadFile(next)
def downloadAll(self):
for filename in self.downloadQueue:
self.downloadFile(filename)
self.downloadQueue = []
@Slot(str, str)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size,
self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(
self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
@Slot(str)
def onUpload(self, filename):
self.uploadQueue.append(filename)
def uploadNext(self):
if len(self.uploadQueue) > 0:
next = self.uploadQueue.pop(0)
self.uploadFile(next)
def uploadAll(self):
for filename in self.uploadQueue:
self.uploadFile(filename)
self.uploadQueue = []
@Slot(str)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'),
1024, handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def lastModified(self, filename):
"""
Uses the MDTM FTP command to find the last modified timestamp
of the file `filename`.
Returns a `datetime.datetime` object in UTC representing the file's
last modified date and time.
:param filename: Relative or absolute path to the file
"""
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
if '213 ' not in timestamp:
# Second chance was found to be needed in some cases.
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
timestamp = timestamp.split(' ')[-1]
dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S'
try:
mtime = dt.strptime(timestamp, dateformat)
except ValueError:
mtime = dt.utcnow()
return mtime
def setLastModified(self, serverpath, newtime):
"""
Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the
file `serverpath` on the server.
:param serverpath: Relative or absolute path to the file
:param newtime: datedatime object holding the required time
"""
cmds = ['MFMT', 'MDTM']
for cmd in cmds:
try:
self.ftp.sendcmd(
'%s %s %s' %
(cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath))
return
except (error_perm, error_reply) as e:
if cmd == cmds[len(cmds) - 1]:
# If is the last comand, re-raise the exception, else
# keep trying.
raise e
else:
continue
def mkpath(self, path):
"""
Creates the path `path` on the server by recursively
created folders, if needed.
:param path: Absolute path on the server to be created
"""
try:
self.ftp.cwd(path)
except error_perm:
# `cwd` call failed. Need to create some folders
make_dir = '/'
steps = path.split('/')
for step in steps:
if len(step) == 0:
continue
make_dir += '%s/' % step
try:
self.ftp.mkd(make_dir)
except error_perm:
# Probably already exists
continue
else:
# `cwd` call succeed. No need to create
# any folders
self.ftp.cwd('/')
return
@Slot(str, str)
def added(self, location, serverpath):
super(ServerWatcher, self).added(location, serverpath)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(
self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD,
ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD,
LocalWatcher.LOCATION)
return action
if self.preemptiveCheck:
if location == ServerWatcher.LOCATION:
localpath = self.localFromServer(serverpath)
if not os.path.exists(localpath):
action = FileAction(serverpath, FileAction.DOWNLOAD,
ServerWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
elif location == LocalWatcher.LOCATION:
try:
self.ftp.sendcmd('SIZE %s' % serverpath)
except (error_reply, error_perm):
exists = False
else:
exists = True
if not exists:
action = FileAction(serverpath, FileAction.UPLOAD,
LocalWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
@Slot(str, str)
def changed(self, location, serverpath):
super(ServerWatcher, self).changed(location, serverpath)
@Slot(str, str)
def deleted(self, location, serverpath):
super(ServerWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inserver = False
class FTPSession(object):
""" Attempt to create some robustness and performance to FTPing """
def __init__(self, server, username, password, tmpdir="/tmp", timeout=60):
"""Build a FTP session """
self.conn = None
self.server = server
self.username = username
self.password = password
self.tmpdir = tmpdir
self.timeout = timeout
def _connect(self):
"""Connect to FTP server """
if self.conn is not None:
return
logging.debug("Creating new connection to server %s", self.server)
not_connected = True
attempt = 1
while not_connected and attempt < 6:
try:
self.conn = FTP_TLS(self.server, timeout=self.timeout)
self.conn.login(self.username, self.password)
self.conn.prot_p()
not_connected = False
except Exception as exp:
logging.debug(exp)
time.sleep(5)
self.close()
attempt += 1
if not_connected is True:
raise Exception("Failed to make FTP connection after 5 tries!")
def _reconnect(self):
""" First attempt to shut down connection and then reconnect """
logging.debug("_reconnect() was called...")
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
self._connect()
def _put(self, path, localfn, remotefn):
""" """
self.chdir(path)
sz = os.path.getsize(localfn)
if sz > 14000000000:
# Step 1 Split this big file into 14GB chunks, each file will have
# suffix .aa then .ab then .ac etc
basefn = os.path.basename(localfn)
cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir, basefn)
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
files = glob.glob("%s/%s.??" % (self.tmpdir, basefn))
for filename in files:
suffix = filename.split(".")[-1]
self.conn.storbinary("STOR %s.%s" % (remotefn, suffix), open(filename))
os.unlink(filename)
else:
logging.debug("_put '%s' to '%s'", localfn, remotefn)
self.conn.storbinary("STOR %s" % (remotefn,), open(localfn))
return True
def close(self):
""" Good bye """
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
def chdir(self, path):
if self.pwd() == path.rstrip("/"):
return
self.conn.cwd("/")
for dirname in path.split("/"):
if dirname == "":
continue
bah = []
self.conn.retrlines("NLST", bah.append)
if dirname not in bah:
logging.debug("Creating directory '%s'", dirname)
self.conn.mkd(dirname)
logging.debug("Changing to directory '%s'", dirname)
self.conn.cwd(dirname)
def pwd(self):
""" Low friction function to get connectivity """
self._connect()
pwd = exponential_backoff(self.conn.pwd)
if pwd is None:
self._reconnect()
pwd = exponential_backoff(self.conn.pwd)
logging.debug("pwd() is currently '%s'", pwd)
return pwd
def put_file(self, path, localfn, remotefn):
""" Put the File """
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
self._reconnect()
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
logging.error("Double Failure to upload filename: '%s'", localfn)
def put_files(self, path, localfns, remotefns):
""" Put the File """
for localfn, remotefn in zip(localfns, remotefns):
self.put_file(path, localfn, remotefn)
class FTPSession(object):
""" Attempt to create some robustness and performance to FTPing """
def __init__(self, server, username, password, tmpdir='/tmp', timeout=60):
"""Build a FTP session """
self.conn = None
self.server = server
self.username = username
self.password = password
self.tmpdir = tmpdir
self.timeout = timeout
def _connect(self):
"""Connect to FTP server """
if self.conn is not None:
return
logging.debug('Creating new connection to server %s', self.server)
not_connected = True
attempt = 1
while not_connected and attempt < 6:
try:
self.conn = FTP_TLS(self.server, timeout=self.timeout)
self.conn.login(self.username, self.password)
self.conn.prot_p()
not_connected = False
except Exception as exp:
logging.debug(exp)
time.sleep(5)
self.close()
attempt += 1
if not_connected is True:
raise Exception("Failed to make FTP connection after 5 tries!")
def _reconnect(self):
""" First attempt to shut down connection and then reconnect """
logging.debug('_reconnect() was called...')
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
self._connect()
def _put(self, path, localfn, remotefn):
""" """
self.chdir(path)
sz = os.path.getsize(localfn)
if sz > 14000000000:
# Step 1 Split this big file into 14GB chunks, each file will have
# suffix .aa then .ab then .ac etc
basefn = os.path.basename(localfn)
cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir,
basefn)
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
files = glob.glob("%s/%s.??" % (self.tmpdir, basefn))
for filename in files:
suffix = filename.split(".")[-1]
self.conn.storbinary('STOR %s.%s' % (remotefn, suffix),
open(filename))
os.unlink(filename)
else:
logging.debug("_put '%s' to '%s'", localfn, remotefn)
self.conn.storbinary('STOR %s' % (remotefn, ), open(localfn))
return True
def close(self):
""" Good bye """
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
def chdir(self, path):
if self.pwd() == path.rstrip("/"):
return
self.conn.cwd("/")
for dirname in path.split("/"):
if dirname == '':
continue
bah = []
self.conn.retrlines('NLST', bah.append)
if dirname not in bah:
logging.debug("Creating directory '%s'", dirname)
self.conn.mkd(dirname)
logging.debug("Changing to directory '%s'", dirname)
self.conn.cwd(dirname)
def pwd(self):
""" Low friction function to get connectivity """
self._connect()
pwd = exponential_backoff(self.conn.pwd)
if pwd is None:
self._reconnect()
pwd = exponential_backoff(self.conn.pwd)
logging.debug("pwd() is currently '%s'", pwd)
return pwd
def put_file(self, path, localfn, remotefn):
""" Put the File """
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
self._reconnect()
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
logging.error("Double Failure to upload filename: '%s'",
localfn)
return False
return True
def put_files(self, path, localfns, remotefns):
""" Put the File """
res = []
for localfn, remotefn in zip(localfns, remotefns):
res.append(self.put_file(path, localfn, remotefn))
return res
def sendFtp(server, port, user, password):
displayOptionsFTP(server, port, user, password)
textPad.configure(state='normal')
if (is_tls_ftp_checked.get() == 1):
try:
textPad.insert('end', "Initializing FTP_TLS\n")
ftps = FTP_TLS()
textPad.insert('end', "\t -> success\n")
textPad.insert('end',
"Connecting to " + server + ":" + port + "\n")
ftps.connect(server, int(port))
textPad.insert('end', "\t -> success\n")
if (is_auth_ftp_checked.get() == 1):
textPad.insert('end',
"Trying to connect with anonymous login\n")
ftps.login()
textPad.insert('end', "\t -> success\n")
textPad.see("end")
else:
textPad.insert(
'end', "Logging in with " + user + ":" + password + "\n")
textPad.see("end")
ftps.login(user=user, passwd=password)
textPad.insert('end', "\t -> success\n")
textPad.see("end")
ftps.prot_p()
textPad.see("end")
listing = []
textPad.insert('end', "--Listing directories:\n")
ftps.retrlines("LIST", listing.append)
textPad.insert('end', "\n".join(listing))
textPad.insert('end', "\n\t -> success\n")
textPad.see("end")
ftps.close()
except Exception as e:
textPad.configure(state='normal')
textPad.insert('end', "Exception raised: {}\n".format(e))
textPad.config(state=DISABLED)
else:
try:
textPad.insert('end', "---Starting FTP checking------------\n")
textPad.insert('end', "Initializing FTP\n")
ftp = FTP()
textPad.insert('end', "\t -> success\n")
textPad.insert('end',
"Connecting to " + server + ":" + port + "\n")
ftp.connect(server, int(port))
textPad.insert('end', "\t -> success\n")
if (is_auth_ftp_checked.get() == 1):
textPad.insert('end',
"Trying to connect with anonymous login\n")
ftp.login()
textPad.insert('end', "\t -> success\n")
textPad.see("end")
else:
textPad.insert(
'end', "Logging in with " + user + ":" + password + "\n")
textPad.see("end")
ftp.login(user=user, passwd=password)
textPad.insert('end', "\t -> success\n")
textPad.see("end")
ftp.prot_p()
textPad.see("end")
listing = []
textPad.insert('end', "--Listing directories:\n")
ftp.retrlines("LIST", listing.append)
textPad.insert('end', "\n".join(listing))
textPad.insert('end', "\n\t -> success\n")
textPad.see("end")
ftp.close()
except Exception as e:
textPad.configure(state='normal')
textPad.insert('end', "Exception raised: {}\n".format(e))
textPad.config(state=DISABLED)
from ftplib import FTP_TLS
ftps = FTP_TLS('127.0.0.1:8021','user','12345',keyfile='privateKey.key')
ftps.login() # login anonymously before securing control channel
ftps.prot_p() # switch to secure data connection
print ftps.retrlines('LIST')
from ftplib import FTP_TLS
ftps = FTP_TLS()
ftps.connect('127.0.0.1', 5000)
ftps.login('user', '12345') # login anonymously before securing control channel
ftps.prot_p() # switch to secure data connection
ftps.retrlines('LIST') # list directory content securely
except (IOError, OSError):
LOG_MAIN.error("Noting variable set")
LOG_MAIN.error("OS ERROR : " + str(OSError))
LOG_MAIN.error("OR OS IOError: " + str(IOError))
if __autopilot__.upper() == 'TRUE':
# region ftp_connection
try:
FTP_TLS.port = int(V_FTPS_PORT)
ftp_conn = FTP_TLS(V_FTPS_SERVER)
ftp_conn.login(V_FTPS_USER, V_FTPS_PASS)
ftp_conn.prot_p()
if ftp_conn:
ftps_chdir(ftp_conn, V_FTPS_RMT_DIR)
ftp_conn.retrlines('LIST')
data = []
ftp_conn.dir(data.append)
coln = []
for line in data:
col = line.split()
coln.append(col[-1])
# print(get_lasted_file_name_glob(coln))
lastFile = get_lasted_file_name_glob(coln)
ftps_download(ftp_conn, lastFile, __dlftpsfiles__,
V_FTPS_RMT_DIR)
__pathlastfile__ = __dlftpsfiles__ + lastFile
try:
url_requested = request.urlopen(url) # Ubuntu
#url_requested = urllib.urlopen(url) # Raspi
if 200 == url_requested.code:
html_content = str(url_requested.read())
# "b'" am Anfang, CR und "'" am Ende entfernen
temp_clean0 = re.sub(r"b'<!DOC", "<!DOC", html_content)
temp_clean1 = re.sub(r"\\n", "", temp_clean0)
temp_clean2 = re.sub(r"ml>'", "ml>", temp_clean1)
print(temp_clean2)
datei = open(htmlfile, 'w')
datei.write(temp_clean2)
datei.close()
#
# ftp/TLS Transfer
#
ftps = FTP_TLS(ftpurl)
ftps.login(ftpuser, ftppass)
ftps.prot_p(
) # switch to secure data connection.. IMPORTANT! Otherwise, only the user and password is encrypted and not all the file data.
ftps.retrlines('LIST')
myfile = open(htmlfile, 'rb')
ftps.storbinary("STOR test.html", myfile, blocksize=8192)
ftps.close()
myfile.close()
class ServerWatcher(Watcher):
downloadProgress = Signal((int, int,))
uploadProgress = Signal((int, int,))
# Si added:
textStatus = Signal((str,))
fileEvent = Signal((str,))
fileEventCompleted = Signal()
loginCompleted = Signal((bool, str,))
badFilenameFound = Signal((str,))
LOCATION = 'server'
TEST_FILE = 'iqbox.test'
def __init__(self, host, ssl, parent=None):
"""
Initializes parent class and attributes. Decides
whether to use `FTP_TLS` or `FTP` based on the `ssl` param.
:param host: Location of the FTP server
:param ssl: Tells whether the FTP needs to support TLS or not
:param parent: Reference to a `QObject` instance a parent
"""
super(ServerWatcher, self).__init__(parent)
self.interval = 5000
self.localdir = ''
self.deleteQueue = []
self.downloadQueue = []
self.uploadQueue = []
self.warnedNames = []
self.ftp = None
self.useSSL = ssl
self.host = host
self.preemptiveCheck = False
self.preemptiveActions = []
self.testFile = 'iqbox.test'
@property
def currentdir(self):
"""Returns the current working directory at the server"""
return self.ftp.pwd()
def setLocalDir(self, localdir):
"""
Sets the local directory used to stored all
downloaded files. Creates the directory if needed.
:param localdir: Absolute path to local directory
"""
self.localdir = localdir
if not os.path.exists(self.localdir):
os.makedirs(self.localdir)
@pause_timer
@Slot()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... '+serverpath+'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
@Slot()
def onLogin(self, username, passwd):
ok = True
msg = ''
error_msg = 'Login failed.'
try:
if not self.ftp:
self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(self.host)
loginResponse = self.ftp.login(username, passwd)
except socket.gaierror:
self.ftp = None
ok = False
msg = 'Server address could not be found.'
except (error_perm, error_reply):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
ok = False
msg = error_msg
else:
if '230' in loginResponse:
ok = True
else:
ok = False
msg = error_msg
if ok:
# Logged in. Now let's do compability tests.
if not self.testPermissions():
# User doesn't have write permissions, don't bother doing next test.
ok = False
msg = 'It seems like you do not have write access to this server.'
else:
# Permissions test passed, now let's test MFMT for timestamp modification.
if not self.testMFMT():
ok = False
msg = 'This server does not support timestamp modification\n \
need by this application.'
self.loginCompleted.emit(ok, msg)
def getFiles(self, path):
"""
This method simply wraps the `nlst` method with an exception handler,
and returns an empty list in case an exception is caught.
:param path: Relative or absolute path on the server
"""
try:
nlst = self.ftp.nlst(path)
dirs = self.getDirs(path)
# Files are items in nlst that are not in dirs
files = [item for item in nlst if os.path.basename(item) not in dirs]
return files
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
def getDirs(self, path):
"""
Retrieves a list of the directories inside `path`,
uses `retrlines` and the LIST command to retrieve the items.
:param path: Relative or absolute path on the server
"""
dirs = list()
def handleLine(line):
"""
Recieves a line from the LIST command.
This function is meant to be used as callback for the `retrlines` method.
:params line: Line from the LIST command
"""
if line.startswith('d'):
# Only lines starting with 'd' are directories
# Parse the directory out of the line; lines look like:
# 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname'
dirname = line[55:].strip()
if dirname != '.' and dirname != '..':
# Ignoring '.' and '..' entries
dirs.append(dirname)
try:
self.ftp.retrlines('LIST %s' % path, handleLine)
return dirs
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
@upload_test
def testPermissions(self):
# For interface purposes. upload_test takes care of everything.
return True
@upload_test
def testMFMT(self):
# Absurd date to test whether the change really happened.
time = dt.utcfromtimestamp(100000000)
try:
self.setLastModified(self.testFile, time)
otherTime = self.lastModified(self.testFile)
diff = (time - otherTime).total_seconds()
if abs(diff) < 2:
# Let's give it a 2 seconds tolerance.
mdtm = True
else:
mdtm = False
except (ValueError, error_reply, error_perm):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
mdtm = False
return mdtm
@Slot(str)
def onDelete(self, filename):
self.deleteQueue.append(filename)
def deleteNext(self):
if len(self.deleteQueue) > 0:
next = self.deleteQueue.pop(0)
self.deleteFile(next)
def deleteAll(self):
for filename in self.deleteQueue:
self.deleteFile(filename)
self.deleteQueue = []
@Slot(str)
def deleteFile(self, filename):
"""
Deletes the file `filename` to the server
:param filename: Absolute or relative path to the file
"""
try:
print 'Deleting %s' % filename
self.ftp.delete(filename)
return True
except (error_reply, error_perm):
print 'Error deleting %s' % filename
return False
self.fileEventCompleted.emit()
@Slot(str)
def onDownload(self, filename):
self.downloadQueue.append(filename)
def downloadNext(self):
if len(self.downloadQueue) > 0:
next = self.downloadQueue.pop(0)
self.downloadFile(next)
def downloadAll(self):
for filename in self.downloadQueue:
self.downloadFile(filename)
self.downloadQueue = []
@Slot(str, str)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size, self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
@Slot(str)
def onUpload(self, filename):
self.uploadQueue.append(filename)
def uploadNext(self):
if len(self.uploadQueue) > 0:
next = self.uploadQueue.pop(0)
self.uploadFile(next)
def uploadAll(self):
for filename in self.uploadQueue:
self.uploadFile(filename)
self.uploadQueue = []
@Slot(str)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename,
open(localpath, 'rb'),
1024,
handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def lastModified(self, filename):
"""
Uses the MDTM FTP command to find the last modified timestamp
of the file `filename`.
Returns a `datetime.datetime` object in UTC representing the file's
last modified date and time.
:param filename: Relative or absolute path to the file
"""
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
if '213 ' not in timestamp:
# Second chance was found to be needed in some cases.
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
timestamp = timestamp.split(' ')[-1]
dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S'
try:
mtime = dt.strptime(timestamp, dateformat)
except ValueError:
mtime = dt.utcnow()
return mtime
def setLastModified(self, serverpath, newtime):
"""
Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the
file `serverpath` on the server.
:param serverpath: Relative or absolute path to the file
:param newtime: datedatime object holding the required time
"""
cmds = ['MFMT', 'MDTM']
for cmd in cmds:
try:
self.ftp.sendcmd(
'%s %s %s' % (cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath))
return
except (error_perm, error_reply) as e:
if cmd == cmds[len(cmds) - 1]:
# If is the last comand, re-raise the exception, else
# keep trying.
raise e
else:
continue
def mkpath(self, path):
"""
Creates the path `path` on the server by recursively
created folders, if needed.
:param path: Absolute path on the server to be created
"""
try:
self.ftp.cwd(path)
except error_perm:
# `cwd` call failed. Need to create some folders
make_dir = '/'
steps = path.split('/')
for step in steps:
if len(step) == 0:
continue
make_dir += '%s/' % step
try:
self.ftp.mkd(make_dir)
except error_perm:
# Probably already exists
continue
else:
# `cwd` call succeed. No need to create
# any folders
self.ftp.cwd('/')
return
@Slot(str, str)
def added(self, location, serverpath):
super(ServerWatcher, self).added(location, serverpath)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION)
return action
if self.preemptiveCheck:
if location == ServerWatcher.LOCATION:
localpath = self.localFromServer(serverpath)
if not os.path.exists(localpath):
action = FileAction(serverpath, FileAction.DOWNLOAD, ServerWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
elif location == LocalWatcher.LOCATION:
try:
self.ftp.sendcmd('SIZE %s' % serverpath)
except (error_reply, error_perm):
exists = False
else:
exists = True
if not exists:
action = FileAction(serverpath, FileAction.UPLOAD, LocalWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
@Slot(str, str)
def changed(self, location, serverpath):
super(ServerWatcher, self).changed(location, serverpath)
@Slot(str, str)
def deleted(self, location, serverpath):
super(ServerWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inserver = False
