def sftpConnect(self, maxCount=200):
count = 0
while count < maxCount:
timex = AlarmFTP('SFTP connection timeout')
# gives "timeout" seconds to open the connection
try:
timex.alarm(self.timeout)
self.t = None
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if self.client.port == '' or self.client.port == None:
self.client.port = 22
if self.client.ssh_keyfile is not None:
self.client.passwd = None
self.ssh.connect( self.client.host, self.client.port, self.client.user, \
self.client.passwd, pkey=None , key_filename=self.client.ssh_keyfile )
else:
self.ssh.connect( self.client.host, self.client.port, self.client.user, \
self.client.passwd )
self.sftp = self.ssh.open_sftp()
# WORKAROUND without going to '.' originalDir was None
self.sftp.chdir('.')
self.originalDir = self.sftp.getcwd()
timex.cancel()
return self.sftp
except FtpTimeoutException:
timex.cancel()
try:
self.sftp.close()
except:
pass
self.logger.error(
"SFTP connection timed out after %d seconds... retrying" %
self.timeout)
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error(
"Unable to connect to %s (user:%s). Type: %s, Value: %s" %
(self.client.host, self.client.user, type, value))
try:
self.sftp.close()
except:
pass
count += 1
time.sleep(5)
self.logger.critical("We exit SenderSFTP after %i unsuccessful try" %
maxCount)
sys.exit(2)
def ftpConnect(self):
timex = AlarmFTP('FTP connection timeout')
try:
# gives 30 seconds to open the connection
timex.alarm(30)
ftp = ftplib.FTP(self.source.host, self.source.user, self.source.passwd)
if self.source.ftp_mode == 'active':
ftp.set_pasv(False)
else:
ftp.set_pasv(True)
try : self.originalDir = ftp.pwd()
except:
(type, value, tb) = sys.exc_info()
self.logger.warning("Unable to ftp.pwd (Type: %s, Value: %s)" % (type ,value))
timex.cancel()
self.connected = True
return ftp
except FtpTimeoutException :
timex.cancel()
self.logger.warning("FTP connection timed out after 30 seconds..." )
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.source.host, self.source.user, type ,value))
def sftp_do_ls(self):
self.ls = {}
timex = AlarmFTP('SFTP retrieving list')
timex.alarm(self.source.timeout_get)
try :
dir_attr = self.sftp.listdir_attr()
for index in range(len(dir_attr)):
attr = dir_attr[index]
line = attr.__str__()
self.callback_line(line)
timex.cancel()
return True
except FtpTimeoutException :
timex.cancel()
self.logger.warning("SFTP doing ls timed out after %s seconds..." % self.source.timeout_get )
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not ls directory %s" % self.destDir)
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
return False
def ftpConnect(self, maxCount=200):
count = 0
while count < maxCount:
timex = AlarmFTP('FTP connection timeout')
try:
# gives "timeout" seconds to open the connection
timex.alarm(self.timeout)
ftp = ftplib.FTP(self.client.host, self.client.user,
self.client.passwd)
if self.client.ftp_mode == 'active':
ftp.set_pasv(False)
else:
ftp.set_pasv(True)
self.originalDir = '.'
try:
self.originalDir = ftp.pwd()
except:
(type, value, tb) = sys.exc_info()
self.logger.warning(
"Unable to ftp.pwd (Type: %s, Value: %s)" %
(type, value))
timex.cancel()
return ftp
except FtpTimeoutException:
timex.cancel()
self.logger.error(
"FTP connection timed out after %d seconds... retrying" %
self.timeout)
except:
timex.cancel()
count += 1
(type, value, tb) = sys.exc_info()
self.logger.error(
"Unable to connect to %s (user:%s). Type: %s, Value: %s" %
(self.client.host, self.client.user, type, value))
time.sleep(5)
self.logger.critical("We exit SenderFTP after %i unsuccessful try" %
maxCount)
sys.exit(2)
def sftpConnect(self, maxCount=200):
count = 0
while count < maxCount:
timex = AlarmFTP('SFTP connection timeout')
# gives "timeout" seconds to open the connection
try:
timex.alarm(self.timeout)
self.t = None
if self.client.port == None :
self.t = paramiko.Transport(self.client.host)
else:
t_args = (self.client.host,self.client.port)
self.t = paramiko.Transport(t_args)
if self.client.ssh_keyfile != None :
#TODO, implement password to use to decrypt the key file, if it's encrypted
#key=DSSKey.from_private_key_file(self.client.ssh_keyfile,password=None)
self.t.connect(username=self.client.user,pkey=None,key_filename=self.client.ssh_keyfile)
else:
self.t.connect(username=self.client.user,password=self.client.passwd)
self.sftp = paramiko.SFTP.from_transport(self.t)
# WORKAROUND without going to '.' originalDir was None
self.sftp.chdir('.')
self.originalDir = self.sftp.getcwd()
timex.cancel()
return self.sftp
except FtpTimeoutException :
timex.cancel()
try : self.sftp.close()
except : pass
try : self.t.close()
except : pass
self.logger.error("SFTP connection timed out after %d seconds... retrying" % self.timeout )
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.client.host, self.client.user, type ,value))
try : self.sftp.close()
except : pass
try : self.t.close()
except : pass
count += 1
time.sleep(5)
self.logger.critical("We exit SenderSFTP after %i unsuccessful try" % maxCount)
sys.exit(2)
def sftpConnect(self):
timex = AlarmFTP('SFTP connection timeout')
# gives 30 seconds to open the connection
try:
timex.alarm(30)
self.t = None
if self.source.port == None :
self.t = paramiko.Transport(self.source.host)
else:
t_args = (self.source.host,self.source.port)
self.t = paramiko.Transport(t_args)
if self.source.ssh_keyfile != None :
#TODO, implement password to use to decrypt the key file, if it's encrypted
key=DSSKey.from_private_key_file(self.source.ssh_keyfile,password=None)
self.t.connect(username=self.source.user,pkey=key)
else:
self.t.connect(username=self.source.user,password=self.source.passwd)
self.sftp = paramiko.SFTP.from_transport(self.t)
# WORKAROUND without going to '.' originalDir was None
self.sftp.chdir('.')
self.originalDir = self.sftp.getcwd()
timex.cancel()
self.connected = True
return self.sftp
except FtpTimeoutException :
timex.cancel()
try : self.sftp.close()
except : pass
try : self.t.close()
except : pass
self.logger.error("SFTP connection timed out after %d seconds... retrying" % 30 )
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.source.host, self.source.user, type ,value))
try : self.sftp.close()
except : pass
try : self.t.close()
except : pass
def ftpConnect(self, maxCount=200):
count = 0
while count < maxCount:
timex = AlarmFTP('FTP connection timeout')
try:
# gives "timeout" seconds to open the connection
timex.alarm(self.timeout)
ftp = ftplib.FTP(self.client.host, self.client.user, self.client.passwd)
if self.client.ftp_mode == 'active':
ftp.set_pasv(False)
else:
ftp.set_pasv(True)
self.originalDir = '.'
try : self.originalDir = ftp.pwd()
except:
(type, value, tb) = sys.exc_info()
self.logger.warning("Unable to ftp.pwd (Type: %s, Value: %s)" % (type ,value))
timex.cancel()
return ftp
except FtpTimeoutException :
timex.cancel()
self.logger.error("FTP connection timed out after %d seconds... retrying" % self.timeout )
except:
timex.cancel()
count += 1
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.client.host, self.client.user, type ,value))
time.sleep(5)
self.logger.critical("We exit SenderFTP after %i unsuccessful try" % maxCount)
sys.exit(2)
def ftpsConnect(self):
timex = AlarmFTP('FTPS connection timeout')
try:
# gives 30 seconds to open the connection
timex.alarm(30)
ftps = ftpslib.FTP_TLS()
ftps.connect(self.source.host,21)
# this should be an option... tls or ssl ftps.auth_ssl()
ftps.auth_tls()
if self.source.ftp_mode == 'active':
ftps.set_pasv(0)
else:
ftps.set_pasv(1)
ftps.login(self.source.user, self.source.passwd)
try : self.originalDir = ftps.pwd()
except:
(type, value, tb) = sys.exc_info()
self.logger.warning("Unable to ftp.pwd (Type: %s, Value: %s)" % (type ,value))
timex.cancel()
self.connected = True
return ftps
except FtpTimeoutException :
timex.cancel()
self.logger.warning("FTPS connection timed out after 30 seconds..." )
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.source.host, self.source.user, type ,value))
def close(self):
if self.client.protocol == 'file': return
timex = AlarmFTP(self.client.protocol + ' connection timeout')
try:
# gives 10 seconds to close the connection
timex.alarm(10)
self.quit()
timex.cancel()
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not close connection")
self.logger.warning(" Type: %s, Value: %s" % (type, value))
def ftp_do_ls(self):
self.ls = {}
timex = AlarmFTP('FTP retrieving list')
timex.alarm(self.source.timeout_get)
try :
ls=self.ftp.retrlines('LIST',self.callback_line )
timex.cancel()
return True
except FtpTimeoutException :
timex.cancel()
self.logger.warning("FTP doing ls timed out after %s seconds..." % self.source.timeout_get )
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not ls directory %s" % self.destDir)
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
return False
def cd(self, path):
timex = AlarmFTP('FTP change working directory timeout')
try :
# gives 10 seconds to get there
timex.alarm(10)
self.chdir(self.originalDir)
self.chdir(path)
timex.cancel()
return True
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not cd to directory %s" % path )
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
return False
def close(self):
if self.client.protocol == 'file': return
timex = AlarmFTP(self.client.protocol + ' connection timeout')
try :
# gives 10 seconds to close the connection
timex.alarm(10)
self.quit()
timex.cancel()
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not close connection")
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
class unittest_AlarmFTP(unittest.TestCase):
def setUp(self):
self.alarm = AlarmFTP('Message AlarmFTP')
def test_AlarmFTP(self):
self.assertEqual(self.alarm.message,'Message AlarmFTP')
self.alarm.alarm(5)
self.assertEqual(self.alarm.state,True)
self.alarm.cancel()
self.assertEqual(self.alarm.state,False)
textFtpTimeoutException = ''
try:
self.alarm.sigalarm('x','x')
except FtpTimeoutException as error:
textFtpTimeoutException = str(error)
self.assertEqual(textFtpTimeoutException,'Message AlarmFTP')
def close(self):
self.connected = False
# connection did not work
if self.ftp == None and self.sftp == None : return
timex = AlarmFTP(self.source.protocol + ' connection timeout')
try :
# gives 10 seconds to close the connection
timex.alarm(10)
self.quit()
timex.cancel()
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not close connection")
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
def send(self, files):
# process with file sending
currentFTPDir = ''
for filex in files:
file = filex
self.partialfile = None
# priority 0 is retransmission and is never suppressed
priority = file.split('/')[-3]
# if in cache than it was already sent... nothing to do
# caching is always done on original file for early check (before fx)
if self.client.nodups and priority != '0' and self.in_cache( True, file ) :
self.logger.info("suppressed duplicate send %s", os.path.basename(file))
os.unlink(file)
continue
# applying the fx_script if defined redefine the file list
if self.client.fx_execfile != None :
fxfile = self.client.run_fx_script(file,self.logger)
if fxfile == None :
self.logger.warning("FX script ignored the file : %s" % os.path.basename(file) )
os.unlink(file)
continue
elif fxfile == file :
self.logger.warning("FX script kept the file as is : %s" % os.path.basename(file) )
else :
self.logger.info("FX script modified %s to %s " % (os.path.basename(file),fxfile) )
os.unlink(file)
file = fxfile
# get files ize
nbBytes = 0
try:
nbBytes = os.stat(file)[stat.ST_SIZE]
except:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to stat %s, Type: %s, Value: %s" % (file, type, value))
continue
# get/check destination Name
basename = os.path.basename(file)
destName, destDir = self.client.getDestInfos(basename)
if not destName :
os.unlink(file)
self.logger.info('No destination name: %s has been erased' % file)
continue
# run destfn
ldestName = self.client.run_destfn_script(destName)
if ldestName != destName :
self.logger.info("destfn_script : %s becomes %s " % (destName,ldestName) )
destName = ldestName
# check protocol
if not self.client.protocol in ['file','ftp','sftp'] :
self.logger.critical("Unknown protocol: %s" % self.client.protocol)
sys.exit(2)
# We remove the first / (if there was only one => relative path, if there was two => absolute path)
destDir = destDir[1:]
if self.client.dir_pattern == True :
destDir = self.dirPattern(file,basename,destDir,destName)
if destDir == '':
destDirString = '/'
elif destDir == '/':
destDirString = '//'
else:
destDirString = '/' + destDir + '/'
# file protocol means file renaming
if self.client.protocol == 'file':
try:
self.logger.start_timer()
if self.client.dir_mkdir == True and not os.path.isdir(destDirString) : os.makedirs(destDirString)
os.rename(file, destDirString + destName)
self.logger.delivered("(%i Bytes) File %s delivered to %s://%s@%s%s%s" % (nbBytes, file, self.client.protocol, self.client.user, self.client.host, destDirString, destName),destDirString + destName)
# add data to cache if needed
if self.client.nodups and self.cacheMD5 != None :
self.cacheManager.find( self.cacheMD5, 'standard' )
except:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to move to: %s, Type: %s, Value:%s" % ((destDirString+destName), type, value))
time.sleep(1)
continue
# ftp or sftp protocol
if self.client.protocol == 'ftp' or self.client.protocol == 'sftp':
try :
self.logger.start_timer()
# the alarm timeout is set at that level
# it means that everything done to a file must be done
# within "timeout_send" seconds
timex = AlarmFTP(self.client.protocol + ' timeout')
if self.client.timeout_send > 0 :
timex.alarm(self.client.timeout_send)
# we are not in the proper directory
if currentFTPDir != destDir:
# create and cd to the directory
if self.client.dir_mkdir:
try:
self.dirMkdir(destDir)
currentFTPDir = destDir
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to mkdir: %s, Type: %s, Value:%s" % (destDir, type, value))
time.sleep(1)
continue
# just cd to the directory
else:
try:
self.chdir(self.originalDir)
self.chdir(destDir)
currentFTPDir = destDir
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to cwd to: %s, Type: %s, Value:%s" % (destDir, type, value))
time.sleep(1)
continue
# try to write the file to the client
try :
d1,d2,d3,d4,tbegin = os.times()
self.partialfile = destName
self.send_file( file, destName )
self.partialfile = None
d1,d2,d3,d4,tend = os.times()
self.logger.delivered("(%i Bytes) File %s delivered to %s://%s@%s%s%s" % \
(nbBytes, file, self.client.protocol, self.client.user, \
self.client.host, destDirString, destName),file)
os.unlink(file)
if self.client.kbytes_ps > 0.0 :
tspan = tend - tbegin
if tspan >= 1.0 :
tspan = tspan * 1024.0
tspeed = nbBytes/tspan
self.logger.debug("Speed %f Kb/s" % tspeed)
# add data to cache if needed
if self.client.nodups and self.cacheMD5 != None :
self.cacheManager.find( self.cacheMD5, 'standard' )
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to deliver to %s://%s@%s%s%s, Type: %s, Value: %s" %
(self.client.protocol, self.client.user, self.client.host, \
destDirString, destName, type, value))
# preventive delete when problem
if self.partialfile != None :
timex.alarm(5)
try :
self.logger.debug("Trying preventive delete")
self.rm(self.partialfile)
except :
self.logger.debug("Preventive delete timed out")
timex.cancel()
time.sleep(1)
return
timex.cancel()
except FtpTimeoutException :
timex.cancel()
self.logger.warning("SEND TIMEOUT (%i Bytes) File %s going to %s://%s@%s%s%s" % \
(nbBytes, file, self.client.protocol, self.client.user, \
self.client.host, destDirString, destName))
return
def sftpConnect(self, maxCount=200):
count = 0
while count < maxCount:
timex = AlarmFTP('SFTP connection timeout')
# gives "timeout" seconds to open the connection
try:
timex.alarm(self.timeout)
self.t = None
if self.client.port == None:
self.t = paramiko.Transport(self.client.host)
else:
t_args = (self.client.host, self.client.port)
self.t = paramiko.Transport(t_args)
if self.client.ssh_keyfile != None:
#TODO, implement password to use to decrypt the key file, if it's encrypted
#key=DSSKey.from_private_key_file(self.client.ssh_keyfile,password=None)
self.t.connect(username=self.client.user,
pkey=None,
key_filename=self.client.ssh_keyfile)
else:
self.t.connect(username=self.client.user,
password=self.client.passwd)
self.sftp = paramiko.SFTP.from_transport(self.t)
# WORKAROUND without going to '.' originalDir was None
self.sftp.chdir('.')
self.originalDir = self.sftp.getcwd()
timex.cancel()
return self.sftp
except FtpTimeoutException:
timex.cancel()
try:
self.sftp.close()
except:
pass
try:
self.t.close()
except:
pass
self.logger.error(
"SFTP connection timed out after %d seconds... retrying" %
self.timeout)
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error(
"Unable to connect to %s (user:%s). Type: %s, Value: %s" %
(self.client.host, self.client.user, type, value))
try:
self.sftp.close()
except:
pass
try:
self.t.close()
except:
pass
count += 1
time.sleep(5)
self.logger.critical("We exit SenderSFTP after %i unsuccessful try" %
maxCount)
sys.exit(2)
def send(self, files):
filelst = None
# process with file sending
for filex in files:
file = filex
# priority 0 is retransmission and is never suppressed
priority = file.split('/')[-3]
# if in cache than it was already sent... nothing to do
# caching is always done on original file for early check (before fx)
if self.client.nodups and priority != '0' and self.in_cache( True, file ) :
self.logger.info("suppressed duplicate send %s", os.path.basename(file))
os.unlink(file)
continue
# applying the fx_script if defined redefine the file list
if self.client.fx_execfile != None :
fxfile = self.client.run_fx_script(file,self.logger)
if fxfile == None :
self.logger.warning("FX script ignored the file : %s" % os.path.basename(file) )
os.unlink(file)
continue
elif fxfile == file :
self.logger.warning("FX script kept the file as is : %s" % os.path.basename(file) )
else :
self.logger.info("FX script modified %s to %s " % (os.path.basename(file),fxfile) )
os.unlink(file)
file = fxfile
# get files ize
try:
nbBytes = os.stat(file)[stat.ST_SIZE]
except:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to stat %s, Type: %s, Value: %s" % (file, type, value))
continue
# get/check destination Name
basename = os.path.basename(file)
destName, destDir = self.client.getDestInfos(basename)
if not destName :
os.unlink(file)
self.logger.info('No destination name: %s has been erased' % file)
continue
# run destfn
ldestName = self.client.run_destfn_script(destName)
if ldestName != destName :
self.logger.info("destfn_script : %s becomes %s " % (destName,ldestName) )
destName = ldestName
# We remove the first / (if there was only one => relative path, if there was two => absolute path)
destDir = destDir[1:]
if self.client.dir_pattern == True :
destDir = self.dirPattern(file,basename,destDir,destName)
if destDir == '':
destDirString = '/'
elif destDir == '/':
destDirString = '//'
else:
destDirString = '/' + destDir + '/'
# add entry to file list
if filelst == None : filelst = []
filelst.append( (file,destDir,destName) )
# no file to process
if filelst == None : return
# the alarm timeout is set at that level
# it means that everything done for a file list under
# within "timeout_send" seconds
timex = AlarmFTP(self.client.protocol + ' timeout')
if self.client.timeout_send > 0 :
timex.alarm(self.client.timeout_send)
# ok using script to send the file list
try :
self.client.send_script( self.client, filelst, self.logger )
for t in filelst :
file,destdir,destName = t
self.logger.info("(%i Bytes) File %s delivered to %s://%s@%s%s%s" % \
(nbBytes, file, self.client.protocol, self.client.user, \
self.client.host, destDir, destName))
os.unlink(file)
# add data to cache if needed
if self.client.nodups and self.cacheMD5 != None :
self.cacheManager.find( self.cacheMD5, 'standard' )
except FtpTimeoutException :
timex.cancel()
self.logger.warning("SEND TIMEOUT (%i Bytes) File %s going to %s://%s@%s" % \
(nbBytes, file, self.client.protocol, self.client.user,self.client.host))
return
except:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to deliver to %s://%s%s, Type: %s, Value: %s" %
(self.client.protocol, self.client.user, self.client.host,type, value))
timex.cancel()
return
timex.cancel()
def get_file_from_request(self,ipath):
# if pull is sleeping and we delete files... nothing to do
# if we don't delete files, we will keep the directory state
if self.sleeping : return
# connection did not work
if self.ftps == None : return None
# log that we are waking up
self.logger.info("request is %s" % os.path.basename(ipath) )
# get request info
try :
f = open(ipath, 'r')
data = f.read()
f.close()
data = data.split('\n')
# line 0 = directory value
lst = data[0].split(' ')
option = lst[0]
if option != "directory" :
self.logger.error("Unable to parse option directory in request file %s" % ipath )
return None
self.destDir = lst[1]
# line 1 = get remote_file local_file
lst = data[1].split(' ')
option = lst[0]
if option != "get" :
self.logger.error("Unable to parse option get in request file %s" % ipath )
return None
remote_file = lst[1]
lfile = lst[1]
if len(lst) == 3 : lfile = lst[2]
local_file = PXPaths.RXQ + self.source.name + '/' + lfile
if remote_file == lfile :
self.logger.info("remote file = %s" % remote_file )
else :
self.logger.info("remote file = %s (local = %s)" % (remote_file,lfile) )
except :
self.logger.error("Problem with request for file %s" % ipath )
return None
# getting our alarm ready
timex = AlarmFTP('FTP timeout')
# cd to that directory
self.logger.debug(" cd %s" % self.destDir)
ok = self.cd(self.destDir)
if not ok :
self.logger.error("Unable to cd to directory %s" % self.destDir )
return None
# get the file
timex.alarm(self.source.timeout_get)
try :
ok = self.retrieve(remote_file, local_file)
if ok :
if self.source.delete : self.rm(remote_file)
timex.cancel()
except FtpTimeoutException :
timex.cancel()
self.logger.warning("FTP timed out retrieving %s " % remote_file )
return None
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable write remote file %s in local file %s" % (remote_file,local_file))
self.logger.error(" Type: %s, Value: %s" % (type ,value))
return None
return local_file
def setUp(self):
self.alarm = AlarmFTP('Message AlarmFTP')
def get(self):
# if pull is sleeping and we delete files... nothing to do
# if we don't delete files, we will keep the directory state
if self.source.delete and self.sleeping : return
# log that we are waking up
if not self.sleeping :
self.logger.info("pull %s is waking up" % self.source.name )
# getting our alarm ready
timex = AlarmFTP('FTP timeout')
# files list to return
files_pulled = []
# connection did not work
if self.ftp == None and self.sftp == None : return files_pulled
# loop on all directories where there are pulls to do
for lst in self.source.pulls :
self.destDir = lst[0]
self.pulllst = lst[1:]
pdir = self.dirPattern(self.destDir)
if pdir != '' : self.destDir = pdir
self.destDir = self.destDir[1:]
# cd to that directory
self.logger.debug(" cd %s" % self.destDir)
ok = self.cd(self.destDir)
if not ok : continue
# create ls filename for that directory
pdir = lst[0]
pdir = pdir.replace('${','')
pdir = pdir.replace('}','')
self.lspath = PXPaths.RXQ + self.source.name + '/.ls' + pdir.replace('/','_')
# ls that directory
ok = self.lsdir()
if not ok : continue
# if we are sleeping and we are here it is because
# this pull is retrieving difference between directory content
# so write the directory content without retrieving files
if self.sleeping :
ok = self.write_ls_file(self.lspath)
continue
# get the file list from the ls
filelst = self.ls.keys()
desclst = self.ls
# if we dont delete, get file list from difference in ls
if not self.source.delete :
filelst,desclst = self.differ()
if len(filelst) == 0 :
ok = self.write_ls_file(self.lspath)
continue
# get remote file time for all files
timelst,filelst,desclst = self.remote_time_sort(filelst,desclst)
# retrieve the files
files_notretrieved = []
for idx,remote_file in enumerate(filelst) :
timex.alarm(self.source.timeout_get)
local_file = self.local_filename(remote_file,desclst,timelst)
try :
ok = self.retrieve(remote_file, local_file)
if ok :
if self.source.delete : self.rm(remote_file)
files_pulled.append(local_file)
# setting access,modify time to remote time
ftime = time.mktime(timelst[remote_file])
os.utime(local_file,(ftime,ftime) )
else :
files_notretrieved.extend(filelst[idx:])
self.logger.warning("problem when retrieving %s " % remote_file )
break
# if batch is reached stop there
if len(files_pulled) == self.source.batch :
if idx != len(filelst)-1 : files_notretrieved.extend(filelst[idx+1:])
break
timex.cancel()
except FtpTimeoutException :
timex.cancel()
files_notretrieved.extend(filelst[idx:])
self.logger.warning("FTP timed out retrieving %s " % remote_file )
break
except :
timex.cancel()
files_notretrieved.extend(filelst[idx:])
(type, value, tb) = sys.exc_info()
self.logger.error("Unable write remote file %s in local file %s" % (remote_file,local_file))
self.logger.error(" Type: %s, Value: %s" % (type ,value))
break
# files not retrieved are removed from the file list
# this allow pull to recover from error on next pass
for f in files_notretrieved :
del self.ls[f]
# save ls file
ok = self.write_ls_file(self.lspath)
# if reached the batch limit : break here
if len(files_pulled) == self.source.batch :
self.logger.warning("break retrieving... batch reached")
break
# if we had a problem break here
if len(files_notretrieved) > 0 :
self.logger.warning("break retrieving... because of error")
break
return files_pulled