class FileLoader(object):
"""Class for loading changed and latest files from the (remove) directory"""
def __init__(self):
"""Constructor"""
#self.script_dir = self.get_script_dir()
self.script_dir = os.getcwd()
self.init_logger()
self.remote_dir = "."
self.host = None
self.port = 22
self.user = None
self.password = None
self.local_dir = "."
self.history_file="%s/history" % (self.script_dir)
self.history = None
self.file_pattern = "^.*$"
self.protocol = 'ftp'
self.debug_level = "INFO"
self._filteredfiles = []
self._files = []
self.donotdownload = False
self.ftpdebug = False
self.pkey = False
self.keysdir = None
self.__start_time = time.time()
self.gzip = False
self.gzip_location = '/usr/contrib/bin/gzip'
self.passphrase = ''
tsre = re.compile('''.*PMSetup startTime="([^\"]+)" ''')
kpire = re.compile('''.*\<M1023C5\>''')
self.regexes = {'ts':tsre,'kpi':kpire}
self.notfoundre = re.compile('''<\/PMMOResult>''')
def main(self):
#Step 1 reads command line arguments
self.parse_options()
self._check_local_dir()
self.set_log_level() #update log level to the one specified in the --debug option (or default value)
#Step 2inits file history object and loads items from the file history if it exists
self.history = FileHistory(file=self.history_file)
if self.donotdownload:
self.history._save_every=10000
#init transport
if (self.protocol=='ftp'):
self.transport = TransportFTP(
host=self.host,
user=self.user,
password=self.password,
remote_dir=self.remote_dir,
)
if self.ftpdebug:
self.transport.ftpdebug=True
elif (self.protocol=='scp'):
self.transport = TransportSCP(
host=self.host,
user=self.user,
password=self.password,
remote_dir=self.remote_dir,
pkey=self.pkey,
keysdir=self.keysdir,
passphrase =self.passphrase
)
else:
print "ERROR: unsupported protocol %s " % self.protocol
sys.exit(1)
#Step 3
#connects to the SFTP server
self.connect()
#Step 4
#get list of files and match with file pattern
self._files = self.get_remote_files()
#pprint.pprint(self._files)
print self.transport.get_remote_files_dir()
#Step 5
#Finds new/changed files using FileHistory and file name regex pattern
if (self.file_pattern is not None):
self._filteredfiles = self.filter_files(self._files,self.history,self.file_pattern)
else:
self._filteredfiles = self.filter_files(self._files,self.history)
#Step 6 download files/update history file
self.download_files(self._filteredfiles)
#Step 7 end: save history file/write log messages
#DONE: think on purge strategry - simply deleting files with TS later then 1 month don't work for old files
#DONE solution would be to have purge history turnable off manually
#DONE have purge history enabled while downloading flow recent files updated every ~5min
#DONE have purge history disabled while using manually
self.history.purge_by_age()
self.history.save_final()
self.disconnect()
#self.lg.info("finished")
self.end()
def init_logger(self):
try:
self.lg = logging.getLogger()
self.lg.setLevel(logging.INFO)
handler = logging.FileHandler("%s/load_sftp.log" % self.script_dir)
#logformat = '%(asctime)s : %(name)s : %(levelname)s : %(message)s'
logformat = '%(asctime)s : %(name)s : %(levelname)s : %(filename)s: %(lineno)d : %(module)s : %(funcName)s : %(message)s'
handler.setFormatter(logging.Formatter(logformat))
self.lg.addHandler(handler)
self.lg.info('Started')
except Exception, e:
print "Failed to init logger %s" % str(e)
traceback.print_exc()
def main(self):
#Step 1 reads command line arguments
self.parse_options()
self._check_local_dir()
self.set_log_level() #update log level to the one specified in the --debug option (or default value)
#Step 2inits file history object and loads items from the file history if it exists
self.history = FileHistory(file=self.history_file)
if self.donotdownload:
self.history._save_every=10000
#init transport
if (self.protocol=='ftp'):
self.transport = TransportFTP(
host=self.host,
user=self.user,
password=self.password,
remote_dir=self.remote_dir,
)
if self.ftpdebug:
self.transport.ftpdebug=True
elif (self.protocol=='scp'):
self.transport = TransportSCP(
host=self.host,
user=self.user,
password=self.password,
remote_dir=self.remote_dir,
pkey=self.pkey,
keysdir=self.keysdir,
passphrase =self.passphrase
)
else:
print "ERROR: unsupported protocol %s " % self.protocol
sys.exit(1)
#Step 3
#connects to the SFTP server
self.connect()
#Step 4
#get list of files and match with file pattern
self._files = self.get_remote_files()
#pprint.pprint(self._files)
print self.transport.get_remote_files_dir()
#Step 5
#Finds new/changed files using FileHistory and file name regex pattern
if (self.file_pattern is not None):
self._filteredfiles = self.filter_files(self._files,self.history,self.file_pattern)
else:
self._filteredfiles = self.filter_files(self._files,self.history)
#Step 6 download files/update history file
self.download_files(self._filteredfiles)
#Step 7 end: save history file/write log messages
#DONE: think on purge strategry - simply deleting files with TS later then 1 month don't work for old files
#DONE solution would be to have purge history turnable off manually
#DONE have purge history enabled while downloading flow recent files updated every ~5min
#DONE have purge history disabled while using manually
self.history.purge_by_age()
self.history.save_final()
self.disconnect()
#self.lg.info("finished")
self.end()
import sys
print "trying user/password login"
logformat = '%(asctime)s : %(name)s : %(levelname)s : %(filename)s: %(lineno)d : %(module)s : %(funcName)s : %(message)s'
logging.basicConfig(level=logging.DEBUG,format=logformat)
_logger = logging.getLogger()
#test main
scp = TransportSCP(
host='osprey.ise.telcordia.com',
port=22,
user="******",
password="******",
remote_dir="."
)
scp.connect()
files = scp.get_remote_files()
pprint.pprint(files)
scp.get_file("/sd/.profile",'out/.profile')
scp.disconnect()
scp.log_error_stats()
print "trying user/publickey login"
scp = TransportSCP(
host='xp4r.com',
