def on_file_received(self, localfile):
logging.debug("file transfer; completed")
profile = self.config["client_map"][self.config_ip][self.username]
sid = profile["sid"]
profile["device_ip"] = self.ip
vc = shelves.VersionChecker(self.db_file, sid, localfile)
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
return
profile["time_received"] = time.time()
profile["cursor"] = cursor
profile["wiring_conf_path"] = self.config.get("wiring_conf_path") or None
profile["col_type"] = self.config["col_type"]
if not self.parser_name_only:
parser_name = profile.get("parser")
charset = profile.get("charset")
parser = None
if parser_name:
try:
parser = GetParser(parser_name, sid, charset,
profile.get("regex_pattern"), profile.get("regexparser_name"))
except InvalidParserException, err:
logging.warn(err)
profile["parser"] = parser
def on_file_received(self, localfile):
profile = self.config['client_map'][self.config_ip][self.username]
sid = profile['sid']
parser = profile['parser']
charset = profile['charset']
device_name = profile['device_name']
device_ip = self.ip
logging.debug('file transfer; completed')
vc = shelves.VersionChecker(self.db_file, sid, localfile)
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
return
file_handler.main(sid,
time.time(),
parser,
localfile,
charset,
device_name,
cursor,
profile.get('regex_pattern'),
profile.get('regexparser_name'),
device_ip,
conf_path=self.config.get('wiring_conf_path')
or None)
def fetch_job(sid, config, db_file):
log.debug('fetching files for sid:%s', sid)
source = config['client_map'][sid]
basedir = config['basedir'].replace('$LOGINSPECT_HOME',
homing.LOGINSPECT_HOME)
ip = source['ip']
port = source['port']
user = source['user']
path = source['path']
password = source['password']
parser = source['parser']
charset = source['charset']
device_name = source['device_name']
localdir = os.path.join(basedir, ip, base64.urlsafe_b64encode(sid))
ftp = ftpclient.login(ip, port, user, password)
for remotefile, mtime in ftpclient.fetch_file_mtime(ftp, path):
disk.prepare_path(localdir + '/')
vc = shelves.VersionChecker(db_file, sid, remotefile, mtime)
if vc.is_older_version():
continue
localfile = os.path.join(localdir,
base64.urlsafe_b64encode(remotefile))
log.info('Downloading remote file %r to %r', remotefile, localfile)
try:
ftpclient.download(ftp, remotefile, localfile)
except Exception, err:
log.warn("fetching failed; remotefile=%s; sid=%s; error=%r",
remotefile, sid, err)
continue
col_ts = time.time()
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
continue
file_handler.main(sid,
col_ts,
parser,
localfile,
charset,
device_name,
source['normalizer'],
source['repo'],
cursor,
source.get('regex_pattern'),
source.get('regexparser_name'),
conf_path=config.get('wiring_conf_path') or None)
class SCPFetcher(Fetcher):
def __init__(self, **args):
super(SCPFetcher, self).__init__(**args)
def fetch_job(self):
log.debug("fetching files for sid:%s", self.sid)
config = self.fetcher_runner.get_config()
try:
source = config["client_map"][self.sid]
except KeyError:
log.debug("source for sid=%s has been deleted" % (self.sid))
return
basedir = config["basedir"].replace('$LOGINSPECT_HOME',
homing.LOGINSPECT_HOME)
scp_shelves_file = os.path.join(basedir, "scp.shelves")
disk.prepare_path(scp_shelves_file)
pd = pdict.PersistentDict(scp_shelves_file)
if pd.get(self.sid):
first_fetch = False
else:
first_fetch = True
pd[self.sid] = True
pd.sync()
db_file = os.path.join(basedir, "checksums.pdict")
remotepath = self.remotepath
if remotepath.startswith('~'):
remotepath = '.' + remotepath[1:]
if '%' in self.device_ip:
old_empty_dir = os.path.join(basedir, self.device_ip)
if os.path.exists(old_empty_dir):
try:
shutil.rmtree(old_empty_dir)
except:
pass
ip_dir = self.device_ip.replace("%", "_")
else:
ip_dir = self.device_ip
localdir = os.path.join(basedir, ip_dir,
base64.urlsafe_b64encode(self.sid))
try:
password = self.get_decrypted_password(self.password)
scp.setup(self.device_ip, self.port, self.user, password)
except (ssh.SSHException, EOFError, SystemExit), err:
log.warn("error while setting up connection; sid=%s", self.sid)
return
try:
for remotefile, mtime in scp.fetch_file_mtime(
remotepath, self.name_pattern):
disk.prepare_path(localdir + '/')
if first_fetch:
vc = shelves.VersionChecker(db_file,
self.sid,
remotefile,
mtime=mtime,
old_logs=self.old_logs)
else:
vc = shelves.VersionChecker(db_file,
self.sid,
remotefile,
mtime=mtime)
if vc.is_older_version():
continue
localfile = os.path.join(localdir,
base64.urlsafe_b64encode(remotefile))
log.info('Downloading remote file %r to %r', remotefile,
localfile)
try:
scp.scp_get(remotefile, localfile)
except (ssh.SSHException, EOFError, SystemExit), err:
log.warn(
"fetching failed; sid=%s; remotefile=%s; error=%r",
self.sid, remotefile, err)
continue
col_ts = time.time()
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
continue
conf_path = self.fetcher_runner.get_field_value_from_config(
"wiring_conf_path") or None
col_type = self.fetcher_runner.get_field_value_from_config(
"col_type")
client_map = self.get_client_map()
file_handler.main(self.sid,
col_type,
col_ts,
self.parser,
localfile,
self.charset,
self.device_name,
client_map["normalizer"],
client_map["repo"],
cursor,
client_map.get("regex_pattern"),
client_map.get("regexparser_name"),
self.device_ip,
conf_path=conf_path,
source_name=remotefile)
except gevent.GreenletExit:
raise
except (Exception, ssh.SSHException, EOFError, SystemExit), err:
log.warn('exception while running job; sid=%s; err=%r', self.sid,
err)
ip_dir = ip.replace("%", "_")
else:
ip_dir = ip
localdir = os.path.join(basedir, ip_dir, base64.urlsafe_b64encode(sid))
try:
scp.setup(ip, port, user, password)
except (ssh.SSHException, EOFError, SystemExit), err:
log.warn('error while setting up connection; sid=%s', sid)
return
try:
for remotefile, mtime in scp.fetch_file_mtime(remotepath):
disk.prepare_path(localdir + '/')
vc = shelves.VersionChecker(db_file, sid, remotefile, mtime=mtime)
if vc.is_older_version():
continue
localfile = os.path.join(localdir,
base64.urlsafe_b64encode(remotefile))
log.info('Downloading remote file %r to %r', remotefile, localfile)
try:
scp.scp_get(remotefile, localfile)
except (ssh.SSHException, EOFError, SystemExit), err:
log.warn("fetching failed; sid=%s; remotefile=%s; error=%r",
sid, remotefile, err)
continue
col_ts = time.time()