def handle_file_received(self, data, config):
"""
data = local file name
"""
home = config["home"]
source_name = data.split(home)
if len(source_name) > 1:
source_name = source_name[1].lstrip("/")
else:
source_name = source_name[0].lstrip("/")
file_handler.main(config["sid"],
config["col_type"],
config["time_received"],
config["parser"],
data,
config["charset"],
config["device_name"],
config["normalizer"],
config["repo"],
config["cursor"],
config.get("regex_pattern"),
config.get("regexparser_name"),
config["device_ip"],
conf_path=config["wiring_conf_path"],
source_name=source_name)
def on_file_received(self, localfile):
profile = self.config['client_map'][self.config_ip][self.username]
sid = profile['sid']
parser = profile['parser']
charset = profile['charset']
device_name = profile['device_name']
device_ip = self.ip
logging.debug('file transfer; completed')
vc = shelves.VersionChecker(self.db_file, sid, localfile)
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
return
file_handler.main(sid,
time.time(),
parser,
localfile,
charset,
device_name,
cursor,
profile.get('regex_pattern'),
profile.get('regexparser_name'),
device_ip,
conf_path=self.config.get('wiring_conf_path')
or None)
def fetch_job(sid, config, db_file):
log.debug('fetching files for sid:%s', sid)
source = config['client_map'][sid]
basedir = config['basedir'].replace('$LOGINSPECT_HOME',
homing.LOGINSPECT_HOME)
ip = source['ip']
port = source['port']
user = source['user']
path = source['path']
password = source['password']
parser = source['parser']
charset = source['charset']
device_name = source['device_name']
localdir = os.path.join(basedir, ip, base64.urlsafe_b64encode(sid))
ftp = ftpclient.login(ip, port, user, password)
for remotefile, mtime in ftpclient.fetch_file_mtime(ftp, path):
disk.prepare_path(localdir + '/')
vc = shelves.VersionChecker(db_file, sid, remotefile, mtime)
if vc.is_older_version():
continue
localfile = os.path.join(localdir,
base64.urlsafe_b64encode(remotefile))
log.info('Downloading remote file %r to %r', remotefile, localfile)
try:
ftpclient.download(ftp, remotefile, localfile)
except Exception, err:
log.warn("fetching failed; remotefile=%s; sid=%s; error=%r",
remotefile, sid, err)
continue
col_ts = time.time()
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
continue
file_handler.main(sid,
col_ts,
parser,
localfile,
charset,
device_name,
source['normalizer'],
source['repo'],
cursor,
source.get('regex_pattern'),
source.get('regexparser_name'),
conf_path=config.get('wiring_conf_path') or None)
def test_file_handler():
col_type = 'ftp'
sid = 'ftp|192.168.1.2'
col_ts = '1234567890'
parser = 'SyslogParser'
file = '/dummy/path'
charset = 'utf-8'
device_name = "makalu"
normalizer = "norm1"
repo = "repo1"
device_ip = "192.168.1.2"
file_handler.main(sid,
col_type,
col_ts,
parser,
file,
charset,
device_name,
normalizer,
repo,
device_ip=device_ip)
batch_processor_in = wiring.Wire('batch_processor_in')
event = batch_processor_in.recv()
eq_(
event,
dict(sid=sid,
col_ts=col_ts,
parser=parser,
file=file,
charset=charset,
device_name=device_name,
cursor=0,
normalizer=normalizer,
repo=repo,
device_ip=device_ip,
regexparser_name=None,
regex_pattern=None,
col_type=col_type))
class SCPFetcher(Fetcher):
def __init__(self, **args):
super(SCPFetcher, self).__init__(**args)
def fetch_job(self):
log.debug("fetching files for sid:%s", self.sid)
config = self.fetcher_runner.get_config()
try:
source = config["client_map"][self.sid]
except KeyError:
log.debug("source for sid=%s has been deleted" % (self.sid))
return
basedir = config["basedir"].replace('$LOGINSPECT_HOME',
homing.LOGINSPECT_HOME)
scp_shelves_file = os.path.join(basedir, "scp.shelves")
disk.prepare_path(scp_shelves_file)
pd = pdict.PersistentDict(scp_shelves_file)
if pd.get(self.sid):
first_fetch = False
else:
first_fetch = True
pd[self.sid] = True
pd.sync()
db_file = os.path.join(basedir, "checksums.pdict")
remotepath = self.remotepath
if remotepath.startswith('~'):
remotepath = '.' + remotepath[1:]
if '%' in self.device_ip:
old_empty_dir = os.path.join(basedir, self.device_ip)
if os.path.exists(old_empty_dir):
try:
shutil.rmtree(old_empty_dir)
except:
pass
ip_dir = self.device_ip.replace("%", "_")
else:
ip_dir = self.device_ip
localdir = os.path.join(basedir, ip_dir,
base64.urlsafe_b64encode(self.sid))
try:
password = self.get_decrypted_password(self.password)
scp.setup(self.device_ip, self.port, self.user, password)
except (ssh.SSHException, EOFError, SystemExit), err:
log.warn("error while setting up connection; sid=%s", self.sid)
return
try:
for remotefile, mtime in scp.fetch_file_mtime(
remotepath, self.name_pattern):
disk.prepare_path(localdir + '/')
if first_fetch:
vc = shelves.VersionChecker(db_file,
self.sid,
remotefile,
mtime=mtime,
old_logs=self.old_logs)
else:
vc = shelves.VersionChecker(db_file,
self.sid,
remotefile,
mtime=mtime)
if vc.is_older_version():
continue
localfile = os.path.join(localdir,
base64.urlsafe_b64encode(remotefile))
log.info('Downloading remote file %r to %r', remotefile,
localfile)
try:
scp.scp_get(remotefile, localfile)
except (ssh.SSHException, EOFError, SystemExit), err:
log.warn(
"fetching failed; sid=%s; remotefile=%s; error=%r",
self.sid, remotefile, err)
continue
col_ts = time.time()
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
continue
conf_path = self.fetcher_runner.get_field_value_from_config(
"wiring_conf_path") or None
col_type = self.fetcher_runner.get_field_value_from_config(
"col_type")
client_map = self.get_client_map()
file_handler.main(self.sid,
col_type,
col_ts,
self.parser,
localfile,
self.charset,
self.device_name,
client_map["normalizer"],
client_map["repo"],
cursor,
client_map.get("regex_pattern"),
client_map.get("regexparser_name"),
self.device_ip,
conf_path=conf_path,
source_name=remotefile)
except gevent.GreenletExit:
raise
except (Exception, ssh.SSHException, EOFError, SystemExit), err:
log.warn('exception while running job; sid=%s; err=%r', self.sid,
err)
scp.scp_get(remotefile, localfile)
except (ssh.SSHException, EOFError, SystemExit), err:
log.warn("fetching failed; sid=%s; remotefile=%s; error=%r",
sid, remotefile, err)
continue
col_ts = time.time()
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
continue
file_handler.main(sid,
col_ts,
parser,
localfile,
charset,
device_name,
source['normalizer'],
source['repo'],
cursor,
source.get('regex_pattern'),
source.get('regexparser_name'),
conf_path=config.get('wiring_conf_path') or None)
except gevent.GreenletExit:
raise
except (Exception, ssh.SSHException, EOFError, SystemExit), err:
log.warn('exception while running job; sid=%s; err=%r', sid, err)
def _run(func, args, seconds):
while True:
func(*args)
gevent.sleep(seconds)