def flush_data(self, data, now, compress=False):
# Collect data until it's time to send it out
self.data.extend(data)
if (not self.data) or (now - self.last_send < self.send_delta):
return
logging.info('Sending data for processing at {}'.format(now))
with NamedTemporaryFile() as f:
if compress:
self._write_compressed(f)
else:
f.writelines(self.data)
f.flush()
fsync(f.fileno())
remote_path = self.api.send_file(DATA_TYPE,
f.name,
now,
suffix=self.log_type)
if remote_path is not None:
data = {
'path': remote_path,
'log_type': self.log_type,
'utcoffset': utcoffset(),
'ip': get_ip(),
}
self.api.send_signal(DATA_TYPE, data)
self.checkpoint(now)
def __init__(self, *args, **kwargs):
kwargs.setdefault('poll_seconds', POLL_SECONDS)
super(SyslogADWatcher, self).__init__(*args, **kwargs)
self.utcoffset = utcoffset()
self.domain_suffix = getenv('OBSRVBL_DOMAIN_SUFFIX', '')
self.data_type = DATA_TYPE
self.log_node = RemoteADLogNode(
log_type=self.data_type,
api=self.api,
log_path=getenv('OBSRVBL_SYSLOG_AD_PATH', DEFAULT_AD_PATH)
)
def _upload(self, now, compress=False):
'''
Upload log files. Hopefully just one, but maybe the last one failed
so we need to pick it up too...
'''
pattern = os.path.join(
self.log_dir, '{}.*.archived'.format(SURICATA_LOGNAME)
)
for file_path in glob.iglob(pattern):
if compress:
file_path = _compress_log(file_path)
path = self.api.send_file(DATA_TYPE, file_path, now,
suffix=self.log_type)
data = {
'path': path,
'log_type': self.log_type,
'utcoffset': utcoffset(),
'ip': get_ip(),
}
self.api.send_signal(DATA_TYPE, data)
os.remove(file_path)
def _upload(self, now, compress=False):
'''
Upload log files. Hopefully just one, but maybe the last one failed
so we need to pick it up too...
'''
pattern = os.path.join(self.log_dir,
'{}.*.archived'.format(SURICATA_LOGNAME))
for file_path in glob.iglob(pattern):
if compress:
file_path = _compress_log(file_path)
path = self.api.send_file(DATA_TYPE,
file_path,
now,
suffix=self.log_type)
data = {
'path': path,
'log_type': self.log_type,
'utcoffset': utcoffset(),
'ip': get_ip(),
}
self.api.send_signal(DATA_TYPE, data)
os.remove(file_path)
def flush_data(self, data, now, compress=False):
# Collect data until it's time to send it out
self.data.extend(data)
if (not self.data) or (now - self.last_send < self.send_delta):
return
logging.info('Sending data for processing at {}'.format(now))
with NamedTemporaryFile() as f:
if compress:
self._write_compressed(f)
else:
f.writelines(self.data)
f.flush()
fsync(f.fileno())
data = {
'path': self.api.send_file(DATA_TYPE, f.name, now,
suffix=self.log_type),
'log_type': self.log_type,
'utcoffset': utcoffset(),
'ip': get_ip(),
}
self.api.send_signal(DATA_TYPE, data)
self.checkpoint(now)