def _build_event(self, events):
if not events:
return None
if not isinstance(events, list):
events = [events]
evts = []
for event in events:
assert event.raw_data, "the raw data of events is empty"
if event.is_unbroken:
evt = unbroken_evt_fmt.format(
event.host or "",
event.source or "",
event.sourcetype or "",
event.time or "",
event.index or "",
scu.escape_cdata(event.raw_data),
"<done/>" if event.is_done else "",
)
else:
evt = evt_fmt.format(
event.host or "",
event.source or "",
event.sourcetype or "",
event.time or "",
event.index or "",
scu.escape_cdata(event.raw_data),
)
evts.append(evt)
return evts
def _write_events(self, events, ckpts):
loader = self._task_config[c.data_loader]
brokers = self._task_config[c.kafka_brokers]
evt_fmt = ("<event><host>{0}</host><source>{1}</source>"
"<sourcetype>kafka:topicEvent</sourcetype>"
"<index>{2}</index><data>{3}</data></event>")
evts = (evt_fmt.format(brokers,
"kafka:{}:{}".format(msg.topic, msg.partition),
self._idx_tbl[msg.topic],
scutil.escape_cdata(msg.value))
for msg in events)
loader.write_events("<stream>{}</stream>".format("".join(evts)))
del events[:]
while 1:
for key, ckpt in ckpts.iteritems():
try:
self._store.update_state(key, ckpt)
except Exception:
time.sleep(2)
logger.error("Failed to update ckpt for key=%s reason=%s",
key, traceback.format_exc())
continue
return
def _do_index_data(self):
if self._api is None:
return
evt_fmt = ("<stream><event>"
"<time>{time}</time>"
"<source>{source}</source>"
"<sourcetype>{sourcetype}</sourcetype>"
"<index>{index}</index>"
"<data>{data}</data>"
"</event></stream>")
task = self._task_config
results = self._api(task)
events = []
size_total = 0
for result in results:
event = evt_fmt.format(source=task[tac.source],
sourcetype=task[tac.sourcetype],
index=task[tac.index],
data=scutil.escape_cdata(result),
time=time.time())
size_total += len(event)
events.append(event)
logger.info("Send data for indexing.",
action="index",
size=size_total,
records=len(events))
task["writer"].write_events("".join(events))
def _encode_to_utf8(self, decoder, chunk):
try:
data = decoder.decode(chunk)
return scutil.escape_cdata(data)
except Exception:
self._logger.exception("Failed to decode data.",
encoding=self._config[asc.character_set])
return None
def _do_format(self, evt, evt_fmt, index, host, source, sourcetype, time):
evt = scutil.escape_cdata(evt)
res = evt_fmt.format(index=index,
host=host,
source=source,
sourcetype=sourcetype,
time=time,
data=evt)
return res
def _encode_to_utf8(self, decoder, chunk):
try:
data = decoder.decode(chunk)
return scutil.escape_cdata(data)
# return xss.escape(data)
except Exception:
self._logger.error(
"Failed to decode data by using encoding=%s, error=%s",
self._config[asc.character_set], traceback.format_exc())
return None
def _do_index(self, source):
logger = self._logger
all_data = [data for data in self._reader]
size = sum((len(data) for data in all_data), 0)
if not all_data:
self.set_eof()
return
try:
all_data = json.loads("".join(all_data))
except ValueError:
logger.error("Invalid key of CloudTrail file.")
self.set_eof()
return
records = all_data.get("Records", [])
blacklist = self._config[asc.ct_blacklist]
if blacklist:
blacklist = re.compile(blacklist)
else:
blacklist = None
loader_service = self._loader_service
events = []
for record in records:
if loader_service.stopped():
break
if blacklist is not None and blacklist.search(record["eventName"]):
continue
data = self.event_fmt.format(
source=source,
sourcetype=self._config[tac.sourcetype],
index=self._config[tac.index],
data=scutil.escape_cdata(json.dumps(record)))
events.append(data)
if events:
logger.info("Indexed cloudtrail records.",
action="index",
num_reocords=len(records),
size=size)
loader_service.write_events("".join(events))
if not loader_service.stopped():
self._key_store.increase_offset(len(all_data))
self.set_eof()
def _index_events(self, results, stream_name):
evt_fmt = self._evt_fmt
task = self._task_config
region = task[tac.region]
log_group_name = task[aclc.log_group_name]
events = []
for result in results:
source = "{region}:{log_group}:{stream}".format(
region=region, log_group=log_group_name, stream=stream_name)
event = evt_fmt.format(
source=source, sourcetype=task[tac.sourcetype],
index=task[tac.index],
data=scutil.escape_cdata(result["message"]),
time=result["timestamp"] / 1000.0)
events.append(event)
task["writer"].write_events("".join(events))
def _do_format(self, evt, evt_fmt, index, host, source, sourcetype, time):
evt = scutil.escape_cdata(evt)
res = evt_fmt.format(index=index, host=host, source=source,
sourcetype=sourcetype, time=time, data=evt)
return res