def export(self):
index_name = audit.get_index_name()
es = ESConnectionFactoryFactory()()
query = self.get_query()
results = es.search(index=index_name,
body=query,
sort='date:desc',
size=3000)
output = StringIO()
writer = csv.writer(output)
writer.writerow(['Action', 'Path', 'User', 'Summary', 'Date'])
for result in results['hits']['hits']:
data = result['_source']
writer.writerow([
data['name'],
self.get_path(data), data['user'], data['summary'],
data['date']
])
resp = self.request.response
resp.setHeader('Content-Disposition',
'attachment; filename=export.csv')
resp.setHeader('Content-Type', 'text/csv')
output.seek(0)
return output.read()
def do_query(self):
index_name = audit.get_index_name()
es = ESConnectionFactoryFactory()()
query = self.get_query()
try:
page = int(self.request.get('page', 1))
except Exception:
page = 1
start = (page - 1) * self.limit
results = es.search(index=index_name,
body=query,
sort='date:desc',
from_=start,
size=self.limit)
return results
def export(args):
if args.indexname is not None:
index_name = args.indexname
else:
es_custom_index_name_enabled = api.portal.get_registry_record(
'castle.es_index_enabled', default=False)
custom_index_value = api.portal.get_registry_record('castle.es_index',
default=None)
index_name = audit.get_index_name(
site_path=None,
es_custom_index_name_enabled=es_custom_index_name_enabled,
custom_index_value=custom_index_value)
logger.info("exporting from ES index `{}`".format(index_name))
starttime = datetime.datetime.now()
hostsoverride = None
optsoverride = None
if args.host is not None:
hostsoverride = args.host
optsoverride = dict(
timeout=args.timeout,
sniff_on_start=False,
sniff_on_connection_fail=False,
)
es = ESConnectionFactoryFactory(hostsoverride=hostsoverride,
optsoverride=optsoverride)()
query = {"query": {'match_all': {}}}
countresult = es.count(index=index_name, body=query)
size = countresult.get("count", -1)
logger.info("{} results need to be exported (-1 is unknown)".format(size))
logger.info("fetching resultset with scroll time of `{}`".format(
args.scrolltime))
results = es.search(
index=index_name,
body=query,
sort='date:desc',
scroll=args.scrolltime,
size=10000, # max per search result
timeout=args.searchtimeout)
logger.info("writing to `{}` (truncated)".format(args.filepath))
with open(args.filepath, 'w') as output:
writer = csv.writer(output, quoting=csv.QUOTE_ALL)
logger.info("writing header row...")
writer.writerow([
'date', 'name', 'object', 'path', 'request_uri', 'summary', 'type',
'user'
])
num = 0
while len(results['hits']['hits']) > 0:
old_scroll_id = results["_scroll_id"]
logger.info("writing {} hits for scroll {}".format(
len(results['hits']['hits']), old_scroll_id))
for result in results['hits']['hits']:
data = result['_source']
rowdata = [
data.get("date", ""),
data.get("name", ""),
data.get("object", ""),
data.get("path", ""),
data.get("request_uri", ""),
data.get("summary", ""),
data.get("type", ""),
data.get("user", ""),
]
rowdata = [convertunicode(a) for a in rowdata]
writer.writerow(rowdata)
num += len(results['hits']['hits'])
logger.info("{} of {} written".format(num, size))
logger.info("fetching next scroll...")
results = es.scroll(scroll_id=old_scroll_id,
scroll=args.scrolltime)
endtime = datetime.datetime.now()
deltatime = endtime - starttime
logger.info("export complete -- took {}s, exported {} records".format(
deltatime.total_seconds(), num))