def index_dupes(hashgroup, cliargs, bot_logger): """This is the ES dupe_md5 tag update function. It updates a file's dupe_md5 field to be md5sum of file if it's marked as a duplicate. """ # create Elasticsearch connection es = diskover.elasticsearch_connect(diskover.config) file_id_list = [] # bulk update data in Elasticsearch index for f in hashgroup['files']: d = { '_op_type': 'update', '_index': cliargs['index'], '_type': 'file', '_id': f['id'], 'doc': { 'dupe_md5': hashgroup['md5sum'] } } file_id_list.append(d) if len(file_id_list) > 0: if cliargs['verbose']: bot_logger.info('Bulk updating %s files in ES index' % len(file_id_list)) diskover.index_bulk_add(es, file_id_list, 'file', diskover.config, cliargs)
def populate_hashgroup(key, cliargs): """Searches ES for all files matching hashgroup key (filehash) and returns dict containing matching files. """ bot_logger = diskover_worker_bot.bot_logger # create Elasticsearch connection es = diskover.elasticsearch_connect(diskover.config) if cliargs['verbose']: bot_logger.info('Searching ES for all files matching hash key %s' % key) hashgroup_files = [] data = { "_source": ["path_parent", "filename"], "query": { "bool": { "must": { "term": { "filehash": key } } } } } # refresh index # ES.indices.refresh(index=cliargs['index']) res = es.search(index=cliargs['index'], doc_type='file', size="1000", body=data, request_timeout=diskover.config['es_timeout']) # add any hits to hashgroups for hit in res['hits']['hits']: hashgroup_files.append({ 'id': hit['_id'], 'filename': hit['_source']['path_parent'] + "/" + hit['_source']['filename'] }) if cliargs['verbose']: bot_logger.info('Found %s files matching hash key %s' % (len(hashgroup_files), key)) # return filehash group and add to queue fhg = {'filehash': key, 'files': hashgroup_files, 'md5sum': ''} return fhg
import os import hashlib import socket import pwd import grp import time import logging # cache uid/gid names uids = [] gids = [] owners = {} groups = {} # create Elasticsearch connection es = diskover.elasticsearch_connect(diskover.config) # create Reddis connection redis_conn = Redis(host=diskover.config['redis_host'], port=diskover.config['redis_port'], password=diskover.config['redis_password']) def parse_cli_args(): """This is the parse CLI arguments function. It parses command line arguments. """ parser = argparse.ArgumentParser() parser.add_argument( "-b", "--burst",