def vectorize_subset(subset): # Checking dataset data_dir = os.path.join(os.getcwd(), 'ember') if subset == 'train': paths = [ os.path.join(data_dir, "train_features_{}.jsonl".format(i)) for i in range(6) ] n_rows = 900000 elif subset == 'test': paths = [ os.path.join(data_dir, "test_features.jsonl"), ] n_rows = 200000 else: logging.error('subset must be "train" or "test"') sys.exit(1) for p in paths: if not os.path.exists(p): logging.info('File not found: {}'.format(p)) sys.exit(1) X_path = os.path.join(data_dir, "X_{}.dat".format(subset)) y_path = os.path.join(data_dir, "y_{}.dat".format(subset)) if os.path.exists(X_path + '.shd256') and os.path.exists(y_path + '.shd256'): with open(X_path + '.shd256', 'r') as f: X_checksum = f.read() with open(y_path + '.shd256', 'r') as f: y_checksum = f.read() if X_checksum == sha256_checksum( X_path) and y_checksum == sha256_checksum(y_path): logging.info('"{}" subset is vectorized'.format(subset)) return # Allocate storage space dim = FeatureExtractor.dim X = np.memmap(X_path, dtype=np.float32, mode="w+", shape=(n_rows, dim)) y = np.memmap(y_path, dtype=np.float32, mode="w+", shape=n_rows) del X, y logging.info('Vectorzing samples in "{}" subset'.format(subset)) pool = mp.Pool() arg_iterator = ( (row, raw_data, X_path, y_path, n_rows) for row, raw_data in enumerate(raw_feature_iterator(paths))) for _ in tqdm(pool.imap_unordered(vectorize_data, arg_iterator), unit='row', unit_scale=True, ncols=96, miniters=1, total=n_rows): pass X_checksum = sha256_checksum(X_path) with open(X_path + '.shd256', 'w') as f: f.write(X_checksum) y_checksum = sha256_checksum(y_path) with open(y_path + '.shd256', 'w') as f: f.write(y_checksum)
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output): """Malice PDF Plugin.""" try: # set up logging init_logging(verbose) # TODO: check if PDF is too big (max size 3000000 ??) # TODO: if PDFiD fails maybe build different response JSON with errors etc. pdfid_results = MalPDFiD(file_path).run() pdf_results = { 'pdfid': pdfid_results, 'streams': MalPdfParser(file_path, pdfid_results, should_dump=dump, dump_path=output, verbose=verbose).run(), } # pdf_dict['pdf']['peepdf'] = MalPeepdf(file_path).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'pdf', 'category': 'document', 'results': pdf_results } malice_scan['results']['markdown'] = json2markdown(pdf_results) # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception( "failed to index malice/pdf results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: print(json.dumps(pdf_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: pdf") return
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output, peid): """Malice PExecutable Scanner""" try: # set up logging init_logging(verbose) # TODO: check if EXE is too big (max size 3000000 ??) pe_results = MalPEFile(file_path, peid_db_path=peid, should_dump=dump, dump_path=output).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'pescan', 'category': 'exe', 'results': pe_results } try: malice_scan['results']['markdown'] = json2markdown(pe_results) except Exception as e: log.exception("failed to render jinja template") malice_scan['results']['markdown'] = e.message # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception( "failed to index malice/pescan results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: pe_results.pop('markdown') print(json.dumps(pe_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: pescan") return
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output, peid): """Malice PExecutable Scanner""" try: # set up logging init_logging(verbose) # TODO: check if EXE is too big (max size 3000000 ??) pe_results = MalPEFile(file_path, peid_db_path=peid, should_dump=dump, dump_path=output).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'pescan', 'category': 'exe', 'results': pe_results } try: malice_scan['results']['markdown'] = json2markdown(pe_results) except Exception as e: log.exception("failed to render jinja template") malice_scan['results']['markdown'] = e.message # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception("failed to index malice/pescan results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: pe_results.pop('markdown') print(json.dumps(pe_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: pescan") return
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output): """Malice Office/OLE/RTF Scanner""" try: # set up logging init_logging(verbose) # TODO: check if DOC is too big (max size 3000000 ??) o_results = MalOffice(file_path, should_dump=dump, dump_path=output).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'office', 'category': 'document', 'results': o_results } malice_scan['results']['markdown'] = json2markdown(o_results) # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception( "failed to index malice/office results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: o_results.pop('markdown') print(json.dumps(o_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: office") return
def addToIndex(cls, **kwargs): image_hash = util.sha256_checksum(kwargs['filename']) picture = session.query(cls).filter_by(file_hash=image_hash) if(picture.scalar()): util.logger.debug( 'Attempted to add existing image to index, skipping...') return False image = ColorThief(kwargs['filename']) dominant_color = image.get_color(quality=6) cls.create( filename=kwargs['filename'], file_hash=util.sha256_checksum(kwargs['filename']), color=json.dumps(dominant_color), ts_indexed=datetime.datetime.now()) session.commit()
def __calc_doubt_dest__(self, factor): if self.doubts == None: return None src_noext, _ext = os.path.splitext(factor.src) hash = sha256_checksum(factor.src) bn = hash + _ext dst = os.path.join(self.doubts, bn) return dst
def save_numpy_file(subset): data_dir = os.path.join(os.getcwd(), 'ember') if subset == 'train': n_rows = 900000 elif subset == 'test': n_rows = 200000 else: logging.error('subset must be "train" or "test"') sys.exit(1) X_npy = os.path.join(data_dir, "X_{}.npy".format(subset)) y_npy = os.path.join(data_dir, "y_{}.npy".format(subset)) if os.path.exists(X_npy + '.shd256') and os.path.exists(y_npy + '.shd256'): with open(X_npy + '.shd256', 'r') as f: X_checksum = f.read() with open(y_npy + '.shd256', 'r') as f: y_checksum = f.read() if X_checksum == sha256_checksum( X_npy) and y_checksum == sha256_checksum(y_npy): logging.info( 'Numpy files of "{}" subset is existed!'.format(subset)) return logging.info( 'Saving numpy files for labeled samples in "{}" subset'.format(subset)) dim = FeatureExtractor.dim X_dat = os.path.join(data_dir, "X_{}.dat".format(subset)) y_dat = os.path.join(data_dir, "y_{}.dat".format(subset)) X = np.memmap(X_dat, dtype=np.float32, mode="r", shape=(n_rows, dim)) y = np.memmap(y_dat, dtype=np.float32, mode="r", shape=n_rows) labeled_rows = (y != -1) np.save(X_npy, X[labeled_rows]) np.save(y_npy, y[labeled_rows]) X_checksum = sha256_checksum(X_npy) with open(X_npy + '.shd256', 'w') as f: f.write(X_checksum) y_checksum = sha256_checksum(y_npy) with open(y_npy + '.shd256', 'w') as f: f.write(y_checksum)
def __determine_dest__(self, src, pref_dest): src_dir = path.dirname(src) src_bn = path.basename(src) src_bn_noext, src_ext = path.splitext(src_bn) dst_dir = path.dirname(pref_dest) dst_bn = path.basename(pref_dest) dst_bn_noext, dst_ext = path.splitext(dst_bn) op_ctx = OpContext(src) if path.exists(pref_dest): if self.hash_check: src_hash = sha256_checksum(src) siblings = [] for sibling in os.listdir(dst_dir): if sibling.startswith(dst_bn_noext) and sibling.endswith( dst_ext): siblings.append(sibling) sibling_hash = sha256_checksum( path.join(dst_dir, sibling)) if sibling_hash == src_hash: op_ctx.duplication = path.join(dst_dir, sibling) return op_ctx aimed = False id = 1 while not aimed: new_dst_bn = dst_bn_noext + self.rename_pattern % ( id) + dst_ext new_dst = path.join(dst_dir, new_dst_bn) if not path.exists(new_dst): op_ctx.dest = new_dst return op_ctx id += 1 raise AssertionError('Error happens') else: op_ctx.dest = pref_dest return op_ctx
def __init__(self, file_path, should_dump=False, dump_path=None): self.file = file_path self.sha256 = sha256_checksum(self.file) self.data = open(file_path, 'rb').read() self.dump = None self.results = {} if not path.exists(self.file): raise Exception("file does not exist: {}".format(self.file)) if should_dump: if path.isdir(dump_path): self.dump = dump_path else: log.error("folder does not exist: {}".format(dump_path)) self.dump = None
def __init__(self, file_path, should_dump=False, dump_path=None): self.file = file_path self.sha256 = sha256_checksum(self.file) self.data = open(file_path, 'rb').read() self.dump = None self.results = {} if not path.exists(self.file): raise Exception("file does not exist: {}".format(self.file)) if should_dump: if path.isdir(dump_path): self.dump = dump_path else: log.error("folder does not exist: {}".format(dump_path)) self.dump = None
print "------------------------------------" print "zookeeper configuratioins : %s %s" %(java64_home,zookeeper_hosts) # zookeeper configuratioins : /usr/jdk64/jdk1.8.0_112 [u'master', u'node1', u'node2'] print "------------------------------------" all_hosts = default("/clusterHostInfo/all_hosts", []) all_racks = default("/clusterHostInfo/all_racks", []) cluster_name = config["clusterName"] # clickhouse-config.xml clickhouse_config_json_template = config['configurations']['clickhouse-config'] tcp_port = config['configurations']['clickhouse-config']['tcp_port'] users_config = config['configurations']['clickhouse-config']['users_config'] clickhouse_data_path = config['configurations']['clickhouse-config']['path'] # clickhouse-metrika cluster configurations clickhouse_metrika_json_template = config['configurations']['clickhouse-metrika'] # clickhouse-user configurations clickhouse_users_json_template = config['configurations']['clickhouse-users']['clickhouse_users'] user_admin = config['configurations']['clickhouse-users']['user_admin'] user_admin_password = config['configurations']['clickhouse-users']['user_admin_password'] user_admin_password_sha256 = utils.sha256_checksum(user_admin_password) user_ck = config['configurations']['clickhouse-users']['user_ck'] user_ck_password = config['configurations']['clickhouse-users']['user_ck_password'] user_ck_password_sha256 = utils.sha256_checksum(user_ck_password)
if args.verbose: print('Scanning %s' % dirName) except UnicodeEncodeError as e: continue for filename in fileList: full_path = os.path.join(dirName, filename) try: p = subprocess.Popen(['file', '--brief', full_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, errors = p.communicate() file_type = str(output.strip().decode()) if "exe" in file_type: file_hash = utils.sha256_checksum(full_path) file_info = dict() file_info["path"] = full_path file_info["type"] = file_type file_info["sha256"] = file_hash status = utils.check_file_status(file_info) if status == utils.FILE_UNKNOWN: utils.add_file_to_db(file_info) elif status == utils.FILE_KNOWN_TOUCHED: utils.add_alert_do_db(file_info) elif status == utils.FILE_KNOWN_UNTOUCHED:
def __init__(self, file_path): self.file = file_path self.sha256 = sha256_checksum(self.file) self.oPDFiD = None