def __init__(self, configfile=CONFIG, config=None, configregen=False): self.storage_lock = threading.Lock() self.storage_counter = ThreadCounter() # Load all storage classes storage_classes = _get_storage_classes() # Read in config if configfile: config_object = configparser.SafeConfigParser() config_object.optionxform = str # Regen the config if needed or wanted if configregen or not os.path.isfile(configfile): _rewrite_config(storage_classes, config_object, configfile) config_object.read(configfile) if config: file_conf = common.parse_config(config_object) for key in config: if key not in file_conf: file_conf[key] = config[key] file_conf[key]['_load_default'] = True else: file_conf[key].update(config[key]) config = file_conf else: config = common.parse_config(config_object) else: if config is None: config = {} for storage_name in storage_classes: config[storage_name] = {} config['_load_default'] = True # Set the config inside of the storage classes for storage_name in storage_classes: if storage_name in config: if '_load_default' in config or '_load_default' in config[storage_name]: # Remove _load_default from config if '_load_default' in config[storage_name]: del config[storage_name]['_load_default'] # Update the default storage config storage_classes[storage_name].config = storage_classes[storage_name].DEFAULTCONF storage_classes[storage_name].config.update(config[storage_name]) else: storage_classes[storage_name].config = config[storage_name] # Call setup for each enabled storage loaded_storage = [] for storage_name in storage_classes: storage = storage_classes[storage_name] if storage.config['ENABLED'] is True: try: if storage.setup(): loaded_storage.append(storage) except Exception as e: print('ERROR:', 'storage', storage_name, 'failed to load.', e) if loaded_storage == []: raise RuntimeError('No storage classes loaded') self.loaded_storage = loaded_storage
def __init__(self, configfile=CONFIG, config=None, configregen=False): self.storage_lock = threading.Lock() self.storage_counter = ThreadCounter() # Load all storage classes storage_classes = _get_storage_classes() # Read in config if configfile: config_object = configparser.SafeConfigParser() config_object.optionxform = str # Regen the config if needed or wanted if configregen or not os.path.isfile(configfile): _rewite_config(storage_classes, config_object, configfile) config_object.read(configfile) if config: file_conf = common.parse_config(config_object) for key in config: if key not in file_conf: file_conf[key] = config[key] file_conf[key]['_load_default'] = True else: file_conf[key].update(config[key]) config = file_conf else: config = common.parse_config(config_object) else: if config is None: config = {} for storage_name in storage_classes: config[storage_name] = {} config['_load_default'] = True # Set the config inside of the storage classes for storage_name in storage_classes: if storage_name in config: if '_load_default' in config or '_load_default' in config[storage_name]: # Remove _load_default from config if '_load_default' in config[storage_name]: del config[storage_name]['_load_default'] # Update the default storage config storage_classes[storage_name].config = storage_classes[storage_name].DEFAULTCONF storage_classes[storage_name].config.update(config[storage_name]) else: storage_classes[storage_name].config = config[storage_name] # Call setup for each enabled storage loaded_storage = [] for storage_name in storage_classes: storage = storage_classes[storage_name] if storage.config['ENABLED'] is True: try: if storage.setup(): loaded_storage.append(storage) except Exception as e: print('ERROR:', 'storage', storage_name, 'failed to load.', e) if loaded_storage == []: raise RuntimeError('No storage classes loaded') self.loaded_storage = loaded_storage
def main(): parser = argparse.ArgumentParser(description='Collect statistics from this server and POST json to a URL.') parser.add_argument ("--diskUsed", action='store_true', help='Collect disk space used at mountpoints specified.') parser.add_argument ("--diskIO", action='store_true', help='Collect disk blocks read / written.') parser.add_argument ("--dirUsed", action='store_true', help='Collect space used in directories (slow).') parser.add_argument ("--cpu", action='store_true', help='Collect CPU utilisation.') parser.add_argument ("--memory", action='store_true', help='Collect memory usage.') parser.add_argument ("--network", action='store_true', help='Collect network usage.') parser.add_argument ("--apacheLog", action='store_true', help='Collect network usage.') parser.add_argument ("--verbosity", nargs=1, type=int, choices=range(0, 3), help='Verbosty of logging, a positive number from 0 (nothing except errors) to 2 (a lot). The default is 1.') args = parser.parse_args() common.setLogLevel (args.verbosity[0] if args.verbosity else 1) common.parse_config() stats = [] if args.diskUsed: stats.extend (disk_stats.disks_used_stats()) if args.diskIO: stats.extend (disk_stats.disk_io_stats()) if args.dirUsed: stats.extend (disk_stats.disk_dirs_size()) if args.cpu: stats.extend (cpu_stats.cpu_stats()) if args.memory: stats.extend (mem_stats.mem_stats()) if args.network: stats.extend (network_stats.network_stats()) if args.apacheLog: stats.extend (apache_logs_stats.apache_stats()) if len (stats) > 0: common.check_config_sections(['api_url',], critical=True) common.check_config_sections(['api_key',], critical=True) common.send_stats(stats) else: parser.print_help() exit(common.EXIT_CODE if common.EXIT_CODE < 256 else 255)
def __init__(self, debug=False): storage_conf = multiscanner.common.get_config_path( multiscanner.CONFIG, 'storage') config_object = configparser.SafeConfigParser() config_object.optionxform = str config_object.read(storage_conf) conf = common.parse_config(config_object) storage_handler = multiscanner.storage.StorageHandler( configfile=storage_conf) es_handler = None for handler in storage_handler.loaded_storage: if isinstance(handler, elasticsearch_storage.ElasticSearchStorage): es_handler = handler break if not es_handler: print( '[!] ERROR: This analytic only works with ES stroage module.') sys.exit(0) # probably not ideal... self.es = es_handler.es self.index = conf['ElasticSearchStorage']['index'] self.doc_type = 'sample' self.debug = debug
def main(): global config config_fn = sys.argv[1] config = parse_config(config_fn) mean_features, std_features = get_mean_std() normalize_chipseq(mean_features, std_features)
def celery_task(files, config=multiscanner.CONFIG): ''' Run multiscanner on the given file and store the results in the storage handler(s) specified in the storage configuration file. ''' # Get the storage config storage_conf = multiscanner.common.get_config_path(config, 'storage') storage_handler = multiscanner.storage.StorageHandler( configfile=storage_conf) resultlist = multiscanner.multiscan(list(files), configfile=config) results = multiscanner.parse_reports(resultlist, python=True) scan_time = datetime.now().isoformat() # Loop through files in a way compatible with Py 2 and 3, and won't be # affected by changing keys to original filenames for file_ in files: original_filename = files[file_]['original_filename'] task_id = files[file_]['task_id'] file_hash = files[file_]['file_hash'] metadata = files[file_]['metadata'] # Get the Scan Config that the task was run with and # add it to the task metadata scan_config_object = configparser.SafeConfigParser() scan_config_object.optionxform = str scan_config_object.read(config) full_conf = common.parse_config(scan_config_object) sub_conf = {} for key in full_conf: if key == 'main': continue sub_conf[key] = {} sub_conf[key]['ENABLED'] = full_conf[key]['ENABLED'] results[file_]['Scan Metadata'] = {} results[file_]['Scan Metadata']['Worker Node'] = gethostname() results[file_]['Scan Metadata']['Scan Config'] = sub_conf # Use the original filename as the value for the filename # in the report (instead of the tmp path assigned to the file # by the REST API) results[original_filename] = results[file_] del results[file_] results[original_filename]['Scan Time'] = scan_time results[original_filename]['Metadata'] = metadata # Update the task DB to reflect that the task is done db.update_task( task_id=task_id, task_status='Complete', timestamp=scan_time, ) # Save the reports to storage storage_handler.store(results, wait=False) storage_handler.close() return results
def main(): global config config_fn = sys.argv[1] config = parse_config(config_fn) method_intra = 'tad' df_edges = get_edge_weights_intermingling(method_intra) df_edges = df_edges[['chr1', 'start row', 'stop row', 'chr2', 'start col', 'stop col', 'correlation']] df_edges.to_csv(config["GRAPH_NEW_DIR"] + 'data.weighted.txt', sep = ' ', header = None, index=None)
def main(event, context): conf = common.parse_config() if not conf: raise Exception("Error: failed to parse config settings file") # reproduce the original format of the paste item to preserve existing logic for retrieval and post processing paste_data_records = common.unpack_ddb_paste_records(event['Records']) for paste_data in paste_data_records:
class JunkfooderBot(irc.IRCClient): # Yuck! # TODO: find a better way config = common.parse_config(CONFIG_FILE)['irc'] nickname = config['nick'] realname = config['realname'] username = config['ident'] def __init__(self): # load plug-ins plug_dir = os.path.dirname(os.path.abspath(__file__)) + '/plugins/' sys.path.append(plug_dir) files = glob.glob(plug_dir + '*.py') plugs = [] for f in files: name = os.path.basename(f) plugs.append(importlib.import_module(name[:-3])) self.plugs = plugs def connectionMade(self): print('Connected') irc.IRCClient.connectionMade(self) def connectionLost(self, reason): print('Connection lost') irc.IRCClient.connectionLost(self, reason) def signedOn(self): print('Online') config = self.factory.config['irc'] nick_password = config.get('nick_password', None) if nick_password: self.msg('nickserv', 'identify ' + nick_password) channel = config['channel'] key = config.get('channel_key', None) self.join(channel, key) def privmsg(self, user, target, msg): if target != self.nickname: plugin.dispatch(self, 'privmsg', user, target, msg) else: print('MSG: %s' % msg) def userJoined(self, user, channel): plugin.dispatch(self, 'join', user, channel) def userLeft(self, user, channel): plugin.dispatch(self, 'part', user, channel) def afterCollideNick(self, nickname): return nickname + random.randint(100, 999)
def main(): global config config_fn = sys.argv[1] config = parse_config(config_fn) # read in file with all names and file names df = pd.read_csv(config["CHIPSEQ_DIR"] + 'feature_filenames.txt', sep='\t') chr_list = config["chrs"] Parallel(n_jobs=config['NUM_PROC'])(delayed(get_matrix_chr)(chrom, df) for chrom in chr_list)
def get_cluster_config(ambari_url, user, password, cluster_name, config_type, config_tag, connection_timeout): r = get( ambari_url, user, password, '/api/v1/clusters/{0}/configurations?type={1}&tag={2}'.format( cluster_name, config_type, config_tag), connection_timeout) assert_return_code(r, 200, 'cluster configurations') config = json.loads(r.content) return parse_config( r, config, lambda config: config['items'][0]['properties'] is not None, lambda config: config['items'][0])
def main(): global config config_fn = sys.argv[1] config = parse_config(config_fn) data_weighted = np.loadtxt(config["GRAPH_NEW_DIR"] + 'data.weighted.txt', dtype= 'S12') G = make_weighted_graph(data_weighted) print 'Number of nodes: ', G.number_of_nodes() print 'Number of edges: ', G.number_of_edges() weighted_correlation_clustering(G)
def get_config_types(ambari_url, user, password, cluster_name, connection_timeout): r = get( ambari_url, user, password, '/api/v1/clusters/{0}?fields=Clusters/desired_configs'.format( cluster_name), connection_timeout) assert_return_code(r, 200, 'cluster config types') config = json.loads(r.content) config_types = parse_config( r, config, lambda conf: conf['Clusters']['desired_configs'] is not None, lambda conf: conf['Clusters']['desired_configs']) return config_types
def main(): global config config_fn = sys.argv[1] config = parse_config(config_fn) #raw_matrices() # construct a dictionary of regions that have more than >50% repeat coverage - these should be filtered out in HiC matrices dic_repeats_tofilter = find_repeat_locations_wg() nonzero_entries = row_col_sums(dic_repeats_tofilter) ##nonzero_entries = np.loadtxt(config["HIC_FILT_DIR"] + 'whole_genome_nonzero.logtrans.txt') mean, std = whole_genome_mean_std(nonzero_entries) z_score_hic_matrix(mean, std) output_blacklist_locations(mean, std)
def main(): global config config_fn = sys.argv[1] config = parse_config(config_fn) #percentile_new = 0.999999999999999 #7.941 iters = 100 threshold_new = scipy.stats.norm.ppf(config["pvalue_threshold"]) print "LAS z-score threshold = ", threshold_new chr_pairs = list(itertools.combinations(config["chrs"], 2)) Parallel(n_jobs=config['NUM_PROC'])(delayed(run_LAS)(pair, threshold_new) for pair in chr_pairs)
def welcome(irc, user, channel, _msg=None): try: welcomes = common.parse_config('plugins/' + WELCOME_FILE) except Exception: # fail gracefully, but uninformatively return for regexp, message in welcomes.items(): regexp = re.compile(regexp, re.I) timestamp = memory.get(regexp, 0) if re.search(regexp, user) and timestamp < time.time() - WELCOME_TIME: line = '[%s] ' % user line += message irc.msg(channel, line) memory[regexp] = time.time()
def get_stack_versions(protocol, host, port, context_path, username, password, cluster_name, connection_timeout=10): ambari_url = get_ambari_url(protocol, host, port, context_path) path = '/api/v1/clusters/{0}/stack_versions/1'.format(cluster_name) resp = get(ambari_url, username, password, path, connection_timeout) assert_return_code(resp, 200, 'cluster stack versions') config = json.loads(resp.content) stack_version = parse_config( resp, config, lambda conf: conf['ClusterStackVersions'] is not None, lambda conf: conf['ClusterStackVersions']) return stack_version['stack'], stack_version['version']
def main(): global config config_fn = sys.argv[1] config = parse_config(config_fn) # get clusters filename = config["GRAPH_NEW_DIR"] + config["filename"] clusters = get_clusters(filename) print len(clusters) features = ['H3K36me3', 'POLR2A', 'H3K9me3', 'H3K9ac', 'H3K4me3', 'H3K4me1', 'H3K27me3'] enrichment_list = Parallel(n_jobs=config["NUM_PROC"])(delayed(run_one_feature)(feature, clusters) for feature in features) df = pd.DataFrame() for df_feat in enrichment_list: df = pd.concat([df, df_feat], axis=1) df.to_csv(config["ENRICHMENT_DIR"] + 'enrichment_all.csv')
def main(event, context): conf = common.parse_config() if not conf: raise Exception("Error: failed to parse config settings file") # we ignore history but it is required for PB inputs script dummy_history = [] pastes, _ = pb.recent_pastes(conf, dummy_history) if len(pastes) > 0: logger.debug(f"Received {len(pastes)} from pastebin.com") prepared_pastes = common.prepare_paste_items(pastes) for paste in prepared_pastes: table.put_item(Item=paste, ConditionExpression='attribute_not_exists(Id)') return {'status_code': 200, 'paste_count': len(pastes)}
def setup_scrape_results(self): self.config = parse_config() scrape_results = [{ "scrape_url": "https://scrape.pastebin.com/api_scrape_item.php?i=abcd1234", "full_url": "https://pastebin.com/abcd1234", "date": "1442911802", "key": "abcd1234", "size": "890", "expire": "1442998159", "title": "Once we all know when we goto function", "syntax": "java", "user": "******" }] responses.add(responses.GET, "https://scrape.pastebin.com/api_scraping.php?limit=200", json=scrape_results, status=404)
def post(self): """Handles requests to change the color configuration.""" logging.info('Received POST request: %s', self.request.body) try: payload = common.parse_config(self.request.body) except ValueError as e: self.response.headers['Content-Type'] = 'application/text' self.response.write(e.message) self.response.set_status(httplib.BAD_REQUEST) return config = ColorConfig( display_duration_ms=payload['display_duration_ms'], fadeout_duration_ms=payload['fadeout_duration_ms'], colors=payload['colors'], user_agent=self.request.user_agent, ip=self.request.remote_addr) config.put() self.response.set_status(httplib.OK)
def _get_main_config(config_object, filepath=CONFIG): """ Reads in config for main script. It will write defaults if not present. Returns dictionary. Config - The config object filepath - The path to the config file """ # Write main defaults if needed ConfNeedsWrite = False if 'main' not in config_object.sections(): ConfNeedsWrite = True maindefaults = DEFAULTCONF config_object.add_section('main') for key in maindefaults: config_object.set('main', key, str(maindefaults[key])) if ConfNeedsWrite: conffile = codecs.open(filepath, 'w', 'utf-8') config_object.write(conffile) conffile.close() # Read in main config return parse_config(config_object)['main']
def __init__(self): """ here we will parse the config file and determine if any global output settings are defined """ self.config = parse_config() self.logger = logging.getLogger('pastehunter') # if globals key is present under "outputs", get each standard output option or set a default self.whitelist_result_type = self.config['outputs'].get( 'globals', {}).get('whitelist_result_type', []) self.blacklist_result_type = self.config['outputs'].get( 'globals', {}).get('blacklist_result_type', []) self.exclude_raw = self.config['outputs'].get('globals', {}).get( 'exclude_row', False) self.raw_only = self.config['outputs'].get('globals', {}).get('raw_only', False) # this is here in case the exclude_raw flag is set. If so, it will get pulled out of paste_data # so that it can be saved to the output without the raw_text. However because python passes dicts by # reference, this raw_text field will not be present in future outputs that may require this field. # This instance var will keep track of that text, and replace it after it's been deleted self.raw_paste = None
def multiscan(Files, recursive=False, configregen=False, configfile=CONFIG, config=None, module_list=None): """ The meat and potatoes. Returns the list of module results Files - A list of files and dirs to be scanned recursive - If true it will search the dirs in Files recursively configregen - If True a new config file will be created overwriting the old configfile - What config file to use. Can be None. config - A dictionary containing the configuration options to be used. """ # Redirect stdout to stderr stdout = sys.stdout sys.stdout = sys.stderr # TODO: Make sure the cleanup from this works is something breaks # Init some vars # If recursive is None we don't parse the file list and take it as is. if recursive is not None: filelist = parseFileList(Files, recursive=recursive) else: filelist = Files # A list of files in the module dir if module_list is None: module_list = parseDir(MODULEDIR, recursive=True) # A dictionary used for the copyfileto parameter filedic = {} # What will be the config file object config_object = None # Read in config if configfile: config_object = configparser.SafeConfigParser() config_object.optionxform = str # Regen the config if needed or wanted if configregen or not os.path.isfile(configfile): _rewite_config(module_list, config_object, filepath=configfile) config_object.read(configfile) main_config = _get_main_config(config_object, filepath=configfile) if config: file_conf = parse_config(config_object) for key in config: if key not in file_conf: file_conf[key] = config[key] file_conf[key]['_load_default'] = True else: file_conf[key].update(config[key]) config = file_conf else: config = parse_config(config_object) else: if config is None: config = {} else: config['_load_default'] = True if 'main' in config: main_config = config['main'] else: main_config = DEFAULTCONF # If none of the files existed if not filelist: sys.stdout = stdout raise ValueError("No valid files") # Copy files to a share if configured if "copyfilesto" not in main_config: main_config["copyfilesto"] = False if main_config["copyfilesto"]: if os.path.isdir(main_config["copyfilesto"]): filelist = _copy_to_share(filelist, filedic, main_config["copyfilesto"]) else: sys.stdout = stdout raise IOError('The copyfilesto dir" ' + main_config["copyfilesto"] + '" is not a valid dir') # Create the global module interface global_module_interface = _GlobalModuleInterface() # Start a thread for each module thread_list = _start_module_threads(filelist, module_list, config, global_module_interface) # Write the default configure settings for missing ones if config_object: _write_missing_module_configs(module_list, config_object, filepath=configfile) # Warn about spaces in file names for f in filelist: if ' ' in f: print('WARNING: You are using file paths with spaces. This may result in modules not reporting correctly.') break # Wait for all threads to finish thread_wait_list = thread_list[:] i = 0 while thread_wait_list: i += 1 for thread in thread_wait_list: if not thread.is_alive(): i = 0 thread_wait_list.remove(thread) if VERBOSE: print(thread.name, "took", thread.endtime-thread.starttime) if i == 15: i = 0 if VERBOSE: p = 'Waiting on' for thread in thread_wait_list: p += ' ' + thread.name p += '...' print(p) time.sleep(1) # Delete copied files if main_config["copyfilesto"]: for item in filelist: os.remove(item) # Get Result list results = [] for thread in thread_list: if thread.ret is not None: results.append(thread.ret) del thread # Translates file names back to the originals if filedic: # I have no idea if this is the best way to do in-place modifications for i in range(0, len(results)): (result, metadata) = results[i] modded = False for j in range(0, len(result)): (filename, hit) = result[j] base = basename(filename) if base in filedic: filename = filedic[base] modded = True result[j] = (filename, hit) if modded: results[i] = (result, metadata) # Scan subfiles if needed subscan_list = global_module_interface._get_subscan_list() if subscan_list: # Translate from_filename back to original if needed if filedic: for i in range(0, len(subscan_list)): file_path, from_filename, module_name = subscan_list[i] base = basename(from_filename) if base in filedic: from_filename = filedic[base] subscan_list[i] = (file_path, from_filename, module_name) results.extend(_subscan(subscan_list, config, main_config, module_list, global_module_interface)) global_module_interface._cleanup() # Return stdout to previous state sys.stdout = stdout return results
def multiscan(Files, recursive=False, configregen=False, configfile=CONFIG, config=None, module_list=None): """ The meat and potatoes. Returns the list of module results Files - A list of files and dirs to be scanned recursive - If true it will search the dirs in Files recursively configregen - If True a new config file will be created overwriting the old configfile - What config file to use. Can be None. config - A dictionary containing the configuration options to be used. """ # Redirect stdout to stderr stdout = sys.stdout sys.stdout = sys.stderr # TODO: Make sure the cleanup from this works is something breaks # Init some vars # If recursive is None we don't parse the file list and take it as is. if recursive is not None: filelist = parseFileList(Files, recursive=recursive) else: filelist = Files # A list of files in the module dir if module_list is None: module_list = parseDir(MODULEDIR, recursive=True) # A dictionary used for the copyfileto parameter filedic = {} # What will be the config file object config_object = None # Read in config if configfile: config_object = configparser.SafeConfigParser() config_object.optionxform = str # Regen the config if needed or wanted if configregen or not os.path.isfile(configfile): _rewrite_config(module_list, config_object, filepath=configfile) config_object.read(configfile) main_config = _get_main_config(config_object, filepath=configfile) if config: file_conf = parse_config(config_object) for key in config: if key not in file_conf: file_conf[key] = config[key] file_conf[key]['_load_default'] = True else: file_conf[key].update(config[key]) config = file_conf else: config = parse_config(config_object) else: if config is None: config = {} else: config['_load_default'] = True if 'main' in config: main_config = config['main'] else: main_config = DEFAULTCONF # If none of the files existed if not filelist: sys.stdout = stdout raise ValueError("No valid files") # Copy files to a share if configured if "copyfilesto" not in main_config: main_config["copyfilesto"] = False if main_config["copyfilesto"]: if os.path.isdir(main_config["copyfilesto"]): filelist = _copy_to_share(filelist, filedic, main_config["copyfilesto"]) else: sys.stdout = stdout raise IOError('The copyfilesto dir" ' + main_config["copyfilesto"] + '" is not a valid dir') # Create the global module interface global_module_interface = _GlobalModuleInterface() # Start a thread for each module thread_list = _start_module_threads(filelist, module_list, config, global_module_interface) # Write the default configure settings for missing ones if config_object: _write_missing_module_configs(module_list, config_object, filepath=configfile) # Warn about spaces in file names for f in filelist: if ' ' in f: print( 'WARNING: You are using file paths with spaces. This may result in modules not reporting correctly.' ) break # Wait for all threads to finish thread_wait_list = thread_list[:] i = 0 while thread_wait_list: i += 1 for thread in thread_wait_list: if not thread.is_alive(): i = 0 thread_wait_list.remove(thread) if VERBOSE: print(thread.name, "took", thread.endtime - thread.starttime) if i == 15: i = 0 if VERBOSE: p = 'Waiting on' for thread in thread_wait_list: p += ' ' + thread.name p += '...' print(p) time.sleep(1) # Delete copied files if main_config["copyfilesto"]: for item in filelist: try: os.remove(item) except OSError: pass # Get Result list results = [] for thread in thread_list: if thread.ret is not None: results.append(thread.ret) del thread # Translates file names back to the originals if filedic: # I have no idea if this is the best way to do in-place modifications for i in range(0, len(results)): (result, metadata) = results[i] modded = False for j in range(0, len(result)): (filename, hit) = result[j] base = basename(filename) if base in filedic: filename = filedic[base] modded = True result[j] = (filename, hit) if modded: results[i] = (result, metadata) # Scan subfiles if needed subscan_list = global_module_interface._get_subscan_list() if subscan_list: # Translate from_filename back to original if needed if filedic: for i in range(0, len(subscan_list)): file_path, from_filename, module_name = subscan_list[i] base = basename(from_filename) if base in filedic: from_filename = filedic[base] subscan_list[i] = (file_path, from_filename, module_name) results.extend( _subscan(subscan_list, config, main_config, module_list, global_module_interface)) global_module_interface._cleanup() # Return stdout to previous state sys.stdout = stdout return results
def _read_conf(file_path): conf = configparser.SafeConfigParser() conf.optionxform = str with codecs.open(file_path, 'r', encoding='utf-8') as fp: conf.readfp(fp) return parse_config(conf)
def compat_check(app_configs=None, **kwargs): errors = [] # Imports first try: import pymongo have_mongo = True except ImportError: have_mongo = False errors.append( Error('Unable to import pymongo', hint='sudo pip install pymongo')) try: from Registry import Registry except ImportError: errors.append( Error('Unable to import python-registry', hint='sudo pip install python-registry')) try: from virus_total_apis import PublicApi except ImportError: errors.append( Warning('Unable to import virustotalapi', hint='sudo pip install virustotal-api')) try: import yara except ImportError: errors.append( Warning('Unable to import Yara', hint='Read the Wiki or google Yara')) try: import distorm3 except ImportError: errors.append( Warning('Unable to import distorm3', hint='sudo pip install distorm3')) # Check Vol Version try: vol_ver = vol_interface.vol_version.split('.') if int(vol_ver[1]) < 5: errors.append( Error( 'Unsupported Volatility version found. Need 2.5 or greater. Found: {0}' .format('.'.join(vol_ver)))) except Exception as error: errors.append( Error('Unable to find Volatility Version Number', hint='Read the installation wiki')) # Config try: from common import parse_config config = parse_config() if config['valid']: pass except: errors.append( Error('Unable to parse a volutility.conf file', hint='Copy volutiltiy.conf.sample to volutitliy.conf')) # Database Connection finally if have_mongo: try: if config['valid']: mongo_uri = config['database']['mongo_uri'] else: mongo_uri = 'mongodb://localhost' connection = pymongo.MongoClient(mongo_uri) # Version Check server_version = connection.server_info()['version'] if int(server_version[0]) < 3: errors.append( Error( str('Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}' .format(server_version)))) connection.close() except Exception as error: errors.append( Error('Unable to connect to MongoDB: {0}'.format(error))) return errors
def init_logging(self): # Setup Default logging self.logger = logging.getLogger('pastehunter') self.logger.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(levelname)s:%(filename)s: %(message)s') ch.setFormatter(formatter) self.logger.addHandler(ch) # Version info self.logger.info("Starting PasteHunter Version: {0}".format(VERSION)) # Parse the config file self.logger.info("Reading Configs") self.conf = parse_config(self.testing) # If the config failed to parse if not self.conf: sys.exit() # Set up the log file if "log" in self.conf and self.conf["log"]["log_to_file"]: if self.conf["log"]["log_path"] != "": logfile = "{0}/{1}.log".format(self.conf["log"]["log_path"], self.conf["log"]["log_file"]) # Assure directory exists try: os.makedirs(self.conf["log"]["log_path"], exist_ok=True) # Python>3.2 except TypeError: try: os.makedirs(self.conf["log"]["log_path"]) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir( self.conf["log"]["log_path"]): pass else: self.logger.error( "Can not create log file {0}: {1}".format( self.conf["log"]["log_path"], exc)) else: logfile = "{0}.log".format(self.conf["log"]["log_file"]) file_handler = handlers.RotatingFileHandler(logfile, mode='a+', maxBytes=(1048576 * 5), backupCount=7) if self.conf["log"]["format"] != "": file_formatter = logging.Formatter("{0}".format( self.conf["log"]["format"])) file_handler.setFormatter(file_formatter) else: file_handler.setFormatter(logFormatter) file_handler.setLevel(self.conf["log"]["logging_level"]) self.logger.addHandler(file_handler) self.logger.info("Enabled Log File: {0}".format(logfile)) else: self.logger.info("Logging to file disabled.") # Override Log level if needed if "logging_level" in self.conf["log"]: log_level = self.conf["log"]["logging_level"] elif "logging_level" in self.conf["general"]: # For old self.configs log_level = self.conf["general"]["logging_level"] else: # For older self.configs self.logger.error( "Log Level not in self.config file. Update your base self.config file!" ) log_level = 20 self.logger.info("Setting Log Level to {0}".format(log_level)) logging.getLogger('requests').setLevel(log_level) logging.getLogger('elasticsearch').setLevel(log_level) logging.getLogger('pastehunter').setLevel(log_level)
import json import pymongo from bson.objectid import ObjectId from gridfs import GridFS from common import parse_config config = parse_config() class Database(): def __init__(self): # Create the connection if config['valid']: mongo_uri = config['database']['mongo_uri'] else: mongo_uri = 'mongodb://localhost' connection = pymongo.MongoClient(mongo_uri) # Version Check server_version = connection.server_info()['version'] if int(server_version[0]) < 3: raise UserWarning('Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}'.format(server_version)) # Connect to Databases. voldb = connection['voldb'] voldbfs = connection['voldbfs'] # Get Collections self.vol_sessions = voldb.sessions self.vol_comments = voldb.comments self.vol_plugins = voldb.plugins
#!/usr/bin/env python import common import chromatics import os import sys from glob import glob config_fn = sys.argv[1] cell_line = config_fn.split('/')[0] config = common.parse_config(config_fn) os.chdir(os.path.expanduser(config['working_dir'])) segmentation_df = chromatics.read_bed(glob('../segmentation/*.bed.gz')[0], usecols = range(4), names = chromatics.generic_bed_columns) enhancer_states = {'E', 'WE', '13_EnhA1', '14_EnhA2', '16_EnhW1', '17_EnhW2'} enhancers_df = segmentation_df.query('name in @enhancer_states and (end - start) > 4').copy() enhancers_df.columns = chromatics.enhancer_bed_columns chromatics.add_names(enhancers_df, cell_line = cell_line) # optionally expand enhancer coordinates enhancers_df['enhancer_start'] -= config['enhancer_extension_size'] if 'enhancer_extension_size' in config else 0 enhancers_df['enhancer_end'] += config['enhancer_extension_size'] if 'enhancer_extension_size' in config else 0 # save assert enhancers_df.duplicated().sum() == 0 chromatics.write_bed(enhancers_df, 'enhancers.bed') print(enhancers_df.eval('enhancer_end - enhancer_start').describe(), '\n')
def multiscanner_celery(file_, original_filename, task_id, file_hash, metadata, config=multiscanner.CONFIG, module_list=None): ''' Queue up multiscanner tasks Usage: from celery_worker import multiscanner_celery multiscanner_celery.delay(full_path, original_filename, task_id, hashed_filename, metadata, config, module_list) ''' # Initialize the connection to the task DB db.init_db() logger.info('\n\n{}{}Got file: {}.\nOriginal filename: {}.\n'.format('='*48, '\n', file_hash, original_filename)) # Get the storage config storage_conf = multiscanner.common.get_config_path(config, 'storage') storage_handler = multiscanner.storage.StorageHandler(configfile=storage_conf) resultlist = multiscanner.multiscan( [file_], configfile=config, module_list=module_list ) results = multiscanner.parse_reports(resultlist, python=True) scan_time = datetime.now().isoformat() # Get the Scan Config that the task was run with and # add it to the task metadata scan_config_object = configparser.SafeConfigParser() scan_config_object.optionxform = str scan_config_object.read(config) full_conf = common.parse_config(scan_config_object) sub_conf = {} # Count number of modules enabled out of total possible # and add it to the Scan Metadata total_enabled = 0 total_modules = 0 for key in full_conf: if key == 'main': continue sub_conf[key] = {} sub_conf[key]['ENABLED'] = full_conf[key]['ENABLED'] total_modules += 1 if sub_conf[key]['ENABLED'] is True: total_enabled += 1 results[file_]['Scan Metadata'] = {} results[file_]['Scan Metadata']['Worker Node'] = gethostname() results[file_]['Scan Metadata']['Scan Config'] = sub_conf results[file_]['Scan Metadata']['Modules Enabled'] = '{} / {}'.format( total_enabled, total_modules ) # Use the original filename as the value for the filename # in the report (instead of the tmp path assigned to the file # by the REST API) results[original_filename] = results[file_] del results[file_] results[original_filename]['Scan Time'] = scan_time results[original_filename]['Metadata'] = metadata # Update the task DB to reflect that the task is done db.update_task( task_id=task_id, task_status='Complete', timestamp=scan_time, ) # Save the reports to storage storage_handler.store(results, wait=False) storage_handler.close() logger.info('Completed Task #{}'.format(task_id)) return results
import json import pymongo from bson.objectid import ObjectId from gridfs import GridFS from common import parse_config config = parse_config() class Database(): def __init__(self): # Create the connection if config['valid']: mongo_uri = config['database']['mongo_uri'] else: mongo_uri = 'mongodb://localhost' connection = pymongo.MongoClient(mongo_uri) # Version Check server_version = connection.server_info()['version'] if int(server_version[0]) < 3: raise UserWarning( 'Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}' .format(server_version)) # Connect to Databases. voldb = connection['voldb'] voldbfs = connection['voldbfs'] # Get Collections
} config_object = configparser.SafeConfigParser() config_object.optionxform = str configfile = common.get_config_path(multiscanner.CONFIG, 'api') config_object.read(configfile) if not config_object.has_section('celery') or not os.path.isfile(configfile): # Write default config config_object.add_section('celery') for key in DEFAULTCONF: config_object.set('celery', key, str(DEFAULTCONF[key])) conffile = codecs.open(configfile, 'w', 'utf-8') config_object.write(conffile) conffile.close() config = common.parse_config(config_object) api_config = config.get('api') worker_config = config.get('celery') db_config = config.get('Database') app = Celery(broker='{0}://{1}:{2}@{3}/{4}'.format( worker_config.get('protocol'), worker_config.get('user'), worker_config.get('password'), worker_config.get('host'), worker_config.get('vhost'), )) app.conf.timezone = worker_config.get('tz') db = database.Database(config=db_config)
def compat_check(app_configs=None, **kwargs): errors = [] # Imports first try: import pymongo have_mongo = True except ImportError: have_mongo = False errors.append(Error('Unable to import pymongo', hint='sudo pip install pymongo')) try: from Registry import Registry except ImportError: errors.append(Error('Unable to import python-registry', hint='sudo pip install python-registry')) try: from virus_total_apis import PublicApi except ImportError: errors.append(Warning('Unable to import virustotalapi', hint='sudo pip install virustotal-api')) try: import yara except ImportError: errors.append(Warning('Unable to import Yara', hint='Read the Wiki or google Yara')) try: import distorm3 except ImportError: errors.append(Warning('Unable to import distorm3', hint='sudo pip install distorm3')) # Check Vol Version try: vol_ver = vol_interface.vol_version.split('.') if int(vol_ver[1]) < 5: errors.append(Error('Unsupported Volatility version found. Need 2.5 or greater. Found: {0}'.format('.'.join(vol_ver)))) except Exception as error: errors.append(Error('Unable to find Volatility Version Number', hint='Read the installation wiki')) # Config try: from common import parse_config config = parse_config() if config['valid']: pass except: errors.append(Error('Unable to parse a volutility.conf file', hint='Copy volutiltiy.conf.sample to volutitliy.conf')) # Database Connection finally if have_mongo: try: if config['valid']: mongo_uri = config['database']['mongo_uri'] else: mongo_uri = 'mongodb://localhost' connection = pymongo.MongoClient(mongo_uri) # Version Check server_version = connection.server_info()['version'] if int(server_version[0]) < 3: errors.append(Error(str('Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}'.format(server_version)))) connection.close() except Exception as error: errors.append(Error('Unable to connect to MongoDB: {0}'.format(error))) return errors
root = logging.getLogger() ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(levelname)s:%(filename)s:%(message)s') ch.setFormatter(formatter) root.addHandler(ch) logger = logging.getLogger('pastehunter') logger.setLevel(logging.INFO) # Version info logger.info("Starting PasteHunter Version: {0}".format(VERSION)) # Parse the config file logger.info("Reading Configs") conf = parse_config() # If the config failed to parse if not conf: sys.exit() class TimeoutError(Exception): pass class timeout: def __init__(self, seconds=1, error_message='Timeout'): self.seconds = seconds self.error_message = error_message
def userLeft(self, user, channel): plugin.dispatch(self, 'part', user, channel) def afterCollideNick(self, nickname): return nickname + random.randint(100, 999) class JunkfooderBotFactory(protocol.ClientFactory): def __init__(self, config): self.config = config def buildProtocol(self, addr): bot = JunkfooderBot() bot.factory = self return bot def clientConnectionLost(self, connector, reason): connector.connect() def clientConnectionFailed(self, connector, reason): reactor.callLater(RECONNECT_TIME, connector.connect) if __name__ == '__main__': config = common.parse_config(CONFIG_FILE) bot = JunkfooderBotFactory(config) server = config['irc']['server'] port = config['irc'].get('port', 6667) reactor.connectTCP(server, port, bot) reactor.run()