def __init__(self, configfile=CONFIG, config=None, configregen=False):
self.storage_lock = threading.Lock()
self.storage_counter = ThreadCounter()
# Load all storage classes
storage_classes = _get_storage_classes()
# Read in config
if configfile:
config_object = configparser.SafeConfigParser()
config_object.optionxform = str
# Regen the config if needed or wanted
if configregen or not os.path.isfile(configfile):
_rewrite_config(storage_classes, config_object, configfile)
config_object.read(configfile)
if config:
file_conf = common.parse_config(config_object)
for key in config:
if key not in file_conf:
file_conf[key] = config[key]
file_conf[key]['_load_default'] = True
else:
file_conf[key].update(config[key])
config = file_conf
else:
config = common.parse_config(config_object)
else:
if config is None:
config = {}
for storage_name in storage_classes:
config[storage_name] = {}
config['_load_default'] = True
# Set the config inside of the storage classes
for storage_name in storage_classes:
if storage_name in config:
if '_load_default' in config or '_load_default' in config[storage_name]:
# Remove _load_default from config
if '_load_default' in config[storage_name]:
del config[storage_name]['_load_default']
# Update the default storage config
storage_classes[storage_name].config = storage_classes[storage_name].DEFAULTCONF
storage_classes[storage_name].config.update(config[storage_name])
else:
storage_classes[storage_name].config = config[storage_name]
# Call setup for each enabled storage
loaded_storage = []
for storage_name in storage_classes:
storage = storage_classes[storage_name]
if storage.config['ENABLED'] is True:
try:
if storage.setup():
loaded_storage.append(storage)
except Exception as e:
print('ERROR:', 'storage', storage_name, 'failed to load.', e)
if loaded_storage == []:
raise RuntimeError('No storage classes loaded')
self.loaded_storage = loaded_storage
def __init__(self, configfile=CONFIG, config=None, configregen=False):
self.storage_lock = threading.Lock()
self.storage_counter = ThreadCounter()
# Load all storage classes
storage_classes = _get_storage_classes()
# Read in config
if configfile:
config_object = configparser.SafeConfigParser()
config_object.optionxform = str
# Regen the config if needed or wanted
if configregen or not os.path.isfile(configfile):
_rewite_config(storage_classes, config_object, configfile)
config_object.read(configfile)
if config:
file_conf = common.parse_config(config_object)
for key in config:
if key not in file_conf:
file_conf[key] = config[key]
file_conf[key]['_load_default'] = True
else:
file_conf[key].update(config[key])
config = file_conf
else:
config = common.parse_config(config_object)
else:
if config is None:
config = {}
for storage_name in storage_classes:
config[storage_name] = {}
config['_load_default'] = True
# Set the config inside of the storage classes
for storage_name in storage_classes:
if storage_name in config:
if '_load_default' in config or '_load_default' in config[storage_name]:
# Remove _load_default from config
if '_load_default' in config[storage_name]:
del config[storage_name]['_load_default']
# Update the default storage config
storage_classes[storage_name].config = storage_classes[storage_name].DEFAULTCONF
storage_classes[storage_name].config.update(config[storage_name])
else:
storage_classes[storage_name].config = config[storage_name]
# Call setup for each enabled storage
loaded_storage = []
for storage_name in storage_classes:
storage = storage_classes[storage_name]
if storage.config['ENABLED'] is True:
try:
if storage.setup():
loaded_storage.append(storage)
except Exception as e:
print('ERROR:', 'storage', storage_name, 'failed to load.', e)
if loaded_storage == []:
raise RuntimeError('No storage classes loaded')
self.loaded_storage = loaded_storage
def main():
parser = argparse.ArgumentParser(description='Collect statistics from this server and POST json to a URL.')
parser.add_argument ("--diskUsed", action='store_true', help='Collect disk space used at mountpoints specified.')
parser.add_argument ("--diskIO", action='store_true', help='Collect disk blocks read / written.')
parser.add_argument ("--dirUsed", action='store_true', help='Collect space used in directories (slow).')
parser.add_argument ("--cpu", action='store_true', help='Collect CPU utilisation.')
parser.add_argument ("--memory", action='store_true', help='Collect memory usage.')
parser.add_argument ("--network", action='store_true', help='Collect network usage.')
parser.add_argument ("--apacheLog", action='store_true', help='Collect network usage.')
parser.add_argument ("--verbosity", nargs=1, type=int, choices=range(0, 3), help='Verbosty of logging, a positive number from 0 (nothing except errors) to 2 (a lot). The default is 1.')
args = parser.parse_args()
common.setLogLevel (args.verbosity[0] if args.verbosity else 1)
common.parse_config()
stats = []
if args.diskUsed:
stats.extend (disk_stats.disks_used_stats())
if args.diskIO:
stats.extend (disk_stats.disk_io_stats())
if args.dirUsed:
stats.extend (disk_stats.disk_dirs_size())
if args.cpu:
stats.extend (cpu_stats.cpu_stats())
if args.memory:
stats.extend (mem_stats.mem_stats())
if args.network:
stats.extend (network_stats.network_stats())
if args.apacheLog:
stats.extend (apache_logs_stats.apache_stats())
if len (stats) > 0:
common.check_config_sections(['api_url',], critical=True)
common.check_config_sections(['api_key',], critical=True)
common.send_stats(stats)
else:
parser.print_help()
exit(common.EXIT_CODE if common.EXIT_CODE < 256 else 255)
def __init__(self, debug=False):
storage_conf = multiscanner.common.get_config_path(
multiscanner.CONFIG, 'storage')
config_object = configparser.SafeConfigParser()
config_object.optionxform = str
config_object.read(storage_conf)
conf = common.parse_config(config_object)
storage_handler = multiscanner.storage.StorageHandler(
configfile=storage_conf)
es_handler = None
for handler in storage_handler.loaded_storage:
if isinstance(handler, elasticsearch_storage.ElasticSearchStorage):
es_handler = handler
break
if not es_handler:
print(
'[!] ERROR: This analytic only works with ES stroage module.')
sys.exit(0)
# probably not ideal...
self.es = es_handler.es
self.index = conf['ElasticSearchStorage']['index']
self.doc_type = 'sample'
self.debug = debug
def main():
global config
config_fn = sys.argv[1]
config = parse_config(config_fn)
mean_features, std_features = get_mean_std()
normalize_chipseq(mean_features, std_features)
def celery_task(files, config=multiscanner.CONFIG):
'''
Run multiscanner on the given file and store the results in the storage
handler(s) specified in the storage configuration file.
'''
# Get the storage config
storage_conf = multiscanner.common.get_config_path(config, 'storage')
storage_handler = multiscanner.storage.StorageHandler(
configfile=storage_conf)
resultlist = multiscanner.multiscan(list(files), configfile=config)
results = multiscanner.parse_reports(resultlist, python=True)
scan_time = datetime.now().isoformat()
# Loop through files in a way compatible with Py 2 and 3, and won't be
# affected by changing keys to original filenames
for file_ in files:
original_filename = files[file_]['original_filename']
task_id = files[file_]['task_id']
file_hash = files[file_]['file_hash']
metadata = files[file_]['metadata']
# Get the Scan Config that the task was run with and
# add it to the task metadata
scan_config_object = configparser.SafeConfigParser()
scan_config_object.optionxform = str
scan_config_object.read(config)
full_conf = common.parse_config(scan_config_object)
sub_conf = {}
for key in full_conf:
if key == 'main':
continue
sub_conf[key] = {}
sub_conf[key]['ENABLED'] = full_conf[key]['ENABLED']
results[file_]['Scan Metadata'] = {}
results[file_]['Scan Metadata']['Worker Node'] = gethostname()
results[file_]['Scan Metadata']['Scan Config'] = sub_conf
# Use the original filename as the value for the filename
# in the report (instead of the tmp path assigned to the file
# by the REST API)
results[original_filename] = results[file_]
del results[file_]
results[original_filename]['Scan Time'] = scan_time
results[original_filename]['Metadata'] = metadata
# Update the task DB to reflect that the task is done
db.update_task(
task_id=task_id,
task_status='Complete',
timestamp=scan_time,
)
# Save the reports to storage
storage_handler.store(results, wait=False)
storage_handler.close()
return results
def main():
global config
config_fn = sys.argv[1]
config = parse_config(config_fn)
method_intra = 'tad'
df_edges = get_edge_weights_intermingling(method_intra)
df_edges = df_edges[['chr1', 'start row', 'stop row', 'chr2', 'start col', 'stop col', 'correlation']]
df_edges.to_csv(config["GRAPH_NEW_DIR"] + 'data.weighted.txt', sep = ' ', header = None, index=None)
def main(event, context):
conf = common.parse_config()
if not conf:
raise Exception("Error: failed to parse config settings file")
# reproduce the original format of the paste item to preserve existing logic for retrieval and post processing
paste_data_records = common.unpack_ddb_paste_records(event['Records'])
for paste_data in paste_data_records:
class JunkfooderBot(irc.IRCClient):
# Yuck!
# TODO: find a better way
config = common.parse_config(CONFIG_FILE)['irc']
nickname = config['nick']
realname = config['realname']
username = config['ident']
def __init__(self):
# load plug-ins
plug_dir = os.path.dirname(os.path.abspath(__file__)) + '/plugins/'
sys.path.append(plug_dir)
files = glob.glob(plug_dir + '*.py')
plugs = []
for f in files:
name = os.path.basename(f)
plugs.append(importlib.import_module(name[:-3]))
self.plugs = plugs
def connectionMade(self):
print('Connected')
irc.IRCClient.connectionMade(self)
def connectionLost(self, reason):
print('Connection lost')
irc.IRCClient.connectionLost(self, reason)
def signedOn(self):
print('Online')
config = self.factory.config['irc']
nick_password = config.get('nick_password', None)
if nick_password:
self.msg('nickserv', 'identify ' + nick_password)
channel = config['channel']
key = config.get('channel_key', None)
self.join(channel, key)
def privmsg(self, user, target, msg):
if target != self.nickname:
plugin.dispatch(self, 'privmsg', user, target, msg)
else:
print('MSG: %s' % msg)
def userJoined(self, user, channel):
plugin.dispatch(self, 'join', user, channel)
def userLeft(self, user, channel):
plugin.dispatch(self, 'part', user, channel)
def afterCollideNick(self, nickname):
return nickname + random.randint(100, 999)
def main():
global config
config_fn = sys.argv[1]
config = parse_config(config_fn)
# read in file with all names and file names
df = pd.read_csv(config["CHIPSEQ_DIR"] + 'feature_filenames.txt', sep='\t')
chr_list = config["chrs"]
Parallel(n_jobs=config['NUM_PROC'])(delayed(get_matrix_chr)(chrom, df)
for chrom in chr_list)
def get_cluster_config(ambari_url, user, password, cluster_name, config_type,
config_tag, connection_timeout):
r = get(
ambari_url, user, password,
'/api/v1/clusters/{0}/configurations?type={1}&tag={2}'.format(
cluster_name, config_type, config_tag), connection_timeout)
assert_return_code(r, 200, 'cluster configurations')
config = json.loads(r.content)
return parse_config(
r, config, lambda config: config['items'][0]['properties'] is not None,
lambda config: config['items'][0])
def main():
global config
config_fn = sys.argv[1]
config = parse_config(config_fn)
data_weighted = np.loadtxt(config["GRAPH_NEW_DIR"] + 'data.weighted.txt', dtype= 'S12')
G = make_weighted_graph(data_weighted)
print 'Number of nodes: ', G.number_of_nodes()
print 'Number of edges: ', G.number_of_edges()
weighted_correlation_clustering(G)
def get_config_types(ambari_url, user, password, cluster_name,
connection_timeout):
r = get(
ambari_url, user, password,
'/api/v1/clusters/{0}?fields=Clusters/desired_configs'.format(
cluster_name), connection_timeout)
assert_return_code(r, 200, 'cluster config types')
config = json.loads(r.content)
config_types = parse_config(
r, config,
lambda conf: conf['Clusters']['desired_configs'] is not None,
lambda conf: conf['Clusters']['desired_configs'])
return config_types
def main():
global config
config_fn = sys.argv[1]
config = parse_config(config_fn)
#raw_matrices()
# construct a dictionary of regions that have more than >50% repeat coverage - these should be filtered out in HiC matrices
dic_repeats_tofilter = find_repeat_locations_wg()
nonzero_entries = row_col_sums(dic_repeats_tofilter)
##nonzero_entries = np.loadtxt(config["HIC_FILT_DIR"] + 'whole_genome_nonzero.logtrans.txt')
mean, std = whole_genome_mean_std(nonzero_entries)
z_score_hic_matrix(mean, std)
output_blacklist_locations(mean, std)
def main():
global config
config_fn = sys.argv[1]
config = parse_config(config_fn)
#percentile_new = 0.999999999999999 #7.941
iters = 100
threshold_new = scipy.stats.norm.ppf(config["pvalue_threshold"])
print "LAS z-score threshold = ", threshold_new
chr_pairs = list(itertools.combinations(config["chrs"], 2))
Parallel(n_jobs=config['NUM_PROC'])(delayed(run_LAS)(pair, threshold_new)
for pair in chr_pairs)
def welcome(irc, user, channel, _msg=None):
try:
welcomes = common.parse_config('plugins/' + WELCOME_FILE)
except Exception:
# fail gracefully, but uninformatively
return
for regexp, message in welcomes.items():
regexp = re.compile(regexp, re.I)
timestamp = memory.get(regexp, 0)
if re.search(regexp, user) and timestamp < time.time() - WELCOME_TIME:
line = '[%s] ' % user
line += message
irc.msg(channel, line)
memory[regexp] = time.time()
def get_stack_versions(protocol,
host,
port,
context_path,
username,
password,
cluster_name,
connection_timeout=10):
ambari_url = get_ambari_url(protocol, host, port, context_path)
path = '/api/v1/clusters/{0}/stack_versions/1'.format(cluster_name)
resp = get(ambari_url, username, password, path, connection_timeout)
assert_return_code(resp, 200, 'cluster stack versions')
config = json.loads(resp.content)
stack_version = parse_config(
resp, config, lambda conf: conf['ClusterStackVersions'] is not None,
lambda conf: conf['ClusterStackVersions'])
return stack_version['stack'], stack_version['version']
def main():
global config
config_fn = sys.argv[1]
config = parse_config(config_fn)
# get clusters
filename = config["GRAPH_NEW_DIR"] + config["filename"]
clusters = get_clusters(filename)
print len(clusters)
features = ['H3K36me3', 'POLR2A', 'H3K9me3', 'H3K9ac', 'H3K4me3', 'H3K4me1', 'H3K27me3']
enrichment_list = Parallel(n_jobs=config["NUM_PROC"])(delayed(run_one_feature)(feature, clusters) for feature in features)
df = pd.DataFrame()
for df_feat in enrichment_list:
df = pd.concat([df, df_feat], axis=1)
df.to_csv(config["ENRICHMENT_DIR"] + 'enrichment_all.csv')
def main(event, context):
conf = common.parse_config()
if not conf:
raise Exception("Error: failed to parse config settings file")
# we ignore history but it is required for PB inputs script
dummy_history = []
pastes, _ = pb.recent_pastes(conf, dummy_history)
if len(pastes) > 0:
logger.debug(f"Received {len(pastes)} from pastebin.com")
prepared_pastes = common.prepare_paste_items(pastes)
for paste in prepared_pastes:
table.put_item(Item=paste,
ConditionExpression='attribute_not_exists(Id)')
return {'status_code': 200, 'paste_count': len(pastes)}
def setup_scrape_results(self):
self.config = parse_config()
scrape_results = [{
"scrape_url":
"https://scrape.pastebin.com/api_scrape_item.php?i=abcd1234",
"full_url": "https://pastebin.com/abcd1234",
"date": "1442911802",
"key": "abcd1234",
"size": "890",
"expire": "1442998159",
"title": "Once we all know when we goto function",
"syntax": "java",
"user": "******"
}]
responses.add(responses.GET,
"https://scrape.pastebin.com/api_scraping.php?limit=200",
json=scrape_results,
status=404)
def post(self):
"""Handles requests to change the color configuration."""
logging.info('Received POST request: %s', self.request.body)
try:
payload = common.parse_config(self.request.body)
except ValueError as e:
self.response.headers['Content-Type'] = 'application/text'
self.response.write(e.message)
self.response.set_status(httplib.BAD_REQUEST)
return
config = ColorConfig(
display_duration_ms=payload['display_duration_ms'],
fadeout_duration_ms=payload['fadeout_duration_ms'],
colors=payload['colors'],
user_agent=self.request.user_agent,
ip=self.request.remote_addr)
config.put()
self.response.set_status(httplib.OK)
def _get_main_config(config_object, filepath=CONFIG):
"""
Reads in config for main script. It will write defaults if not present. Returns dictionary.
Config - The config object
filepath - The path to the config file
"""
# Write main defaults if needed
ConfNeedsWrite = False
if 'main' not in config_object.sections():
ConfNeedsWrite = True
maindefaults = DEFAULTCONF
config_object.add_section('main')
for key in maindefaults:
config_object.set('main', key, str(maindefaults[key]))
if ConfNeedsWrite:
conffile = codecs.open(filepath, 'w', 'utf-8')
config_object.write(conffile)
conffile.close()
# Read in main config
return parse_config(config_object)['main']
def _get_main_config(config_object, filepath=CONFIG):
"""
Reads in config for main script. It will write defaults if not present. Returns dictionary.
Config - The config object
filepath - The path to the config file
"""
# Write main defaults if needed
ConfNeedsWrite = False
if 'main' not in config_object.sections():
ConfNeedsWrite = True
maindefaults = DEFAULTCONF
config_object.add_section('main')
for key in maindefaults:
config_object.set('main', key, str(maindefaults[key]))
if ConfNeedsWrite:
conffile = codecs.open(filepath, 'w', 'utf-8')
config_object.write(conffile)
conffile.close()
# Read in main config
return parse_config(config_object)['main']
def __init__(self):
"""
here we will parse the config file and determine if any global output settings are defined
"""
self.config = parse_config()
self.logger = logging.getLogger('pastehunter')
# if globals key is present under "outputs", get each standard output option or set a default
self.whitelist_result_type = self.config['outputs'].get(
'globals', {}).get('whitelist_result_type', [])
self.blacklist_result_type = self.config['outputs'].get(
'globals', {}).get('blacklist_result_type', [])
self.exclude_raw = self.config['outputs'].get('globals', {}).get(
'exclude_row', False)
self.raw_only = self.config['outputs'].get('globals',
{}).get('raw_only', False)
# this is here in case the exclude_raw flag is set. If so, it will get pulled out of paste_data
# so that it can be saved to the output without the raw_text. However because python passes dicts by
# reference, this raw_text field will not be present in future outputs that may require this field.
# This instance var will keep track of that text, and replace it after it's been deleted
self.raw_paste = None
def multiscan(Files, recursive=False, configregen=False, configfile=CONFIG, config=None, module_list=None):
"""
The meat and potatoes. Returns the list of module results
Files - A list of files and dirs to be scanned
recursive - If true it will search the dirs in Files recursively
configregen - If True a new config file will be created overwriting the old
configfile - What config file to use. Can be None.
config - A dictionary containing the configuration options to be used.
"""
# Redirect stdout to stderr
stdout = sys.stdout
sys.stdout = sys.stderr
# TODO: Make sure the cleanup from this works is something breaks
# Init some vars
# If recursive is None we don't parse the file list and take it as is.
if recursive is not None:
filelist = parseFileList(Files, recursive=recursive)
else:
filelist = Files
# A list of files in the module dir
if module_list is None:
module_list = parseDir(MODULEDIR, recursive=True)
# A dictionary used for the copyfileto parameter
filedic = {}
# What will be the config file object
config_object = None
# Read in config
if configfile:
config_object = configparser.SafeConfigParser()
config_object.optionxform = str
# Regen the config if needed or wanted
if configregen or not os.path.isfile(configfile):
_rewite_config(module_list, config_object, filepath=configfile)
config_object.read(configfile)
main_config = _get_main_config(config_object, filepath=configfile)
if config:
file_conf = parse_config(config_object)
for key in config:
if key not in file_conf:
file_conf[key] = config[key]
file_conf[key]['_load_default'] = True
else:
file_conf[key].update(config[key])
config = file_conf
else:
config = parse_config(config_object)
else:
if config is None:
config = {}
else:
config['_load_default'] = True
if 'main' in config:
main_config = config['main']
else:
main_config = DEFAULTCONF
# If none of the files existed
if not filelist:
sys.stdout = stdout
raise ValueError("No valid files")
# Copy files to a share if configured
if "copyfilesto" not in main_config:
main_config["copyfilesto"] = False
if main_config["copyfilesto"]:
if os.path.isdir(main_config["copyfilesto"]):
filelist = _copy_to_share(filelist, filedic, main_config["copyfilesto"])
else:
sys.stdout = stdout
raise IOError('The copyfilesto dir" ' + main_config["copyfilesto"] + '" is not a valid dir')
# Create the global module interface
global_module_interface = _GlobalModuleInterface()
# Start a thread for each module
thread_list = _start_module_threads(filelist, module_list, config, global_module_interface)
# Write the default configure settings for missing ones
if config_object:
_write_missing_module_configs(module_list, config_object, filepath=configfile)
# Warn about spaces in file names
for f in filelist:
if ' ' in f:
print('WARNING: You are using file paths with spaces. This may result in modules not reporting correctly.')
break
# Wait for all threads to finish
thread_wait_list = thread_list[:]
i = 0
while thread_wait_list:
i += 1
for thread in thread_wait_list:
if not thread.is_alive():
i = 0
thread_wait_list.remove(thread)
if VERBOSE:
print(thread.name, "took", thread.endtime-thread.starttime)
if i == 15:
i = 0
if VERBOSE:
p = 'Waiting on'
for thread in thread_wait_list:
p += ' ' + thread.name
p += '...'
print(p)
time.sleep(1)
# Delete copied files
if main_config["copyfilesto"]:
for item in filelist:
os.remove(item)
# Get Result list
results = []
for thread in thread_list:
if thread.ret is not None:
results.append(thread.ret)
del thread
# Translates file names back to the originals
if filedic:
# I have no idea if this is the best way to do in-place modifications
for i in range(0, len(results)):
(result, metadata) = results[i]
modded = False
for j in range(0, len(result)):
(filename, hit) = result[j]
base = basename(filename)
if base in filedic:
filename = filedic[base]
modded = True
result[j] = (filename, hit)
if modded:
results[i] = (result, metadata)
# Scan subfiles if needed
subscan_list = global_module_interface._get_subscan_list()
if subscan_list:
# Translate from_filename back to original if needed
if filedic:
for i in range(0, len(subscan_list)):
file_path, from_filename, module_name = subscan_list[i]
base = basename(from_filename)
if base in filedic:
from_filename = filedic[base]
subscan_list[i] = (file_path, from_filename, module_name)
results.extend(_subscan(subscan_list, config, main_config, module_list, global_module_interface))
global_module_interface._cleanup()
# Return stdout to previous state
sys.stdout = stdout
return results
def multiscan(Files,
recursive=False,
configregen=False,
configfile=CONFIG,
config=None,
module_list=None):
"""
The meat and potatoes. Returns the list of module results
Files - A list of files and dirs to be scanned
recursive - If true it will search the dirs in Files recursively
configregen - If True a new config file will be created overwriting the old
configfile - What config file to use. Can be None.
config - A dictionary containing the configuration options to be used.
"""
# Redirect stdout to stderr
stdout = sys.stdout
sys.stdout = sys.stderr
# TODO: Make sure the cleanup from this works is something breaks
# Init some vars
# If recursive is None we don't parse the file list and take it as is.
if recursive is not None:
filelist = parseFileList(Files, recursive=recursive)
else:
filelist = Files
# A list of files in the module dir
if module_list is None:
module_list = parseDir(MODULEDIR, recursive=True)
# A dictionary used for the copyfileto parameter
filedic = {}
# What will be the config file object
config_object = None
# Read in config
if configfile:
config_object = configparser.SafeConfigParser()
config_object.optionxform = str
# Regen the config if needed or wanted
if configregen or not os.path.isfile(configfile):
_rewrite_config(module_list, config_object, filepath=configfile)
config_object.read(configfile)
main_config = _get_main_config(config_object, filepath=configfile)
if config:
file_conf = parse_config(config_object)
for key in config:
if key not in file_conf:
file_conf[key] = config[key]
file_conf[key]['_load_default'] = True
else:
file_conf[key].update(config[key])
config = file_conf
else:
config = parse_config(config_object)
else:
if config is None:
config = {}
else:
config['_load_default'] = True
if 'main' in config:
main_config = config['main']
else:
main_config = DEFAULTCONF
# If none of the files existed
if not filelist:
sys.stdout = stdout
raise ValueError("No valid files")
# Copy files to a share if configured
if "copyfilesto" not in main_config:
main_config["copyfilesto"] = False
if main_config["copyfilesto"]:
if os.path.isdir(main_config["copyfilesto"]):
filelist = _copy_to_share(filelist, filedic,
main_config["copyfilesto"])
else:
sys.stdout = stdout
raise IOError('The copyfilesto dir" ' +
main_config["copyfilesto"] + '" is not a valid dir')
# Create the global module interface
global_module_interface = _GlobalModuleInterface()
# Start a thread for each module
thread_list = _start_module_threads(filelist, module_list, config,
global_module_interface)
# Write the default configure settings for missing ones
if config_object:
_write_missing_module_configs(module_list,
config_object,
filepath=configfile)
# Warn about spaces in file names
for f in filelist:
if ' ' in f:
print(
'WARNING: You are using file paths with spaces. This may result in modules not reporting correctly.'
)
break
# Wait for all threads to finish
thread_wait_list = thread_list[:]
i = 0
while thread_wait_list:
i += 1
for thread in thread_wait_list:
if not thread.is_alive():
i = 0
thread_wait_list.remove(thread)
if VERBOSE:
print(thread.name, "took",
thread.endtime - thread.starttime)
if i == 15:
i = 0
if VERBOSE:
p = 'Waiting on'
for thread in thread_wait_list:
p += ' ' + thread.name
p += '...'
print(p)
time.sleep(1)
# Delete copied files
if main_config["copyfilesto"]:
for item in filelist:
try:
os.remove(item)
except OSError:
pass
# Get Result list
results = []
for thread in thread_list:
if thread.ret is not None:
results.append(thread.ret)
del thread
# Translates file names back to the originals
if filedic:
# I have no idea if this is the best way to do in-place modifications
for i in range(0, len(results)):
(result, metadata) = results[i]
modded = False
for j in range(0, len(result)):
(filename, hit) = result[j]
base = basename(filename)
if base in filedic:
filename = filedic[base]
modded = True
result[j] = (filename, hit)
if modded:
results[i] = (result, metadata)
# Scan subfiles if needed
subscan_list = global_module_interface._get_subscan_list()
if subscan_list:
# Translate from_filename back to original if needed
if filedic:
for i in range(0, len(subscan_list)):
file_path, from_filename, module_name = subscan_list[i]
base = basename(from_filename)
if base in filedic:
from_filename = filedic[base]
subscan_list[i] = (file_path, from_filename, module_name)
results.extend(
_subscan(subscan_list, config, main_config, module_list,
global_module_interface))
global_module_interface._cleanup()
# Return stdout to previous state
sys.stdout = stdout
return results
def _read_conf(file_path):
conf = configparser.SafeConfigParser()
conf.optionxform = str
with codecs.open(file_path, 'r', encoding='utf-8') as fp:
conf.readfp(fp)
return parse_config(conf)
def compat_check(app_configs=None, **kwargs):
errors = []
# Imports first
try:
import pymongo
have_mongo = True
except ImportError:
have_mongo = False
errors.append(
Error('Unable to import pymongo', hint='sudo pip install pymongo'))
try:
from Registry import Registry
except ImportError:
errors.append(
Error('Unable to import python-registry',
hint='sudo pip install python-registry'))
try:
from virus_total_apis import PublicApi
except ImportError:
errors.append(
Warning('Unable to import virustotalapi',
hint='sudo pip install virustotal-api'))
try:
import yara
except ImportError:
errors.append(
Warning('Unable to import Yara',
hint='Read the Wiki or google Yara'))
try:
import distorm3
except ImportError:
errors.append(
Warning('Unable to import distorm3',
hint='sudo pip install distorm3'))
# Check Vol Version
try:
vol_ver = vol_interface.vol_version.split('.')
if int(vol_ver[1]) < 5:
errors.append(
Error(
'Unsupported Volatility version found. Need 2.5 or greater. Found: {0}'
.format('.'.join(vol_ver))))
except Exception as error:
errors.append(
Error('Unable to find Volatility Version Number',
hint='Read the installation wiki'))
# Config
try:
from common import parse_config
config = parse_config()
if config['valid']:
pass
except:
errors.append(
Error('Unable to parse a volutility.conf file',
hint='Copy volutiltiy.conf.sample to volutitliy.conf'))
# Database Connection finally
if have_mongo:
try:
if config['valid']:
mongo_uri = config['database']['mongo_uri']
else:
mongo_uri = 'mongodb://localhost'
connection = pymongo.MongoClient(mongo_uri)
# Version Check
server_version = connection.server_info()['version']
if int(server_version[0]) < 3:
errors.append(
Error(
str('Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}'
.format(server_version))))
connection.close()
except Exception as error:
errors.append(
Error('Unable to connect to MongoDB: {0}'.format(error)))
return errors
def init_logging(self):
# Setup Default logging
self.logger = logging.getLogger('pastehunter')
self.logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(levelname)s:%(filename)s: %(message)s')
ch.setFormatter(formatter)
self.logger.addHandler(ch)
# Version info
self.logger.info("Starting PasteHunter Version: {0}".format(VERSION))
# Parse the config file
self.logger.info("Reading Configs")
self.conf = parse_config(self.testing)
# If the config failed to parse
if not self.conf:
sys.exit()
# Set up the log file
if "log" in self.conf and self.conf["log"]["log_to_file"]:
if self.conf["log"]["log_path"] != "":
logfile = "{0}/{1}.log".format(self.conf["log"]["log_path"],
self.conf["log"]["log_file"])
# Assure directory exists
try:
os.makedirs(self.conf["log"]["log_path"],
exist_ok=True) # Python>3.2
except TypeError:
try:
os.makedirs(self.conf["log"]["log_path"])
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(
self.conf["log"]["log_path"]):
pass
else:
self.logger.error(
"Can not create log file {0}: {1}".format(
self.conf["log"]["log_path"], exc))
else:
logfile = "{0}.log".format(self.conf["log"]["log_file"])
file_handler = handlers.RotatingFileHandler(logfile,
mode='a+',
maxBytes=(1048576 * 5),
backupCount=7)
if self.conf["log"]["format"] != "":
file_formatter = logging.Formatter("{0}".format(
self.conf["log"]["format"]))
file_handler.setFormatter(file_formatter)
else:
file_handler.setFormatter(logFormatter)
file_handler.setLevel(self.conf["log"]["logging_level"])
self.logger.addHandler(file_handler)
self.logger.info("Enabled Log File: {0}".format(logfile))
else:
self.logger.info("Logging to file disabled.")
# Override Log level if needed
if "logging_level" in self.conf["log"]:
log_level = self.conf["log"]["logging_level"]
elif "logging_level" in self.conf["general"]:
# For old self.configs
log_level = self.conf["general"]["logging_level"]
else:
# For older self.configs
self.logger.error(
"Log Level not in self.config file. Update your base self.config file!"
)
log_level = 20
self.logger.info("Setting Log Level to {0}".format(log_level))
logging.getLogger('requests').setLevel(log_level)
logging.getLogger('elasticsearch').setLevel(log_level)
logging.getLogger('pastehunter').setLevel(log_level)
import json
import pymongo
from bson.objectid import ObjectId
from gridfs import GridFS
from common import parse_config
config = parse_config()
class Database():
def __init__(self):
# Create the connection
if config['valid']:
mongo_uri = config['database']['mongo_uri']
else:
mongo_uri = 'mongodb://localhost'
connection = pymongo.MongoClient(mongo_uri)
# Version Check
server_version = connection.server_info()['version']
if int(server_version[0]) < 3:
raise UserWarning('Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}'.format(server_version))
# Connect to Databases.
voldb = connection['voldb']
voldbfs = connection['voldbfs']
# Get Collections
self.vol_sessions = voldb.sessions
self.vol_comments = voldb.comments
self.vol_plugins = voldb.plugins
#!/usr/bin/env python
import common
import chromatics
import os
import sys
from glob import glob
config_fn = sys.argv[1]
cell_line = config_fn.split('/')[0]
config = common.parse_config(config_fn)
os.chdir(os.path.expanduser(config['working_dir']))
segmentation_df = chromatics.read_bed(glob('../segmentation/*.bed.gz')[0], usecols = range(4), names = chromatics.generic_bed_columns)
enhancer_states = {'E', 'WE', '13_EnhA1', '14_EnhA2', '16_EnhW1', '17_EnhW2'}
enhancers_df = segmentation_df.query('name in @enhancer_states and (end - start) > 4').copy()
enhancers_df.columns = chromatics.enhancer_bed_columns
chromatics.add_names(enhancers_df, cell_line = cell_line)
# optionally expand enhancer coordinates
enhancers_df['enhancer_start'] -= config['enhancer_extension_size'] if 'enhancer_extension_size' in config else 0
enhancers_df['enhancer_end'] += config['enhancer_extension_size'] if 'enhancer_extension_size' in config else 0
# save
assert enhancers_df.duplicated().sum() == 0
chromatics.write_bed(enhancers_df, 'enhancers.bed')
print(enhancers_df.eval('enhancer_end - enhancer_start').describe(), '\n')
def multiscanner_celery(file_, original_filename, task_id, file_hash, metadata, config=multiscanner.CONFIG, module_list=None):
'''
Queue up multiscanner tasks
Usage:
from celery_worker import multiscanner_celery
multiscanner_celery.delay(full_path, original_filename, task_id,
hashed_filename, metadata, config, module_list)
'''
# Initialize the connection to the task DB
db.init_db()
logger.info('\n\n{}{}Got file: {}.\nOriginal filename: {}.\n'.format('='*48, '\n', file_hash, original_filename))
# Get the storage config
storage_conf = multiscanner.common.get_config_path(config, 'storage')
storage_handler = multiscanner.storage.StorageHandler(configfile=storage_conf)
resultlist = multiscanner.multiscan(
[file_],
configfile=config,
module_list=module_list
)
results = multiscanner.parse_reports(resultlist, python=True)
scan_time = datetime.now().isoformat()
# Get the Scan Config that the task was run with and
# add it to the task metadata
scan_config_object = configparser.SafeConfigParser()
scan_config_object.optionxform = str
scan_config_object.read(config)
full_conf = common.parse_config(scan_config_object)
sub_conf = {}
# Count number of modules enabled out of total possible
# and add it to the Scan Metadata
total_enabled = 0
total_modules = 0
for key in full_conf:
if key == 'main':
continue
sub_conf[key] = {}
sub_conf[key]['ENABLED'] = full_conf[key]['ENABLED']
total_modules += 1
if sub_conf[key]['ENABLED'] is True:
total_enabled += 1
results[file_]['Scan Metadata'] = {}
results[file_]['Scan Metadata']['Worker Node'] = gethostname()
results[file_]['Scan Metadata']['Scan Config'] = sub_conf
results[file_]['Scan Metadata']['Modules Enabled'] = '{} / {}'.format(
total_enabled, total_modules
)
# Use the original filename as the value for the filename
# in the report (instead of the tmp path assigned to the file
# by the REST API)
results[original_filename] = results[file_]
del results[file_]
results[original_filename]['Scan Time'] = scan_time
results[original_filename]['Metadata'] = metadata
# Update the task DB to reflect that the task is done
db.update_task(
task_id=task_id,
task_status='Complete',
timestamp=scan_time,
)
# Save the reports to storage
storage_handler.store(results, wait=False)
storage_handler.close()
logger.info('Completed Task #{}'.format(task_id))
return results
import json
import pymongo
from bson.objectid import ObjectId
from gridfs import GridFS
from common import parse_config
config = parse_config()
class Database():
def __init__(self):
# Create the connection
if config['valid']:
mongo_uri = config['database']['mongo_uri']
else:
mongo_uri = 'mongodb://localhost'
connection = pymongo.MongoClient(mongo_uri)
# Version Check
server_version = connection.server_info()['version']
if int(server_version[0]) < 3:
raise UserWarning(
'Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}'
.format(server_version))
# Connect to Databases.
voldb = connection['voldb']
voldbfs = connection['voldbfs']
# Get Collections
}
config_object = configparser.SafeConfigParser()
config_object.optionxform = str
configfile = common.get_config_path(multiscanner.CONFIG, 'api')
config_object.read(configfile)
if not config_object.has_section('celery') or not os.path.isfile(configfile):
# Write default config
config_object.add_section('celery')
for key in DEFAULTCONF:
config_object.set('celery', key, str(DEFAULTCONF[key]))
conffile = codecs.open(configfile, 'w', 'utf-8')
config_object.write(conffile)
conffile.close()
config = common.parse_config(config_object)
api_config = config.get('api')
worker_config = config.get('celery')
db_config = config.get('Database')
app = Celery(broker='{0}://{1}:{2}@{3}/{4}'.format(
worker_config.get('protocol'),
worker_config.get('user'),
worker_config.get('password'),
worker_config.get('host'),
worker_config.get('vhost'),
))
app.conf.timezone = worker_config.get('tz')
db = database.Database(config=db_config)
def compat_check(app_configs=None, **kwargs):
errors = []
# Imports first
try:
import pymongo
have_mongo = True
except ImportError:
have_mongo = False
errors.append(Error('Unable to import pymongo', hint='sudo pip install pymongo'))
try:
from Registry import Registry
except ImportError:
errors.append(Error('Unable to import python-registry', hint='sudo pip install python-registry'))
try:
from virus_total_apis import PublicApi
except ImportError:
errors.append(Warning('Unable to import virustotalapi', hint='sudo pip install virustotal-api'))
try:
import yara
except ImportError:
errors.append(Warning('Unable to import Yara', hint='Read the Wiki or google Yara'))
try:
import distorm3
except ImportError:
errors.append(Warning('Unable to import distorm3', hint='sudo pip install distorm3'))
# Check Vol Version
try:
vol_ver = vol_interface.vol_version.split('.')
if int(vol_ver[1]) < 5:
errors.append(Error('Unsupported Volatility version found. Need 2.5 or greater. Found: {0}'.format('.'.join(vol_ver))))
except Exception as error:
errors.append(Error('Unable to find Volatility Version Number', hint='Read the installation wiki'))
# Config
try:
from common import parse_config
config = parse_config()
if config['valid']:
pass
except:
errors.append(Error('Unable to parse a volutility.conf file', hint='Copy volutiltiy.conf.sample to volutitliy.conf'))
# Database Connection finally
if have_mongo:
try:
if config['valid']:
mongo_uri = config['database']['mongo_uri']
else:
mongo_uri = 'mongodb://localhost'
connection = pymongo.MongoClient(mongo_uri)
# Version Check
server_version = connection.server_info()['version']
if int(server_version[0]) < 3:
errors.append(Error(str('Incompatible MongoDB Version detected. Requires 3 or higher. Found {0}'.format(server_version))))
connection.close()
except Exception as error:
errors.append(Error('Unable to connect to MongoDB: {0}'.format(error)))
return errors
root = logging.getLogger()
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(levelname)s:%(filename)s:%(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
logger = logging.getLogger('pastehunter')
logger.setLevel(logging.INFO)
# Version info
logger.info("Starting PasteHunter Version: {0}".format(VERSION))
# Parse the config file
logger.info("Reading Configs")
conf = parse_config()
# If the config failed to parse
if not conf:
sys.exit()
class TimeoutError(Exception):
pass
class timeout:
def __init__(self, seconds=1, error_message='Timeout'):
self.seconds = seconds
self.error_message = error_message
def userLeft(self, user, channel):
plugin.dispatch(self, 'part', user, channel)
def afterCollideNick(self, nickname):
return nickname + random.randint(100, 999)
class JunkfooderBotFactory(protocol.ClientFactory):
def __init__(self, config):
self.config = config
def buildProtocol(self, addr):
bot = JunkfooderBot()
bot.factory = self
return bot
def clientConnectionLost(self, connector, reason):
connector.connect()
def clientConnectionFailed(self, connector, reason):
reactor.callLater(RECONNECT_TIME, connector.connect)
if __name__ == '__main__':
config = common.parse_config(CONFIG_FILE)
bot = JunkfooderBotFactory(config)
server = config['irc']['server']
port = config['irc'].get('port', 6667)
reactor.connectTCP(server, port, bot)
reactor.run()
