def run_network(inputs, publish=False):
"""
Pipes functions from the different massoc modules to run complete network inference.
:param inputs: Dictionary of inputs.
:param publish: If True, publishes messages to be received by GUI.
:return:
"""
_create_logger(inputs['fp'])
old_inputs = read_settings(inputs['fp'] + '/settings.json')
old_inputs.update(inputs)
inputs = old_inputs
# handler to file
filestore = read_bioms(inputs['procbioms'])
bioms = Batch(filestore, inputs)
bioms = Nets(bioms)
if inputs['tools'] is not None:
logger.info('Tools to run with default settings: ' + str(inputs['tools']) + ' ')
bioms.inputs['network'] = list()
network_names = list()
for tool in bioms.inputs['tools']:
for level in bioms.inputs['levels']:
for name in bioms.inputs['name']:
filename = bioms.inputs['fp'] + '/' + tool + '_' + level + '_' + name + '.txt'
network_names.append(filename)
bioms.inputs['network'] = network_names
if publish:
pub.sendMessage('update', msg='Starting network inference. This may take some time!')
try:
logger.info('Running network inference... ')
networks = run_parallel(bioms)
networks.write_networks()
except Exception:
logger.warning('Failed to complete network inference. ', exc_info=True)
write_settings(networks.inputs)
if publish:
pub.sendMessage('update', msg="Finished running network inference!")
logger.info('Finished running network inference. ')
def save_settings(self, event):
"""
Takes self.settings file to write to disk.
Source: wxpython FileDialog docs
"""
with wx.FileDialog(self,
"Save settings file",
wildcard="json files (*.json)|*.json",
style=wx.FD_SAVE
| wx.FD_OVERWRITE_PROMPT) as fileDialog:
if fileDialog.ShowModal() == wx.ID_CANCEL:
return # the user changed their mind
# save the current contents in the file
pathname = fileDialog.GetPath()
try:
write_settings(self.settings, pathname)
except IOError:
wx.LogError("Cannot save current data in file '%s'." %
pathname)
logger.error("Cannot save current data in file. ",
exc_info=True)
def run_metastats(inputs, publish=False):
"""
Module that carries out analysis of metadata on the database.
This module also interfaces with external APIs to pull in additional metadata.
:param inputs: Dictionary of inputs.
:param publish: If True, publishes messages to be received by GUI.
:return:
"""
old_inputs = read_settings(inputs['fp'] + '/settings.json')
old_inputs.update(inputs)
inputs = old_inputs
# handler to file
_create_logger(inputs['fp'])
checks = str()
try:
if publish:
pub.sendMessage('update', msg='Starting database drivers.')
# sys.stdout.write('Starting database drivers.')
metadriver = MetaDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'],
filepath=inputs['fp'])
importdriver = ImportDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'],
filepath=inputs['fp'])
except Exception:
logger.warning("Failed to start database worker. ", exc_info=True)
if inputs['sequence']:
try:
logger.info('Uploading sequences to database...')
if publish:
pub.sendMessage('update', msg='Uploading sequences to database...')
importdriver.include_sequences(inputs['sequence'])
except Exception:
logger.warning("Failed to upload sequences to database. ", exc_info=True)
if inputs['add']:
try:
logger.info('Uploading additional properties... ')
if publish:
pub.sendMessage('update', msg='Uploading files to database...')
# create dictionary from file
# first check if this is an abundance table
for k in range(len(inputs['add'])):
filepath = inputs['add'][k]
with open(filepath, 'r') as file:
# Second column name is type
# Newline is cutoff
colnames = file.readline().split(sep="\t")
lines = file.readlines()[1:]
if not inputs['type']:
label = colnames[0].rstrip()
else:
label = inputs['type']
# if the supplied file is a dataframe,
# treat first column as source and rest as target
logger.info('Found ' + str(len(colnames)) + ' properties.')
for i in range(1, len(colnames)):
# give a logger update every 5th property
node_dict = dict()
name = colnames[i].rstrip()
if i % 5 == 0:
logger.info('Working on the ' + str(i) + 'th property.')
for line in lines:
source = line.split(sep="\t")[0].rstrip()
weight = None
if inputs['abundance']:
target = colnames[i].rstrip()
name = inputs['abundance'][k]
weight = line.split(sep="\t")[i].rstrip()
else:
target = line.split(sep="\t")[i].rstrip()
if weight != 0:
node_dict[source] = {'target': target, 'weight': weight}
importdriver.include_nodes(nodes=node_dict, name=name, label=label)
except Exception:
logger.warning("Failed to upload properties to database. ", exc_info=True)
inputs['add'] = None
inputs['type'] = None
# prevents reuploading
try:
# write operations here
if inputs['agglom']:
tax_list = ['Species', 'Genus', 'Family', 'Order', 'Class', 'Phylum', 'Kingdom']
level_id = tax_list.index(inputs['agglom'].capitalize())
if inputs['weight']:
mode = inputs['weight']
else:
mode = 'Ignore weight'
for level in range(0, level_id+1):
# pub.sendMessage('update', msg="Agglomerating edges...")
logger.info("Agglomerating edges...")
metadriver.agglomerate_network(level=tax_list[level], mode=mode)
checks += 'Successfully agglomerated edges. \n'
except Exception:
logger.warning("Failed to carry out edge agglomeration. ", exc_info=True)
checks += 'Failed to carry out edge agglomeration. \n'
try:
if inputs['variable']:
logger.info("Associating samples... ")
pub.sendMessage('update', msg="Associating samples...")
# sys.stdout.write("Associating samples...")
if inputs['variable'][0] == 'all':
properties = set([x[y] for x in metadriver.custom_query("MATCH (n:Property) RETURN n.type") for y in x])
for prop in properties:
metadriver.associate_samples(label=prop)
else:
for var in inputs['variable']:
metadriver.associate_samples(label=var)
checks += 'Completed associations. \n'
except Exception:
logger.warning("Failed to compute metadata associations. ", exc_info=True)
checks += 'Failed to compute metadata associations. \n'
if publish:
pub.sendMessage('database_log', msg=checks)
# functions to include:
# include_sequences
metadriver.close()
importdriver.close()
logger.info('Completed metastats operations! ')
write_settings(inputs)
def get_input(inputs, publish=False):
"""
Takes all input and returns a dictionary of biom files.
If tab-delimited files are supplied, these are combined
into a biom file. File names are used as keys.
This is mostly a utility wrapper, as all biom-related functions
are from biom-format.org.
At the moment, rarefaction is performed after sample splitting.
This means that samples with uneven sequence counts will not
be rarefied to equal depths.
All files are written to BIOM files, while a settings file is also written to disk
for use by other massoc commands.
:param inputs: Dictionary of inputs.
:param publish: If True, publishes messages to be received by GUI.
:return:
"""
# handler to file
# construct logger after filepath is provided
_create_logger(inputs['fp'])
if inputs['biom_file'] is not None:
logger.info('BIOM file(s) to process: ' + ", ".join(inputs['biom_file']))
if inputs['otu_table'] is not None:
logger.info('Tab-delimited OTU table(s) to process: ' + ", ".join(inputs['otu_table']))
if inputs['tax_table'] is not None:
if len(inputs['otu_table']) is not len(inputs['tax_table']):
logger.error("Add a taxonomy table for every OTU table!", exc_info=True)
raise ValueError("Add a taxonomy table for every OTU table!")
if inputs['sample_data'] is not None:
if len(inputs['otu_table']) is not len(inputs['sample_data']):
logger.error("Add a sample data table for every OTU table!", exc_info=True)
raise ValueError("Add a sample data table for every OTU table!")
if inputs['otu_meta'] is not None:
if len(inputs['otu_table']) is not len(inputs['otu_meta']):
logger.error("Add a metadata table for every OTU table!", exc_info=True)
raise ValueError("Add a metadata table for every OTU table!")
filestore = {}
if inputs['biom_file'] is None and inputs['network'] is None:
if inputs['otu_table'] is None and inputs['network'] is None:
logger.error("Please supply either a biom file"
", a tab-delimited OTU table or a network!", exc_info=True)
raise ValueError("Please supply either a biom file"
", a tab-delimited OTU table or a network!")
# Only process count files if present
i = 0
if inputs['name'] is None:
inputs['name'] = list()
inputs['name'].append('file_')
if inputs['biom_file'] is not None:
try:
for x in inputs['biom_file']:
biomtab = load_table(x)
filestore[inputs['name'][i]] = biomtab
i += 1
except Exception:
logger.error("Failed to import BIOM files.", exc_info=True)
if inputs['otu_table'] is not None:
try:
j = 0 # j is used to match sample + tax data to OTU data
for x in inputs['otu_table']:
input_fp = x
sample_metadata_fp = None
observation_metadata_fp = None
obs_data = None
sample_data = None
biomtab = load_table(input_fp)
try:
sample_metadata_fp = inputs['sample_data'][j]
observation_metadata_fp = inputs['tax_table'][j]
except TypeError or KeyError:
pass
if sample_metadata_fp is not None:
sample_f = open(sample_metadata_fp, 'r')
sample_data = MetadataMap.from_file(sample_f)
sample_f.close()
biomtab.add_metadata(sample_data, axis='sample')
if observation_metadata_fp is not None:
obs_f = open(observation_metadata_fp, 'r')
obs_data = MetadataMap.from_file(obs_f)
obs_f.close()
# for taxonomy collapsing,
# metadata variable needs to be a complete list
# not separate entries for each tax level
for b in list(obs_data):
tax = list()
for l in list(obs_data[b]):
tax.append(obs_data[b][l])
obs_data[b].pop(l, None)
obs_data[b]['taxonomy'] = tax
biomtab.add_metadata(obs_data, axis='observation')
filestore[inputs['name'][j]] = biomtab
j += 1
except Exception:
logger.warning("Failed to combine input files.", exc_info=True)
bioms = Batch({'otu': filestore}, inputs)
# it is possible that there are forbidden characters in the OTU identifiers
# we can forbid people from using those, or replace those with an underscore
if inputs['biom_file'] or inputs['otu_table']:
for name in bioms.otu:
biomfile = bioms.otu[name]
taxon_ids = biomfile._observation_ids # need to be careful with these operations
taxon_index = biomfile._obs_index # likely to corrupt BIOM file if done wrong
new_ids = deepcopy(taxon_ids)
new_indexes = deepcopy(taxon_index)
for i in range(0, len(taxon_ids)):
id = taxon_ids[i]
new_id = id.replace(" ", "_")
new_ids[i] = new_id
new_indexes[new_id] = new_indexes.pop(id)
biomfile._observation_ids = new_ids
biomfile._obs_index = new_indexes
bioms.otu[name] = biomfile
logger.info('Collapsing taxonomy... ')
bioms.collapse_tax()
if inputs['cluster'] is not None:
if publish:
pub.sendMessage('update', msg='Clustering BIOM files...')
logger.info('Clustering BIOM files... ')
bioms.cluster_biom()
if inputs['split'] is not None and inputs['split'] is not 'TRUE':
bioms.split_biom()
if inputs['min'] is not None:
if publish:
pub.sendMessage('update', msg='Setting minimum mean abundance...')
logger.info('Removing taxa below minimum count... ')
bioms.prev_filter(mode='min')
if inputs['prev'] is not None:
if publish:
pub.sendMessage('update', msg='Setting prevalence filter...')
logger.info('Setting prevalence filter... ')
bioms.prev_filter(mode='prev')
if inputs['rar'] is not None:
if publish:
pub.sendMessage('update', msg='Rarefying counts...')
logger.info('Rarefying counts... ')
bioms.rarefy()
bioms.inputs['procbioms'] = dict()
if inputs['biom_file'] or inputs['otu_table']:
if 'otu' not in bioms.inputs['levels']: # add otu level always
bioms.inputs['procbioms']['otu'] = dict()
for name in bioms.inputs['name']:
biomname = bioms.inputs['fp'] + '/' + name + '_' + 'otu' + '.hdf5'
bioms.inputs['procbioms']['otu'][name] = biomname
for level in bioms.inputs['levels']:
bioms.inputs['procbioms'][level] = dict()
for name in bioms.inputs['name']:
biomname = bioms.inputs['fp'] + '/' + name + '_' + level + '.hdf5'
bioms.inputs['procbioms'][level][name] = biomname
all_bioms = {**bioms.otu, **bioms.genus, **bioms.family, **bioms.order,
**bioms.class_, **bioms.phylum}
for biomfile in all_bioms:
if all_bioms[biomfile].shape[0] == 1:
logger.error("The current preprocessing steps resulted in BIOM files with only 1 row.", exc_info=True)
if inputs['network'] is not None:
if publish:
pub.sendMessage('update', msg='Checking previously generated networks...')
logger.info('Checking previously generated networks...')
filelist = deepcopy(inputs['network'])
for file in filelist:
network = _read_network(file)
nodes = len(network.nodes)
edges = len(network.edges)
logger.info("This network has " + str(nodes) + \
" nodes and " + str(edges) + " edges.")
weight = nx.get_edge_attributes(network, 'weight')
if len(weight) > 0:
logger.info('This is a weighted network.')
else:
logger.info('This is an unweighted network.')
try:
if inputs['biom_file'] or inputs['otu_table']:
bioms.write_bioms()
logger.info('BIOM files written to disk. ')
except Exception:
logger.warning('Failed to write BIOM files to disk. ', exc_info=True)
write_settings(bioms.inputs)
logger.info('Settings file written to disk. ')
def run_netstats(inputs, publish=False):
"""
Runs statistical analyses on the Neo4j database, as well as logic operations.
To do: null models.
:param inputs: Dictionary of inputs.
:param publish: If True, publishes messages to be received by GUI.
:return:
"""
old_inputs = read_settings(inputs['fp'] + '/settings.json')
old_inputs.update(inputs)
inputs = old_inputs
# handler to file
_create_logger(inputs['fp'])
checks = str()
if 'pid' in inputs:
existing_pid = pid_exists(inputs['pid'])
else:
existing_pid = False
if not existing_pid:
start_database(inputs, publish)
existing_pid = True
try:
if publish:
pub.sendMessage('update', msg='Starting database drivers.')
# sys.stdout.write('Starting database drivers.')
netdriver = NetDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'], filepath=inputs['fp'])
importdriver = ImportDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'], filepath=inputs['fp'])
except Exception:
logger.warning("Failed to start database worker. ", exc_info=True)
try:
# write operations here
if inputs['logic']:
if not inputs['networks']:
networks = list()
hits = importdriver.custom_query("MATCH (n:Network) RETURN n")
for hit in hits:
networks.append(hit['n'].get('name'))
else:
networks = inputs['networks']
if 'union' in inputs['logic']:
netdriver.graph_union(networks=networks)
if 'intersection' in inputs['logic']:
for n in inputs['num']:
netdriver.graph_intersection(networks=networks,
weight=inputs['weight'], n=int(n))
if 'difference' in inputs['logic']:
netdriver.graph_difference(networks=networks,
weight=inputs['weight'])
checks += 'Logic operations completed. \n'
if publish:
pub.sendMessage('update', msg="Exporting network...")
if inputs['networks'] is not None:
names = [x.split('.')[0] for x in inputs['networks']]
importdriver.export_network(path=inputs['fp'] + '/' +
"_".join(names) + '.graphml')
logger.info("Exporting networks to: " + inputs['fp'] + '/' +
"_".join(names) + '.graphml')
checks += "Exporting networks to: " + inputs['fp'] + '/' +\
"_".join(names) + '.graphml' "\n"
else:
importdriver.export_network(path=inputs['fp'] + '/' +
'_complete.graphml')
logger.info("Exporting networks to: " + inputs['fp'] + '/' +
'_complete.graphml')
checks += "Exporting networks to: " + inputs['fp'] + '/' +\
'_complete.graphml' "\n"
else:
logger.warning("No logic operation specified!")
if publish:
pub.sendMessage('update', msg="Completed database operations!")
# sys.stdout.write("Completed database operations!")
checks += 'Completed database operations! \n'
except Exception:
logger.warning("Failed to run database worker. ", exc_info=True)
checks += 'Failed to run database worker. \n'
if publish:
pub.sendMessage('database_log', msg=checks)
importdriver.close()
netdriver.close()
logger.info('Completed netstats operations! ')
write_settings(inputs)
def run_neo4j(inputs, publish=False):
"""
Starts and carries out operations on the Neo4j database.
:param inputs: Dictionary of inputs.
:param publish: If True, publishes messages to be received by GUI.
:return:
"""
_create_logger(inputs['fp'])
# overwritten settings should be retained
old_inputs = read_settings(inputs['fp'] + '/settings.json')
# handler to file
# check if password etc is already there
if 'username' in old_inputs:
logins = dict((k, old_inputs[k]) for k in ('username', 'password', 'address', 'neo4j'))
old_inputs.update(inputs)
inputs = old_inputs
if 'pid' in inputs:
existing_pid = pid_exists(inputs['pid'])
else:
existing_pid = False
if not inputs['neo4j']:
inputs.update(logins)
checks = str()
if inputs['job'] == 'start':
if not existing_pid:
start_database(inputs, publish)
existing_pid = True
else:
logger.info("Database is already running. ")
elif inputs['job'] == 'quit':
if not existing_pid:
logger.info("No database open. ")
else:
try:
if publish:
pub.sendMessage('update', msg='Getting PID...')
# there is a lingering Java process that places a lock on the database.
# terminating the subprocess does NOT terminate the Java process,
# so the store lock has to be deleted manually.
# This is different for Linux & Windows machines and may not be trivial
# however, PID solution may be platform-independent
# CURRENT SOLUTION:
# get parent PID of subprocess
# use psutil to get child PIDs
# kill child PIDs too
parent_pid = inputs['pid']
parent = Process(parent_pid)
children = parent.children(recursive=True)
for child in children:
child.kill()
# apparently killing the children also kills the parent
except Exception:
logger.warning("Failed to close database. ", exc_info=True)
elif inputs['job'] == 'clear':
if not existing_pid:
start_database(inputs, publish)
existing_pid = True
try:
if publish:
pub.sendMessage('update', msg='Clearing database...')
importdriver = ImportDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'], filepath=inputs['fp'])
importdriver.clear_database()
importdriver.close()
except Exception:
logger.warning("Failed to clear database. ", exc_info=True)
elif inputs['job'] == 'write':
if not existing_pid:
start_database(inputs, publish)
existing_pid = True
try:
if publish:
pub.sendMessage('update', msg='Accessing database...')
importdriver = ImportDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'], filepath=inputs['fp'])
importdriver.export_network(path=inputs['fp'])
importdriver.close()
except Exception:
logger.warning("Failed to write database to graphml file. ", exc_info=True)
elif inputs['job'] == 'cyto':
if not existing_pid:
start_database(inputs, publish)
existing_pid = True
try:
if publish:
pub.sendMessage('update', msg='Accessing database...')
importdriver = ImportDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'], filepath=inputs['fp'])
importdriver.export_cyto()
importdriver.close()
except Exception:
logger.warning("Failed to export networks to Cytoscape. ", exc_info=True)
else:
if not existing_pid:
start_database(inputs, publish)
existing_pid = True
if publish:
pub.sendMessage('update', msg='Uploading files to database...')
filestore = None
if inputs['procbioms']:
filestore = read_bioms(inputs['procbioms'])
# ask users for additional input
bioms = Batch(filestore, inputs)
bioms = Nets(bioms)
for file in inputs['network']:
network = _read_network(file)
bioms.add_networks(network, file)
importdriver = None
sleep(12)
importdriver = ImportDriver(user=inputs['username'],
password=inputs['password'],
uri=inputs['address'], filepath=inputs['fp'])
# importdriver.clear_database()
try:
# pub.sendMessage('update', msg='Uploading BIOM files...')
logger.info("Uploading BIOM files...")
itemlist = list()
for level in inputs['procbioms']:
for item in inputs['procbioms'][level]:
name = inputs['procbioms'][level][item]
biomfile = load_table(name)
importdriver.convert_biom(biomfile=biomfile, exp_id=name)
itemlist.append(name)
checks += 'Successfully uploaded the following items and networks to the database: \n'
for item in itemlist:
checks += (item + '\n')
checks += '\n'
logger.info(checks)
except Exception:
logger.warning("Failed to upload BIOM files to Neo4j database. ", exc_info=True)
try:
# pub.sendMessage('update', msg='Uploading network files...')
logger.info('Uploading network files... ')
for item in bioms.networks:
network = bioms.networks[item]
# try to split filename to make a nicer network id
subnames = item.split('/')
if len(subnames) == 1:
subnames = item.split('\\')
name = subnames[-1].split('.')[0]
importdriver.convert_networkx(network=network, network_id=name, mode='weight')
itemlist.append(item)
except Exception:
logger.warning('Unable to upload network files to Neo4j database. ', exc_info=True)
checks += 'Unable to upload network files to Neo4j database.\n'
if publish:
pub.sendMessage('database_log', msg=checks)
importdriver.close()
logger.info('Completed database operations! ')
write_settings(inputs)