def get_data(in_param_datasource, in_param_searchQuery, in_param_maxResults): result = None error = None response_status = 200 qryStr = "MATCH (ds:Datasource {label: '" + in_param_datasource + "'})-[:DATASOURCE_OF]->(:Feature)-[:PARENT_OF*]->(f:Feature) WHERE f.label contains '" + in_param_searchQuery + "' " \ "RETURN f.label as gene, f.start as start, f.end as end, 'neo4j' as seqName, f.id as nodeId, f.taxonomy as level " \ "ORDER BY f.depth limit " + in_param_maxResults try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) result = [] for index, row in df.iterrows(): temp = row.to_dict() result.append(temp) except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str( error_info[2]) response_status = 500 return result, error, response_status
def run(train_x, train_y, test_x, test_y, share=0.4, count=100): best_c = find_best_c(train_x, train_y, share, count) print(best_c) w1, w2 = train(train_x, train_y, best_c, count) print(w2) res = test(test_x, test_y, w1, w2) print(res) return utils.process_result(res)
def run(train_x, train_y, test_x, test_y, share=0.4, count=100): best_c = find_best_c(train_x, train_y, share, count) print(best_c) w1, w2 = train(train_x, train_y, best_c, count) print(w2) res = test(test_x, test_y, w1, w2) print(res) return utils.process_result(res)
def find_best_c(x, y, share): x_train, x_check = utils.split_data(x, share) y_train, y_check = utils.split_data(y, share) best_c = 2**-7 best_f1 = 0 for i in range(-7, 7): c = 2**i v = train(x_train, y_train, c) p, r = utils.process_result(test(x_check, y_check, v)) f1 = utils.f1(p, r) if f1 > best_f1: best_f1 = f1 best_c = c return best_c
def find_best_c(x, y, share): x_train, x_check = utils.split_data(x, share) y_train, y_check = utils.split_data(y, share) best_c = 2 ** -7 best_f1 = 0 for i in range(-7, 7): c = 2 ** i v = train(x_train, y_train, c) p, r = utils.process_result(test(x_check, y_check, v)) f1 = utils.f1(p, r) if f1 > best_f1: best_f1 = f1 best_c = c return best_c
def find_best_c(x, y, share, count): x_train, x_check = utils.split_data(x, share) y_train, y_check = utils.split_data(y, share) best_f1 = 0 best_c = -1 c = 10 while c <= 40: w1, w2 = train(x_train, y_train, c, count) p, r = utils.process_result(test(x_check, y_check, w1, w2)) f1 = utils.f1(p, r) if f1 > best_f1: best_f1 = f1 best_c = c c += 10 return best_c
def find_best_c(x, y, share, count): x_train, x_check = utils.split_data(x, share) y_train, y_check = utils.split_data(y, share) best_f1 = 0 best_c = -1 c = 10 while c <= 40: w1, w2 = train(x_train, y_train, c, count) p, r = utils.process_result(test(x_check, y_check, w1, w2)) f1 = utils.f1(p, r) if f1 > best_f1: best_f1 = f1 best_c = c c += 10 return best_c
def get_data(in_datasource): """ Returns the range of features in the database. The cypher query finds the root of the Neo4j feature hierarchy and retrieves the start and end values which denote the range of features. Args: in_datasource: namspace to query Returns: arr: Feature range under root of tree """ qryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'})-[:DATASOURCE_OF]->(f:Feature {id:'0-0'}) RETURN f.start as start, f.end as end" rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) arr = [] arr.append([None, df['start'][0], df['end'][0]]) return arr
def run(train_x, train_y, test_x, test_y, test_share=0.4): best_c = find_best_c(train_x, train_y, test_share) vector = train(train_x, train_y, best_c) # print(vector) res = test(test_x, test_y, vector) return utils.process_result(res)
def get_data(in_params_selectedLevels, in_params_samples, in_datasource): """ Computes Alpha Diversity using the specified samples and level of hierarchy :param in_params_selectedLevels: Hierarchy level to compute Alpha Diversity :param in_params_samples: Samples to use for computing Alpha Diversity :return: Args: in_params_selectedLevels: Hierarchy level to compute Alpha Diversity in_params_samples: Samples to use for computing Alpha Diversity in_datasource: datasource to query Returns: resRowsCols: Alpha diversity for the samples at the selected level """ tick_samples = in_params_samples.replace("\"", "\'") diversity_type = "shannon" # get the min selected Level if aggregated at multiple levels result = None error = None response_status = 200 qryStr = "MATCH (s:Sample)-[:COUNT]->(f:Feature)<-[:LEAF_OF]-(:Feature)<-[:PARENT_OF*]-(:Feature)<-[:DATASOURCE_OF]-(ds:Datasource {label: '" + in_datasource + "'}) RETURN f.depth as depth LIMIT 1" try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) minSelectedLevel = int(df['depth'].values[0]) if minSelectedLevel is None: minSelectedLevel = 6 for level in in_params_selectedLevels.keys(): if in_params_selectedLevels[level] == 2 and int( level) < minSelectedLevel: minSelectedLevel = int(level) except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str( error_info[2]) response_status = 500 return result, error, response_status qryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'})-[:DATASOURCE_OF]->(:Feature)-[:PARENT_OF*]->(f:Feature)-[:LEAF_OF]->()<-[v:COUNT]-(s:Sample) WHERE (f.depth=" + str(minSelectedLevel) + ") " \ "AND s.id IN " + tick_samples + " with distinct f, s, SUM(v.val) as agg RETURN distinct agg, s.id, " \ "f.label as label, f.leafIndex as index, f.end as end, f.start as start, f.id as id, f.lineage as lineage, " \ "f.lineageLabel as lineageLabel, f.order as order" try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) forDiversityDF = df[["agg", "s.id", "label"]] forDiversityMat = pandas.pivot_table(df, index=["label"], columns="s.id", values="agg", fill_value=0) alphaDiversityVals = [] cols = {} sample_ids = list(set(forDiversityDF["s.id"])) if diversity_type == "shannon": for i in range(0, len(sample_ids)): col_vals = forDiversityMat.ix[:, i].get_values() props = list() totalSum = col_vals.sum() for k in range(0, len(col_vals)): temp_prop = float(col_vals[k] / totalSum) if temp_prop != 0.0: props.append(float((temp_prop * math.log(temp_prop)))) else: props.append(0.0) nd_props = numpy.asarray(props, dtype=float) alphaDiversity = -(nd_props.sum()) alphaDiversityVals.append(alphaDiversity) cols[forDiversityMat.columns.values[i]] = alphaDiversity sampleQryStr = "MATCH (s:Sample) WHERE s.id IN " + tick_samples + " RETURN s" sample_rq_res = utils.cypher_call(sampleQryStr) sample_df = utils.process_result_graph(sample_rq_res) vals = [] for index, row in sample_df.iterrows(): temp = {} for key in row.keys().values: temp[key] = row[key] temp['alphaDiversity'] = cols[row['id']] temp['sample_id'] = temp['id'] del temp['id'] vals.append(temp) result = {"data": vals} except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str( error_info[2]) response_status = 500 return result, error, response_status
def run(train_x, train_y, test_x, test_y, steps=1000): vector = train(train_x, train_y, steps) # print(vector) res = test(test_x, test_y, vector) return utils.process_result(res)
def get_data(in_params_selection, in_params_order, in_params_selected_levels, in_params_nodeId, in_params_depth, in_datasource): """ Finds and returns the hierarchy of the taxonomic features in the database. The hierarchy is traversed starting at the root node by using the PARENT_OF relationships the paths to until all leaf nodes are discovered. The results are formatted according the the metaviz API specification. Args: in_params_selection: The samples selected in_params_order: The order of the features in_params_selected_levels: The levels for aggregation of each feature node or all nodes by default in_params_nodeId: The id of the root node in_params_depth: level depth to query at in_datasource: namespace to query Returns: result: Heirachy of levels in database """ root_node = in_params_nodeId root_node = root_node.replace('"', "") taxonomy = False result = None error = None response_status = 200 if len(root_node) == 0 or root_node == "0-0": root_node = "0-0" qryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'})-[:DATASOURCE_OF]->(f:Feature {id:'" + root_node + "'})-[:PARENT_OF*0..3]->(f2:Feature) " \ "with collect(f2) + f as nodesFeat unwind nodesFeat as ff " \ "return distinct ff.lineage as lineage, ff.start as start, ff.label as label, " \ "ff.leafIndex as leafIndex, ff.parentId as parentId, ff.depth as depth, ff.partition as partition, " \ "ff.end as end, ff.id as id, ff.lineageLabel as lineageLabel, ff.nchildren as nchildren, " \ "ff.taxonomy as taxonomy, ff.nleaves as nleaves, ff.order as order ORDER by ff.depth, ff.leafIndex, ff.order" tQryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'})-[:DATASOURCE_OF]->(f:Feature) RETURN DISTINCT f.taxonomy as taxonomy, f.depth as depth ORDER BY f.depth" taxonomy = True else: qryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'})-[:DATASOURCE_OF]->(:Feature)-[:PARENT_OF*]->(f:Feature {id:'" + root_node + "'})-[:PARENT_OF*0..3]->(f2:Feature) " \ "OPTIONAL MATCH (f)<-[:PARENT_OF]-(fParent:Feature) with collect(f2) + f + fParent as nodesFeat " \ "unwind nodesFeat as ff return distinct ff.lineage as lineage, ff.start as start, " \ "ff.label as label, ff.leafIndex as leafIndex, ff.parentId as parentId, ff.depth as depth, " \ "ff.partition as partition, ff.end as end, ff.id as id, ff.lineageLabel as lineageLabel, " \ "ff.nchildren as nchildren, ff.taxonomy as taxonomy, ff.nleaves as nleaves, ff.order as order " \ "ORDER by ff.depth, ff.leafIndex, ff.order" try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) if len(df) > 0: # convert columns to int df['start'] = df['start'].astype(int) df['end'] = df['end'].astype(int) df['order'] = df['order'].astype(int) df['leafIndex'] = df['leafIndex'].astype(int) df['nchildren'] = df['nchildren'].astype(int) df['nleaves'] = df['nleaves'].astype(int) df['depth'] = df['depth'].astype(int) df['depth'] = df['depth'].astype(int) # restore current order, selection and levels from input params for key in in_params_order.keys(): df.loc[df['id'] == key, 'order'] = in_params_order[key] for key in in_params_selection.keys(): df.loc[df['id'] == key, 'selectionType'] = in_params_selection[key] for key in in_params_selected_levels.keys(): df.loc[df['depth'] == int(key), 'selectionType'] = in_params_selected_levels[key] root = df.iloc[0] other = df.loc[1:, ] rootDict = row_to_dict(root) result = df_to_tree(rootDict, other) if taxonomy: trq_res = utils.cypher_call(tQryStr) tdf = utils.process_result(trq_res) result['rootTaxonomies'] = tdf['taxonomy'].values.tolist() except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str( error_info[2]) response_status = 500 return result, error, response_status
def get_data(in_params_selectedLevels, in_params_samples, in_datasource): """ Computes PCA over the selected samples and the given level of the hierarchy Args: in_params_selectedLevels: Level of hierarchy of features to compute PCA in_params_samples: Samples to use to compute PCA in_datasource: datasource to query Returns: resRowsCols: PCA for the samples at the selected level """ tick_samples = in_params_samples.replace("\"", "\'") # get the min selected Level if aggregated at multiple levels qryStr = "MATCH (s:Sample)-[:COUNT]->(f:Feature)<-[:LEAF_OF]-(:Feature)<-[:PARENT_OF*]-(:Feature)<-[:DATASOURCE_OF]-(ds:Datasource {label: '" + in_datasource + "'}) RETURN f.depth as depth LIMIT 1" result = None error = None response_status = 200 try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str( error_info[2]) response_status = 500 return result, error, response_status minSelectedLevel = int(df['depth'].values[0]) if minSelectedLevel is None: minSelectedLevel = 6 for level in in_params_selectedLevels.keys(): if in_params_selectedLevels[level] == 2 and int( level) < minSelectedLevel: minSelectedLevel = int(level) qryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'})-[:DATASOURCE_OF]->(:Feature)-[:PARENT_OF*]->(f:Feature)-[:LEAF_OF]->()<-[v:COUNT]-(s:Sample) WHERE (f.depth=" + str(minSelectedLevel) + ") " \ "AND s.id IN " + tick_samples + " with distinct f, s, SUM(v.val) as agg RETURN distinct agg, s.id, f.label " \ "as label, f.leafIndex as index, f.end as end, f.start as start, f.id as id, f.lineage as lineage, " \ "f.lineageLabel as lineageLabel, f.order as order" try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) forPCAmat = pandas.pivot_table(df, index=["label"], columns="s.id", values="agg", fill_value=0) pca = PCA(n_components=2) pca.fit(forPCAmat) variance_explained = pca.explained_variance_ratio_ cols = {} cols['PC1'] = pca.components_[0] cols['PC2'] = pca.components_[1] samplesQryStr = "MATCH (s:Sample) WHERE s.id IN " + tick_samples + " RETURN s" samples_rq_res = utils.cypher_call(samplesQryStr) samples_df = utils.process_result_graph(samples_rq_res) vals = [] for index, row in samples_df.iterrows(): temp = {} for key in row.keys().values: temp[key] = row[key] temp['PC1'] = cols['PC1'][index] temp['PC2'] = cols['PC2'][index] temp['sample_id'] = temp['id'] del temp['id'] vals.append(temp) result = {"data": vals} variance_explained[0] = round(variance_explained[0] * 100.0, 2) variance_explained[1] = round(variance_explained[1] * 100.0, 2) result['pca_variance_explained'] = variance_explained except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str( error_info[2]) response_status = 500 return result, error, response_status
def get_data(in_params_start, in_params_end, in_params_order, in_params_selection, in_params_selectedLevels, in_params_samples, in_datasource): """ Aggregates counts to the selected nodes in the feature hierarchy and returns the counts for the samples selected. Args: in_params_start: Start of range for features to use during aggregation in_params_end: End of range for features to use during aggregation in_params_order: Order of features in_params_selection: Features nodes and the selection type of expanded, aggregated, or removed in_params_selectedLevels: Level of the hierarchy to use in_params_samples: Samples to compute aggregation with Returns: resRowsCols: Aggregated counts for the selected Features over the selected Samples """ tick_samples = in_params_samples.replace("\"", "\'") result = None error = None response_status = 200 # get the min selected Level if aggregated at multiple levels qryStr = "MATCH (s:Sample)-[:COUNT]->(f:Feature)<-[:LEAF_OF]-(:Feature)<-[:PARENT_OF*]-(:Feature)<-[:DATASOURCE_OF]-(ds:Datasource {label: '" + in_datasource + "'}) RETURN f.depth as depth LIMIT 1" try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) minSelectedLevel = int(df['depth'].values[0]) if minSelectedLevel is None: minSelectedLevel = 6 for level in in_params_selectedLevels.keys(): if in_params_selectedLevels[level] == 2 and int(level) < minSelectedLevel: minSelectedLevel = int(level) # user selection nodes for custom aggregation - decides the cut selNodes = "[" selFlag = 0 for node in in_params_selection.keys(): if in_params_selection[node] == 2: selNodes += "'" + node + "'," selFlag = 1 if selFlag == 1: selNodes = selNodes[:-1] selNodes += "]" except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str(error_info[2]) response_status = 500 return result, error, response_status qryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'}) " \ "MATCH (ds)-[:DATASOURCE_OF]->(:Feature)-[:PARENT_OF*]->(f:Feature) MATCH (f)-[:LEAF_OF]->()<-[v:COUNT]-(s:Sample)" \ "USING INDEX s:Sample(id) WHERE (f.depth=" + str(minSelectedLevel) + " OR f.id IN " + selNodes + ") AND " \ "(f.start >= " + in_params_start + " AND " \ "f.end <= " + in_params_end + ") AND s.id IN " + tick_samples + " with distinct f, s, SUM(v.val) as agg " \ "RETURN distinct agg, s.id, f.label as label, f.leafIndex as index, f.end as end, f.start as start, " \ "f.id as id, f.lineage as lineage, f.lineageLabel as lineageLabel, f.order as order" try: rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) if len(df) > 0: # change column type df['index'] = df['index'].astype(int) df['start'] = df['start'].astype(int) df['end'] = df['end'].astype(int) df['order'] = df['order'].astype(int) # update order based on req for key in in_params_order.keys(): df.loc[df['id'] == key, 'order'] = in_params_order[key] for key in in_params_selection.keys(): lKey = key.split('-') if int(lKey[0]) <= minSelectedLevel: if in_params_selection[key] == 0: # user selected nodes to ignore! df = df[~df['lineage'].str.contains(key)] elif in_params_selection[key] == 2: df = df[~(df['lineage'].str.contains(key) & ~df['id'].str.contains(key))] # create a pivot_table where columns are samples and rows are features # for pandas > 0.17 df_pivot = pandas.pivot_table(df, index=["id", "label", "index", "lineage", "lineageLabel", "start", "end", "order"], columns="s.id", values="agg", fill_value=0).sortlevel("index") cols = {} for col in df_pivot: cols[col] = df_pivot[col].values.tolist() rows = {} rows['metadata'] = {} metadata_row = ["end", "start", "index"] for row in df_pivot.index.names: if row in metadata_row: rows[row] = df_pivot.index.get_level_values(row).values.tolist() else: rows['metadata'][row] = df_pivot.index.get_level_values(row).values.tolist() result = {"cols": cols, "rows": rows, "globalStartIndex": (min(rows['start']))} else: cols = {} samples = eval(in_params_samples) for sa in samples: cols[sa] = [] rows = { "end": [], "start": [], "index": [], "metadata": {} } result = {"cols": cols, "rows": rows, "globalStartIndex": None} except: error_info = sys.exc_info() error = str(error_info[0]) + " " + str(error_info[1]) + " " + str(error_info[2]) response_status = 500 return result, error, response_status
def get_data(in_datasource): """ This function returns the set of all samples in the database. The first cypher query is finding all samples in the database. The second cypher query is used to find the mix and max count value for all features across all samples. This is return along with data source information including name and taxonomic hierarchy level names. Args: in_datasource: namespace to query Returns: result: Sample nodes information in database """ qryStr = "MATCH (ds:Datasource {label: '" + in_datasource + "'})-[:DATASOURCE_OF]->()-[LEAF_OF]->()<-[:COUNT]-(s:Sample)" \ "RETURN DISTINCT ds,s" rq_res = utils.cypher_call(qryStr) df = utils.process_result(rq_res) measurements = [] anno = [] df.fillna(0, inplace=True) dsGroup = [] dsId = [] dsDescription = [] for index, row in df.iterrows(): temp = row['s'] measurements.append(temp['id']) del temp['id'] anno.append(temp) dsGroup.append(row['ds']['label']) dsId.append(row['ds']['label']) dsDescription.append(row['ds']['description']) rowQryStr = "MATCH ()-[r]-() WHERE EXISTS(r.val) RETURN min(r.val) as minVal, max(r.val) as maxVal" rq_res2 = utils.cypher_call(rowQryStr) df2 = utils.process_result(rq_res2) result = { "id": measurements, "name": measurements, "datasourceGroup": dsGroup, "datasourceId": dsId, "datasourceDescription": dsDescription, "defaultChartType": "", "type": "feature", "minValue": df2['minVal'][0], "maxValue": df2['maxVal'][0], "annotation": anno, "metadata": [ "label", "id", "taxonomy1", "taxonomy2", "taxonomy3", "taxonomy4", "taxonomy5", "taxonomy6", "taxonomy7", "lineage" ] } return result
def run(train_x, train_y, test_x, test_y, test_share=0.4): best_c = find_best_c(train_x, train_y, test_share) vector = train(train_x, train_y, best_c) # print(vector) res = test(test_x, test_y, vector) return utils.process_result(res)