def testExternalURL(self): pa = PhylesystemAPI(self.domains, get_from='api') u = pa.get_external_url('pg_10') re = requests.get(u).json() sid = find_val_literal_meta_first(re['nexml'], 'ot:studyId', detect_nexson_version(re)) self.assertTrue(sid in ['10', 'pg_10'])
def testExternalURL(self): # N.B. that the URL for this API call is an odd one, e.g. # http://devapi.opentreeoflife.org/phylesystem/external_url/pg_10 pa = PhylesystemAPI(self.domains, get_from='api') u = pa.get_external_url('pg_10') re = requests.get(u).json() sid = find_val_literal_meta_first(re['nexml'], 'ot:studyId', detect_nexson_version(re)) self.assertTrue(sid in ['10', 'pg_10'])
def phyleystem_api(self): if self._phylesystem_api is None: # trigger helpful message if config is not found self._get_config_var('phylesystem_api_parent', 'phylesystem', 'parent', None) pa = PhylesystemAPI(get_from='local') self._phylesystem_api = pa return self._phylesystem_api
def testRemoteTransSugar(self): pa = PhylesystemAPI(self.domains, get_from='api', transform='server') self._do_sugar_tests(pa)
def testConfig(self): pa = PhylesystemAPI(self.domains, get_from='api') x = pa.phylesystem_config self.assertTrue('repo_nexml2json' in x.keys())
def testLocalSugar(self): pa = PhylesystemAPI(self.domains, get_from='local') self._do_sugar_tests(pa)
def testExternalSugar(self): pa = PhylesystemAPI(self.domains, get_from='external') self._do_sugar_tests(pa)
def testRemoteStudyList(self): pa = PhylesystemAPI(self.domains, get_from='api') sl = pa.study_list # dev/production repos should have hundreds of studies self.assertTrue(len(sl) > 100)
parser.add_argument('collection', default=None, type=str, help='filepath for the collections JSON') args = parser.parse_args(sys.argv[1:]) if not os.path.isfile(args.collection): sys.exit('Input collection "{}" does not exist.\n'.format(args.collection)) try: # get the list of trees that are 'included' included = collection_to_included_trees(args.collection) except: sys.stderr.write('JSON parse error when reading collection "{}".\n'.format(args.collection)) raise # work off the local copy of the phylesystem phy = PhylesystemAPI(get_from='local') # looking for the tree properties: ot:unrootedTree, ot:inGroupClade # and the study property ot:candidateTreeForSynthesis print("studyid,treeid,preferred,ingroup,unrooted") for d in included: studyid = d['studyID'] treeid = d['treeID'] # expect these defaults for synthesis trees unrootedTree = False ingroupSpecified = True preferredTree = True check = True nx = phy.get_study(studyid)['data']
#!/usr/bin/env python from peyotl.api import PhylesystemAPI pa = PhylesystemAPI(get_from='external') print(pa.get('pg_10', tree_id='tree3', subtree_id='ingroup', format='newick'))
def addStudy(session, study_id): # get latest version of nexson print "adding study {s}".format(s=study_id) phy = PhylesystemAPI(get_from="local") studyobj = phy.get_study(study_id)["data"] nexml = get_nexml_el(studyobj) year = nexml.get("^ot:studyYear") proposedTrees = nexml.get("^ot:candidateTreeForSynthesis") if proposedTrees is None: proposedTrees = [] # create a new Study object new_study = Study(id=study_id, year=year) session.add(new_study) # session.commit() # get curator(s), noting that ot:curators might be a # string or a list c = nexml.get("^ot:curatorName") print " ot:curatorName: ", c # create list of curator objects curator_list = [] if isinstance(c, basestring): curator_list.append(c) else: curator_list = c for curator in curator_list: test_c = session.query(Curator).filter(Curator.name == curator).first() if test_c: print "curator {c} already exists".format(c=curator) # session.add(curator) new_study.curators.append(test_c) else: print "curator {c} does no exist".format(c=curator) new_study.curators.append(Curator(name=curator)) # mapped otus in this study otu_dict = gen_otu_dict(studyobj) # iterate over the OTUs in the study, collecting the mapped # ones (oid to ott_id mapping held at the study level) mapped_otus = {} for oid, o in otu_dict.items(): ottID = o.get("^ot:ottId") if ottID is not None: mapped_otus[oid] = ottID # iterate over trees and insert tree data for trees_group_id, tree_id, tree in iter_trees(studyobj): print " tree :", tree_id proposedForSynth = False if tree_id in proposedTrees: proposedForSynth = True treejson = json.dumps(tree) new_tree = Tree(tree_id=tree_id, study_id=study_id, proposed=proposedForSynth, data=treejson) # get otus ottIDs = set() # ott ids for this tree ntips = 0 for node_id, node in iter_node(tree): oid = node.get("@otu") # no @otu property on internal nodes if oid is not None: ntips += 1 # ottID = mapped_otus[oid] if oid in mapped_otus: ottID = mapped_otus[oid] # check that this exists in the taxonomy # (it might not, if the ID has been deprecated) taxon = session.query(Taxonomy).filter(Taxonomy.id == ottID).first() if taxon: new_tree.otus.append(taxon) ottIDs.add(ottID) new_tree.ntips = ntips # need to write function for recursive query of Taxonomy table # ottIDs = parent_closure(ottIDs,taxonomy) # update with treebase id, if exists datadeposit = nexml.get("^ot:dataDeposit") if datadeposit: url = datadeposit["@href"] pattern = re.compile(u".+TB2:(.+)$") matchobj = re.match(pattern, url) if matchobj: tb_id = matchobj.group(1) new_tree.treebase_id = tb_id session.add(new_tree) # now that we have added the tree info, update the study record # with the json data (minus the tree info) del nexml["treesById"] studyjson = json.dumps(nexml) new_study.data = studyjson session.commit()
#!/usr/bin/env python """Takes meta properties for a tree requested""" from __future__ import absolute_import, division, print_function, unicode_literals from peyotl.api import PhylesystemAPI from peyotl.utility import propinquity_fn_to_study_tree import codecs import sys out = codecs.getwriter('utf-8')(sys.stdout) NON_META = frozenset([u'^ot:rootNodeId', u'nodeById', u'edgeBySourceId', u'^ot:inGroupClade', u'@xsi:type', u'^ot:branchLengthTimeUnit', u'^ot:branchLengthDescription', u'^ot:tag', u'^ot:branchLengthMode']) for arg in sys.argv[1:]: study_id, tree_id = propinquity_fn_to_study_tree(arg, strip_extension=False) pa = PhylesystemAPI(get_from='local') try: tree = pa.get(study_id, tree_id=tree_id)[tree_id] print('Tree "{}" in study "{}":'.format(tree_id, study_id)) for k, v in tree.items(): if (v is not None) and (v is not '') and (k not in NON_META): print(k, v) except: sys.stderr.write('WARNING: did not find tree "{}" in study "{}"'.format(tree_id, study_id))
def testFetchStudyRemote(self): pa = PhylesystemAPI(self.domains, get_from='api') x = pa.get_study('pg_10')['data'] sid = find_val_literal_meta_first(x['nexml'], 'ot:studyId', detect_nexson_version(x)) self.assertTrue(sid in ['10', 'pg_10'])
def testExternalSugar(self): pa = PhylesystemAPI(self.domains, get_from='external') test_phylesystem_api_for_study(self, pa)
def testRemoteSugar(self): pa = PhylesystemAPI(self.domains, get_from='api') test_phylesystem_api_for_study(self, pa)
def testStudyList(self): pa = PhylesystemAPI(self.domains, get_from='local') sl = pa.study_list self.assertTrue(len(sl) > 100)
def testPushFailureState(self): pa = PhylesystemAPI(self.domains, get_from='api') sl = pa.push_failure_state self.assertTrue(sl[0] is True)
def scream_about_new_failure(resp, OAUTH_TOKEN): params = {'access_token': OAUTH_TOKEN} template = "A push to GitHub has failed:\n date: {d}\n study: {s}\n commit: {c}" msg = template.format(d=resp['date'], s=resp['study'], c=resp['commit']) payload = {"body": msg} resp = requests.post(URL, params=params, data=json.dumps(payload)) resp.raise_for_status() def write_known_failures(fail_list): with codecs.open(KNOWN_FAILURE_FILE, 'w', encoding='utf-8') as kff: json.dump(fail_list, kff, sort_keys=True, indent=2) if __name__ == '__main__': if len(sys.argv) < 2: sys.exit( 'expecting a GitHub OAUTH token as the first (and only) argument') OAUTH_TOKEN = sys.argv[1] pa = PhylesystemAPI() succeeding, push_fail_dict = pa.push_failure_state if not succeeding: known_failures_d, kf_list = read_known_failures() c = push_fail_dict['commit'] if c not in known_failures_d: scream_about_new_failure(push_fail_dict, OAUTH_TOKEN) kf_list.append(push_fail_dict) write_known_failures(kf_list)
#!/usr/bin/env python from peyotl.api import PhylesystemAPI pa = PhylesystemAPI(get_from='api', transform='client') print(pa.get('pg_10', tree_id='tree3', subtree_id='ingroup', format='newick'))
def testRemoteSugar(self): pa = PhylesystemAPI(self.domains, get_from='api') self._do_sugar_tests(pa)
def setUp(self): self.pa = PhylesystemAPI(None, get_from='local', locals_repos_dict=test_repos)
def testRemoteTransSugar(self): pa = PhylesystemAPI(self.domains, get_from='api', transform='server') test_phylesystem_api_for_study(self, pa)