def openapi(): yield '5f5141cbae5ca099d3f420f9c42c94cf' refresh() try: # teardown APIDoc.get('5f5141cbae5ca099d3f420f9c42c94cf').delete() except elasticsearch.exceptions.NotFoundError: pass
def delete(self): try: APIDoc.get(self._id).delete() except ESNotFoundError as err: raise NotFoundError() from err return self._id
def setup(): """ Setup Elasticsearch Index with dynamic template. Run it on an open index to update dynamic mapping. """ _dirname = os.path.dirname(__file__) with open(os.path.join(_dirname, 'mapping.json'), 'r') as file: mapping = json.load(file) if not exists(): APIDoc.init() elastic = Elasticsearch() elastic.indices.put_mapping(index=APIDoc.Index.name, body=mapping)
def get(cls, _id): try: doc = APIDoc.get(_id) except ESNotFoundError as err: raise NotFoundError from err obj = cls(doc._meta.url) obj.raw = decoder.decompress(doc._raw) obj.username = doc._meta.username obj.slug = doc._meta.slug obj.date_created = doc._meta.date_created obj.last_updated = doc._meta.last_updated obj.uptime = APIMonitorStatus( obj, ( doc._status.uptime_status, doc._status.uptime_msg ), doc._status.uptime_ts ) obj.webdoc = APIRefreshStatus( obj, doc._status.refresh_status, doc._status.refresh_ts ) return obj
def setup_fixture(): """ Index 2 documents. """ reset() # save initial docs with paths already transformed mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES) mygene._raw = decoder.compress(MYGENE_RAW) mygene.save() mychem = APIDoc(meta={'id': MYCHEM_ID}, **MYCHEM_ES) mychem._raw = decoder.compress(MYCHEM_RAW) mychem.save() # refresh index refresh()
def test_aggregation(): assert 'Chunlei Wu' in APIDoc.aggregate('info.contact.name') assert 'gene' in APIDoc.aggregate() assert 'annotation' in APIDoc.aggregate() assert 'query' in APIDoc.aggregate() assert 'query' in APIDoc.aggregate('tags.name.raw') assert 'query' in APIDoc.aggregate('tags.name') assert 'tester' in APIDoc.aggregate('_meta.username') assert 'mygene' in APIDoc.aggregate('_meta.slug')
def test_uptime_status(): mygene = SmartAPI.get(MYGENE_ID) assert mygene.uptime.status[0] is None mygene.uptime.update(('good', None)) assert mygene.uptime.status[0] == 'good' mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.uptime_status == 'good' mygene.uptime.update((None, None)) assert mygene.uptime.status[0] is None mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.uptime_status is None
def test_get_all_from(): """ SmartAPI.get_all(from_=1) """ search = APIDoc.search() assert search.count() == 2 docs = list(SmartAPI.get_all(from_=1)) assert len(docs) == 1
def exists(cls, _id): """ If a SmartAPI document exists in database. Do NOT fully rely on this for other operations. """ # Data can change in between calls. # Use try-catch blocks in follow up ops. return bool(APIDoc.exists(_id))
def test_uptime_update(): mygene = SmartAPI.get(MYGENE_ID) mygene.check() # minimum api document assert mygene.uptime.status[0] == 'unknown' # TODO VERIFY THIS IS IN FACT CORRECT mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.uptime_status == 'unknown' mychem = SmartAPI.get(MYCHEM_ID) mychem.check() # full api document assert mychem.uptime.status[0] == 'unknown' mychem.save() refresh() mychem_doc = APIDoc.get(MYCHEM_ID) assert mychem_doc._status.uptime_status == 'unknown'
def find(cls, val, field='slug'): """ Find a SmartAPI by a field other than _id. Return the first _id or None if no match. """ # Data can change in between calls. # Use try-catch blocks in follow up ops. if field in ('slug', 'username', 'url'): field = '_meta.' + field return APIDoc.exists(val, field)
def get_tags(field='info.contact.name'): """ Perform aggregations on a given field. For example: generate list of tags and authors. Result looks like: { "tagC" : 60, "tagA" : 10, "tagB" : 2, ... } """ return APIDoc.aggregate(field)
def test_delete(self): # setup assert SmartAPI.exists(MYGENE_ID) self.request("/api/metadata/" + MYGENE_ID, method='DELETE', expect=401) self.request("/api/metadata/" + MYGENE_ID, method='DELETE', headers=self.evil_user, expect=403) self.request("/api/metadata/" + MYGENE_ID, method='DELETE', headers=self.auth_user) refresh() assert not SmartAPI.exists(MYGENE_ID) # teardown refresh() if not SmartAPI.exists(MYGENE_ID): # recover the deleted file mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES) mygene._raw = decoder.compress(MYGENE_RAW) mygene.save() refresh()
def test_delete(myvariant): mv = SmartAPI.get(myvariant) mv.delete() refresh() assert not APIDoc.exists(myvariant) URL = "http://example.com/valid.json" with open(os.path.join(dirname, './validate/openapi-pass.json'), 'rb') as file: smartapi = SmartAPI(URL) smartapi.raw = file.read() with pytest.raises(NotFoundError): smartapi.delete()
def get_all(cls, size=10, from_=0): """ Returns a list of SmartAPIs. Size is the at-most number. """ search = APIDoc.search() search = search.source(False) search = search[from_: from_ + size] for hit in search: try: # unlikely but possible doc = cls.get(hit.meta.id) except ESNotFoundError: pass # inconsistent else: # consistent yield doc
def test_refresh_update(): mychem = SmartAPI.get(MYCHEM_ID) assert mychem.webdoc.status is None # NOTE # the following update and the updates thereafter will be applied # https://github.com/NCATS-Tangerine/translator-api-registry/commit/b01baa5 mychem.refresh() assert mychem.webdoc.status == 299 # new version assert mychem.webdoc.timestamp > datetime(2020, 1, 1, tzinfo=timezone.utc) _ts0 = mychem.webdoc.timestamp mychem.refresh() assert mychem.webdoc.status == 200 # already latest assert mychem.webdoc.timestamp >= _ts0 # could be cached internally mychem.save() refresh() mychem_doc = APIDoc.get(MYCHEM_ID) assert mychem_doc._status.refresh_status == 200
def myvariant(): with open(os.path.join(dirname, 'myvariant.es.json'), 'r') as file: MYVARIANT_ES = json.load(file) with open(os.path.join(dirname, 'myvariant.yml'), 'rb') as file: MYVARIANT_RAW = file.read() MYVARIANT_ID = MYVARIANT_ES.pop("_id") myvariant = APIDoc(meta={'id': MYVARIANT_ID}, **MYVARIANT_ES) myvariant._raw = decoder.compress(MYVARIANT_RAW) myvariant.save() refresh() yield MYVARIANT_ID refresh() try: APIDoc.get(MYVARIANT_ID).delete() except elasticsearch.exceptions.NotFoundError: pass
def test_refresh_status(): with open(os.path.join(dirname, 'mygene_full.yml'), 'rb') as file: MYGENE_FULL = file.read() mygene = SmartAPI.get(MYGENE_ID) # minimum assert mygene.webdoc.status is None assert 'components' not in mygene mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # new content assert mygene.webdoc.status == 299 # updated assert 'components' in mygene assert mygene.webdoc.timestamp > datetime(2020, 1, 1) _ts0 = mygene.webdoc.timestamp mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.refresh_status == 299 assert 'components' in mygene_doc mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # no change assert mygene.webdoc.status == 200 # latest assert 'components' in mygene assert mygene.webdoc.timestamp > _ts0 mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.refresh_status == 200 assert 'components' in mygene_doc mygene.webdoc.update(File(404, None, None, None)) # link broken assert mygene.webdoc.status == 404 assert 'components' in mygene # do not affect main copy mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.refresh_status == 404 assert 'components' in mygene_doc mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # link back working assert mygene.webdoc.status == 200 # latest assert 'components' in mygene mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.refresh_status == 200 assert 'components' in mygene_doc mygene.webdoc.update(File(200, b'{"openapi":"3.0.0"}', None, None)) # invalid assert mygene.webdoc.status == 499 # invalid assert 'components' in mygene # do not affect main copy mygene.save() refresh() mygene_doc = APIDoc.get(MYGENE_ID) assert mygene_doc._status.refresh_status == 499 assert 'components' in mygene_doc
def test_exists(): # info.title : "MyDisease.info API" assert not APIDoc.exists('doc0') assert APIDoc.exists('doc1') assert APIDoc.exists('3.0.0', 'openapi') assert APIDoc.exists('mygene', '_meta.slug') assert not APIDoc.exists('mygene', 'info.title') assert APIDoc.exists('mygene.info', 'info.title') assert APIDoc.exists('mygene.info api', 'info.title') assert APIDoc.exists('api', 'info.title') assert not APIDoc.exists('mygene', 'info.title.raw') assert not APIDoc.exists('mygene.info', 'info.title.raw') assert not APIDoc.exists('mygene.info api', 'info.title.raw') assert not APIDoc.exists('api', 'info.title.raw') assert APIDoc.exists('MyGene.info API', 'info.title.raw') assert APIDoc.exists('mygene.info', 'info.description')
def save(self): # TODO DOCSTRING if not self.raw: raise ControllerError("No content.") if not self.username: raise ControllerError("Username is required.") if self.url is self.VALIDATION_ONLY: raise ControllerError("In validation-only mode.") if not self.last_updated: self.last_updated = datetime.now(timezone.utc) warn("Filling in date_updated with current time.") if not self.date_created: self.date_created = self.last_updated warn("Filling in date_created with current time.") if not isinstance(self.date_created, datetime): raise ControllerError("Invalid created time.") if not isinstance(self.last_updated, datetime): raise ControllerError("Invalid updated time.") if self.date_created > self.last_updated: raise ControllerError("Invalid timestamps.") # NOTE # why not enforce validation here? # we add additional constraints to the application from time to time # it's actually hard to retrospectively make sure all previously # submitted API document always meet our latest requirements _doc = self._validate_dispatch() _doc.transform() if self.slug: _id = self.find(self.slug) if _id and _id != self._id: # another doc same slug. raise ConflictError("Slug is already registered.") # NOTE # if the slug of another document changed at this point # it's possible to have two documents with the same slug # registered. but it should be rare enough in reality. doc = APIDoc(**_doc) doc.meta.id = self._id doc._meta.url = self.url doc._meta.username = self.username doc._meta.slug = self.slug doc._meta.date_created = self.date_created doc._meta.last_updated = self.last_updated if self.uptime.status: doc._status.uptime_status = self.uptime.status[0] doc._status.uptime_msg = self.uptime.status[1] doc._status.uptime_ts = self.uptime.timestamp doc._status.refresh_status = self.webdoc.status doc._status.refresh_ts = self.webdoc.timestamp doc._raw = decoder.compress(self.raw) doc.save(skip_empty=False) return self._id
def test_update_slug(self): mygene = SmartAPI.get(MYGENE_ID) assert mygene.slug == "mygene" self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "mygeeni"}, expect=401) self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "mygeeni"}, headers=self.evil_user, expect=403) self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "my"}, headers=self.auth_user, expect=400) self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "www"}, headers=self.auth_user, expect=400) self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "MYGENE"}, headers=self.auth_user, expect=400) self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "mygene!!"}, headers=self.auth_user, expect=400) self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "mygeeni"}, headers=self.auth_user) refresh() assert not SmartAPI.find("mygene") assert SmartAPI.find("mygeeni") self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": ""}, headers=self.auth_user) refresh() assert not SmartAPI.find("mygeeni") assert not SmartAPI.find("mygene") self.request("/api/metadata/" + MYGENE_ID, method='PUT', data={"slug": "mygene"}, headers=self.auth_user) refresh() assert not SmartAPI.find("mygeeni") assert SmartAPI.find("mygene") # teardown refresh() if not SmartAPI.find("mygene"): mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES) mygene._raw = decoder.compress(MYGENE_RAW) mygene.save()
def test_save(openapi): """ SmartAPI.slug.validate(slug) smartapi.slug smartapi.save() """ _t0 = datetime.now(timezone.utc) time.sleep(0.1) URL = "http://example.com/valid.json" with pytest.raises(ValueError): SmartAPI.slug.validate("a") with pytest.raises(ValueError): SmartAPI.slug.validate("AAA") with pytest.raises(ValueError): SmartAPI.slug.validate("www") with pytest.raises(ValueError): SmartAPI.slug.validate("^_^") with open(os.path.join(dirname, './validate/openapi-pass.json'), 'rb') as file: raw = file.read() smartapi = SmartAPI(URL) with pytest.raises(ControllerError): smartapi.raw = None smartapi.raw = raw smartapi.slug = "mygene" smartapi.validate() with pytest.raises(ControllerError): smartapi.save() smartapi.username = "******" with pytest.raises(ConflictError): smartapi.save() smartapi.slug = "mychem" with pytest.raises(ConflictError): smartapi.save() smartapi.slug = "openapi" smartapi.save() refresh() assert SmartAPI.find("openapi") == smartapi._id assert smartapi.date_created > _t0 assert smartapi.last_updated > _t0 assert smartapi.date_created == smartapi.last_updated apidoc = APIDoc.get(smartapi._id) assert apidoc._meta.date_created == smartapi.date_created assert apidoc._meta.last_updated == smartapi.last_updated _t1 = smartapi.date_created smartapi.save() # no change refresh() assert SmartAPI.find("openapi") == smartapi._id assert smartapi.date_created == _t1 assert smartapi.last_updated == _t1 assert smartapi.date_created == smartapi.last_updated apidoc = APIDoc.get(smartapi._id) assert apidoc._meta.date_created == smartapi.date_created assert apidoc._meta.last_updated == smartapi.last_updated smartapi.slug = None smartapi.save() refresh() assert not SmartAPI.find("openapi") found = SmartAPI.get(openapi) assert dict(smartapi) == dict(found) assert smartapi.username == found.username assert smartapi.slug == found.slug assert smartapi.url == found.url assert smartapi.date_created == _t1 assert smartapi.last_updated == _t1 assert smartapi.date_created == smartapi.last_updated apidoc = APIDoc.get(smartapi._id) assert apidoc._meta.date_created == smartapi.date_created assert apidoc._meta.last_updated == smartapi.last_updated smartapi.raw = raw # should trigger ts update smartapi.save() refresh() assert smartapi.date_created == _t1 assert smartapi.last_updated > _t1 apidoc = APIDoc.get(smartapi._id) assert apidoc._meta.date_created == smartapi.date_created assert apidoc._meta.last_updated == smartapi.last_updated