def openapi():
yield '5f5141cbae5ca099d3f420f9c42c94cf'
refresh()
try: # teardown
APIDoc.get('5f5141cbae5ca099d3f420f9c42c94cf').delete()
except elasticsearch.exceptions.NotFoundError:
pass
def delete(self):
try:
APIDoc.get(self._id).delete()
except ESNotFoundError as err:
raise NotFoundError() from err
return self._id
def setup():
"""
Setup Elasticsearch Index with dynamic template.
Run it on an open index to update dynamic mapping.
"""
_dirname = os.path.dirname(__file__)
with open(os.path.join(_dirname, 'mapping.json'), 'r') as file:
mapping = json.load(file)
if not exists():
APIDoc.init()
elastic = Elasticsearch()
elastic.indices.put_mapping(index=APIDoc.Index.name, body=mapping)
def get(cls, _id):
try:
doc = APIDoc.get(_id)
except ESNotFoundError as err:
raise NotFoundError from err
obj = cls(doc._meta.url)
obj.raw = decoder.decompress(doc._raw)
obj.username = doc._meta.username
obj.slug = doc._meta.slug
obj.date_created = doc._meta.date_created
obj.last_updated = doc._meta.last_updated
obj.uptime = APIMonitorStatus(
obj, (
doc._status.uptime_status,
doc._status.uptime_msg
),
doc._status.uptime_ts
)
obj.webdoc = APIRefreshStatus(
obj, doc._status.refresh_status,
doc._status.refresh_ts
)
return obj
def setup_fixture():
"""
Index 2 documents.
"""
reset()
# save initial docs with paths already transformed
mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES)
mygene._raw = decoder.compress(MYGENE_RAW)
mygene.save()
mychem = APIDoc(meta={'id': MYCHEM_ID}, **MYCHEM_ES)
mychem._raw = decoder.compress(MYCHEM_RAW)
mychem.save()
# refresh index
refresh()
def test_aggregation():
assert 'Chunlei Wu' in APIDoc.aggregate('info.contact.name')
assert 'gene' in APIDoc.aggregate()
assert 'annotation' in APIDoc.aggregate()
assert 'query' in APIDoc.aggregate()
assert 'query' in APIDoc.aggregate('tags.name.raw')
assert 'query' in APIDoc.aggregate('tags.name')
assert 'tester' in APIDoc.aggregate('_meta.username')
assert 'mygene' in APIDoc.aggregate('_meta.slug')
def test_uptime_status():
mygene = SmartAPI.get(MYGENE_ID)
assert mygene.uptime.status[0] is None
mygene.uptime.update(('good', None))
assert mygene.uptime.status[0] == 'good'
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.uptime_status == 'good'
mygene.uptime.update((None, None))
assert mygene.uptime.status[0] is None
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.uptime_status is None
def test_get_all_from():
"""
SmartAPI.get_all(from_=1)
"""
search = APIDoc.search()
assert search.count() == 2
docs = list(SmartAPI.get_all(from_=1))
assert len(docs) == 1
def exists(cls, _id):
"""
If a SmartAPI document exists in database.
Do NOT fully rely on this for other operations.
"""
# Data can change in between calls.
# Use try-catch blocks in follow up ops.
return bool(APIDoc.exists(_id))
def test_uptime_update():
mygene = SmartAPI.get(MYGENE_ID)
mygene.check() # minimum api document
assert mygene.uptime.status[0] == 'unknown' # TODO VERIFY THIS IS IN FACT CORRECT
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.uptime_status == 'unknown'
mychem = SmartAPI.get(MYCHEM_ID)
mychem.check() # full api document
assert mychem.uptime.status[0] == 'unknown'
mychem.save()
refresh()
mychem_doc = APIDoc.get(MYCHEM_ID)
assert mychem_doc._status.uptime_status == 'unknown'
def find(cls, val, field='slug'):
"""
Find a SmartAPI by a field other than _id.
Return the first _id or None if no match.
"""
# Data can change in between calls.
# Use try-catch blocks in follow up ops.
if field in ('slug', 'username', 'url'):
field = '_meta.' + field
return APIDoc.exists(val, field)
def get_tags(field='info.contact.name'):
"""
Perform aggregations on a given field.
For example: generate list of tags and authors.
Result looks like:
{
"tagC" : 60,
"tagA" : 10,
"tagB" : 2,
...
}
"""
return APIDoc.aggregate(field)
def test_delete(self):
# setup
assert SmartAPI.exists(MYGENE_ID)
self.request("/api/metadata/" + MYGENE_ID, method='DELETE', expect=401)
self.request("/api/metadata/" + MYGENE_ID,
method='DELETE',
headers=self.evil_user,
expect=403)
self.request("/api/metadata/" + MYGENE_ID,
method='DELETE',
headers=self.auth_user)
refresh()
assert not SmartAPI.exists(MYGENE_ID)
# teardown
refresh()
if not SmartAPI.exists(MYGENE_ID): # recover the deleted file
mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES)
mygene._raw = decoder.compress(MYGENE_RAW)
mygene.save()
refresh()
def test_delete(myvariant):
mv = SmartAPI.get(myvariant)
mv.delete()
refresh()
assert not APIDoc.exists(myvariant)
URL = "http://example.com/valid.json"
with open(os.path.join(dirname, './validate/openapi-pass.json'), 'rb') as file:
smartapi = SmartAPI(URL)
smartapi.raw = file.read()
with pytest.raises(NotFoundError):
smartapi.delete()
def get_all(cls, size=10, from_=0):
"""
Returns a list of SmartAPIs.
Size is the at-most number.
"""
search = APIDoc.search()
search = search.source(False)
search = search[from_: from_ + size]
for hit in search:
try: # unlikely but possible
doc = cls.get(hit.meta.id)
except ESNotFoundError:
pass # inconsistent
else: # consistent
yield doc
def test_refresh_update():
mychem = SmartAPI.get(MYCHEM_ID)
assert mychem.webdoc.status is None
# NOTE
# the following update and the updates thereafter will be applied
# https://github.com/NCATS-Tangerine/translator-api-registry/commit/b01baa5
mychem.refresh()
assert mychem.webdoc.status == 299 # new version
assert mychem.webdoc.timestamp > datetime(2020, 1, 1, tzinfo=timezone.utc)
_ts0 = mychem.webdoc.timestamp
mychem.refresh()
assert mychem.webdoc.status == 200 # already latest
assert mychem.webdoc.timestamp >= _ts0 # could be cached internally
mychem.save()
refresh()
mychem_doc = APIDoc.get(MYCHEM_ID)
assert mychem_doc._status.refresh_status == 200
def myvariant():
with open(os.path.join(dirname, 'myvariant.es.json'), 'r') as file:
MYVARIANT_ES = json.load(file)
with open(os.path.join(dirname, 'myvariant.yml'), 'rb') as file:
MYVARIANT_RAW = file.read()
MYVARIANT_ID = MYVARIANT_ES.pop("_id")
myvariant = APIDoc(meta={'id': MYVARIANT_ID}, **MYVARIANT_ES)
myvariant._raw = decoder.compress(MYVARIANT_RAW)
myvariant.save()
refresh()
yield MYVARIANT_ID
refresh()
try:
APIDoc.get(MYVARIANT_ID).delete()
except elasticsearch.exceptions.NotFoundError:
pass
def test_refresh_status():
with open(os.path.join(dirname, 'mygene_full.yml'), 'rb') as file:
MYGENE_FULL = file.read()
mygene = SmartAPI.get(MYGENE_ID) # minimum
assert mygene.webdoc.status is None
assert 'components' not in mygene
mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # new content
assert mygene.webdoc.status == 299 # updated
assert 'components' in mygene
assert mygene.webdoc.timestamp > datetime(2020, 1, 1)
_ts0 = mygene.webdoc.timestamp
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 299
assert 'components' in mygene_doc
mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # no change
assert mygene.webdoc.status == 200 # latest
assert 'components' in mygene
assert mygene.webdoc.timestamp > _ts0
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 200
assert 'components' in mygene_doc
mygene.webdoc.update(File(404, None, None, None)) # link broken
assert mygene.webdoc.status == 404
assert 'components' in mygene # do not affect main copy
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 404
assert 'components' in mygene_doc
mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # link back working
assert mygene.webdoc.status == 200 # latest
assert 'components' in mygene
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 200
assert 'components' in mygene_doc
mygene.webdoc.update(File(200, b'{"openapi":"3.0.0"}', None, None)) # invalid
assert mygene.webdoc.status == 499 # invalid
assert 'components' in mygene # do not affect main copy
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 499
assert 'components' in mygene_doc
def test_exists():
# info.title : "MyDisease.info API"
assert not APIDoc.exists('doc0')
assert APIDoc.exists('doc1')
assert APIDoc.exists('3.0.0', 'openapi')
assert APIDoc.exists('mygene', '_meta.slug')
assert not APIDoc.exists('mygene', 'info.title')
assert APIDoc.exists('mygene.info', 'info.title')
assert APIDoc.exists('mygene.info api', 'info.title')
assert APIDoc.exists('api', 'info.title')
assert not APIDoc.exists('mygene', 'info.title.raw')
assert not APIDoc.exists('mygene.info', 'info.title.raw')
assert not APIDoc.exists('mygene.info api', 'info.title.raw')
assert not APIDoc.exists('api', 'info.title.raw')
assert APIDoc.exists('MyGene.info API', 'info.title.raw')
assert APIDoc.exists('mygene.info', 'info.description')
def save(self):
# TODO DOCSTRING
if not self.raw:
raise ControllerError("No content.")
if not self.username:
raise ControllerError("Username is required.")
if self.url is self.VALIDATION_ONLY:
raise ControllerError("In validation-only mode.")
if not self.last_updated:
self.last_updated = datetime.now(timezone.utc)
warn("Filling in date_updated with current time.")
if not self.date_created:
self.date_created = self.last_updated
warn("Filling in date_created with current time.")
if not isinstance(self.date_created, datetime):
raise ControllerError("Invalid created time.")
if not isinstance(self.last_updated, datetime):
raise ControllerError("Invalid updated time.")
if self.date_created > self.last_updated:
raise ControllerError("Invalid timestamps.")
# NOTE
# why not enforce validation here?
# we add additional constraints to the application from time to time
# it's actually hard to retrospectively make sure all previously
# submitted API document always meet our latest requirements
_doc = self._validate_dispatch()
_doc.transform()
if self.slug:
_id = self.find(self.slug)
if _id and _id != self._id: # another doc same slug.
raise ConflictError("Slug is already registered.")
# NOTE
# if the slug of another document changed at this point
# it's possible to have two documents with the same slug
# registered. but it should be rare enough in reality.
doc = APIDoc(**_doc)
doc.meta.id = self._id
doc._meta.url = self.url
doc._meta.username = self.username
doc._meta.slug = self.slug
doc._meta.date_created = self.date_created
doc._meta.last_updated = self.last_updated
if self.uptime.status:
doc._status.uptime_status = self.uptime.status[0]
doc._status.uptime_msg = self.uptime.status[1]
doc._status.uptime_ts = self.uptime.timestamp
doc._status.refresh_status = self.webdoc.status
doc._status.refresh_ts = self.webdoc.timestamp
doc._raw = decoder.compress(self.raw)
doc.save(skip_empty=False)
return self._id
def test_update_slug(self):
mygene = SmartAPI.get(MYGENE_ID)
assert mygene.slug == "mygene"
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygeeni"},
expect=401)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygeeni"},
headers=self.evil_user,
expect=403)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "my"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "www"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "MYGENE"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygene!!"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygeeni"},
headers=self.auth_user)
refresh()
assert not SmartAPI.find("mygene")
assert SmartAPI.find("mygeeni")
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": ""},
headers=self.auth_user)
refresh()
assert not SmartAPI.find("mygeeni")
assert not SmartAPI.find("mygene")
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygene"},
headers=self.auth_user)
refresh()
assert not SmartAPI.find("mygeeni")
assert SmartAPI.find("mygene")
# teardown
refresh()
if not SmartAPI.find("mygene"):
mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES)
mygene._raw = decoder.compress(MYGENE_RAW)
mygene.save()
def test_save(openapi):
"""
SmartAPI.slug.validate(slug)
smartapi.slug
smartapi.save()
"""
_t0 = datetime.now(timezone.utc)
time.sleep(0.1)
URL = "http://example.com/valid.json"
with pytest.raises(ValueError):
SmartAPI.slug.validate("a")
with pytest.raises(ValueError):
SmartAPI.slug.validate("AAA")
with pytest.raises(ValueError):
SmartAPI.slug.validate("www")
with pytest.raises(ValueError):
SmartAPI.slug.validate("^_^")
with open(os.path.join(dirname, './validate/openapi-pass.json'), 'rb') as file:
raw = file.read()
smartapi = SmartAPI(URL)
with pytest.raises(ControllerError):
smartapi.raw = None
smartapi.raw = raw
smartapi.slug = "mygene"
smartapi.validate()
with pytest.raises(ControllerError):
smartapi.save()
smartapi.username = "******"
with pytest.raises(ConflictError):
smartapi.save()
smartapi.slug = "mychem"
with pytest.raises(ConflictError):
smartapi.save()
smartapi.slug = "openapi"
smartapi.save()
refresh()
assert SmartAPI.find("openapi") == smartapi._id
assert smartapi.date_created > _t0
assert smartapi.last_updated > _t0
assert smartapi.date_created == smartapi.last_updated
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
_t1 = smartapi.date_created
smartapi.save() # no change
refresh()
assert SmartAPI.find("openapi") == smartapi._id
assert smartapi.date_created == _t1
assert smartapi.last_updated == _t1
assert smartapi.date_created == smartapi.last_updated
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
smartapi.slug = None
smartapi.save()
refresh()
assert not SmartAPI.find("openapi")
found = SmartAPI.get(openapi)
assert dict(smartapi) == dict(found)
assert smartapi.username == found.username
assert smartapi.slug == found.slug
assert smartapi.url == found.url
assert smartapi.date_created == _t1
assert smartapi.last_updated == _t1
assert smartapi.date_created == smartapi.last_updated
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
smartapi.raw = raw # should trigger ts update
smartapi.save()
refresh()
assert smartapi.date_created == _t1
assert smartapi.last_updated > _t1
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
