def test_get():
"""
smartapi = SmartAPI.get(_id)
smartapi._id
smartapi.url
smartapi.version
...
"""
mygene = SmartAPI.get(MYGENE_ID)
assert mygene._id == MYGENE_ID
assert mygene.version == 'openapi'
assert mygene.username == 'tester'
assert mygene.slug == 'mygene'
assert mygene['info']['title'] == 'MyGene.info API'
assert mygene.raw == MYGENE_RAW
assert mygene.url == MYGENE_URL
with pytest.raises(AttributeError):
mygene._id = "NEWID"
with pytest.raises(AttributeError):
mygene.url = "http://example.org/"
with pytest.raises(ValueError):
mygene.slug = "a"
with pytest.raises(ValueError):
mygene.slug = "AAA"
with pytest.raises(ValueError):
mygene.slug = "www"
with pytest.raises(ValueError):
mygene.slug = "^_^"
with pytest.raises(NotFoundError):
SmartAPI.get("NOTEXIST")
def test_get_tags():
"""
SmartAPI.get_tags()
SmartAPI.get_tags(field)
"""
aggs = SmartAPI.get_tags()
assert aggs == {'Chunlei Wu': 2}
aggs = SmartAPI.get_tags('tags.name')
assert 'annotation' in aggs
assert 'translator' in aggs
assert 'biothings' in aggs
def test_post(self):
_ID = '1ad2cba40cb25cd70d00aa8fba9cfaf3'
VALID_V3_URL = \
'https://raw.githubusercontent.com/schurerlab/'\
'smartAPIs/master/LINCS_Data_Portal_smartAPIs.yml'
try:
smartapi = SmartAPI.get(_ID)
smartapi.delete()
refresh()
except NotFoundError:
pass
self.request("/api/metadata/",
method='POST',
data={'url': VALID_V3_URL},
expect=401)
self.request("/api/metadata/",
method='POST',
data={'url': MYGENE_URL},
headers=self.auth_user,
expect=409)
self.request("/api/metadata/",
method='POST',
headers=self.auth_user,
expect=400)
self.request("/api/metadata/",
method='POST',
data={'url': "http://invalidhost/file"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/",
method='POST',
data={
'url': VALID_V3_URL,
'dryrun': True
},
headers=self.auth_user)
refresh()
assert not SmartAPI.exists(_ID)
self.request("/api/metadata/",
method='POST',
data={'url': VALID_V3_URL},
headers=self.auth_user)
refresh()
assert SmartAPI.exists(_ID)
try:
smartapi = SmartAPI.get(_ID)
smartapi.delete()
except NotFoundError:
pass
def refresh_document():
logger = logging.getLogger("refresh")
for smartapi in SmartAPI.get_all(1000):
logger.info(smartapi._id)
_status = smartapi.refresh()
logger.info(_status)
smartapi.save()
def validate(self, raw):
try:
smartapi = SmartAPI(SmartAPI.VALIDATION_ONLY)
smartapi.raw = raw
smartapi.validate()
except (ControllerError, AssertionError) as err:
raise BadRequest(details=str(err))
else:
self.finish({
'success':
True,
'details':
f'valid SmartAPI ({smartapi.version}) metadata.'
})
def check_uptime():
logger = logging.getLogger("uptime")
for smartapi in SmartAPI.get_all(1000):
logger.info(smartapi._id)
_status = smartapi.check()
logger.info(_status)
smartapi.save()
def test_get_all():
"""
SmartAPI.get_all()
"""
docs = list(SmartAPI.get_all())
assert len(docs) == 2
assert docs[0]["info"]["title"] in ["MyGene.info API", "MyChem.info API"]
assert docs[1]["info"]["title"] in ["MyGene.info API", "MyChem.info API"]
def get(self):
"""
/api/suggestion?field=
Returns aggregations for any field provided
Used for tag:count on registry
"""
res = SmartAPI.get_tags(self.args.field)
self.finish(res)
def test_get_all_from():
"""
SmartAPI.get_all(from_=1)
"""
search = APIDoc.search()
assert search.count() == 2
docs = list(SmartAPI.get_all(from_=1))
assert len(docs) == 1
def test_search():
"""
SmartAPI.exists(_id)
SmartAPI.find(slug)
SmartAPI.find(val, field)
"""
assert SmartAPI.exists(MYGENE_ID)
assert SmartAPI.exists(MYCHEM_ID)
assert SmartAPI.find('mygene') == MYGENE_ID
assert SmartAPI.find('mychem') == MYCHEM_ID
assert SmartAPI.find('tester', 'username')
assert SmartAPI.find('gene', 'tags.name') == MYGENE_ID
assert SmartAPI.find('drug', 'tags.name') == MYCHEM_ID
assert SmartAPI.find(MYGENE_URL, 'url') == MYGENE_ID
assert SmartAPI.find(MYCHEM_URL, 'url') == MYCHEM_ID
def _restore(smartapis):
if indices.exists():
logging.error("Cannot write to an existing index.")
return
indices.reset()
for smartapi in smartapis:
logging.info(smartapi["url"])
_smartapi = SmartAPI(smartapi["url"])
_smartapi.username = smartapi["username"]
_smartapi.slug = smartapi["slug"]
_smartapi.date_created = datetime.fromisoformat(
smartapi["date_created"])
_smartapi.last_updated = datetime.fromisoformat(
smartapi["last_updated"])
_smartapi.raw = smartapi["raw"].encode() # to bytes
_smartapi.save()
def _backup():
smartapis = []
for smartapi in SmartAPI.get_all(1000):
smartapis.append({
"url": smartapi.url,
"username": smartapi.username,
"slug": smartapi.slug,
"date_created": smartapi.date_created.isoformat(),
"last_updated": smartapi.last_updated.isoformat(),
"raw": smartapi.raw.decode() # to string
})
return smartapis
def test_uptime_update():
mygene = SmartAPI.get(MYGENE_ID)
mygene.check() # minimum api document
assert mygene.uptime.status[0] == 'unknown' # TODO VERIFY THIS IS IN FACT CORRECT
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.uptime_status == 'unknown'
mychem = SmartAPI.get(MYCHEM_ID)
mychem.check() # full api document
assert mychem.uptime.status[0] == 'unknown'
mychem.save()
refresh()
mychem_doc = APIDoc.get(MYCHEM_ID)
assert mychem_doc._status.uptime_status == 'unknown'
async def post(self):
"""
Add an API document
"""
if SmartAPI.find(self.args.url, "url"):
raise HTTPError(409)
try:
file = await download_async(self.args.url)
except DownloadError as err:
raise BadRequest(details=str(err)) from err
try:
smartapi = SmartAPI(self.args.url)
smartapi.raw = file.raw
smartapi.validate()
except (ControllerError, AssertionError) as err:
raise BadRequest(details=str(err)) from err
if self.args.dryrun:
raise Finish({
'success':
True,
'details':
f"[Dryrun] Valid {smartapi.version} Metadata"
})
try:
smartapi.username = self.current_user['login']
smartapi.refresh(file) # populate webdoc meta
_id = smartapi.save()
except ControllerError as err:
raise BadRequest(details=str(err)) from err
else:
self.finish({'success': True, '_id': _id})
await self._notify(smartapi)
def test_delete(self):
# setup
assert SmartAPI.exists(MYGENE_ID)
self.request("/api/metadata/" + MYGENE_ID, method='DELETE', expect=401)
self.request("/api/metadata/" + MYGENE_ID,
method='DELETE',
headers=self.evil_user,
expect=403)
self.request("/api/metadata/" + MYGENE_ID,
method='DELETE',
headers=self.auth_user)
refresh()
assert not SmartAPI.exists(MYGENE_ID)
# teardown
refresh()
if not SmartAPI.exists(MYGENE_ID): # recover the deleted file
mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES)
mygene._raw = decoder.compress(MYGENE_RAW)
mygene.save()
refresh()
async def put(self, _id):
"""
Add/Update the URL slug:
PUT {"slug": "new_slug"}
Remove a URL slug:
PUT {"slug": "" }
Refresh a document:
PUT {}
"""
try:
smartapi = SmartAPI.get(_id)
except NotFoundError:
raise HTTPError(404)
if smartapi.username != self.current_user['login']:
raise HTTPError(403)
if self.args.slug is not None:
if self.args.slug in {'api'}: #reserved
raise BadRequest(details='slug is reserved')
try: # update slug
smartapi.slug = self.args.slug or None
smartapi.save()
except (ControllerError, ValueError) as err:
raise BadRequest(details=str(err)) from err
self.finish({'success': True})
else: # refresh
file = await download_async(smartapi.url, raise_error=False)
code = smartapi.refresh(file)
smartapi.save()
try:
status = smartapi.webdoc.STATUS(code)
status = status.name.lower()
except ValueError:
status = 'nofile' # keep the original copy
self.finish({
'success': code in (200, 299),
'status': status,
'code': code
})
def update():
for doc in scan(Elasticsearch(ES_DESTINATION),
query={"query": {
"match_all": {}
}},
index="smartapi_docs",
scroll="60m"):
print(doc["_id"])
smartapi = SmartAPI.get(doc["_id"])
print(smartapi.check())
print(smartapi.refresh())
if smartapi.webdoc.status == 299:
smartapi.webdoc._status = 200 # change status not reliable during migration
smartapi.save()
print()
def test_uptime_status():
mygene = SmartAPI.get(MYGENE_ID)
assert mygene.uptime.status[0] is None
mygene.uptime.update(('good', None))
assert mygene.uptime.status[0] == 'good'
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.uptime_status == 'good'
mygene.uptime.update((None, None))
assert mygene.uptime.status[0] is None
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.uptime_status is None
def delete(self, _id):
"""
Delete API
"""
try:
smartapi = SmartAPI.get(_id)
except NotFoundError:
raise HTTPError(404)
if smartapi.username != self.current_user['login']:
raise HTTPError(403)
try:
_id = smartapi.delete()
except ControllerError as err:
raise BadRequest(details=str(err)) from err
self.finish({'success': True, '_id': _id})
def test_delete(myvariant):
mv = SmartAPI.get(myvariant)
mv.delete()
refresh()
assert not APIDoc.exists(myvariant)
URL = "http://example.com/valid.json"
with open(os.path.join(dirname, './validate/openapi-pass.json'), 'rb') as file:
smartapi = SmartAPI(URL)
smartapi.raw = file.read()
with pytest.raises(NotFoundError):
smartapi.delete()
def migrate():
for doc in scan(Elasticsearch(ES_ORIGIN),
query={"query": {
"match_all": {}
}},
index="smartapi_oas3",
doc_type="api"):
print(doc["_id"])
if not doc['_source']['_meta'].get('_archived'):
url = doc['_source']['_meta']['url']
raw = decoder.decompress(
base64.urlsafe_b64decode(doc['_source']['~raw']))
smartapi = SmartAPI(url)
smartapi.raw = raw
smartapi.date_created = parser.parse(
doc['_source']['_meta']['timestamp']).replace(
tzinfo=timezone.utc)
smartapi.username = doc['_source']['_meta']['github_username']
smartapi.slug = doc['_source']['_meta'].get('slug')
smartapi.save()
print()
def test_refresh_update():
mychem = SmartAPI.get(MYCHEM_ID)
assert mychem.webdoc.status is None
# NOTE
# the following update and the updates thereafter will be applied
# https://github.com/NCATS-Tangerine/translator-api-registry/commit/b01baa5
mychem.refresh()
assert mychem.webdoc.status == 299 # new version
assert mychem.webdoc.timestamp > datetime(2020, 1, 1, tzinfo=timezone.utc)
_ts0 = mychem.webdoc.timestamp
mychem.refresh()
assert mychem.webdoc.status == 200 # already latest
assert mychem.webdoc.timestamp >= _ts0 # could be cached internally
mychem.save()
refresh()
mychem_doc = APIDoc.get(MYCHEM_ID)
assert mychem_doc._status.refresh_status == 200
def test_save(openapi):
"""
SmartAPI.slug.validate(slug)
smartapi.slug
smartapi.save()
"""
_t0 = datetime.now(timezone.utc)
time.sleep(0.1)
URL = "http://example.com/valid.json"
with pytest.raises(ValueError):
SmartAPI.slug.validate("a")
with pytest.raises(ValueError):
SmartAPI.slug.validate("AAA")
with pytest.raises(ValueError):
SmartAPI.slug.validate("www")
with pytest.raises(ValueError):
SmartAPI.slug.validate("^_^")
with open(os.path.join(dirname, './validate/openapi-pass.json'), 'rb') as file:
raw = file.read()
smartapi = SmartAPI(URL)
with pytest.raises(ControllerError):
smartapi.raw = None
smartapi.raw = raw
smartapi.slug = "mygene"
smartapi.validate()
with pytest.raises(ControllerError):
smartapi.save()
smartapi.username = "******"
with pytest.raises(ConflictError):
smartapi.save()
smartapi.slug = "mychem"
with pytest.raises(ConflictError):
smartapi.save()
smartapi.slug = "openapi"
smartapi.save()
refresh()
assert SmartAPI.find("openapi") == smartapi._id
assert smartapi.date_created > _t0
assert smartapi.last_updated > _t0
assert smartapi.date_created == smartapi.last_updated
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
_t1 = smartapi.date_created
smartapi.save() # no change
refresh()
assert SmartAPI.find("openapi") == smartapi._id
assert smartapi.date_created == _t1
assert smartapi.last_updated == _t1
assert smartapi.date_created == smartapi.last_updated
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
smartapi.slug = None
smartapi.save()
refresh()
assert not SmartAPI.find("openapi")
found = SmartAPI.get(openapi)
assert dict(smartapi) == dict(found)
assert smartapi.username == found.username
assert smartapi.slug == found.slug
assert smartapi.url == found.url
assert smartapi.date_created == _t1
assert smartapi.last_updated == _t1
assert smartapi.date_created == smartapi.last_updated
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
smartapi.raw = raw # should trigger ts update
smartapi.save()
refresh()
assert smartapi.date_created == _t1
assert smartapi.last_updated > _t1
apidoc = APIDoc.get(smartapi._id)
assert apidoc._meta.date_created == smartapi.date_created
assert apidoc._meta.last_updated == smartapi.last_updated
def resave():
# when index mappings are changed
logger = logging.getLogger("resave")
for smartapi in SmartAPI.get_all(1000):
logger.info(smartapi._id)
smartapi.save()
def test_refresh_status():
with open(os.path.join(dirname, 'mygene_full.yml'), 'rb') as file:
MYGENE_FULL = file.read()
mygene = SmartAPI.get(MYGENE_ID) # minimum
assert mygene.webdoc.status is None
assert 'components' not in mygene
mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # new content
assert mygene.webdoc.status == 299 # updated
assert 'components' in mygene
assert mygene.webdoc.timestamp > datetime(2020, 1, 1)
_ts0 = mygene.webdoc.timestamp
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 299
assert 'components' in mygene_doc
mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # no change
assert mygene.webdoc.status == 200 # latest
assert 'components' in mygene
assert mygene.webdoc.timestamp > _ts0
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 200
assert 'components' in mygene_doc
mygene.webdoc.update(File(404, None, None, None)) # link broken
assert mygene.webdoc.status == 404
assert 'components' in mygene # do not affect main copy
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 404
assert 'components' in mygene_doc
mygene.webdoc.update(File(200, MYGENE_FULL, None, None)) # link back working
assert mygene.webdoc.status == 200 # latest
assert 'components' in mygene
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 200
assert 'components' in mygene_doc
mygene.webdoc.update(File(200, b'{"openapi":"3.0.0"}', None, None)) # invalid
assert mygene.webdoc.status == 499 # invalid
assert 'components' in mygene # do not affect main copy
mygene.save()
refresh()
mygene_doc = APIDoc.get(MYGENE_ID)
assert mygene_doc._status.refresh_status == 499
assert 'components' in mygene_doc
def test_update_slug(self):
mygene = SmartAPI.get(MYGENE_ID)
assert mygene.slug == "mygene"
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygeeni"},
expect=401)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygeeni"},
headers=self.evil_user,
expect=403)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "my"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "www"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "MYGENE"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygene!!"},
headers=self.auth_user,
expect=400)
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygeeni"},
headers=self.auth_user)
refresh()
assert not SmartAPI.find("mygene")
assert SmartAPI.find("mygeeni")
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": ""},
headers=self.auth_user)
refresh()
assert not SmartAPI.find("mygeeni")
assert not SmartAPI.find("mygene")
self.request("/api/metadata/" + MYGENE_ID,
method='PUT',
data={"slug": "mygene"},
headers=self.auth_user)
refresh()
assert not SmartAPI.find("mygeeni")
assert SmartAPI.find("mygene")
# teardown
refresh()
if not SmartAPI.find("mygene"):
mygene = APIDoc(meta={'id': MYGENE_ID}, **MYGENE_ES)
mygene._raw = decoder.compress(MYGENE_RAW)
mygene.save()
def setup_fixture():
"""
Index 2 documents.
"""
reset()
# save initial docs with paths already transformed
mygene = SmartAPI(MYGENE_URL)
mygene.raw = MYGENE_RAW
mygene.username = '******'
mygene.slug = 'mygene'
mygene.save()
mychem = SmartAPI(MYCHEM_URL)
mychem.raw = MYCHEM_RAW
mychem.username = '******'
mychem.slug = 'mychem'
mychem.save()
# refresh index
refresh()
def setup():
"""
setup state called once for the class
"""
reset()
mygene = SmartAPI(MYGENE_URL)
mygene.username = '******'
mygene.refresh()
mygene.check()
mygene.save()
mychem = SmartAPI(MYCHEM_URL)
mychem.username = '******'
mychem.refresh()
mychem.check()
mychem.save()
refresh()
def test_validation():
"""
smartapi.validate()
"""
URL = "http://example.com/invalid.json"
with open(os.path.join(dirname, './validate/openapi-pass.json'), 'rb') as file:
smartapi = SmartAPI(URL)
smartapi.raw = file.read()
smartapi.validate()
with open(os.path.join(dirname, './validate/swagger-pass.json'), 'rb') as file:
smartapi = SmartAPI(URL)
smartapi.raw = file.read()
smartapi.validate()
with open(os.path.join(dirname, './validate/x-translator-pass.json'), 'rb') as file:
smartapi = SmartAPI(URL)
smartapi.raw = file.read()
smartapi.validate()
with open(os.path.join(dirname, './validate/x-translator-fail-1.yml'), 'rb') as file:
smartapi = SmartAPI(URL)
smartapi.raw = file.read()
with pytest.raises(ControllerError):
smartapi.validate()
with open(os.path.join(dirname, './validate/x-translator-fail-2.yml'), 'rb') as file:
smartapi = SmartAPI(URL)
smartapi.raw = file.read()
with pytest.raises(ControllerError):
smartapi.validate()
smartapi = SmartAPI(URL)
smartapi.raw = b'{}'
with pytest.raises(ControllerError):
smartapi.validate()
def test_update_doc(self):
self.request("/api/metadata/" + MYCHEM_ID, method="PUT", expect=401)
self.request("/api/metadata/notexists",
method="PUT",
headers=self.auth_user,
expect=404)
self.request("/api/metadata/" + MYCHEM_ID,
method="PUT",
headers=self.evil_user,
expect=403)
res = self.request("/api/metadata/" + MYCHEM_ID,
method="PUT",
headers=self.auth_user).json()
assert res["success"]
assert res["code"] == 299
assert res["status"] == "updated"
mychem = SmartAPI.get(MYCHEM_ID)
assert mychem.webdoc.status == 299
assert mychem.webdoc.timestamp
ts0 = mychem.webdoc.timestamp
res = self.request("/api/metadata/" + MYCHEM_ID,
method="PUT",
headers=self.auth_user).json()
assert res["success"]
assert res["code"] == 200
assert res["status"] == "not_modified"
mychem = SmartAPI.get(MYCHEM_ID)
assert mychem.webdoc.status == 200
assert mychem.webdoc.timestamp >= ts0
# setup
mygene_ref = SmartAPI(self.get_url('/test/mygene.yml'))
mygene_ref.raw = MYGENE_RAW
mygene_ref.username = '******'
mygene_ref.save()
# first request, same file, minimul version
res = self.request("/api/metadata/" + mygene_ref._id,
method="PUT",
headers=self.auth_user).json()
assert res["success"]
assert res["code"] == 200
assert res["status"] == "not_modified"
mygene = SmartAPI.get(mygene_ref._id)
assert mygene.webdoc.status == 200
# second request, full version, updated
res = self.request("/api/metadata/" + mygene_ref._id,
method="PUT",
headers=self.auth_user).json()
assert res["success"]
assert res["code"] == 299
assert res["status"] == "updated"
mygene = SmartAPI.get(mygene_ref._id)
assert mygene.webdoc.status == 299
# third request, link temporarily unavailable
res = self.request("/api/metadata/" + mygene_ref._id,
method="PUT",
headers=self.auth_user).json()
assert not res["success"]
assert res["code"] == 404
assert res["status"] == "nofile"
mygene = SmartAPI.get(mygene_ref._id)
assert mygene.webdoc.status == 404
# fourth request, link back up, new version doesn't pass validation
res = self.request("/api/metadata/" + mygene_ref._id,
method="PUT",
headers=self.auth_user).json()
assert not res["success"]
assert res["code"] == 499
assert res["status"] == "invalid"
mygene = SmartAPI.get(mygene_ref._id)
assert mygene.webdoc.status == 499
# fifth request, link restored to a working version
res = self.request("/api/metadata/" + mygene_ref._id,
method="PUT",
headers=self.auth_user).json()
assert res["success"]
assert res["code"] == 200
assert res["status"] == "not_modified"
mygene = SmartAPI.get(mygene_ref._id)
assert mygene.webdoc.status == 200
# setup
mygene_ref_2 = SmartAPI('http://invalidhost/mygene.yml')
mygene_ref_2.raw = MYGENE_RAW
mygene_ref_2.username = '******'
mygene_ref_2.save()
res = self.request("/api/metadata/" + mygene_ref_2._id,
method="PUT",
headers=self.auth_user).json()
assert not res["success"]
assert res["code"] == 599
assert res["status"] == "nofile"
mygene = SmartAPI.get(mygene_ref_2._id)
assert mygene.webdoc.status == 599