def _get_model(ModelName, MINTconfiguration, MINTusername):
configuration = MINTconfiguration
username = MINTusername
try:
api_instance = mint_client.ModelApi(
mint_client.ApiClient(configuration))
api_response = api_instance.get_model(ModelName, username=username)
model = {
'name': api_response.id,
'description': api_response.description,
'maintainer': '',
'category': api_response.has_model_category,
'versions': [v['id'] for v in api_response.has_software_version]
}
return model
except ApiException as e:
return "Exception when calling ModelApi->get_model: %s\n" % e
def model_config_model_name_get(ModelName): # noqa: E501
"""Obtain an example model configuration.
Submit a model name and receive a model configuration for the given model. # noqa: E501
:param model_name: The name of a model.
:type model_name: str
:rtype: ModelConfig
"""
# get model
try:
api_instance = mint_client.ModelApi(
mint_client.ApiClient(configuration))
model = api_instance.get_model(ModelName, username=username)
versions = [v['id'] for v in model.has_software_version]
# for each model version, obtain configuration ids
configuration_ids = []
api_instance = mint_client.ModelversionApi(
mint_client.ApiClient(configuration))
for v in versions:
version = api_instance.get_model_version(v, username=username)
c_ids = [c.id for c in version.has_configuration]
configuration_ids.extend(c_ids)
# get configurations
configurations = []
api_instance = mint_client.ModelconfigurationApi(
mint_client.ApiClient(configuration))
for _id in configuration_ids:
config = api_instance.get_model_configuraton(_id,
username=username)
configurations.append({
'name': ModelName,
'config': config.to_dict()
})
return configurations
except ApiException as e:
return "Exception when calling MINT: %s\n" % e
def list_models_post(): # noqa: E501
"""Obtain a list of current models
Request a list of currently available models. # noqa: E501
:rtype: List[str]
"""
api_instance = mint_client.ModelApi(mint_client.ApiClient(configuration))
try:
api_response = api_instance.get_models(username=username)
models = [m.id for m in api_response]
# obtain model info:
models_infos = []
for m in models:
models_infos.append(model_info_model_name_get(m))
return models_infos
except ApiException as e:
return "Exception when calling ModelApi->get_models: %s\n" % e
def _find_model_by_dataset_id(dataset_id, MINTconfiguration, MINTuser):
print(f"Searching for models associated with the dataset {dataset_id}")
configuration = MINTconfiguration
username = MINTuser
# Step 1: Obtain all MINT Model configurations
# from these configurations, extract the input/output dataset IDs
# Step 1 output: a set of Configuration IDs associated with the dataset_id
api_instance = mint_client.ModelconfigurationApi(
mint_client.ApiClient(configuration))
try:
# List modelconfiguration
api_response = api_instance.get_model_configurations(username=username)
dataset_configs = set()
for config in api_response:
for io in config.has_input:
_id = io.id
if _id == dataset_id:
dataset_configs.add(config.id)
for io in config.has_output:
_id = io.id
if _id == dataset_id:
dataset_configs.add(config.id)
print(
f"Found {len(dataset_configs)} Model Configuration associated with {dataset_id}"
)
except ApiException as e:
print(
"Exception when calling ModelconfigurationApi->get_model_configurations: %s\n"
% e)
# Step 2: Obtain all Model Versions and their associated Model Configuration IDs
# Step 2 output: a set of Model Version IDs associated with the dataset_id
api_instance = mint_client.ModelversionApi(
mint_client.ApiClient(configuration))
try:
# List All ModelVersions
version_ids = set()
api_response = api_instance.get_model_versions(username=username)
for v in api_response:
c_ = v.has_configuration
for conf in c_:
if conf.id in dataset_configs:
# this is a config associated with the dataset_id
# so add the version to version_ids
version_ids.add(v.id)
print(
f"Found {len(version_ids)} Model Version associated with {dataset_id}"
)
except ApiException as e:
print(
"Exception when calling ModelversionApi->get_model_versions: %s\n"
% e)
# Step 3: Obtain all Models and check if their versions match the Version IDs of interest
# Step 3 output:
api_instance = mint_client.ModelApi(mint_client.ApiClient(configuration))
try:
# List All models
api_response = api_instance.get_models(username=username)
models = set()
for m in api_response:
for v in m.has_software_version:
if v['id'] in version_ids:
# this is a model associated with the dataset_id
# obtain its name
models.add(m.label)
print(f"Found {len(models)} Models associated with {dataset_id}")
except ApiException as e:
print("Exception when calling ModelApi->get_models: %s\n" % e)
try:
models_output = []
for m in list(models):
m_ = _get_model(m, configuration, username)
models_output.append(m_)
return models_output
except Exception as e:
print(f"Exception when processing model info: {e}")
def _execute_text_query(TextQuery, url, request_headers, MINTconfiguration,
MINTusername):
###### STANDARD NAME QUERY #######
if TextQuery.type == 'standard name':
q = {"standard_variable_names__in": [TextQuery.term]}
print(q)
resp = requests.post(f"{url}/datasets/find",
headers=request_headers,
json=q).json()
if resp['result'] == 'success':
found_resources = resp['resources']
print(f"Found {len(found_resources)} resources")
else:
found_resources = []
if TextQuery.result_type == 'datasets':
return found_resources
elif TextQuery.result_type == 'models':
# We add the JSON string representation of the model to a set
# so that we can avoid returning to the user duplicate models
models = set()
for d in found_resources:
found_models = _find_model_by_dataset_id(
d["dataset_id"], MINTconfiguration, MINTusername)
for model in found_models:
models.add(json.dumps(model))
models_output = []
for m in list(models):
models_output.append(json.loads(m))
return models_output
###### KEYWORD QUERY #######
elif TextQuery.type == 'keyword':
if TextQuery.result_type == 'datasets':
payload = {
"provenance_id": "3831a57f-a372-424a-b310-525b5441581b",
"search_query": [TextQuery.term]
}
response = requests.post(
'http://api.mint-data-catalog.org/datasets/jataware_search',
data=json.dumps(payload))
return response.json()['datasets']
elif TextQuery.result_type == 'models':
params = (
('text', TextQuery.term),
('endpoint', 'https://endpoint.mint.isi.edu/ds/query'),
)
response = requests.get(
'https://query.mint.isi.edu/api/dgarijo/jatawareAPI/searchModels',
params=params)
# obtain model labels from result set
models_ = response.json()['results']['bindings']
models = [
res['w']['value'].split('instance/')[1] for res in models_
]
api_instance = mint_client.ModelApi()
models_output = []
for model_id in models:
try:
# Get a Model
api_response = api_instance.get_model(
model_id, username=MINTusername)
except ApiException as e:
print("Exception when calling ModelApi->get_model: %s\n" %
e)
models_output.append(api_response.to_dict())
return models_output
elif TextQuery.result_type == 'variables':
params = (
('text', TextQuery.term),
('endpoint', 'https://endpoint.mint.isi.edu/ds/query'),
)
response = requests.get(
'https://query.mint.isi.edu/api/dgarijo/jatawareAPI/searchVariables',
params=params)
results = response.json()['results']['bindings']
variables = []
for res in results:
var = {}
var['name'] = res['desc']['value']
var['id'] = res['w']['value'].split('instance/')[1]
variables.append(var)
# Get standard variable information from DCAT
# This is clunky to have to use both catalogs
# But it seems that the standard variable information is
# Not readily available in MCAT so we got to DCAT
# TODO: ensure that the standard variables are actually compliant
# TODO: right now standard variables do not have URIs returned by DCAT
for var in variables:
q = {"variable_ids__in": [var['id']]}
resp = requests.post(
f"{url}/variables/variables_standard_variables",
headers=request_headers,
json=q).json()
dcat_var = resp['variables'][0]
var['metadata'] = dcat_var['metadata']
for std in dcat_var['standard_variables']:
# TODO: address these blanks
if 'standard_variable_uri' not in std:
std['standard_variable_uri'] = ''
var['standard_variables'] = dcat_var['standard_variables']
return variables
def model_io_post(): # noqa: E501
"""Obtain information on a given model's inputs or outputs.
Submit a model name and receive information about the input or output files required by this model. # noqa: E501
Note that this includes all inputs and all outputs for a given model, irrespective of the
configuration. In the future this could be subset to just a specific configuration.
:param io_request: The name of a model and an IO type.
:type io_request: dict | bytes
:rtype: List[IOFile]
"""
if connexion.request.is_json:
io_request = IORequest.from_dict(
connexion.request.get_json()) # noqa: E501
name_ = io_request.name
type_ = io_request.iotype
try:
# get model configuration ids
api_instance = mint_client.ModelApi(
mint_client.ApiClient(configuration))
model = api_instance.get_model(name_, username=username)
versions = [v['id'] for v in model.has_software_version]
# for each model version, obtain configuration ids
configuration_ids = []
api_instance = mint_client.ModelversionApi(
mint_client.ApiClient(configuration))
for v in versions:
version = api_instance.get_model_version(v, username=username)
c_ids = [c.id for c in version.has_configuration]
configuration_ids.extend(c_ids)
# get IO
api_instance = mint_client.ModelconfigurationApi(
mint_client.ApiClient(configuration))
inputs_outputs = []
if type_ == 'input':
for config_id in configuration_ids:
response = requests.get(
f"https://api.models.mint.isi.edu/v0.0.2/modelconfiguration/{config_id}/inputs?username=modelservice"
)
api_response = response.json()
for io in api_response:
io_ = util._parse_io(io, url, request_headers)
if io_:
inputs_outputs.append(io_)
elif type_ == 'output':
outputs = []
for config_id in configuration_ids:
response = requests.get(
f"https://api.models.mint.isi.edu/v0.0.2/modelconfiguration/{config_id}/outputs?username=modelservice"
)
api_response = response.json()
for io in api_response:
io_ = util._parse_io(io, url, request_headers)
if io_:
inputs_outputs.append(io_)
return inputs_outputs
except ApiException as e:
return "Exception when calling MINT API: %s\n" % e
#
# try:
# # Create user
# api_instance.create_user(user)
# except ApiException as e:
# print("Exception when calling UserApi->create_user: %s\n" % e)
try:
# Logs user into the system
configuration.access_token = api_instance.login_user(username, password)
print("Log in success! Token: %s\n" % configuration.access_token)
except ApiException as e:
print("Exception when calling UserApi->login_user: %s\n" % e)
# create model
api_instance = mint_client.ModelApi(mint_client.ApiClient(configuration))
# create an instance of the API class
model = {
"description": "SSCYP is built upon deep learning of past harvest data and satellite images of a certain area, it is able to predict the yield of a given crop in a given region based on its current satellite images of at least 6 days of time span. The current model is trained for South Sudan and pre-trained on Ethiopia, the prediction can be of the same region, or it can be of neighboring countries or regions with similar underlying distributions.",
"hasDocumentation": [
"https://cloud.docker.com/repository/docker/minyinsri/crop-yield-transfer-learning-cpu"
],
"hasModelCategory": [
"Agriculture"
],
"hasSoftwareVersion": [
{
"id": "SSCYP_1.0"
}
],
"id": "SSCYP",