def batch_create(self, metadata_list, overwrite=True, **kwargs):
"""
Create the list of metadata associated with the list of guids
Args:
metadata_list (List[Dict{"guid": "", "data": {}}]): list of metadata
objects in a specific format. Expects a dict with "guid" and "data"
fields where "data" is another JSON blob to add to the mds
overwrite (bool, optional): whether or not to overwrite existing data
"""
url = self.admin_endpoint + f"/metadata"
if len(metadata_list) > 1 and ("guid" not in metadata_list[0]
and "data" not in metadata_list[0]):
logging.warning(
"it looks like your metadata list for bulk create is malformed. "
"the expected format is a list of dicts that have 2 keys: 'guid' "
"and 'data', where 'guid' is a string and 'data' is another dict. "
f"The first element doesn't match that pattern: {metadata_list[0]}"
)
url_with_params = append_query_params(url,
overwrite=overwrite,
**kwargs)
logging.debug(f"hitting: {url_with_params}")
logging.debug(f"data: {metadata_list}")
response = requests.post(url_with_params,
json=metadata_list,
auth=self._auth_provider)
raise_for_status(response)
return response.json()
async def async_update(self, guid, metadata, _ssl=None, **kwargs):
"""
Asynchronous function to update metadata
Args:
guid (str): guid to use
metadata (Dict): dictionary representing what will end up a JSON blob
attached to the provided GUID as metadata
_ssl (None, optional): whether or not to use ssl
"""
async with aiohttp.ClientSession() as session:
url = self.admin_endpoint + f"/metadata/{guid}"
url_with_params = append_query_params(url, **kwargs)
# aiohttp only allows basic auth with their built in auth, so we
# need to manually add JWT auth header
headers = {"Authorization": self._auth_provider._get_auth_value()}
async with session.put(url_with_params,
json=metadata,
headers=headers,
ssl=_ssl) as response:
raise_for_status(response)
response = await response.json()
return response
async def async_get_output(self, job_id, _ssl=None, **kwargs):
async with aiohttp.ClientSession() as session:
url = self.endpoint + f"/output?UID={job_id}"
url_with_params = append_query_params(url, **kwargs)
# aiohttp only allows basic auth with their built in auth, so we
# need to manually add JWT auth header
headers = {"Authorization": self._auth_provider._get_auth_value()}
async with session.get(url_with_params, headers=headers,
ssl=_ssl) as response:
raise_for_status(response)
response = await response.json(content_type=None)
return response
def delete(self, guid, **kwargs):
"""
Delete the metadata associated with the guid
Args:
guid (str): guid to use
"""
url = self.admin_endpoint + f"/metadata/{guid}"
url_with_params = append_query_params(url, **kwargs)
logging.debug(f"hitting: {url_with_params}")
response = requests.delete(url_with_params, auth=self._auth_provider)
raise_for_status(response)
return response.json()
async def async_create_job(self, job_name, job_input, _ssl=None, **kwargs):
async with aiohttp.ClientSession() as session:
url = self.endpoint + f"/dispatch"
url_with_params = append_query_params(url, **kwargs)
data = json.dumps({"action": job_name, "input": job_input})
# aiohttp only allows basic auth with their built in auth, so we
# need to manually add JWT auth header
headers = {"Authorization": self._auth_provider._get_auth_value()}
async with session.post(
url_with_params, data=data, headers=headers, ssl=_ssl
) as response:
raise_for_status(response)
response = await response.json(content_type=None)
return response
def get(self, guid, **kwargs):
"""
Get the metadata associated with the guid
Args:
guid (str): guid to use
Returns:
Dict: metadata for given guid
"""
url = self.endpoint + f"/metadata/{guid}"
url_with_params = append_query_params(url, **kwargs)
logging.debug(f"hitting: {url_with_params}")
response = requests.get(url_with_params, auth=self._auth_provider)
raise_for_status(response)
return response.json()
def update(self, guid, metadata, **kwargs):
"""
Update the metadata associated with the guid
Args:
guid (str): guid to use
metadata (Dict): dictionary representing what will end up a JSON blob
attached to the provided GUID as metadata
"""
url = self.admin_endpoint + f"/metadata/{guid}"
url_with_params = append_query_params(url, **kwargs)
logging.debug(f"hitting: {url_with_params}")
logging.debug(f"data: {metadata}")
response = requests.put(url_with_params,
json=metadata,
auth=self._auth_provider)
raise_for_status(response)
return response.json()
async def async_get(self, guid, _ssl=None, **kwargs):
"""
Asynchronous function to get metadata
Args:
guid (str): guid to use
_ssl (None, optional): whether or not to use ssl
Returns:
Dict: metadata for given guid
"""
async with aiohttp.ClientSession() as session:
url = self.endpoint + f"/metadata/{guid}"
url_with_params = append_query_params(url, **kwargs)
logging.debug(f"hitting: {url_with_params}")
async with session.get(url_with_params, ssl=_ssl) as response:
raise_for_status(response)
response = await response.json()
return response
def create(self, guid, metadata, overwrite=False, **kwargs):
"""
Create the metadata associated with the guid
Args:
guid (str): guid to use
metadata (Dict): dictionary representing what will end up a JSON blob
attached to the provided GUID as metadata
overwrite (bool, optional): whether or not to overwrite existing data
"""
url = self.admin_endpoint + f"/metadata/{guid}"
url_with_params = append_query_params(url,
overwrite=overwrite,
**kwargs)
logging.debug(f"hitting: {url_with_params}")
logging.debug(f"data: {metadata}")
response = requests.post(url_with_params,
json=metadata,
auth=self._auth_provider)
response.raise_for_status()
return response.json()
def query(
self,
query,
return_full_metadata=False,
limit=10,
offset=0,
use_agg_mds=False,
**kwargs,
):
"""
Query the metadata given a query.
Query format is based off the logic used in the service:
'''
Without filters, this will return all data. Add filters as query strings like this:
GET /metadata?a=1&b=2
This will match all records that have metadata containing all of:
{"a": 1, "b": 2}
The values are always treated as strings for filtering. Nesting is supported:
GET /metadata?a.b.c=3
Matching records containing:
{"a": {"b": {"c": 3}}}
Providing the same key with more than one value filters records whose value of the given key matches any of the given values. But values of different keys must all match. For example:
GET /metadata?a.b.c=3&a.b.c=33&a.b.d=4
Matches these:
{"a": {"b": {"c": 3, "d": 4}}}
{"a": {"b": {"c": 33, "d": 4}}}
{"a": {"b": {"c": "3", "d": 4, "e": 5}}}
But won't match these:
{"a": {"b": {"c": 3}}}
{"a": {"b": {"c": 3, "d": 5}}}
{"a": {"b": {"d": 5}}}
{"a": {"b": {"c": "333", "d": 4}}}
'''
Args:
query (str): mds query as defined by the metadata api
return_full_metadata (bool, optional): if False will just return a list of guids
limit (int, optional): max num records to return
offset (int, optional): offset for output
Returns:
List: list of guids matching query
OR if return_full_metadata=True
Dict{guid: {metadata}}: Dictionary with GUIDs as keys and associated
metadata JSON blobs as values
"""
url = self.endpoint + f"/metadata?{query}"
url_with_params = append_query_params(url,
data=return_full_metadata,
limit=limit,
offset=offset,
**kwargs)
logging.debug(f"hitting: {url_with_params}")
response = requests.get(url_with_params, auth=self._auth_provider)
raise_for_status(response)
return response.json()
