def batch_create(self, metadata_list, overwrite=True, **kwargs): """ Create the list of metadata associated with the list of guids Args: metadata_list (List[Dict{"guid": "", "data": {}}]): list of metadata objects in a specific format. Expects a dict with "guid" and "data" fields where "data" is another JSON blob to add to the mds overwrite (bool, optional): whether or not to overwrite existing data """ url = self.admin_endpoint + f"/metadata" if len(metadata_list) > 1 and ("guid" not in metadata_list[0] and "data" not in metadata_list[0]): logging.warning( "it looks like your metadata list for bulk create is malformed. " "the expected format is a list of dicts that have 2 keys: 'guid' " "and 'data', where 'guid' is a string and 'data' is another dict. " f"The first element doesn't match that pattern: {metadata_list[0]}" ) url_with_params = append_query_params(url, overwrite=overwrite, **kwargs) logging.debug(f"hitting: {url_with_params}") logging.debug(f"data: {metadata_list}") response = requests.post(url_with_params, json=metadata_list, auth=self._auth_provider) raise_for_status(response) return response.json()
async def async_update(self, guid, metadata, _ssl=None, **kwargs): """ Asynchronous function to update metadata Args: guid (str): guid to use metadata (Dict): dictionary representing what will end up a JSON blob attached to the provided GUID as metadata _ssl (None, optional): whether or not to use ssl """ async with aiohttp.ClientSession() as session: url = self.admin_endpoint + f"/metadata/{guid}" url_with_params = append_query_params(url, **kwargs) # aiohttp only allows basic auth with their built in auth, so we # need to manually add JWT auth header headers = {"Authorization": self._auth_provider._get_auth_value()} async with session.put(url_with_params, json=metadata, headers=headers, ssl=_ssl) as response: raise_for_status(response) response = await response.json() return response
async def async_get_output(self, job_id, _ssl=None, **kwargs): async with aiohttp.ClientSession() as session: url = self.endpoint + f"/output?UID={job_id}" url_with_params = append_query_params(url, **kwargs) # aiohttp only allows basic auth with their built in auth, so we # need to manually add JWT auth header headers = {"Authorization": self._auth_provider._get_auth_value()} async with session.get(url_with_params, headers=headers, ssl=_ssl) as response: raise_for_status(response) response = await response.json(content_type=None) return response
def delete(self, guid, **kwargs): """ Delete the metadata associated with the guid Args: guid (str): guid to use """ url = self.admin_endpoint + f"/metadata/{guid}" url_with_params = append_query_params(url, **kwargs) logging.debug(f"hitting: {url_with_params}") response = requests.delete(url_with_params, auth=self._auth_provider) raise_for_status(response) return response.json()
async def async_create_job(self, job_name, job_input, _ssl=None, **kwargs): async with aiohttp.ClientSession() as session: url = self.endpoint + f"/dispatch" url_with_params = append_query_params(url, **kwargs) data = json.dumps({"action": job_name, "input": job_input}) # aiohttp only allows basic auth with their built in auth, so we # need to manually add JWT auth header headers = {"Authorization": self._auth_provider._get_auth_value()} async with session.post( url_with_params, data=data, headers=headers, ssl=_ssl ) as response: raise_for_status(response) response = await response.json(content_type=None) return response
def get(self, guid, **kwargs): """ Get the metadata associated with the guid Args: guid (str): guid to use Returns: Dict: metadata for given guid """ url = self.endpoint + f"/metadata/{guid}" url_with_params = append_query_params(url, **kwargs) logging.debug(f"hitting: {url_with_params}") response = requests.get(url_with_params, auth=self._auth_provider) raise_for_status(response) return response.json()
def update(self, guid, metadata, **kwargs): """ Update the metadata associated with the guid Args: guid (str): guid to use metadata (Dict): dictionary representing what will end up a JSON blob attached to the provided GUID as metadata """ url = self.admin_endpoint + f"/metadata/{guid}" url_with_params = append_query_params(url, **kwargs) logging.debug(f"hitting: {url_with_params}") logging.debug(f"data: {metadata}") response = requests.put(url_with_params, json=metadata, auth=self._auth_provider) raise_for_status(response) return response.json()
async def async_get(self, guid, _ssl=None, **kwargs): """ Asynchronous function to get metadata Args: guid (str): guid to use _ssl (None, optional): whether or not to use ssl Returns: Dict: metadata for given guid """ async with aiohttp.ClientSession() as session: url = self.endpoint + f"/metadata/{guid}" url_with_params = append_query_params(url, **kwargs) logging.debug(f"hitting: {url_with_params}") async with session.get(url_with_params, ssl=_ssl) as response: raise_for_status(response) response = await response.json() return response
def create(self, guid, metadata, overwrite=False, **kwargs): """ Create the metadata associated with the guid Args: guid (str): guid to use metadata (Dict): dictionary representing what will end up a JSON blob attached to the provided GUID as metadata overwrite (bool, optional): whether or not to overwrite existing data """ url = self.admin_endpoint + f"/metadata/{guid}" url_with_params = append_query_params(url, overwrite=overwrite, **kwargs) logging.debug(f"hitting: {url_with_params}") logging.debug(f"data: {metadata}") response = requests.post(url_with_params, json=metadata, auth=self._auth_provider) response.raise_for_status() return response.json()
def query( self, query, return_full_metadata=False, limit=10, offset=0, use_agg_mds=False, **kwargs, ): """ Query the metadata given a query. Query format is based off the logic used in the service: ''' Without filters, this will return all data. Add filters as query strings like this: GET /metadata?a=1&b=2 This will match all records that have metadata containing all of: {"a": 1, "b": 2} The values are always treated as strings for filtering. Nesting is supported: GET /metadata?a.b.c=3 Matching records containing: {"a": {"b": {"c": 3}}} Providing the same key with more than one value filters records whose value of the given key matches any of the given values. But values of different keys must all match. For example: GET /metadata?a.b.c=3&a.b.c=33&a.b.d=4 Matches these: {"a": {"b": {"c": 3, "d": 4}}} {"a": {"b": {"c": 33, "d": 4}}} {"a": {"b": {"c": "3", "d": 4, "e": 5}}} But won't match these: {"a": {"b": {"c": 3}}} {"a": {"b": {"c": 3, "d": 5}}} {"a": {"b": {"d": 5}}} {"a": {"b": {"c": "333", "d": 4}}} ''' Args: query (str): mds query as defined by the metadata api return_full_metadata (bool, optional): if False will just return a list of guids limit (int, optional): max num records to return offset (int, optional): offset for output Returns: List: list of guids matching query OR if return_full_metadata=True Dict{guid: {metadata}}: Dictionary with GUIDs as keys and associated metadata JSON blobs as values """ url = self.endpoint + f"/metadata?{query}" url_with_params = append_query_params(url, data=return_full_metadata, limit=limit, offset=offset, **kwargs) logging.debug(f"hitting: {url_with_params}") response = requests.get(url_with_params, auth=self._auth_provider) raise_for_status(response) return response.json()