def send_request_json(
endpoint: Endpoint,
params: Dict = None,
n_try: int = 3,
server_url: str = "http://amda.irap.omp.eu") -> str or None:
"""Send a request on the AMDA_Webservice REST service to the given endpoint with given parameters. We expect the result to be JSON data.
Parameters
----------
endpoint: Endpoint
target API endpoint on which the request will be performed
params: dict
request parameters
n_try: int
the number of retry in case of failure
server_url: str
the base server URL
Returns
-------
str or None
request result parsed as json object
"""
url = request_url(endpoint, server_url=server_url)
params = params or {}
params['token'] = token(server_url=server_url)
for _ in [None] * n_try: # in case of failure
# add token now ? does it change
log.debug(f"Send request on AMDA_Webservice server {url}")
r = http.get(url, params=params)
js = r.json()
if 'success' in js and \
js['success'] is True and \
'dataFileURLs' in js:
log.debug(f"success: {js['dataFileURLs']}")
return js['dataFileURLs']
elif "success" in js and \
js["success"] is True and \
"status" in js and \
js["status"]=="in progress":
log.warning(
"This request duration is too long, consider reducing time range"
)
while True:
default_sleep_time = 10.
time.sleep(default_sleep_time)
url = request_url(Endpoint.GETSTATUS, server_url=server_url)
status = http.get(url, params=js).json()
if status is not None and status["status"] == "done":
return status["dataFileURLs"]
else:
log.debug(f"Failed: {r.text}")
return None
def get_dataviews(self):
resp = http.get(self.__url + '/dataviews',
headers={"Accept": "application/json"})
if not resp.ok:
return None
dataviews = [dv['Id'] for dv in resp.json()['DataviewDescription']]
return dataviews
def send_request(endpoint: Endpoint,
params: dict = None,
n_try: int = 3,
server_url: str = "http://amda.irap.omp.eu") -> str or None:
"""Send a request on the AMDA_Webservice REST service to the given endpoint with given parameters. Retry up to :data:`n_try` times upon failure.
Parameters
----------
endpoint: Endpoint
target API endpoint on which the request will be performed
params: dict
request parameters
n_try: int
the number of retry in case of failure
server_url: str
the base server URL
Returns
-------
str or None
request result text, stripped of spaces and newlines
"""
url = request_url(endpoint, server_url=server_url)
params = params or {}
params['token'] = token(server_url=server_url)
for _ in [None] * n_try: # in case of failure
# add token now ? does it change
log.debug(f"Send request on AMDA_Webservice server {url}")
r = http.get(url, params=params)
if r is None:
# try again
continue
return r.text.strip()
return None
def get_user_parameters_xml_tree(username: str,
password: str,
server_url: str = "http://amda.irap.omp.eu",
**kwargs: Dict) -> str or None:
"""Get private list of parameters.
Parameters
----------
username: str
AMDA username
password:
AMDA password
server_url: str
the base server URL
kwargs: dict
extra request arguments
Returns
-------
str or None
request result, XML formatted text
"""
xml_resp = send_request(Endpoint.LISTPARAM,
params=pack_kwargs(**kwargs,
**auth_args(username,
password)),
server_url=server_url).strip()
node = Et.fromstring(f'<root>{xml_resp}</root>').find(
"UserDefinedParameters")
if node is not None:
return http.get(node.text).text
return None
def get_variables(self, dataset, dataview='sp_phys'):
resp = http.get(self.__url +
f'/dataviews/{dataview}/datasets/{dataset}/variables',
headers={"Accept": "application/json"})
if not resp.ok:
return None
variables = [
varaible for varaible in resp.json()['VariableDescription']
]
return variables
def get_instruments(self,
dataview='sp_phys',
observatory=None,
instrumentType=None):
args = []
if observatory is not None:
args.append(f'observatory={observatory}')
if instrumentType is not None:
args.append(f'instrumentType={instrumentType}')
resp = http.get(self.__url + f'/dataviews/{dataview}/instruments?' +
"&".join(args),
headers={"Accept": "application/json"})
if not resp.ok:
return None
instruments = [
instrument for instrument in resp.json()['InstrumentDescription']
if instrument['Name'] != ''
]
return instruments
def token(server_url: str = "http://amda.irap.omp.eu") -> str:
"""Returns authentication token.
Parameters
----------
server_url:str
server base URL on which the API token will be generated
Returns
-------
str
the generated token
"""
# url = "{0}/php/rest/auth.php?".format(self.server_url)
r = http.get(request_url(Endpoint.AUTH, server_url=server_url))
if r.status_code == 200:
return r.text.strip()
else:
raise RuntimeError("Failed to get auth token")
def _dl_variable(self,
dataset: str,
variable: str,
start_time: datetime,
stop_time: datetime,
fmt: str = None) -> Optional[SpeasyVariable]:
start_time, stop_time = start_time.strftime(
'%Y%m%dT%H%M%SZ'), stop_time.strftime('%Y%m%dT%H%M%SZ')
loader = _read_csv
fmt = "csv"
url = f"{self.__url}/dataviews/sp_phys/datasets/{dataset}/data/{start_time},{stop_time}/{variable}?format={fmt}"
headers = {"Accept": "application/json"}
log.debug(url)
resp = http.get(url, headers=headers)
if resp.status_code != 200:
raise CdaWebException(
f'Failed to get data with request: {url}, got {resp.status_code} HTTP response'
)
if not resp.ok or 'FileDescription' not in resp.json():
return None
return loader(resp.json()['FileDescription'][0]['Name'], variable)
def get_datasets(self,
dataview='sp_phys',
observatoryGroup=None,
instrumentType=None,
observatory=None,
instrument=None,
startDate=None,
stopDate=None,
idPattern=None,
labelPattern=None,
notesPattern=None):
args = []
if observatory is not None:
args.append(f'observatory={observatory}')
if observatoryGroup is not None:
args.append(f'observatoryGroup={observatoryGroup}')
if instrumentType is not None:
args.append(f'instrumentType={instrumentType}')
if instrument is not None:
args.append(f'instrument={instrument}')
if startDate is not None:
args.append(f'startDate={startDate}')
if stopDate is not None:
args.append(f'stopDate={stopDate}')
if idPattern is not None:
args.append(f'idPattern={idPattern}')
if labelPattern is not None:
args.append(f'labelPattern={labelPattern}')
if notesPattern is not None:
args.append(f'notesPattern={notesPattern}')
resp = http.get(self.__url + f'/dataviews/{dataview}/datasets?' +
"&".join(args),
headers={"Accept": "application/json"})
if not resp.ok:
return None
datasets = [dataset for dataset in resp.json()['DatasetDescription']]
return datasets
def send_indirect_request(
endpoint: Endpoint,
params: dict = None,
n_try: int = 3,
server_url: str = "http://amda.irap.omp.eu") -> str or None:
"""Send a request on the AMDA_Webservice REST service to the given endpoint with given parameters. The request is special in that the result
is the URL to an XML file containing the actual data we are interested in. That is why
we call :data:`requests.get()` twice in a row.
Parameters
----------
endpoint: Endpoint
target API endpoint on which the request will be performed
params: dict
request parameters
n_try: int
the number of retry in case of failure
server_url: str
the base server URL
Returns
-------
str or None
request result text, stripped of spaces and newlines
"""
next_url = send_request(endpoint=endpoint,
params=params,
n_try=n_try,
server_url=server_url)
if '<' in next_url and '>' in next_url:
next_url = next_url.split(">")[1].split("<")[0]
r = http.get(next_url)
if r.status_code == 200:
return r.text.strip()
return None