def test_cycle1(self, temp_dir):
# About 500 MB
alma = Alma()
alma.cache_location = temp_dir
target = 'NGC4945'
project_code = '2012.1.00912.S'
payload = {
'project_code': project_code,
'source_name_alma': target,
}
result = alma.query(payload=payload)
assert len(result) == 1
# Need new Alma() instances each time
a1 = alma()
uid_url_table_mous = a1.stage_data(result['Member ous id'])
a2 = alma()
uid_url_table_asdm = a2.stage_data(result['Asdm uid'])
# I believe the fixes as part of #495 have resulted in removal of a
# redundancy in the table creation, so a 1-row table is OK here.
# A 2-row table may not be OK any more, but that's what it used to
# be...
assert len(uid_url_table_asdm) == 1
assert len(uid_url_table_mous) >= 2 # now is len=3 (Nov 17, 2016)
# URL should look like:
# https://almascience.eso.org/dataPortal/requests/anonymous/944120962/ALMA/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar
# https://almascience.eso.org/rh/requests/anonymous/944222597/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar
small = uid_url_table_mous['size'] < 1
urls_to_download = uid_url_table_mous[small]['URL']
uri = urlparse(urls_to_download[0])
assert uri.path == (
'/dataPortal/requests/anonymous/{0}/ALMA/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar' # noqa
.format(a1._staging_log['staging_page_id']))
# THIS IS FAIL
# '2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar'
left = uid_url_table_mous['URL'][0].split("/")[-1]
assert left == '2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar'
right = uid_url_table_mous['uid'][0]
assert right == 'uid://A002/X5a9a13/X528'
assert left[15:-15] == right.replace(":", "_").replace("/", "_")
data = alma.download_and_extract_files(urls_to_download)
assert len(data) == 6
def test_cycle1(self, temp_dir):
# About 500 MB
alma = Alma()
alma.cache_location = temp_dir
target = "NGC4945"
project_code = "2012.1.00912.S"
payload = {"project_code": project_code, "source_name_alma": target}
result = alma.query(payload=payload)
assert len(result) == 1
# Need new Alma() instances each time
a1 = alma()
uid_url_table_mous = a1.stage_data(result["Member ous id"])
a2 = alma()
uid_url_table_asdm = a2.stage_data(result["Asdm uid"])
# I believe the fixes as part of #495 have resulted in removal of a
# redundancy in the table creation, so a 1-row table is OK here.
# A 2-row table may not be OK any more, but that's what it used to
# be...
assert len(uid_url_table_asdm) == 1
assert len(uid_url_table_mous) == 2
# URL should look like:
# https://almascience.eso.org/dataPortal/requests/anonymous/944120962/ALMA/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar
# https://almascience.eso.org/rh/requests/anonymous/944222597/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar
small = uid_url_table_mous["size"] < 1
urls_to_download = uid_url_table_mous[small]["URL"]
uri = urlparse(urls_to_download[0])
assert uri.path == (
"/dataPortal/requests/anonymous/{0}/ALMA/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar/2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar".format(
a1._staging_log["staging_page_id"]
)
)
# THIS IS FAIL
# '2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar'
left = uid_url_table_mous["URL"][0].split("/")[-1]
assert left == "2012.1.00912.S_uid___A002_X5a9a13_X528_001_of_001.tar"
right = uid_url_table_mous["uid"][0]
assert right == "uid://A002/X5a9a13/X528"
assert left[15:-15] == right.replace(":", "_").replace("/", "_")
data = alma.download_and_extract_files(urls_to_download)
assert len(data) == 6
def stage_data(self, uids):
"""
Stage ALMA data
Parameters
----------
uids : list or str
A list of valid UIDs or a single UID.
UIDs should have the form: 'uid://A002/X391d0b/X7b'
Returns
-------
data_file_table : Table
A table containing 3 columns: the UID, the file URL (for future
downloading), and the file size
"""
"""
With log.set_level(10)
INFO: Staging files... [astroquery.alma.core]
DEBUG: First request URL: https://almascience.eso.org/rh/submission [astroquery.alma.core]
DEBUG: First request payload: {'dataset': [u'ALMA+uid___A002_X3b3400_X90f']} [astroquery.alma.core]
DEBUG: First response URL: https://almascience.eso.org/rh/checkAuthenticationStatus/3f98de33-197e-4692-9afa-496842032ea9/submission [astroquery.alma.core]
DEBUG: Request ID: 3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
DEBUG: Submission URL: https://almascience.eso.org/rh/submission/3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
.DEBUG: Data list URL: https://almascience.eso.org/rh/requests/anonymous/786823226 [astroquery.alma.core]
"""
if isinstance(uids, six.string_types):
uids = [uids]
if not isinstance(uids, (list, tuple, np.ndarray)):
raise TypeError("Datasets must be given as a list of strings.")
log.info("Staging files...")
self._get_dataarchive_url()
url = urljoin(self.dataarchive_url, 'rh/submission')
log.debug("First request URL: {0}".format(url))
# 'ALMA+uid___A002_X391d0b_X7b'
payload = {'dataset': ['ALMA+' + clean_uid(uid) for uid in uids]}
log.debug("First request payload: {0}".format(payload))
self._staging_log = {'first_post_url': url}
# Request staging for the UIDs
# This component cannot be cached, since the returned data can change
# if new data are uploaded
response = self._request('POST', url, data=payload,
timeout=self.TIMEOUT, cache=False)
self._staging_log['initial_response'] = response
log.debug("First response URL: {0}".format(response.url))
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
time.sleep(1)
# CANNOT cache this stage: it not a real data page! results in
# infinite loops
response = self._request('POST', url, data=payload,
timeout=self.TIMEOUT, cache=False)
self._staging_log['initial_response'] = response
if 'j_spring_cas_security_check' in response.url:
log.warn("Staging request was not successful. Try again?")
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
raise RemoteServiceError("Could not access data. This error "
"can arise if the data are private and "
"you do not have access rights or are "
"not logged in.")
request_id = response.url.split("/")[-2]
assert len(request_id) == 36
self._staging_log['request_id'] = request_id
log.debug("Request ID: {0}".format(request_id))
# Submit a request for the specific request ID identified above
submission_url = urljoin(self.dataarchive_url,
os.path.join('rh/submission', request_id))
log.debug("Submission URL: {0}".format(submission_url))
self._staging_log['submission_url'] = submission_url
staging_submission = self._request('GET', submission_url, cache=True)
self._staging_log['staging_submission'] = staging_submission
staging_submission.raise_for_status()
data_page_url = staging_submission.url
self._staging_log['data_page_url'] = data_page_url
dpid = data_page_url.split("/")[-1]
assert len(dpid) == 9
self._staging_log['staging_page_id'] = dpid
# CANNOT cache this step: please_wait will happen infinitely
data_page = self._request('GET', data_page_url, cache=False)
self._staging_log['data_page'] = data_page
data_page.raise_for_status()
has_completed = False
while not has_completed:
time.sleep(1)
summary = self._request('GET', os.path.join(data_page_url,
'summary'),
cache=False)
summary.raise_for_status()
print(".", end='')
sys.stdout.flush()
has_completed = summary.json()['complete']
self._staging_log['summary'] = summary
summary.raise_for_status()
self._staging_log['json_data'] = json_data = summary.json()
username = self._username if hasattr(self, '_username') else 'anonymous'
# templates:
# https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
# 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar
# uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar
url_decomposed = urlparse(data_page_url)
base_url = ('{uri.scheme}://{uri.netloc}/'
'dataPortal/requests/{username}/'
'{staging_page_id}/ALMA'.format(uri=url_decomposed,
staging_page_id=dpid,
username=username,
))
tbl = self._json_summary_to_table(json_data, base_url=base_url)
# staging_root = BeautifulSoup(data_page.content)
# downloadFileURL = staging_root.find('form').attrs['action']
# data_list_url = os.path.split(downloadFileURL)[0]
# # Old version, unreliable: data_list_url = staging_submission.url
# log.debug("Data list URL: {0}".format(data_list_url))
# self._staging_log['data_list_url'] = data_list_url
# time.sleep(1)
# data_list_page = self._request('GET', data_list_url, cache=True)
# self._staging_log['data_list_page'] = data_list_page
# data_list_page.raise_for_status()
# if 'Error' in data_list_page.text:
# errormessage = staging_root.find(
# 'div', id='errorContent').string.strip()
# raise RemoteServiceError(errormessage)
# tbl = self._parse_staging_request_page(data_list_page)
return tbl
def stage_data(self, uids):
"""
Stage ALMA data
Parameters
----------
uids : list or str
A list of valid UIDs or a single UID.
UIDs should have the form: 'uid://A002/X391d0b/X7b'
Returns
-------
data_file_table : Table
A table containing 3 columns: the UID, the file URL (for future
downloading), and the file size
"""
"""
With log.set_level(10)
INFO: Staging files... [astroquery.alma.core]
DEBUG: First request URL: https://almascience.eso.org/rh/submission [astroquery.alma.core]
DEBUG: First request payload: {'dataset': [u'ALMA+uid___A002_X3b3400_X90f']} [astroquery.alma.core]
DEBUG: First response URL: https://almascience.eso.org/rh/checkAuthenticationStatus/3f98de33-197e-4692-9afa-496842032ea9/submission [astroquery.alma.core]
DEBUG: Request ID: 3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
DEBUG: Submission URL: https://almascience.eso.org/rh/submission/3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
.DEBUG: Data list URL: https://almascience.eso.org/rh/requests/anonymous/786823226 [astroquery.alma.core]
"""
if isinstance(uids, six.string_types + (np.bytes_, )):
uids = [uids]
if not isinstance(uids, (list, tuple, np.ndarray)):
raise TypeError("Datasets must be given as a list of strings.")
log.info("Staging files...")
self._get_dataarchive_url()
url = urljoin(self.dataarchive_url, 'rh/submission')
log.debug("First request URL: {0}".format(url))
# 'ALMA+uid___A002_X391d0b_X7b'
payload = {'dataset': ['ALMA+' + clean_uid(uid) for uid in uids]}
log.debug("First request payload: {0}".format(payload))
self._staging_log = {'first_post_url': url}
# Request staging for the UIDs
# This component cannot be cached, since the returned data can change
# if new data are uploaded
response = self._request('POST',
url,
data=payload,
timeout=self.TIMEOUT,
cache=False)
self._staging_log['initial_response'] = response
log.debug("First response URL: {0}".format(response.url))
if 'login' in response.url:
raise ValueError(
"You must login before downloading this data set.")
if response.status_code == 405:
if hasattr(self, '_last_successful_staging_log'):
log.warning(
"Error 405 received. If you have previously staged "
"the same UIDs, the result returned is probably "
"correct, otherwise you may need to create a fresh "
"astroquery.Alma instance.")
return self._last_successful_staging_log['result']
else:
raise HTTPError(
"Received an error 405: this may indicate you "
"have already staged the data. Try downloading "
"the file URLs directly with download_files.")
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
time.sleep(1)
# CANNOT cache this stage: it not a real data page! results in
# infinite loops
response = self._request('POST',
url,
data=payload,
timeout=self.TIMEOUT,
cache=False)
self._staging_log['initial_response'] = response
if 'j_spring_cas_security_check' in response.url:
log.warning("Staging request was not successful. Try again?")
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
raise RemoteServiceError("Could not access data. This error "
"can arise if the data are private and "
"you do not have access rights or are "
"not logged in.")
request_id = response.url.split("/")[-2]
self._staging_log['request_id'] = request_id
log.debug("Request ID: {0}".format(request_id))
# Submit a request for the specific request ID identified above
submission_url = urljoin(self.dataarchive_url,
url_helpers.join('rh/submission', request_id))
log.debug("Submission URL: {0}".format(submission_url))
self._staging_log['submission_url'] = submission_url
staging_submission = self._request('GET', submission_url, cache=True)
self._staging_log['staging_submission'] = staging_submission
staging_submission.raise_for_status()
data_page_url = staging_submission.url
self._staging_log['data_page_url'] = data_page_url
dpid = data_page_url.split("/")[-1]
self._staging_log['staging_page_id'] = dpid
# CANNOT cache this step: please_wait will happen infinitely
data_page = self._request('GET', data_page_url, cache=False)
self._staging_log['data_page'] = data_page
data_page.raise_for_status()
has_completed = False
while not has_completed:
time.sleep(1)
summary = self._request('GET',
url_helpers.join(data_page_url, 'summary'),
cache=False)
summary.raise_for_status()
print(".", end='')
sys.stdout.flush()
has_completed = summary.json()['complete']
self._staging_log['summary'] = summary
summary.raise_for_status()
self._staging_log['json_data'] = json_data = summary.json()
username = self.USERNAME if self.USERNAME else 'anonymous'
# templates:
# https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
# 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar
# uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar
url_decomposed = urlparse(data_page_url)
base_url = ('{uri.scheme}://{uri.netloc}/'
'dataPortal/requests/{username}/'
'{staging_page_id}/ALMA'.format(
uri=url_decomposed,
staging_page_id=dpid,
username=username,
))
tbl = self._json_summary_to_table(json_data, base_url=base_url)
self._staging_log['result'] = tbl
self._staging_log['file_urls'] = tbl['URL']
self._last_successful_staging_log = self._staging_log
return tbl
def stage_data(self, uids):
"""
Stage ALMA data
Parameters
----------
uids : list or str
A list of valid UIDs or a single UID.
UIDs should have the form: 'uid://A002/X391d0b/X7b'
Returns
-------
data_file_table : Table
A table containing 3 columns: the UID, the file URL (for future
downloading), and the file size
"""
"""
With log.set_level(10)
INFO: Staging files... [astroquery.alma.core]
DEBUG: First request URL: https://almascience.eso.org/rh/submission [astroquery.alma.core]
DEBUG: First request payload: {'dataset': [u'ALMA+uid___A002_X3b3400_X90f']} [astroquery.alma.core]
DEBUG: First response URL: https://almascience.eso.org/rh/checkAuthenticationStatus/3f98de33-197e-4692-9afa-496842032ea9/submission [astroquery.alma.core]
DEBUG: Request ID: 3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
DEBUG: Submission URL: https://almascience.eso.org/rh/submission/3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
.DEBUG: Data list URL: https://almascience.eso.org/rh/requests/anonymous/786823226 [astroquery.alma.core]
"""
if isinstance(uids, six.string_types + (np.bytes_,)):
uids = [uids]
if not isinstance(uids, (list, tuple, np.ndarray)):
raise TypeError("Datasets must be given as a list of strings.")
log.info("Staging files...")
self._get_dataarchive_url()
url = urljoin(self.dataarchive_url, 'rh/submission')
log.debug("First request URL: {0}".format(url))
# 'ALMA+uid___A002_X391d0b_X7b'
payload = {'dataset': ['ALMA+' + clean_uid(uid) for uid in uids]}
log.debug("First request payload: {0}".format(payload))
self._staging_log = {'first_post_url': url}
# Request staging for the UIDs
# This component cannot be cached, since the returned data can change
# if new data are uploaded
response = self._request('POST', url, data=payload,
timeout=self.TIMEOUT, cache=False)
self._staging_log['initial_response'] = response
log.debug("First response URL: {0}".format(response.url))
if response.status_code == 405:
if hasattr(self,'_last_successful_staging_log'):
log.warning("Error 405 received. If you have previously staged "
"the same UIDs, the result returned is probably "
"correct, otherwise you may need to create a fresh "
"astroquery.Alma instance.")
return self._last_successful_staging_log['result']
else:
raise HTTPError("Received an error 405: this may indicate you "
"have already staged the data. Try downloading "
"the file URLs directly with download_files.")
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
time.sleep(1)
# CANNOT cache this stage: it not a real data page! results in
# infinite loops
response = self._request('POST', url, data=payload,
timeout=self.TIMEOUT, cache=False)
self._staging_log['initial_response'] = response
if 'j_spring_cas_security_check' in response.url:
log.warning("Staging request was not successful. Try again?")
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
raise RemoteServiceError("Could not access data. This error "
"can arise if the data are private and "
"you do not have access rights or are "
"not logged in.")
request_id = response.url.split("/")[-2]
self._staging_log['request_id'] = request_id
log.debug("Request ID: {0}".format(request_id))
# Submit a request for the specific request ID identified above
submission_url = urljoin(self.dataarchive_url,
url_helpers.join('rh/submission', request_id))
log.debug("Submission URL: {0}".format(submission_url))
self._staging_log['submission_url'] = submission_url
staging_submission = self._request('GET', submission_url, cache=True)
self._staging_log['staging_submission'] = staging_submission
staging_submission.raise_for_status()
data_page_url = staging_submission.url
self._staging_log['data_page_url'] = data_page_url
dpid = data_page_url.split("/")[-1]
self._staging_log['staging_page_id'] = dpid
# CANNOT cache this step: please_wait will happen infinitely
data_page = self._request('GET', data_page_url, cache=False)
self._staging_log['data_page'] = data_page
data_page.raise_for_status()
has_completed = False
while not has_completed:
time.sleep(1)
summary = self._request('GET', url_helpers.join(data_page_url,
'summary'),
cache=False)
summary.raise_for_status()
print(".", end='')
sys.stdout.flush()
has_completed = summary.json()['complete']
self._staging_log['summary'] = summary
summary.raise_for_status()
self._staging_log['json_data'] = json_data = summary.json()
username = self.USERNAME if self.USERNAME else 'anonymous'
# templates:
# https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
# 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar
# uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar
url_decomposed = urlparse(data_page_url)
base_url = ('{uri.scheme}://{uri.netloc}/'
'dataPortal/requests/{username}/'
'{staging_page_id}/ALMA'.format(uri=url_decomposed,
staging_page_id=dpid,
username=username,
))
tbl = self._json_summary_to_table(json_data, base_url=base_url)
self._staging_log['result'] = tbl
self._staging_log['file_urls'] = tbl['URL']
self._last_successful_staging_log = self._staging_log
return tbl
def stage_data(self, uids):
"""
Stage ALMA data
Parameters
----------
uids : list or str
A list of valid UIDs or a single UID.
UIDs should have the form: 'uid://A002/X391d0b/X7b'
Returns
-------
data_file_table : Table
A table containing 3 columns: the UID, the file URL (for future
downloading), and the file size
"""
"""
With log.set_level(10)
INFO: Staging files... [astroquery.alma.core]
DEBUG: First request URL: https://almascience.eso.org/rh/submission [astroquery.alma.core]
DEBUG: First request payload: {'dataset': [u'ALMA+uid___A002_X3b3400_X90f']} [astroquery.alma.core]
DEBUG: First response URL: https://almascience.eso.org/rh/checkAuthenticationStatus/3f98de33-197e-4692-9afa-496842032ea9/submission [astroquery.alma.core]
DEBUG: Request ID: 3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
DEBUG: Submission URL: https://almascience.eso.org/rh/submission/3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
.DEBUG: Data list URL: https://almascience.eso.org/rh/requests/anonymous/786823226 [astroquery.alma.core]
"""
if isinstance(uids, six.string_types):
uids = [uids]
if not isinstance(uids, (list, tuple, np.ndarray)):
raise TypeError("Datasets must be given as a list of strings.")
log.info("Staging files...")
self._get_dataarchive_url()
url = urljoin(self.dataarchive_url, 'rh/submission')
log.debug("First request URL: {0}".format(url))
#'ALMA+uid___A002_X391d0b_X7b'
#payload = [('dataset','ALMA+'+clean_uid(uid)) for uid in uids]
payload = {'dataset': ['ALMA+' + clean_uid(uid) for uid in uids]}
log.debug("First request payload: {0}".format(payload))
self._staging_log = {'first_post_url': url}
# Request staging for the UIDs
# This component cannot be cached, since the returned data can change
# if new data are uploaded
response = self._request('POST',
url,
data=payload,
timeout=self.TIMEOUT,
cache=False)
self._staging_log['initial_response'] = response
log.debug("First response URL: {0}".format(response.url))
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
time.sleep(1)
# CANNOT cache this stage: it not a real data page! results in
# infinite loops
response = self._request('POST',
url,
data=payload,
timeout=self.TIMEOUT,
cache=False)
self._staging_log['initial_response'] = response
if 'j_spring_cas_security_check' in response.url:
log.warn("Staging request was not successful. Try again?")
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
raise RemoteServiceError("Could not access data. This error "
"can arise if the data are private and "
"you do not have access rights or are "
"not logged in.")
request_id = response.url.split("/")[-2]
assert len(request_id) == 36
self._staging_log['request_id'] = request_id
log.debug("Request ID: {0}".format(request_id))
# Submit a request for the specific request ID identified above
submission_url = urljoin(self.dataarchive_url,
os.path.join('rh/submission', request_id))
log.debug("Submission URL: {0}".format(submission_url))
self._staging_log['submission_url'] = submission_url
staging_submission = self._request('GET', submission_url, cache=True)
self._staging_log['staging_submission'] = staging_submission
staging_submission.raise_for_status()
data_page_url = staging_submission.url
self._staging_log['data_page_url'] = data_page_url
dpid = data_page_url.split("/")[-1]
assert len(dpid) == 9
self._staging_log['staging_page_id'] = dpid
# CANNOT cache this step: please_wait will happen infinitely
data_page = self._request('GET', data_page_url, cache=False)
self._staging_log['data_page'] = data_page
data_page.raise_for_status()
has_completed = False
while not has_completed:
time.sleep(1)
summary = self._request('GET',
os.path.join(data_page_url, 'summary'),
cache=False)
summary.raise_for_status()
print(".", end='')
sys.stdout.flush()
has_completed = summary.json()['complete']
self._staging_log['summary'] = summary
summary.raise_for_status()
self._staging_log['json_data'] = json_data = summary.json()
username = self._username if hasattr(self,
'_username') else 'anonymous'
# templates:
# https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
# 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar
# uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar
url_decomposed = urlparse(data_page_url)
base_url = ('{uri.scheme}://{uri.netloc}/'
'dataPortal/requests/{username}/'
'{staging_page_id}/ALMA'.format(
uri=url_decomposed,
staging_page_id=dpid,
username=username,
))
tbl = self._json_summary_to_table(json_data, base_url=base_url)
# staging_root = BeautifulSoup(data_page.content)
# downloadFileURL = staging_root.find('form').attrs['action']
# data_list_url = os.path.split(downloadFileURL)[0]
# # Old version, unreliable: data_list_url = staging_submission.url
# log.debug("Data list URL: {0}".format(data_list_url))
# self._staging_log['data_list_url'] = data_list_url
# time.sleep(1)
# data_list_page = self._request('GET', data_list_url, cache=True)
# self._staging_log['data_list_page'] = data_list_page
# data_list_page.raise_for_status()
# if 'Error' in data_list_page.text:
# errormessage = staging_root.find('div', id='errorContent').string.strip()
# raise RemoteServiceError(errormessage)
# tbl = self._parse_staging_request_page(data_list_page)
return tbl