def download_cutouts(sbid, username, password, destination_dir, catalogue_query, do_cutouts, cutout_radius_degrees=0.1):
# 2) Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id
print ("\n\n** Finding images and image cubes for scheduling block {} ... \n\n".format(sbid))
data_product_id_query = "select * from ivoa.obscore where obs_id = '" + str(
sbid) + "' and dataproduct_type = 'cube' and dataproduct_subtype in ('cont.restored.t0', 'spectral.restored.3d')"
filename = destination_dir + "image_cubes_" + str(sbid) + ".xml"
casda.sync_tap_query(data_product_id_query, filename, username, password)
image_cube_votable = parse(filename, pedantic=False)
results_array = image_cube_votable.get_table_by_id('results').array
service = 'cutout_service' if do_cutouts else 'async_service'
# 3) For each of the image cubes, query datalink to get the secure datalink details
print ("\n\n** Retrieving datalink for each image and image cube...\n\n")
authenticated_id_tokens = []
for image_cube_result in results_array:
image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8')
async_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username,
password,
service=service,
destination_dir=destination_dir)
if authenticated_id_token is not None and len(authenticated_id_tokens) < 10:
authenticated_id_tokens.append(authenticated_id_token)
if len(authenticated_id_tokens) == 0:
print ("No image cubes for scheduling_block_id " + str(sbid))
return 1
# Run the catalogue_query to find catalogue entries that are of interest
if do_cutouts:
print ("\n\n** Finding components in each image and image cube...\n\n")
filename = destination_dir + "catalogue_query_" + str(sbid) + ".xml"
casda.sync_tap_query(catalogue_query, filename, username, password)
catalogue_vo_table = parse(filename, pedantic=False)
catalogue_results_array = catalogue_vo_table.get_table_by_id('results').array
print ("\n\n** Found %d components...\n\n" % (len(catalogue_results_array)))
if len(catalogue_results_array) == 0:
print ("No catalogue entries matching the criteria found for scheduling_block_id " + str(sbid))
return 1
# For each source found in the catalogue query, create a position filter
pos_list = []
for entry in catalogue_results_array:
ra = entry['ra_deg_cont']
dec = entry['dec_deg_cont']
circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(cutout_radius_degrees)
pos_list.append(circle)
# Generate cutouts from each image around each source
# where there is no overlap an error file is generated but can be ignored.
job_location = casda.create_async_soda_job(authenticated_id_tokens)
if do_cutouts:
casda.add_params_to_async_job(job_location, 'pos', pos_list)
job_status = casda.run_async_job(job_location)
print ('\nJob finished with status %s address is %s\n\n' % (job_status, job_location))
if job_status != 'ERROR':
casda.download_all(job_location, destination_dir)
return 0
def download_images(ra,
dec,
username,
password,
destination_dir,
max_images=3):
if ra.find(':') > -1 or ra.find('h') > -1:
sky_loc = SkyCoord(ra,
dec,
frame='icrs',
unit=(units.hourangle, units.deg))
else:
sky_loc = SkyCoord(ra, dec, frame='icrs', unit=(units.deg, units.deg))
sky_region_query = 'CIRCLE %f %f %f' % (
sky_loc.ra.degree, sky_loc.dec.degree, search_radius_degrees)
# 2) Use CASDA SIA2 (secure) to query for the images associated with the given sky location
print("\n\n** Finding images and image cubes ... \n\n")
image_cube_votable = casda.find_images([
sky_region_query,
], username, password)
results_array = image_cube_votable.get_table_by_id('results').array
# print results_array
# 3) For each of the image cubes, query datalink to get the secure datalink details
print("\n\n** Retrieving datalink for each image and image cube...\n\n")
authenticated_id_tokens = []
for image_cube_result in results_array:
image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8')
# A test like the following can be used to further filter images retrieved.
# if row['dataproduct_subtype'].decode() == 'cont.restored.t0':
sync_url, authenticated_id_token = casda.get_service_link_and_id(
image_cube_id,
username,
password,
service='spectrum_generation_service',
destination_dir=destination_dir)
if authenticated_id_token is not None and (
max_images <= 0 or len(authenticated_id_tokens) < max_images):
authenticated_id_tokens.append(authenticated_id_token)
if len(authenticated_id_tokens) == 0:
print("\n\nNo image cubes available in sky location %f %f" %
(sky_loc.ra.degree, sky_loc.dec.degree))
return 1
# 4) Create the async job
job_location = casda.create_async_soda_job(authenticated_id_tokens)
# 5) Run the async job
print("\n\n** Starting the retrieval job...\n\n")
job_status = casda.run_async_job(job_location)
print('\nJob finished with status %s address is %s\n\n' %
(job_status, job_location))
if job_status != 'ERROR':
casda.download_all(job_location, destination_dir)
return 0
def main():
args = parseargs()
password = casda.get_opal_password(args.opal_password, args.password_file)
# Change this to choose which environment to use, prod is the default
casda.use_at()
start = time.time()
if args.destination_directory is not None and not os.path.exists(
args.destination_directory):
os.makedirs(args.destination_directory)
# Read cube dimensions
cube_dim = get_dimensions(args.cubeid)
# print("DIM=", cube_dim)
# Generate random locations in the cutout
pos_params, band_params = generate_random_cutouts(args, cube_dim)
# print("POS=", pos_params)
# print("BAND=", band_params)
# Get access to the cube - sia call then datalink
async_url, authenticated_id_token = casda.get_service_link_and_id(
args.cubeid,
args.opal_username,
password,
destination_dir=args.destination_directory)
print(async_url, authenticated_id_token)
# Create a job to retrieve the cutouts
job_location = casda.create_async_soda_job([authenticated_id_token],
soda_url=async_url)
casda.add_params_to_async_job(job_location, 'POS', pos_params)
casda.add_params_to_async_job(job_location, 'BAND', band_params)
print('\n\n Job will have %d cutouts.\n\n' %
(len(pos_params) * len(band_params)))
# Run and time the job
run_start = time.time()
status = casda.run_async_job(job_location)
run_end = time.time()
print('Job finished with status %s in %.02f s\n\n' %
(status, run_end - run_start))
# Optionally download
print("Job result available at ", casda.get_results_page(job_location))
if args.download:
casda.download_all(job_location, args.destination_directory)
# Report
end = time.time()
print('#### Cutout processing completed at %s ####' %
(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end))))
print('Job was processed in %.02f s' % (run_end - run_start))
print('Full run took %.02f s' % (end - start))
return 0
def produce_cutouts(source_list, image_id, username, password,
destination_dir):
# Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id
print("\n\n** Retreiving image details for %s ... \n\n" % image_id)
filename = destination_dir + str(image_id) + ".xml"
data_product_id_query = "select * from ivoa.obscore where obs_publisher_did = '" + image_id + \
"' and dataproduct_type = 'cube'"
casda.sync_tap_query(data_product_id_query,
filename,
username=username,
password=password)
image_cube_votable = votable.parse(filename, pedantic=False)
results_array = image_cube_votable.get_table_by_id('results').array
# For each of the image cubes, query datalink to get the secure datalink details
print("\n\n** Retrieving datalink for each image and image cube...\n\n")
authenticated_id_tokens = []
for image_cube_result in results_array:
image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8')
async_url, authenticated_id_token = casda.get_service_link_and_id(
image_cube_id,
username,
password,
service='cutout_service',
destination_dir=destination_dir)
if authenticated_id_token is not None:
authenticated_id_tokens.append(authenticated_id_token)
if len(authenticated_id_tokens) == 0:
print("No image cubes found")
return 1
# Create the async job
job_location = casda.create_async_soda_job(authenticated_id_tokens)
# For each entry in the results of the catalogue query, add the position filter as a parameter to the async job
cutout_filters = []
for sky_loc in source_list:
ra = sky_loc.ra.degree
dec = sky_loc.dec.degree
circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(
cutout_radius_degrees)
cutout_filters.append(circle)
casda.add_params_to_async_job(job_location, 'pos', cutout_filters)
# Run the job
status = casda.run_async_job(job_location)
# Download all of the files, or alert if it didn't complete
if status == 'COMPLETED':
print("\n\n** Downloading results...\n\n")
casda.download_all(job_location, destination_dir)
else:
print("Job did not complete: Status was %s." % status)
return 1
return 0
def main():
args = parseargs()
password = casda.get_opal_password(args.opal_password, args.password_file)
# Change this to choose which environment to use, prod is the default
casda.use_at()
start = time.time()
if args.destination_directory is not None and not os.path.exists(args.destination_directory):
os.makedirs(args.destination_directory)
# Read cube dimensions
cube_dim = get_dimensions(args.cubeid)
# print("DIM=", cube_dim)
# Generate random locations in the cutout
pos_params, band_params = generate_random_cutouts(args, cube_dim)
# print("POS=", pos_params)
# print("BAND=", band_params)
# Get access to the cube - sia call then datalink
async_url, authenticated_id_token = casda.get_service_link_and_id(
args.cubeid,
args.opal_username,
password,
destination_dir=args.destination_directory)
print (async_url, authenticated_id_token)
# Create a job to retrieve the cutouts
job_location = casda.create_async_soda_job([authenticated_id_token], soda_url=async_url)
casda.add_params_to_async_job(job_location, 'POS', pos_params)
casda.add_params_to_async_job(job_location, 'BAND', band_params)
print ('\n\n Job will have %d cutouts.\n\n' % (len(pos_params)*len(band_params)))
# Run and time the job
run_start = time.time()
status = casda.run_async_job(job_location)
run_end = time.time()
print('Job finished with status %s in %.02f s\n\n' % (status, run_end - run_start))
# Optionally download
print ("Job result available at ", casda.get_results_page(job_location))
if args.download:
casda.download_all(job_location, args.destination_directory)
# Report
end = time.time()
print('#### Cutout processing completed at %s ####'
% (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end))))
print('Job was processed in %.02f s' % (run_end - run_start))
print('Full run took %.02f s' % (end - start))
return 0
def produce_cutouts(source_list, image_id, username, password, destination_dir):
# Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id
print ("\n\n** Retreiving image details for %s ... \n\n" % image_id)
filename = destination_dir + str(image_id) + ".xml"
data_product_id_query = "select * from ivoa.obscore where obs_publisher_did = '" + image_id + \
"' and dataproduct_type = 'cube'"
casda.sync_tap_query(data_product_id_query, filename, username=username, password=password)
image_cube_votable = votable.parse(filename, pedantic=False)
results_array = image_cube_votable.get_table_by_id('results').array
# For each of the image cubes, query datalink to get the secure datalink details
print ("\n\n** Retrieving datalink for each image and image cube...\n\n")
authenticated_id_tokens = []
for image_cube_result in results_array:
image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8')
async_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username,
password,
service='cutout_service',
destination_dir=destination_dir)
if authenticated_id_token is not None:
authenticated_id_tokens.append(authenticated_id_token)
if len(authenticated_id_tokens) == 0:
print ("No image cubes found")
return 1
# Create the async job
job_location = casda.create_async_soda_job(authenticated_id_tokens)
# For each entry in the results of the catalogue query, add the position filter as a parameter to the async job
cutout_filters = []
for sky_loc in source_list:
ra = sky_loc.ra.degree
dec = sky_loc.dec.degree
circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(cutout_radius_degrees)
cutout_filters.append(circle)
casda.add_params_to_async_job(job_location, 'pos', cutout_filters)
# Run the job
status = casda.run_async_job(job_location)
# Download all of the files, or alert if it didn't complete
if status == 'COMPLETED':
print ("\n\n** Downloading results...\n\n")
casda.download_all(job_location, destination_dir)
else:
print ("Job did not complete: Status was %s." % status)
return 1
return 0
def extract_spectra(source_list, cutout_radius_degrees, opal_username,
opal_password, destination_directory):
"""
Extract spectra at the specified locations from ASKAP image cubes. Only cubes of subtype spectral.restored.3d will be
used in the extraction. Cubes that are either released or in a project that the opal user has pre-release access to
will be included.
:param source_list: The list of SkyCoord objects specifying source positions
:param cutout_radius_degrees: The number of degrees around the central point to extract or find.
:param opal_username: The user's user name on the ATNF's online proposal system OPAL (normally an email address)
:param opal_password: The user's OPAL password
:param destination_directory: The directory where the resulting files will be stored
:return: None
"""
# Build query to produce list of cubes for the sources.
pos_params = build_pos_criteria(source_list, cutout_radius_degrees)
# Run an immediate sia2 job to get the list of target cubes
votable = casda.find_images(pos_params, opal_username, opal_password)
table = votable.get_first_table()
authenticated_ids = []
for row in table.array:
# We are only interested in the restored spectral line cubes
if row['dataproduct_subtype'].decode() == 'spectral.restored.3d':
data_product_id = row['obs_publisher_did'].decode('utf-8')
async_url, authenticated_id_token = casda.get_service_link_and_id(
data_product_id,
opal_username,
opal_password,
service='spectrum_generation_service',
destination_dir=destination_directory)
authenticated_ids.append(authenticated_id_token)
if len(authenticated_ids) == 0:
print(
'\n\nNo image cubes were found which matched any of your sources.')
return
# Generate spectra at each location for each cube - no spectra file is generated where there is no overlap
job_location = casda.create_async_soda_job(authenticated_ids)
casda.add_params_to_async_job(job_location, 'pos', pos_params)
job_status = casda.run_async_job(job_location)
print('\nJob finished with status %s address is %s\n\n' %
(job_status, job_location))
if job_status != 'ERROR':
casda.download_all(job_location, destination_directory)
def extract_spectra(source_list, cutout_radius_degrees, opal_username, opal_password, destination_directory):
"""
Extract spectra at the specified locations from ASKAP image cubes. Only cubes of subtype spectral.restored.3d will be
used in the extraction. Cubes that are either released or in a project that the opal user has pre-release access to
will be included.
:param source_list: The list of SkyCoord objects specifying source positions
:param cutout_radius_degrees: The number of degrees around the central point to extract or find.
:param opal_username: The user's user name on the ATNF's online proposal system OPAL (normally an email address)
:param opal_password: The user's OPAL password
:param destination_directory: The directory where the resulting files will be stored
:return: None
"""
# Build query to produce list of cubes for the sources.
pos_params = build_pos_criteria(source_list, cutout_radius_degrees)
# Run an immediate sia2 job to get the list of target cubes
votable = casda.find_images(pos_params, opal_username, opal_password)
table = votable.get_first_table()
authenticated_ids = []
for row in table.array:
# We are only interested in the restored spectral line cubes
if row['dataproduct_subtype'].decode() == 'spectral.restored.3d':
data_product_id = row['obs_publisher_did'].decode('utf-8')
async_url, authenticated_id_token = casda.get_service_link_and_id(data_product_id, opal_username,
opal_password,
service='spectrum_generation_service',
destination_dir=destination_directory)
authenticated_ids.append(authenticated_id_token)
if len(authenticated_ids) == 0:
print('\n\nNo image cubes were found which matched any of your sources.')
return
# Generate spectra at each location for each cube - no spectra file is generated where there is no overlap
job_location = casda.create_async_soda_job(authenticated_ids)
casda.add_params_to_async_job(job_location, 'pos', pos_params)
job_status = casda.run_async_job(job_location)
print('\nJob finished with status %s address is %s\n\n' % (job_status, job_location))
if job_status != 'ERROR':
casda.download_all(job_location, destination_directory)
def download_images(ra, dec, username, password, destination_dir, max_images=3):
if ra.find(':') > -1 or ra.find('h') > -1:
sky_loc = SkyCoord(ra, dec, frame='icrs', unit=(units.hourangle, units.deg))
else:
sky_loc = SkyCoord(ra, dec, frame='icrs', unit=(units.deg, units.deg))
sky_region_query = 'CIRCLE %f %f %f' % (sky_loc.ra.degree, sky_loc.dec.degree, search_radius_degrees)
# 2) Use CASDA SIA2 (secure) to query for the images associated with the given sky location
print ("\n\n** Finding images and image cubes ... \n\n")
image_cube_votable = casda.find_images([sky_region_query, ], username, password)
results_array = image_cube_votable.get_table_by_id('results').array
# print results_array
# 3) For each of the image cubes, query datalink to get the secure datalink details
print ("\n\n** Retrieving datalink for each image and image cube...\n\n")
authenticated_id_tokens = []
for image_cube_result in results_array:
image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8')
# A test like the following can be used to further filter images retrieved.
# if row['dataproduct_subtype'].decode() == 'cont.restored.t0':
sync_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username,
password,
service='spectrum_generation_service',
destination_dir=destination_dir)
if authenticated_id_token is not None and (max_images <= 0 or len(authenticated_id_tokens) < max_images):
authenticated_id_tokens.append(authenticated_id_token)
if len(authenticated_id_tokens) == 0:
print ("\n\nNo image cubes available in sky location %f %f" % (sky_loc.ra.degree, sky_loc.dec.degree))
return 1
# 4) Create the async job
job_location = casda.create_async_soda_job(authenticated_id_tokens)
# 5) Run the async job
print ("\n\n** Starting the retrieval job...\n\n")
job_status = casda.run_async_job(job_location)
print ('\nJob finished with status %s address is %s\n\n' % (job_status, job_location))
if job_status != 'ERROR':
casda.download_all(job_location, destination_dir)
return 0
def download_catalogue(catalogue_type, work_folder):
query = queries[catalogue_type]
job_location = casda.create_async_tap_job()
print("Job is ", job_location)
casda.add_param_to_async_job(job_location, 'query', query)
casda.add_param_to_async_job(job_location, 'RESPONSEFORMAT', 'votable')
casda.add_param_to_async_job(job_location, 'request', 'doQuery')
casda.add_param_to_async_job(job_location, 'lang', 'ADQL')
job_status = casda.run_async_job(job_location)
print('\nJob finished with status %s address is %s\n\n' % (job_status, job_location))
if job_status != 'ERROR':
filenames = casda.download_all(job_location, work_folder)
return filenames[0]
def download_cutouts(sbid,
username,
password,
destination_dir,
catalogue_query,
do_cutouts,
cutout_radius_degrees=0.1):
# 2) Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id
print(
"\n\n** Finding images and image cubes for scheduling block {} ... \n\n"
.format(sbid))
data_product_id_query = "select * from ivoa.obscore where obs_id = '" + str(
sbid
) + "' and dataproduct_type = 'cube' and dataproduct_subtype in ('cont.restored.t0', 'spectral.restored.3d')"
filename = destination_dir + "image_cubes_" + str(sbid) + ".xml"
casda.sync_tap_query(data_product_id_query, filename, username, password)
image_cube_votable = parse(filename, pedantic=False)
results_array = image_cube_votable.get_table_by_id('results').array
service = 'cutout_service' if do_cutouts else 'async_service'
# 3) For each of the image cubes, query datalink to get the secure datalink details
print("\n\n** Retrieving datalink for each image and image cube...\n\n")
authenticated_id_tokens = []
for image_cube_result in results_array:
image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8')
async_url, authenticated_id_token = casda.get_service_link_and_id(
image_cube_id,
username,
password,
service=service,
destination_dir=destination_dir)
if authenticated_id_token is not None and len(
authenticated_id_tokens) < 10:
authenticated_id_tokens.append(authenticated_id_token)
if len(authenticated_id_tokens) == 0:
print("No image cubes for scheduling_block_id " + str(sbid))
return 1
# Run the catalogue_query to find catalogue entries that are of interest
if do_cutouts:
print("\n\n** Finding components in each image and image cube...\n\n")
filename = destination_dir + "catalogue_query_" + str(sbid) + ".xml"
casda.sync_tap_query(catalogue_query, filename, username, password)
catalogue_vo_table = parse(filename, pedantic=False)
catalogue_results_array = catalogue_vo_table.get_table_by_id(
'results').array
print("\n\n** Found %d components...\n\n" %
(len(catalogue_results_array)))
if len(catalogue_results_array) == 0:
print(
"No catalogue entries matching the criteria found for scheduling_block_id "
+ str(sbid))
return 1
# For each source found in the catalogue query, create a position filter
pos_list = []
for entry in catalogue_results_array:
ra = entry['ra_deg_cont']
dec = entry['dec_deg_cont']
circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(
cutout_radius_degrees)
pos_list.append(circle)
# Generate cutouts from each image around each source
# where there is no overlap an error file is generated but can be ignored.
job_location = casda.create_async_soda_job(authenticated_id_tokens)
if do_cutouts:
casda.add_params_to_async_job(job_location, 'pos', pos_list)
job_status = casda.run_async_job(job_location)
print('\nJob finished with status %s address is %s\n\n' %
(job_status, job_location))
if job_status != 'ERROR':
casda.download_all(job_location, destination_dir)
return 0
def produce_cutouts(source_list, proj, username, password, destination_dir,
cutout_radius_degrees):
# Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id
print("\n\n** Retreiving image details for %s ... \n\n" % proj)
filename = destination_dir + str(proj) + ".xml"
#Do initial filter of images, allow for 3 deg cone around position (get ASKAP image which is ~30 sq deg).
src_num = 0
for sky_loc in source_list:
src_num = src_num + 1
ra = sky_loc.ra.degree
dec = sky_loc.dec.degree
data_product_id_query = "select * from ivoa.obscore where obs_collection LIKE '%" + proj + \
"%' and dataproduct_subtype = 'cont.restored.t0' and pol_states = '/I/' and 1 = CONTAINS(POINT('ICRS',"+ str(ra) + ","+ str(dec) + "),s_region)"
casda.sync_tap_query(data_product_id_query,
filename,
username=username,
password=password)
image_cube_votable = votable.parse(filename, pedantic=False)
results_array = image_cube_votable.get_table_by_id('results').array
# For each of the image cubes, query datalink to get the secure datalink details
print(
"\n\n** Retrieving datalink for each image containing source number "
+ str(src_num) + " ...\n\n")
authenticated_id_tokens = []
for image_cube_result in results_array:
image_cube_id = image_cube_result['obs_publisher_did'].decode(
'utf-8')
async_url, authenticated_id_token = casda.get_service_link_and_id(
image_cube_id,
username,
password,
service='cutout_service',
destination_dir=destination_dir)
if authenticated_id_token is not None:
authenticated_id_tokens.append(authenticated_id_token)
if len(authenticated_id_tokens) == 0:
print("No image cubes found")
return 1
# Create the async job
job_location = casda.create_async_soda_job(authenticated_id_tokens)
# For each entry in the results of the catalogue query, add the position filter as a parameter to the async job
cutout_filters = []
circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(
cutout_radius_degrees)
cutout_filters.append(circle)
casda.add_params_to_async_job(job_location, 'pos', cutout_filters)
# Run the job
status = casda.run_async_job(job_location)
# Download all of the files, or alert if it didn't complete
if status == 'COMPLETED':
print("\n\n** Downloading results...\n\n")
casda.download_all(job_location, destination_dir)
returnflag = 0
else:
print("Job did not complete: Status was %s." % status)
returnflag = 1
if returnflag == 0:
return 0
else:
return 1