def download_cutouts(sbid, username, password, destination_dir, catalogue_query, do_cutouts, cutout_radius_degrees=0.1): # 2) Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id print ("\n\n** Finding images and image cubes for scheduling block {} ... \n\n".format(sbid)) data_product_id_query = "select * from ivoa.obscore where obs_id = '" + str( sbid) + "' and dataproduct_type = 'cube' and dataproduct_subtype in ('cont.restored.t0', 'spectral.restored.3d')" filename = destination_dir + "image_cubes_" + str(sbid) + ".xml" casda.sync_tap_query(data_product_id_query, filename, username, password) image_cube_votable = parse(filename, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array service = 'cutout_service' if do_cutouts else 'async_service' # 3) For each of the image cubes, query datalink to get the secure datalink details print ("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username, password, service=service, destination_dir=destination_dir) if authenticated_id_token is not None and len(authenticated_id_tokens) < 10: authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print ("No image cubes for scheduling_block_id " + str(sbid)) return 1 # Run the catalogue_query to find catalogue entries that are of interest if do_cutouts: print ("\n\n** Finding components in each image and image cube...\n\n") filename = destination_dir + "catalogue_query_" + str(sbid) + ".xml" casda.sync_tap_query(catalogue_query, filename, username, password) catalogue_vo_table = parse(filename, pedantic=False) catalogue_results_array = catalogue_vo_table.get_table_by_id('results').array print ("\n\n** Found %d components...\n\n" % (len(catalogue_results_array))) if len(catalogue_results_array) == 0: print ("No catalogue entries matching the criteria found for scheduling_block_id " + str(sbid)) return 1 # For each source found in the catalogue query, create a position filter pos_list = [] for entry in catalogue_results_array: ra = entry['ra_deg_cont'] dec = entry['dec_deg_cont'] circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(cutout_radius_degrees) pos_list.append(circle) # Generate cutouts from each image around each source # where there is no overlap an error file is generated but can be ignored. job_location = casda.create_async_soda_job(authenticated_id_tokens) if do_cutouts: casda.add_params_to_async_job(job_location, 'pos', pos_list) job_status = casda.run_async_job(job_location) print ('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': casda.download_all(job_location, destination_dir) return 0
def download_images(ra, dec, username, password, destination_dir, max_images=3): if ra.find(':') > -1 or ra.find('h') > -1: sky_loc = SkyCoord(ra, dec, frame='icrs', unit=(units.hourangle, units.deg)) else: sky_loc = SkyCoord(ra, dec, frame='icrs', unit=(units.deg, units.deg)) sky_region_query = 'CIRCLE %f %f %f' % ( sky_loc.ra.degree, sky_loc.dec.degree, search_radius_degrees) # 2) Use CASDA SIA2 (secure) to query for the images associated with the given sky location print("\n\n** Finding images and image cubes ... \n\n") image_cube_votable = casda.find_images([ sky_region_query, ], username, password) results_array = image_cube_votable.get_table_by_id('results').array # print results_array # 3) For each of the image cubes, query datalink to get the secure datalink details print("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') # A test like the following can be used to further filter images retrieved. # if row['dataproduct_subtype'].decode() == 'cont.restored.t0': sync_url, authenticated_id_token = casda.get_service_link_and_id( image_cube_id, username, password, service='spectrum_generation_service', destination_dir=destination_dir) if authenticated_id_token is not None and ( max_images <= 0 or len(authenticated_id_tokens) < max_images): authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print("\n\nNo image cubes available in sky location %f %f" % (sky_loc.ra.degree, sky_loc.dec.degree)) return 1 # 4) Create the async job job_location = casda.create_async_soda_job(authenticated_id_tokens) # 5) Run the async job print("\n\n** Starting the retrieval job...\n\n") job_status = casda.run_async_job(job_location) print('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': casda.download_all(job_location, destination_dir) return 0
def main(): args = parseargs() password = casda.get_opal_password(args.opal_password, args.password_file) # Change this to choose which environment to use, prod is the default casda.use_at() start = time.time() if args.destination_directory is not None and not os.path.exists( args.destination_directory): os.makedirs(args.destination_directory) # Read cube dimensions cube_dim = get_dimensions(args.cubeid) # print("DIM=", cube_dim) # Generate random locations in the cutout pos_params, band_params = generate_random_cutouts(args, cube_dim) # print("POS=", pos_params) # print("BAND=", band_params) # Get access to the cube - sia call then datalink async_url, authenticated_id_token = casda.get_service_link_and_id( args.cubeid, args.opal_username, password, destination_dir=args.destination_directory) print(async_url, authenticated_id_token) # Create a job to retrieve the cutouts job_location = casda.create_async_soda_job([authenticated_id_token], soda_url=async_url) casda.add_params_to_async_job(job_location, 'POS', pos_params) casda.add_params_to_async_job(job_location, 'BAND', band_params) print('\n\n Job will have %d cutouts.\n\n' % (len(pos_params) * len(band_params))) # Run and time the job run_start = time.time() status = casda.run_async_job(job_location) run_end = time.time() print('Job finished with status %s in %.02f s\n\n' % (status, run_end - run_start)) # Optionally download print("Job result available at ", casda.get_results_page(job_location)) if args.download: casda.download_all(job_location, args.destination_directory) # Report end = time.time() print('#### Cutout processing completed at %s ####' % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end)))) print('Job was processed in %.02f s' % (run_end - run_start)) print('Full run took %.02f s' % (end - start)) return 0
def produce_cutouts(source_list, image_id, username, password, destination_dir): # Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id print("\n\n** Retreiving image details for %s ... \n\n" % image_id) filename = destination_dir + str(image_id) + ".xml" data_product_id_query = "select * from ivoa.obscore where obs_publisher_did = '" + image_id + \ "' and dataproduct_type = 'cube'" casda.sync_tap_query(data_product_id_query, filename, username=username, password=password) image_cube_votable = votable.parse(filename, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array # For each of the image cubes, query datalink to get the secure datalink details print("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id( image_cube_id, username, password, service='cutout_service', destination_dir=destination_dir) if authenticated_id_token is not None: authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print("No image cubes found") return 1 # Create the async job job_location = casda.create_async_soda_job(authenticated_id_tokens) # For each entry in the results of the catalogue query, add the position filter as a parameter to the async job cutout_filters = [] for sky_loc in source_list: ra = sky_loc.ra.degree dec = sky_loc.dec.degree circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str( cutout_radius_degrees) cutout_filters.append(circle) casda.add_params_to_async_job(job_location, 'pos', cutout_filters) # Run the job status = casda.run_async_job(job_location) # Download all of the files, or alert if it didn't complete if status == 'COMPLETED': print("\n\n** Downloading results...\n\n") casda.download_all(job_location, destination_dir) else: print("Job did not complete: Status was %s." % status) return 1 return 0
def main(): args = parseargs() password = casda.get_opal_password(args.opal_password, args.password_file) # Change this to choose which environment to use, prod is the default casda.use_at() start = time.time() if args.destination_directory is not None and not os.path.exists(args.destination_directory): os.makedirs(args.destination_directory) # Read cube dimensions cube_dim = get_dimensions(args.cubeid) # print("DIM=", cube_dim) # Generate random locations in the cutout pos_params, band_params = generate_random_cutouts(args, cube_dim) # print("POS=", pos_params) # print("BAND=", band_params) # Get access to the cube - sia call then datalink async_url, authenticated_id_token = casda.get_service_link_and_id( args.cubeid, args.opal_username, password, destination_dir=args.destination_directory) print (async_url, authenticated_id_token) # Create a job to retrieve the cutouts job_location = casda.create_async_soda_job([authenticated_id_token], soda_url=async_url) casda.add_params_to_async_job(job_location, 'POS', pos_params) casda.add_params_to_async_job(job_location, 'BAND', band_params) print ('\n\n Job will have %d cutouts.\n\n' % (len(pos_params)*len(band_params))) # Run and time the job run_start = time.time() status = casda.run_async_job(job_location) run_end = time.time() print('Job finished with status %s in %.02f s\n\n' % (status, run_end - run_start)) # Optionally download print ("Job result available at ", casda.get_results_page(job_location)) if args.download: casda.download_all(job_location, args.destination_directory) # Report end = time.time() print('#### Cutout processing completed at %s ####' % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end)))) print('Job was processed in %.02f s' % (run_end - run_start)) print('Full run took %.02f s' % (end - start)) return 0
def produce_cutouts(source_list, image_id, username, password, destination_dir): # Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id print ("\n\n** Retreiving image details for %s ... \n\n" % image_id) filename = destination_dir + str(image_id) + ".xml" data_product_id_query = "select * from ivoa.obscore where obs_publisher_did = '" + image_id + \ "' and dataproduct_type = 'cube'" casda.sync_tap_query(data_product_id_query, filename, username=username, password=password) image_cube_votable = votable.parse(filename, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array # For each of the image cubes, query datalink to get the secure datalink details print ("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username, password, service='cutout_service', destination_dir=destination_dir) if authenticated_id_token is not None: authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print ("No image cubes found") return 1 # Create the async job job_location = casda.create_async_soda_job(authenticated_id_tokens) # For each entry in the results of the catalogue query, add the position filter as a parameter to the async job cutout_filters = [] for sky_loc in source_list: ra = sky_loc.ra.degree dec = sky_loc.dec.degree circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(cutout_radius_degrees) cutout_filters.append(circle) casda.add_params_to_async_job(job_location, 'pos', cutout_filters) # Run the job status = casda.run_async_job(job_location) # Download all of the files, or alert if it didn't complete if status == 'COMPLETED': print ("\n\n** Downloading results...\n\n") casda.download_all(job_location, destination_dir) else: print ("Job did not complete: Status was %s." % status) return 1 return 0
def extract_spectra(source_list, cutout_radius_degrees, opal_username, opal_password, destination_directory): """ Extract spectra at the specified locations from ASKAP image cubes. Only cubes of subtype spectral.restored.3d will be used in the extraction. Cubes that are either released or in a project that the opal user has pre-release access to will be included. :param source_list: The list of SkyCoord objects specifying source positions :param cutout_radius_degrees: The number of degrees around the central point to extract or find. :param opal_username: The user's user name on the ATNF's online proposal system OPAL (normally an email address) :param opal_password: The user's OPAL password :param destination_directory: The directory where the resulting files will be stored :return: None """ # Build query to produce list of cubes for the sources. pos_params = build_pos_criteria(source_list, cutout_radius_degrees) # Run an immediate sia2 job to get the list of target cubes votable = casda.find_images(pos_params, opal_username, opal_password) table = votable.get_first_table() authenticated_ids = [] for row in table.array: # We are only interested in the restored spectral line cubes if row['dataproduct_subtype'].decode() == 'spectral.restored.3d': data_product_id = row['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id( data_product_id, opal_username, opal_password, service='spectrum_generation_service', destination_dir=destination_directory) authenticated_ids.append(authenticated_id_token) if len(authenticated_ids) == 0: print( '\n\nNo image cubes were found which matched any of your sources.') return # Generate spectra at each location for each cube - no spectra file is generated where there is no overlap job_location = casda.create_async_soda_job(authenticated_ids) casda.add_params_to_async_job(job_location, 'pos', pos_params) job_status = casda.run_async_job(job_location) print('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': casda.download_all(job_location, destination_directory)
def extract_spectra(source_list, cutout_radius_degrees, opal_username, opal_password, destination_directory): """ Extract spectra at the specified locations from ASKAP image cubes. Only cubes of subtype spectral.restored.3d will be used in the extraction. Cubes that are either released or in a project that the opal user has pre-release access to will be included. :param source_list: The list of SkyCoord objects specifying source positions :param cutout_radius_degrees: The number of degrees around the central point to extract or find. :param opal_username: The user's user name on the ATNF's online proposal system OPAL (normally an email address) :param opal_password: The user's OPAL password :param destination_directory: The directory where the resulting files will be stored :return: None """ # Build query to produce list of cubes for the sources. pos_params = build_pos_criteria(source_list, cutout_radius_degrees) # Run an immediate sia2 job to get the list of target cubes votable = casda.find_images(pos_params, opal_username, opal_password) table = votable.get_first_table() authenticated_ids = [] for row in table.array: # We are only interested in the restored spectral line cubes if row['dataproduct_subtype'].decode() == 'spectral.restored.3d': data_product_id = row['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id(data_product_id, opal_username, opal_password, service='spectrum_generation_service', destination_dir=destination_directory) authenticated_ids.append(authenticated_id_token) if len(authenticated_ids) == 0: print('\n\nNo image cubes were found which matched any of your sources.') return # Generate spectra at each location for each cube - no spectra file is generated where there is no overlap job_location = casda.create_async_soda_job(authenticated_ids) casda.add_params_to_async_job(job_location, 'pos', pos_params) job_status = casda.run_async_job(job_location) print('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': casda.download_all(job_location, destination_directory)
def download_images(ra, dec, username, password, destination_dir, max_images=3): if ra.find(':') > -1 or ra.find('h') > -1: sky_loc = SkyCoord(ra, dec, frame='icrs', unit=(units.hourangle, units.deg)) else: sky_loc = SkyCoord(ra, dec, frame='icrs', unit=(units.deg, units.deg)) sky_region_query = 'CIRCLE %f %f %f' % (sky_loc.ra.degree, sky_loc.dec.degree, search_radius_degrees) # 2) Use CASDA SIA2 (secure) to query for the images associated with the given sky location print ("\n\n** Finding images and image cubes ... \n\n") image_cube_votable = casda.find_images([sky_region_query, ], username, password) results_array = image_cube_votable.get_table_by_id('results').array # print results_array # 3) For each of the image cubes, query datalink to get the secure datalink details print ("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') # A test like the following can be used to further filter images retrieved. # if row['dataproduct_subtype'].decode() == 'cont.restored.t0': sync_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username, password, service='spectrum_generation_service', destination_dir=destination_dir) if authenticated_id_token is not None and (max_images <= 0 or len(authenticated_id_tokens) < max_images): authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print ("\n\nNo image cubes available in sky location %f %f" % (sky_loc.ra.degree, sky_loc.dec.degree)) return 1 # 4) Create the async job job_location = casda.create_async_soda_job(authenticated_id_tokens) # 5) Run the async job print ("\n\n** Starting the retrieval job...\n\n") job_status = casda.run_async_job(job_location) print ('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': casda.download_all(job_location, destination_dir) return 0
def download_catalogue(catalogue_type, work_folder): query = queries[catalogue_type] job_location = casda.create_async_tap_job() print("Job is ", job_location) casda.add_param_to_async_job(job_location, 'query', query) casda.add_param_to_async_job(job_location, 'RESPONSEFORMAT', 'votable') casda.add_param_to_async_job(job_location, 'request', 'doQuery') casda.add_param_to_async_job(job_location, 'lang', 'ADQL') job_status = casda.run_async_job(job_location) print('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': filenames = casda.download_all(job_location, work_folder) return filenames[0]
def download_cutouts(sbid, username, password, destination_dir, catalogue_query, do_cutouts, cutout_radius_degrees=0.1): # 2) Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id print( "\n\n** Finding images and image cubes for scheduling block {} ... \n\n" .format(sbid)) data_product_id_query = "select * from ivoa.obscore where obs_id = '" + str( sbid ) + "' and dataproduct_type = 'cube' and dataproduct_subtype in ('cont.restored.t0', 'spectral.restored.3d')" filename = destination_dir + "image_cubes_" + str(sbid) + ".xml" casda.sync_tap_query(data_product_id_query, filename, username, password) image_cube_votable = parse(filename, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array service = 'cutout_service' if do_cutouts else 'async_service' # 3) For each of the image cubes, query datalink to get the secure datalink details print("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id( image_cube_id, username, password, service=service, destination_dir=destination_dir) if authenticated_id_token is not None and len( authenticated_id_tokens) < 10: authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print("No image cubes for scheduling_block_id " + str(sbid)) return 1 # Run the catalogue_query to find catalogue entries that are of interest if do_cutouts: print("\n\n** Finding components in each image and image cube...\n\n") filename = destination_dir + "catalogue_query_" + str(sbid) + ".xml" casda.sync_tap_query(catalogue_query, filename, username, password) catalogue_vo_table = parse(filename, pedantic=False) catalogue_results_array = catalogue_vo_table.get_table_by_id( 'results').array print("\n\n** Found %d components...\n\n" % (len(catalogue_results_array))) if len(catalogue_results_array) == 0: print( "No catalogue entries matching the criteria found for scheduling_block_id " + str(sbid)) return 1 # For each source found in the catalogue query, create a position filter pos_list = [] for entry in catalogue_results_array: ra = entry['ra_deg_cont'] dec = entry['dec_deg_cont'] circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str( cutout_radius_degrees) pos_list.append(circle) # Generate cutouts from each image around each source # where there is no overlap an error file is generated but can be ignored. job_location = casda.create_async_soda_job(authenticated_id_tokens) if do_cutouts: casda.add_params_to_async_job(job_location, 'pos', pos_list) job_status = casda.run_async_job(job_location) print('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': casda.download_all(job_location, destination_dir) return 0
def produce_cutouts(source_list, proj, username, password, destination_dir, cutout_radius_degrees): # Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id print("\n\n** Retreiving image details for %s ... \n\n" % proj) filename = destination_dir + str(proj) + ".xml" #Do initial filter of images, allow for 3 deg cone around position (get ASKAP image which is ~30 sq deg). src_num = 0 for sky_loc in source_list: src_num = src_num + 1 ra = sky_loc.ra.degree dec = sky_loc.dec.degree data_product_id_query = "select * from ivoa.obscore where obs_collection LIKE '%" + proj + \ "%' and dataproduct_subtype = 'cont.restored.t0' and pol_states = '/I/' and 1 = CONTAINS(POINT('ICRS',"+ str(ra) + ","+ str(dec) + "),s_region)" casda.sync_tap_query(data_product_id_query, filename, username=username, password=password) image_cube_votable = votable.parse(filename, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array # For each of the image cubes, query datalink to get the secure datalink details print( "\n\n** Retrieving datalink for each image containing source number " + str(src_num) + " ...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode( 'utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id( image_cube_id, username, password, service='cutout_service', destination_dir=destination_dir) if authenticated_id_token is not None: authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print("No image cubes found") return 1 # Create the async job job_location = casda.create_async_soda_job(authenticated_id_tokens) # For each entry in the results of the catalogue query, add the position filter as a parameter to the async job cutout_filters = [] circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str( cutout_radius_degrees) cutout_filters.append(circle) casda.add_params_to_async_job(job_location, 'pos', cutout_filters) # Run the job status = casda.run_async_job(job_location) # Download all of the files, or alert if it didn't complete if status == 'COMPLETED': print("\n\n** Downloading results...\n\n") casda.download_all(job_location, destination_dir) returnflag = 0 else: print("Job did not complete: Status was %s." % status) returnflag = 1 if returnflag == 0: return 0 else: return 1