def doquery_from_table(table):
#Execute a TAP+ query using information from a single row of an astropy table
server = TapPlus(url = table['queryurl'], \
default_protocol_is_https = True, \
verbose = True)
if table['upload_table_name'] == '':
result = server.launch_job_async(query = table['query'], verbose = True, \
dump_to_file = True, output_file = table['outfile'])
else:
result = server.launch_job_async(query = table['query'], verbose = True, \
dump_to_file = True, output_file = table['outfile'], \
upload_table_name = table['upload_table_name'], \
upload_resource = table['upload_resource'])
return result
def _download_ps_table():
exoarch = TapPlus(url="https://exoplanetarchive.ipac.caltech.edu/TAP")
job = exoarch.launch_job_async("select * from ps")
# TODO: fix dtype conversion
df = job.get_results().to_pandas()
setattr(df, "_is_ps_table", True)
return df
def getColumns(catalog):
from astroquery.utils.tap.core import TapPlus
tap = TapPlus(url="http://TAPVizieR.u-strasbg.fr/TAPVizieR/tap")
query_string = """SELECT Top 1 * FROM {}""".format(catalog)
job = tap.launch_job_async(query_string, dump_to_file=False)
res = job.get_results()
return res.columns
def master_table_query(args):
import numpy as np
import pandas as pd
import time
import datetime
from astroquery.utils.tap.core import TapPlus
if args.verbose > 0:
print
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print 'master_table_query'
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print
print datetime.datetime.now()
# Extract info from argument list...
outputFile = args.outputFile
object_id_lo = args.object_id_lo
object_id_hi = args.object_id_hi
# Establish TAP connection to SkyMapper database...
skymapper = TapPlus(url="http://skymappertap.asvo.nci.org.au/ncitap/tap")
# Formulate the query...
query = """SELECT * FROM dr1.master where object_id between %d and %d""" \
% (object_id_lo, object_id_hi)
if args.verbose > 0:
print "Query: ", query
# Submit the query as an asynchronous job...
if args.verbose > 0:
print "Query start: ", datetime.datetime.now()
job = skymapper.launch_job_async(query)
if args.verbose > 0:
print "Query completed: ", datetime.datetime.now()
# Retrieve the results as an astropy Table...
if args.verbose > 0:
print "Retrieval start: ", datetime.datetime.now()
master_table = job.get_results()
if args.verbose > 0:
print "Retrieval completed: ", datetime.datetime.now()
# Sort table by image_id
master_table.sort('object_id')
# Save table to outputFile...
if args.verbose > 0:
print "File output start: ", datetime.datetime.now()
master_table.write(outputFile)
if args.verbose > 0:
print "File output completed: ", datetime.datetime.now()
if args.verbose > 0: print
return 0
def test_abort_job(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
jobid = '12345'
# Phase POST response
responsePhase = DummyResponse()
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
responsePhase.set_data(method='POST',
context=None,
body=None,
headers=None)
req = "async/" + jobid + "/phase?PHASE=ABORT"
connHandler.set_response(req, responsePhase)
# Launch response
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(303)
responseLaunchJob.set_message("OK")
# list of list (httplib implementation for headers in response)
launchResponseHeaders = [[
'location', 'http://test:1111/tap/async/' + jobid
]]
responseLaunchJob.set_data(method='POST',
context=None,
body=None,
headers=launchResponseHeaders)
query = 'query'
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"QUERY": str(query)
}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
req = "async?" + sortedKey
connHandler.set_response(req, responseLaunchJob)
job = tap.launch_job_async(query, autorun=False)
assert job is not None, "Expected a valid job"
assert job.get_phase() == 'PENDING', \
"Wrong job phase. Expected: %s, found %s" % \
('PENDING', job.get_phase())
# abort job
job.abort()
assert job.get_phase() == 'ABORT', \
"Wrong job phase. Expected: %s, found %s" % \
('ABORT', job.get_phase())
# try to abort again
with pytest.raises(Exception):
job.abort()
def download_extended_catalog(self, merge=True):
""" Dowloads smaller catalog source from PS:
https://outerspace.stsci.edu/display/PANSTARRS/PS1+Source+extraction+and+catalogs
Here merging:
StackModelFitDeV:
Contains the de Vaucouleurs fit parameters for stack
detections brighter than some limit (is this a S/N cut or a mag limit?)
outside the galactic plane.
All filters are matched into a single row.
Given are mag, radius, axial ratio, position angle, RA, Dec, chisq of fit.
StackModelFitSer:
Contains the Sersic fit parameters for stack
detections brighter than magnitude 21.5 outside the galactic plane.
All filters are matched into a single row.
Given are mag, radius, axial ratio, Sersic index, position angle, RA, Dec, chisq of fit.
"""
if self.catdata is None:
self.download_catalog(update=True)
objid = ",".join(["%s" % s for s in self.catdata["objID"].values])
from astroquery.utils.tap.core import TapPlus
tap_service = TapPlus(
url="http://vao.stsci.edu/PS1DR2/tapservice.aspx")
job = tap_service.launch_job_async("""
SELECT *
FROM dbo.StackModelFitDeV AS dev
LEFT JOIN dbo.StackModelFitSer AS ser
ON dev.objID = ser.objID
WHERE
dev.objID IN ({})
""".format(objid))
results = job.get_results().to_pandas()
if merge:
self._catdata = pandas.merge(self.catdata, results, on="objID")
self._is_extended_cat_set = True
else:
self._is_extended_cat_set = False
return results
def query_gaia():
gaia = TapPlus(url='http://gea.esac.esa.int/tap-server/tap')
job = gaia.launch_job_async('select top 100000 ra, dec, phot_g_mean_mag \
from gaiadr1.gaia_source order by source_id')
results = job.get_results()
results = np.array(
[result.as_void().view((float, 3)) for result in results])
x = results[:, 2] * np.cos(results[:, 0]) * np.cos(results[:, 1])
y = results[:, 2] * np.sin(results[:, 0]) * np.cos(results[:, 1])
z = results[:, 2] * np.sin(results[:, 1])
sample_size = 1000
random_idns = np.random.choice(results.shape[0], sample_size)
ax.scatter(x[random_idns], y[random_idns], z[random_idns], s=3)
plt.show()
class ESAHubbleClass(BaseQuery):
"""
Class to init ESA Hubble Module and communicate with eHST TAP
"""
data_url = conf.DATA_ACTION
metadata_url = conf.METADATA_ACTION
TIMEOUT = conf.TIMEOUT
calibration_levels = {
0: "AUXILIARY",
1: "RAW",
2: "CALIBRATED",
3: "PRODUCT"
}
copying_string = "Copying file to {0}..."
def __init__(self, tap_handler=None):
super(ESAHubbleClass, self).__init__()
if tap_handler is None:
self._tap = TapPlus(url="http://hst.esac.esa.int"
"/tap-server/tap/")
else:
self._tap = tap_handler
def download_product(self,
observation_id,
calibration_level="RAW",
filename=None,
verbose=False):
"""
Download products from EHST
Parameters
----------
observation_id : string
id of the observation to be downloaded, mandatory
The identifier of the observation we want to retrieve, regardless
of whether it is simple or composite.
calibration_level : string
calibration level, optional, default 'RAW'
The identifier of the data reduction/processing applied to the
data. By default, the most scientifically relevant level will be
chosen. RAW, CALIBRATED, PRODUCT or AUXILIARY
filename : string
file name to be used to store the artifact, optional, default
None
File name for the observation.
verbose : bool
optional, default 'False'
flag to display information about the process
Returns
-------
None. It downloads the observation indicated
"""
params = {
"OBSERVATION_ID": observation_id,
"CALIBRATION_LEVEL": calibration_level
}
if filename is None:
filename = observation_id + ".tar"
response = self._request('GET',
self.data_url,
save=True,
cache=True,
params=params)
if verbose:
log.info(self.data_url + "?OBSERVATION_ID=" + observation_id +
"&CALIBRATION_LEVEL=" + calibration_level)
log.info(self.copying_string.format(filename))
shutil.move(response, filename)
def get_artifact(self, artifact_id, filename=None, verbose=False):
"""
Download artifacts from EHST. Artifact is a single Hubble product file.
Parameters
----------
artifact_id : string
id of the artifact to be downloaded, mandatory
The identifier of the physical product (file) we want to retrieve.
filename : string
file name to be used to store the artifact, optional, default None
File name for the artifact
verbose : bool
optional, default 'False'
flag to display information about the process
Returns
-------
None. It downloads the artifact indicated
"""
params = {"ARTIFACT_ID": artifact_id}
response = self._request('GET',
self.data_url,
save=True,
cache=True,
params=params)
if filename is None:
filename = artifact_id
if verbose:
log.info(self.data_url + "?ARTIFACT_ID=" + artifact_id)
log.info(self.copying_string.format(filename))
shutil.move(response, filename)
def get_postcard(self,
observation_id,
calibration_level="RAW",
resolution=256,
filename=None,
verbose=False):
"""
Download postcards from EHST
Parameters
----------
observation_id : string
id of the observation for which download the postcard, mandatory
The identifier of the observation we want to retrieve, regardless
of whether it is simple or composite.
calibration_level : string
calibration level, optional, default 'RAW'
The identifier of the data reduction/processing applied to the
data. By default, the most scientifically relevant level will be
chosen. RAW, CALIBRATED, PRODUCT or AUXILIARY
resolution : integer
postcard resolution, optional, default 256
Resolution of the retrieved postcard. 256 or 1024
filename : string
file name to be used to store the postcard, optional, default None
File name for the artifact
verbose : bool
optional, default 'False'
Flag to display information about the process
Returns
-------
None. It downloads the observation postcard indicated
"""
params = {
"RETRIEVAL_TYPE": "POSTCARD",
"OBSERVATION_ID": observation_id,
"CALIBRATION_LEVEL": calibration_level,
"RESOLUTION": resolution
}
response = self._request('GET',
self.data_url,
save=True,
cache=True,
params=params)
if filename is None:
filename = observation_id
if verbose:
log.info(self.data_url + "&".join([
"?RETRIEVAL_TYPE=POSTCARD", "OBSERVATION_ID=" +
observation_id, "CALIBRATION_LEVEL=" +
calibration_level, "RESOLUTION=" + str(resolution)
]))
log.info(self.copying_string.format(filename))
shutil.move(response, filename)
def cone_search(self,
coordinates,
radius=0.0,
filename=None,
output_format='votable',
cache=True):
"""
"""
coord = self._getCoordInput(coordinates, "coordinate")
radiusInGrades = float(radius / 60) # Converts to degrees
raHours, dec = commons.coord_to_radec(coord)
ra = raHours * 15.0 # Converts to degrees
payload = {
"RESOURCE_CLASS":
"OBSERVATION",
"ADQLQUERY":
"SELECT DISTINCT OBSERVATION,OBSERVATION.TYPE,"
"TARGET.MOVING_TARGET"
",TARGET.TARGET_NAME,TARGET.TARGET_DESCRIPTION,PROPOSAL."
"PROPOSAL_ID,PROPOSAL.PI_"
"NAME,PROPOSAL.PROPOSAL_TITLE,INSTRUMENT.INSTRUMENT_NAME,"
"PLANE.METADATA_PROVENANCE"
",PLANE.DATA_PRODUCT_TYPE,PLANE.SOFTWARE_VERSION,POSITION"
".RA,POSITION.DEC,POSITION."
"GAL_LAT,POSITION.GAL_LON,POSITION.ECL_LAT,POSITION.ECL_LON"
",POSITION.FOV_SIZE,ENERGY."
"WAVE_CENTRAL,ENERGY.WAVE_BANDWIDTH,ENERGY.WAVE_MAX,ENERGY"
".WAVE_MIN,ENERGY.FILTER FROM"
" FIELD_NOT_USED WHERE OBSERVATION.COLLECTION='HST' AND "
"PLANE.MAIN_SCIENCE_PLANE="
"'true' AND (OBSERVATION.TYPE='HST Composite' OR "
"OBSERVATION.TYPE='HST Singleton')"
" AND INTERSECTS(CIRCLE('ICRS', {0}, {1}, {2}"
"),POSITION)=1 AND PLANE.MAIN_SCIENCE_PLANE='true' "
"ORDER BY PROPOSAL.PROPOSAL_ID "
"DESC".format(str(ra), str(dec), str(radiusInGrades)),
"RETURN_TYPE":
str(output_format)
}
response = self._request('GET',
self.metadata_url,
params=payload,
cache=cache,
timeout=self.TIMEOUT)
if filename is None:
filename = "cone." + str(output_format)
if response is None:
table = None
else:
fileobj = BytesIO(response.content)
table = Table.read(fileobj, format=output_format)
# TODO: add "correct units" material here
return table
def query_metadata(self, output_format='votable', verbose=False):
return
def query_target(self,
name,
filename=None,
output_format='votable',
verbose=False):
"""
It executes a query over EHST and download the xml with the results.
Parameters
----------
name : string
target name to be requested, mandatory
filename : string
file name to be used to store the metadata, optional, default None
output_format : string
optional, default 'votable'
output format of the query
verbose : bool
optional, default 'False'
Flag to display information about the process
Returns
-------
Table with the result of the query. It downloads metadata as a file.
"""
params = {
"RESOURCE_CLASS": "OBSERVATION",
"SELECTED_FIELDS": "OBSERVATION",
"QUERY": "(TARGET.TARGET_NAME=='" + name + "')",
"RETURN_TYPE": str(output_format)
}
response = self._request('GET',
self.metadata_url,
save=True,
cache=True,
params=params)
if verbose:
log.info(self.metadata_url + "?RESOURCE_CLASS=OBSERVATION&"
"SELECTED_FIELDS=OBSERVATION&QUERY=(TARGET.TARGET_NAME"
"=='" + name + "')&RETURN_TYPE=" + str(output_format))
log.info(self.copying_string.format(filename))
if filename is None:
filename = "target.xml"
shutil.move(response, filename)
return modelutils.read_results_table_from_file(filename,
str(output_format))
def query_hst_tap(self,
query,
async_job=False,
output_file=None,
output_format="votable",
verbose=False):
"""Launches a synchronous or asynchronous job to query the HST tap
Parameters
----------
query : str, mandatory
query (adql) to be executed
async_job : bool, optional, default 'False'
executes the query (job) in asynchronous/synchronous mode (default
synchronous)
output_file : str, optional, default None
file name where the results are saved if dumpToFile is True.
If this parameter is not provided, the jobid is used instead
output_format : str, optional, default 'votable'
results format
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
A table object
"""
if async_job:
job = self._tap.launch_job_async(query=query,
output_file=output_file,
output_format=output_format,
verbose=False,
dump_to_file=output_file
is not None)
else:
job = self._tap.launch_job(query=query,
output_file=output_file,
output_format=output_format,
verbose=False,
dump_to_file=output_file is not None)
table = job.get_results()
return table
def query_criteria(self,
calibration_level=None,
data_product_type=None,
intent=None,
obs_collection=None,
instrument_name=None,
filters=None,
async_job=True,
output_file=None,
output_format="votable",
verbose=False,
get_query=False):
"""
Launches a synchronous or asynchronous job to query the HST tap
using calibration level, data product type, intent, collection,
instrument name, and filters as criteria to create and execute the
associated query.
Parameters
----------
calibration_level : str or int, optional
The identifier of the data reduction/processing applied to the
data. RAW (1), CALIBRATED (2), PRODUCT (3) or AUXILIARY (0)
data_product_type : str, optional
High level description of the product.
image, spectrum or timeseries.
intent : str, optional
The intent of the original observer in acquiring this observation.
SCIENCE or CALIBRATION
collection : list of str, optional
List of collections that are available in eHST catalogue.
HLA, HST
instrument_name : list of str, optional
Name(s) of the instrument(s) used to generate the dataset
filters : list of str, optional
Name(s) of the filter(s) used to generate the dataset
async_job : bool, optional, default 'True'
executes the query (job) in asynchronous/synchronous mode (default
synchronous)
output_file : str, optional, default None
file name where the results are saved if dumpToFile is True.
If this parameter is not provided, the jobid is used instead
output_format : str, optional, default 'votable'
results format
verbose : bool, optional, default 'False'
flag to display information about the process
get_query : bool, optional, default 'False'
flag to return the query associated to the criteria as the result
of this function.
Returns
-------
A table object
"""
parameters = []
if calibration_level is not None:
parameters.append("p.calibration_level LIKE '%{}%'".format(
self.__get_calibration_level(calibration_level)))
if data_product_type is not None:
if isinstance(data_product_type, str):
parameters.append("p.data_product_type LIKE '%{}%'".format(
data_product_type))
else:
raise ValueError("data_product_type must be a string")
if intent is not None:
if isinstance(intent, str):
parameters.append("o.intent LIKE '%{}%'".format(intent))
else:
raise ValueError("intent must be a string")
if self.__check_list_strings(obs_collection):
parameters.append("(o.collection LIKE '%{}%')".format(
"%' OR o.collection LIKE '%".join(obs_collection)))
if self.__check_list_strings(instrument_name):
parameters.append("(o.instrument_name LIKE '%{}%')".format(
"%' OR o.instrument_name LIKE '%".join(instrument_name)))
if self.__check_list_strings(filters):
parameters.append(
"(o.instrument_configuration LIKE '%{}%')".format(
"%' OR o.instrument_configuration "
"LIKE '%".join(filters)))
query = "select o.*, p.calibration_level, p.data_product_type "\
"from ehst.observation AS o LEFT JOIN ehst.plane as p "\
"on o.observation_uuid=p.observation_uuid"
if parameters:
query += " where({})".format(" AND ".join(parameters))
table = self.query_hst_tap(query=query,
async_job=async_job,
output_file=output_file,
output_format=output_format,
verbose=verbose)
if verbose:
log.info(query)
if get_query:
return query
return table
def __get_calibration_level(self, calibration_level):
condition = ""
if (calibration_level is not None):
if isinstance(calibration_level, str):
condition = calibration_level
elif isinstance(calibration_level, int):
if calibration_level < 4:
condition = self.calibration_levels[calibration_level]
else:
raise KeyError("Calibration level must be between 0 and 3")
else:
raise KeyError("Calibration level must be either "
"a string or an integer")
return condition
def __check_list_strings(self, list):
if list is None:
return False
if list and all(isinstance(elem, str) for elem in list):
return True
else:
raise ValueError("One of the lists is empty or there are "
"elements that are not strings")
def get_tables(self, only_names=True, verbose=False):
"""Get the available table in EHST TAP service
Parameters
----------
only_names : bool, TAP+ only, optional, default 'False'
True to load table names only
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
A list of tables
"""
tables = self._tap.load_tables(only_names=only_names,
include_shared_tables=False,
verbose=verbose)
if only_names is True:
table_names = []
for t in tables:
table_names.append(t.name)
return table_names
else:
return tables
def get_columns(self, table_name, only_names=True, verbose=False):
"""Get the available columns for a table in EHST TAP service
Parameters
----------
table_name : string, mandatory, default None
table name of which, columns will be returned
only_names : bool, TAP+ only, optional, default 'False'
True to load table names only
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
A list of columns
"""
tables = self._tap.load_tables(only_names=False,
include_shared_tables=False,
verbose=verbose)
columns = None
for t in tables:
if str(t.name) == str(table_name):
columns = t.columns
break
if columns is None:
raise ValueError("table name specified is not found in "
"EHST TAP service")
if only_names is True:
column_names = []
for c in columns:
column_names.append(c.name)
return column_names
else:
return columns
def _getCoordInput(self, value, msg):
if not (isinstance(value, str)
or isinstance(value, commons.CoordClasses)):
raise ValueError(
str(msg) + ""
" must be either a string or astropy.coordinates")
if isinstance(value, str):
coords = commons.parse_coordinates(value)
return coords
else:
return value
from astroquery.utils.tap.core import TapPlus
# Select CSC2 sources within 1 arcsec from qso_farfrom_xsources.
# I'm using the CSC2 catalogue hosted in Vizier because
# I cannot make it work using the CXC TAP service
csc = TapPlus(url="http://tapvizier.u-strasbg.fr/TAPVizieR/tap")
query = (
"SELECT t.SRCID_sdss, t.RA, t.DEC, "
"DISTANCE(POINT('ICRS', t.RA, t.DEC), POINT('ICRS', m.RAICRS, m.DEICRS)) as d, "
'm."2CXO", m.RAICRS, m.DEICRS, m."b_Fluxb", m.Fluxb, m."B_Fluxb", m.ExpAC, m.ExpHRC '
'FROM "IX/57/csc2master" as m, tap_upload.qso_farfrom_xsources as t '
"WHERE 1=CONTAINS(POINT('ICRS', t.RA, t.DEC), CIRCLE('ICRS', m.RAICRS, m.DEICRS, 1/3600.))"
)
# upload_resource is an Astropy table with the coordinates (columns RA, DEC)
# of all positions you want to query
job = csc.launch_job_async(
query=query,
upload_resource=upload_resource,
upload_table_name="qso_farfrom_xsources",
)
csc_data = job.get_results()
csc_data["SRCID_sdss"] = csc_data["SRCID_sdss"].astype(str)
csc_data["_2CXO"] = csc_data["_2CXO"].astype(str)
csc_data.rename_column("_2CXO", "2CXO")
filename = data_folder.joinpath("qso_sdss_farfrom_4xmm_csc2.fits")
csc_data.write(str(filename), format="fits", overwrite=True)
def nobs_fs_table_query(args):
import numpy as np
import pandas as pd
import time
import datetime
from astroquery.utils.tap.core import TapPlus
if args.verbose>0:
print
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print 'nobs_fs_table_query'
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print
print datetime.datetime.now()
# Extract info from argument list...
outputFile=args.outputFile
# Establish TAP connection to SkyMapper database...
skymapper = TapPlus(url="http://skymappertap.asvo.nci.org.au/ncitap/tap")
# Formulate the query...
query = """SELECT image_id, count(*) as nobj FROM dr1.fs_photometry group by image_id"""
if args.verbose>0:
print "Query: ", query
# Submit the query as an asynchronous job...
if args.verbose>0:
print "Query start: ", datetime.datetime.now()
job = skymapper.launch_job_async(query)
if args.verbose>0:
print "Query completed: ", datetime.datetime.now()
# Retrieve the results as an astropy Table...
if args.verbose>0:
print "Retrieval start: ", datetime.datetime.now()
nobs_fs_table = job.get_results()
if args.verbose>0:
print "Retrieval completed: ", datetime.datetime.now()
# Sort table by image_id
nobs_fs_table.sort('image_id')
# Convert the results into a pandas data frame...
# (Could also use Table.to_pandas, but, unless careful,
# the image_id gets converted from an int64 to a float.)
d = {'nobj':np.array(nobs_fs_table['nobj']), 'image_id':np.array(nobs_fs_table['image_id'])}
df = pd.DataFrame(d)
# Save dataframe to outputFile...
if args.verbose>0:
print "File output start: ", datetime.datetime.now()
df.to_csv(outputFile,index=False)
if args.verbose>0:
print "File output completed: ", datetime.datetime.now()
if args.verbose>0: print
return 0
class ESAHubbleClass(BaseQuery):
"""
Class to init ESA Hubble Module and communicate with eHST TAP
"""
data_url = conf.DATA_ACTION
metadata_url = conf.METADATA_ACTION
target_url = conf.TARGET_ACTION
TIMEOUT = conf.TIMEOUT
calibration_levels = {
0: "AUXILIARY",
1: "RAW",
2: "CALIBRATED",
3: "PRODUCT"
}
product_types = ["PRODUCT", "SCIENCE_PRODUCT", "POSTCARD"]
copying_string = "Copying file to {0}..."
def __init__(self, tap_handler=None):
super().__init__()
if tap_handler is None:
self._tap = TapPlus(url="http://hst.esac.esa.int"
"/tap-server/tap/")
else:
self._tap = tap_handler
def download_product(self,
observation_id,
*,
calibration_level=None,
filename=None,
verbose=False,
product_type=None):
"""
Download products from EHST based on their observation ID and the
calibration level or the product type.
Parameters
----------
observation_id : string
id of the observation to be downloaded, mandatory
The identifier of the observation we want to retrieve, regardless
of whether it is simple or composite.
calibration_level : string
calibration level, optional
The identifier of the data reduction/processing applied to the
data. By default, the most scientifically relevant level will be
chosen. RAW, CALIBRATED, PRODUCT or AUXILIARY
filename : string
file name to be used to store the artifact, optional, default
None
File name for the observation.
verbose : bool
optional, default 'False'
flag to display information about the process
product_type : string
type of product retrieval, optional
PRODUCT, SCIENCE_PRODUCT or POSTCARD
Returns
-------
None. It downloads the observation indicated
"""
params = {
"OBSERVATION_ID": observation_id,
"USERNAME": "******"
}
url = self.data_url + "?OBSERVATION_ID=" + observation_id
url += "&USERNAME="******"ehst-astroquery"
if filename is None:
filename = observation_id + ".tar"
if calibration_level:
params["CALIBRATION_LEVEL"] = calibration_level
url += "&CALIBRATION_LEVEL=" + calibration_level
if product_type:
self.__validate_product_type(product_type)
params["RETRIEVAL_TYPE"] = product_type
filename = self._get_product_filename(product_type, filename)
url += "&RETRIEVAL_TYPE=" + params["RETRIEVAL_TYPE"]
response = self._request('GET',
self.data_url,
save=True,
cache=True,
params=params)
if verbose:
log.info(url)
log.info(self.copying_string.format(filename))
shutil.move(response, filename)
def get_member_observations(self, observation_id):
"""
Returns the related members of simple and composite observations
Parameters
----------
observation_id : str
Observation identifier.
Returns
-------
A list of strings with the observation_id of the associated
observations
"""
observation_type = self.get_observation_type(observation_id)
if 'Composite' in observation_type:
oids = self._select_related_members(observation_id)
elif 'Simple' in observation_type:
oids = self._select_related_composite(observation_id)
else:
raise ValueError("Invalid observation id")
return oids
def get_hap_hst_link(self, observation_id):
"""
Returns the related members of hap and hst observations
Parameters
----------
observation_id : string
id of the observation to be downloaded, mandatory
The identifier of the observation we want to retrieve, regardless
of whether it is simple or composite.
Returns
-------
A list of strings with the observation_id of the associated
observations
"""
observation_type = self.get_observation_type(observation_id)
if 'Composite' in observation_type:
raise ValueError(
"HAP-HST link is only available for simple observations. Input observation is Composite."
)
elif 'HAP' in observation_type:
oids = self._select_related_members(observation_id)
elif 'HST' in observation_type:
query = f"select observation_id from ehst.observation where obs_type='HAP Simple' and members like '%{observation_id}%'"
job = self.query_hst_tap(query=query)
oids = job["observation_id"].pformat(show_name=False)
else:
raise ValueError("Invalid observation id")
return oids
def get_observation_type(self, observation_id):
"""
Returns the type of an observation
Parameters
----------
observation_id : string
id of the observation to be downloaded, mandatory
The identifier of the observation we want to retrieve, regardless
of whether it is simple or composite.
Returns
-------
String with the observation type
"""
if observation_id is None:
raise ValueError("Please input an observation id")
query = f"select obs_type from ehst.observation where observation_id='{observation_id}'"
job = self.query_hst_tap(query=query)
if any(job["obs_type"]):
obs_type = self._get_decoded_string(string=job["obs_type"][0])
else:
raise ValueError("Invalid Observation ID")
return obs_type
def _select_related_members(self, observation_id):
query = f"select members from ehst.observation where observation_id='{observation_id}'"
job = self.query_hst_tap(query=query)
oids = self._get_decoded_string(string=job["members"][0]).replace(
"caom:HST/", "").split(" ")
return oids
def _select_related_composite(self, observation_id):
query = f"select observation_id from ehst.observation where members like '%{observation_id}%'"
job = self.query_hst_tap(query=query)
oids = job["observation_id"].pformat(show_name=False)
return oids
def __validate_product_type(self, product_type):
if (product_type not in self.product_types):
raise ValueError("This product_type is not allowed")
def _get_product_filename(self, product_type, filename):
if (product_type == "PRODUCT"):
return filename
elif (product_type == "SCIENCE_PRODUCT"):
log.info("This is a SCIENCE_PRODUCT, the filename will be "
"renamed to " + filename + ".fits.gz")
return filename + ".fits.gz"
else:
log.info("This is a POSTCARD, the filename will be "
"renamed to " + filename + ".jpg")
return filename + ".jpg"
def get_artifact(self, artifact_id, filename=None, verbose=False):
"""
Download artifacts from EHST. Artifact is a single Hubble product file.
Parameters
----------
artifact_id : string
id of the artifact to be downloaded, mandatory
The identifier of the physical product (file) we want to retrieve.
filename : string
file name to be used to store the artifact, optional, default None
File name for the artifact
verbose : bool
optional, default 'False'
flag to display information about the process
Returns
-------
None. It downloads the artifact indicated
"""
params = {"ARTIFACT_ID": artifact_id, "USERNAME": "******"}
response = self._request('GET',
self.data_url,
save=True,
cache=True,
params=params)
if filename is None:
filename = artifact_id
if verbose:
log.info(self.data_url + "?ARTIFACT_ID=" + artifact_id +
"&USERNAME=ehst-astroquery")
log.info(self.copying_string.format(filename))
shutil.move(response, filename)
def get_postcard(self,
observation_id,
calibration_level="RAW",
resolution=256,
filename=None,
verbose=False):
"""
Download postcards from EHST
Parameters
----------
observation_id : string
id of the observation for which download the postcard, mandatory
The identifier of the observation we want to retrieve, regardless
of whether it is simple or composite.
calibration_level : string
calibration level, optional, default 'RAW'
The identifier of the data reduction/processing applied to the
data. By default, the most scientifically relevant level will be
chosen. RAW, CALIBRATED, PRODUCT or AUXILIARY
resolution : integer
postcard resolution, optional, default 256
Resolution of the retrieved postcard. 256 or 1024
filename : string
file name to be used to store the postcard, optional, default None
File name for the artifact
verbose : bool
optional, default 'False'
Flag to display information about the process
Returns
-------
None. It downloads the observation postcard indicated
"""
params = {
"RETRIEVAL_TYPE": "POSTCARD",
"OBSERVATION_ID": observation_id,
"CALIBRATION_LEVEL": calibration_level,
"RESOLUTION": resolution,
"USERNAME": "******"
}
response = self._request('GET',
self.data_url,
save=True,
cache=True,
params=params)
if filename is None:
filename = observation_id
if verbose:
log.info(self.data_url + "&".join([
"?RETRIEVAL_TYPE=POSTCARD", "OBSERVATION_ID=" +
observation_id, "CALIBRATION_LEVEL=" +
calibration_level, "RESOLUTION=" +
str(resolution), "USERNAME=ehst-astroquery"
]))
log.info(self.copying_string.format(filename))
shutil.move(response, filename)
def cone_search(self,
coordinates,
radius,
filename=None,
output_format='votable',
cache=True,
async_job=False,
verbose=False):
"""
To execute a cone search defined by a coordinate and a radius
Parameters
----------
coordinates : astropy.coordinate, mandatory
coordinates of the center in the cone search
radius : float or quantity
radius in arcmin (int, float) or quantity of the cone_search
filename : str, default None
Path and name of the file to store the results.
If the filename is defined, the file will be
automatically saved
output_format : string
results format. Options are:
'votable': str, binary VOTable format
'csv': str, comma-separated values format
async_job : bool, optional, default 'False'
executes the query (job) in asynchronous/synchronous mode (default
synchronous)
cache : bool
optional, default 'True'
Flag to save the results in the local cache
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
astropy.table.Table with the result of the cone_search
"""
coord = self._getCoordInput(coordinates)
if type(radius) == int or type(radius) == float:
radius_in_grades = Angle(radius, units.arcmin).deg
else:
radius_in_grades = radius.to(units.deg).value
ra = coord.ra.deg
dec = coord.dec.deg
query = (
"select o.observation_id, "
"o.start_time, o.end_time, o.start_time_mjd, "
"o.end_time_mjd, o.exposure_duration, o.release_date, "
"o.run_id, o.program_id, o.set_id, o.collection, "
"o.members_number, o.instrument_configuration, "
"o.instrument_name, o.obs_type, o.target_moving, "
"o.target_name, o.target_description, o.proposal_id, "
"o.pi_name, prop.title, pl.metadata_provenance, "
"pl.data_product_type, pl.software_version, pos.ra, "
"pos.dec, pos.gal_lat, pos.gal_lon, pos.ecl_lat, "
"pos.ecl_lon, pos.fov_size, en.wave_central, "
"en.wave_bandwidth, en.wave_max, en.wave_min, "
"en.filter from ehst.observation o join ehst.proposal "
"prop on o.proposal_id=prop.proposal_id join ehst.plane "
"pl on pl.observation_id=o.observation_id join "
"ehst.position pos on pos.plane_id = pl.plane_id join "
"ehst.energy en on en.plane_id=pl.plane_id where "
"pl.main_science_plane='true' and 1=CONTAINS(POINT('ICRS', "
f"pos.ra, pos.dec),CIRCLE('ICRS', {str(ra)}, {str(dec)}, {str(radius_in_grades)})) order "
"by prop.proposal_id desc")
print("type: " + str(type(query)))
if verbose:
log.info(query)
table = self.query_hst_tap(query=query,
async_job=async_job,
output_file=filename,
output_format=output_format,
verbose=verbose)
return table
def cone_search_criteria(self,
radius,
target=None,
coordinates=None,
calibration_level=None,
data_product_type=None,
intent=None,
obs_collection=None,
instrument_name=None,
filters=None,
async_job=True,
filename=None,
output_format='votable',
save=False,
cache=True,
verbose=False):
"""
To execute a cone search defined by a coordinate (an
astropy.coordinate element or a target name which is resolved),
a radius and a set of criteria to filter the results. This function
comprises the outputs of query_target, cone_search and query_criteria
methods.
Parameters
----------
radius : float or quantity
radius in arcmin (int, float) or quantity of the cone_search
target : str, mandatory if no coordinates is provided
name of the target, that will act as center in the cone search
coordinates : astropy.coordinate, mandatory if no target is provided
coordinates of the center in the cone search
calibration_level : str or int, optional
The identifier of the data reduction/processing applied to the
data. RAW (1), CALIBRATED (2), PRODUCT (3) or AUXILIARY (0)
data_product_type : str, optional
High level description of the product.
image, spectrum or timeseries.
intent : str, optional
The intent of the original observer in acquiring this observation.
SCIENCE or CALIBRATION
collection : list of str, optional
List of collections that are available in eHST catalogue.
HLA, HST
instrument_name : list of str, optional
Name(s) of the instrument(s) used to generate the dataset
filters : list of str, optional
Name(s) of the filter(s) used to generate the dataset
async_job : bool, optional, default 'False'
executes the query (job) in asynchronous/synchronous mode (default
synchronous)
filename : str, default None
Path and name of the file to store the results.
If the filename is defined, the file will be
automatically saved
output_format : string
results format. Options are:
'votable': str, binary VOTable format
'csv': str, comma-separated values format
save : bool
optional, default 'False'
Flag to save the result in a file. If the filename
is not defined, it will use a formatted name to save
the file
cache : bool
optional, default 'True'
Flag to save the results in the local cache
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
astropy.table.Table with the result of the cone_search
"""
crit_query = self.query_criteria(calibration_level=calibration_level,
data_product_type=data_product_type,
intent=intent,
obs_collection=obs_collection,
instrument_name=instrument_name,
filters=filters,
async_job=True,
get_query=True)
if crit_query.endswith(")"):
crit_query = crit_query[:-1] + " AND "
else:
crit_query = crit_query + " WHERE ("
if target and coordinates:
raise TypeError("Please use only target or coordinates as"
"parameter.")
if target:
coord = self._query_tap_target(target)
else:
coord = self._getCoordInput(coordinates)
ra = coord.ra.deg
dec = coord.dec.deg
if type(radius) == int or type(radius) == float:
radius_in_grades = Angle(radius, units.arcmin).deg
else:
radius_in_grades = radius.to(units.deg).value
cone_query = "1=CONTAINS(POINT('ICRS', pos.ra, pos.dec)," \
"CIRCLE('ICRS', {0}, {1}, {2}))". \
format(str(ra), str(dec), str(radius_in_grades))
query = "{}{})".format(crit_query, cone_query)
if verbose:
log.info(query)
table = self.query_hst_tap(query=query,
async_job=async_job,
output_file=filename,
output_format=output_format,
verbose=verbose)
return table
def _query_tap_target(self, target):
try:
params = {
"TARGET_NAME": target,
"RESOLVER_TYPE": "SN",
"FORMAT": "json"
}
target_response = self._request('GET',
self.target_url,
cache=True,
params=params)
target_result = target_response.json()['data'][0]
ra = target_result['RA_DEGREES']
dec = target_result['DEC_DEGREES']
return SkyCoord(ra=ra, dec=dec, unit="deg")
except KeyError as e:
raise ValueError("This target cannot be resolved")
def query_metadata(self, output_format='votable', verbose=False):
return
def query_target(self,
name,
*,
filename=None,
output_format='votable',
verbose=False,
async_job=False,
radius=7):
"""
It executes a query over EHST and download the xml with the results.
Parameters
----------
name : string
target name to be requested, mandatory
filename : string
file name to be used to store the metadata, optional, default None
output_format : string
optional, default 'votable'
output format of the query
verbose : bool
optional, default 'False'
Flag to display information about the process
async_job : bool, optional, default 'False'
executes the query (job) in asynchronous/synchronous mode (default
synchronous)
radius : int
optional, default 7
radius in arcmin (int, float) or quantity of the cone_search
Returns
-------
Table with the result of the query. It downloads metadata as a file.
"""
coordinates = self._query_tap_target(name)
table = self.cone_search(coordinates,
radius,
filename=filename,
output_format=output_format,
verbose=verbose,
async_job=async_job)
return table
def query_hst_tap(self,
query,
async_job=False,
output_file=None,
output_format="votable",
verbose=False):
"""Launches a synchronous or asynchronous job to query the HST tap
Parameters
----------
query : str, mandatory
query (adql) to be executed
async_job : bool, optional, default 'False'
executes the query (job) in asynchronous/synchronous mode (default
synchronous)
output_file : str, optional, default None
file name where the results are saved if dumpToFile is True.
If this parameter is not provided, the jobid is used instead
output_format : str, optional, default 'votable'
results format
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
A table object
"""
if async_job:
job = self._tap.launch_job_async(query=query,
output_file=output_file,
output_format=output_format,
verbose=verbose,
dump_to_file=output_file
is not None)
else:
job = self._tap.launch_job(query=query,
output_file=output_file,
output_format=output_format,
verbose=verbose,
dump_to_file=output_file is not None)
table = job.get_results()
return table
def query_criteria(self,
calibration_level=None,
data_product_type=None,
intent=None,
obs_collection=None,
instrument_name=None,
filters=None,
async_job=True,
output_file=None,
output_format="votable",
verbose=False,
get_query=False):
"""
Launches a synchronous or asynchronous job to query the HST tap
using calibration level, data product type, intent, collection,
instrument name, and filters as criteria to create and execute the
associated query.
Parameters
----------
calibration_level : str or int, optional
The identifier of the data reduction/processing applied to the
data. RAW (1), CALIBRATED (2), PRODUCT (3) or AUXILIARY (0)
data_product_type : str, optional
High level description of the product.
image, spectrum or timeseries.
intent : str, optional
The intent of the original observer in acquiring this observation.
SCIENCE or CALIBRATION
collection : list of str, optional
List of collections that are available in eHST catalogue.
HLA, HST
instrument_name : list of str, optional
Name(s) of the instrument(s) used to generate the dataset
filters : list of str, optional
Name(s) of the filter(s) used to generate the dataset
async_job : bool, optional, default 'True'
executes the query (job) in asynchronous/synchronous mode (default
synchronous)
output_file : str, optional, default None
file name where the results are saved if dumpToFile is True.
If this parameter is not provided, the jobid is used instead
output_format : str, optional, default 'votable'
results format
verbose : bool, optional, default 'False'
flag to display information about the process
get_query : bool, optional, default 'False'
flag to return the query associated to the criteria as the result
of this function.
Returns
-------
A table object
"""
parameters = []
if calibration_level is not None:
parameters.append("p.calibration_level LIKE '%{}%'".format(
self.__get_calibration_level(calibration_level)))
if data_product_type is not None:
if isinstance(data_product_type, str):
parameters.append("p.data_product_type LIKE '%{}%'".format(
data_product_type))
else:
raise ValueError("data_product_type must be a string")
if intent is not None:
if isinstance(intent, str):
parameters.append("o.intent LIKE '%{}%'".format(intent))
else:
raise ValueError("intent must be a string")
if self.__check_list_strings(obs_collection):
parameters.append("(o.collection LIKE '%{}%')".format(
"%' OR o.collection LIKE '%".join(obs_collection)))
if self.__check_list_strings(instrument_name):
parameters.append("(o.instrument_name LIKE '%{}%')".format(
"%' OR o.instrument_name LIKE '%".join(instrument_name)))
if self.__check_list_strings(filters):
parameters.append(
"(o.instrument_configuration LIKE '%{}%')".format(
"%' OR o.instrument_configuration "
"LIKE '%".join(filters)))
query = "select o.*, p.calibration_level, p.data_product_type, " \
"pos.ra, pos.dec from ehst.observation AS o JOIN " \
"ehst.plane as p on o.observation_uuid=p.observation_uuid " \
"JOIN ehst.position as pos on p.plane_id = pos.plane_id"
if parameters:
query += " where({})".format(" AND ".join(parameters))
if verbose:
log.info(query)
if get_query:
return query
table = self.query_hst_tap(query=query,
async_job=async_job,
output_file=output_file,
output_format=output_format,
verbose=verbose)
return table
def __get_calibration_level(self, calibration_level):
condition = ""
if (calibration_level is not None):
if isinstance(calibration_level, str):
condition = calibration_level
elif isinstance(calibration_level, int):
if calibration_level < 4:
condition = self.calibration_levels[calibration_level]
else:
raise KeyError("Calibration level must be between 0 and 3")
else:
raise KeyError("Calibration level must be either "
"a string or an integer")
return condition
def __check_list_strings(self, list):
if list is None:
return False
if list and all(isinstance(elem, str) for elem in list):
return True
else:
raise ValueError("One of the lists is empty or there are "
"elements that are not strings")
def get_tables(self, only_names=True, verbose=False):
"""Get the available table in EHST TAP service
Parameters
----------
only_names : bool, TAP+ only, optional, default 'False'
True to load table names only
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
A list of tables
"""
tables = self._tap.load_tables(only_names=only_names,
include_shared_tables=False,
verbose=verbose)
if only_names is True:
table_names = []
for t in tables:
table_names.append(t.name)
return table_names
else:
return tables
def get_columns(self, table_name, only_names=True, verbose=False):
"""Get the available columns for a table in EHST TAP service
Parameters
----------
table_name : string, mandatory, default None
table name of which, columns will be returned
only_names : bool, TAP+ only, optional, default 'False'
True to load table names only
verbose : bool, optional, default 'False'
flag to display information about the process
Returns
-------
A list of columns
"""
tables = self._tap.load_tables(only_names=False,
include_shared_tables=False,
verbose=verbose)
columns = None
for t in tables:
if str(t.name) == str(table_name):
columns = t.columns
break
if columns is None:
raise ValueError("table name specified is not found in "
"EHST TAP service")
if only_names is True:
column_names = []
for c in columns:
column_names.append(c.name)
return column_names
else:
return columns
def _getCoordInput(self, value):
if not (isinstance(value, str) or isinstance(value, SkyCoord)):
raise ValueError("Coordinates" +
" must be either a string or astropy.coordinates")
if isinstance(value, str):
return SkyCoord(value)
else:
return value
def _get_decoded_string(self, string):
try:
return string.decode('utf-8')
except (UnicodeDecodeError, AttributeError):
return string
def find_outliers(vis, refant, dopol, nloops, loop, cell, robust, imsize,
wprojplanes, niter, threshold, uvrange, nterms, gridder,
deconvolver, solint, calmode, discard_nloops, gaintype,
outlier_threshold, flag, step):
local = locals()
local.pop('step')
imbase, imagename, outimage, pixmask, rmsfile, caltable, prev_caltables, threshold, outlierfile, cfcache, thresh, maskfile = bookkeeping.get_selfcal_args(
vis,
loop,
nloops,
nterms,
deconvolver,
discard_nloops,
calmode,
outlier_threshold,
threshold,
step=step)
cat = imagename + ".catalog.fits"
outlierfile_all = 'outliers.txt'
fitsname = imagename + '.fits'
outlier_imsize = 128
outlier_snr = 5
sky_model_radius = 2 #degrees
if step != 'sky':
pybdsf(imbase, rmsfile, imagename, outimage, thresh, maskfile, cat)
#Store before potentially updating to mJy
orig_threshold = outlier_threshold
if outlier_threshold != '' and outlier_threshold != 0:
#Take sky positions from RACS
if step == 'sky':
from astroquery.utils.tap.core import TapPlus
from astropy.table import vstack
#Open MS and extract first target centre; use first sub-MS for speed if MMS
if os.path.exists('{0}/SUBMSS'.format(vis)):
tmpvis = glob.glob('{0}/SUBMSS/*'.format(vis))[0]
else:
tmpvis = vis
msmd.open(tmpvis)
dir = msmd.sourcedirs()[str(msmd.fieldsforintent('TARGET')[0])]
ra = qa.convert(dir['m0'], 'deg')['value']
dec = qa.convert(dir['m1'], 'deg')['value']
if ra < 0:
ra += 360
phasecenter = SkyCoord(ra=ra, dec=dec, unit='deg,deg')
msmd.done()
cat = 'RACS_local.fits'
fluxcol = 'total_flux_source'
efluxcol = 'e_total_flux_source'
racol = 'ra'
deccol = 'dec'
outlier_threshold *= 1e3 #convert from mJy to Jy
#Extract RACS positions within sky_model_radius
try:
casdatap = TapPlus(
url="https://casda.csiro.au/casda_vo_tools/tap")
query = "SELECT * FROM AS110.racs_dr1_sources_galacticcut_v2021_08_v01 where 1=CONTAINS(POINT('ICRS', ra, dec),CIRCLE('ICRS',{0},{1},{2}))".format(
ra, dec, sky_model_radius)
job = casdatap.launch_job_async(query)
galcut = job.get_results()
job = casdatap.launch_job_async(query.replace('cut', 'region'))
galregion = job.get_results()
RACS = vstack([galcut, galregion], join_type='exact')
RACS.write(cat, overwrite=True)
except:
RACS = '{0}/RACS.fits.gz'.format(processMeerKAT.SCRIPT_DIR)
tmp = fits.open(RACS)
all_positions = SkyCoord(ra=tmp[1].data[racol],
dec=tmp[1].data[deccol],
unit='deg,deg')
tmp[1].data = tmp[1].data[phasecenter.separation(all_positions)
< Quantity(sky_model_radius, 'deg')]
tmp.writeto(cat, overwrite=True)
tmp.close()
del all_positions #Don't store millions of positions beyond here
index = loop
else:
fluxcol = 'Total_flux'
efluxcol = 'E_Total_flux'
racol = 'RA'
deccol = 'Dec'
index = loop + 1
# Write initial outlier file
if step == 'sky':
tab = fits.open(cat)
data = tab[1].data
tab.close()
if orig_threshold > 1.0:
metric = data[fluxcol] / data[efluxcol]
else:
metric = data[fluxcol]
outliers = data[metric > outlier_threshold]
out = open(outlierfile_all, 'w')
mask = 'mask={0}'.format(pixmask) # if pixmask != '' else ''
for i, row in enumerate(outliers):
SkyPos = SkyCoord(ra=row[racol],
dec=row[deccol],
unit='deg,deg')
position = 'J2000 {0}'.format(SkyPos.to_string('hmsdms'))
out.write("""
imagename={0}_outlier{1}
imsize=[{2},{2}]
cell=[1.0arcsec,1.0arcsec]
phasecenter={3}
nterms={4}
gridder=standard
{5}\n""".format(imbase % (index), i, outlier_imsize, position,
nterms[loop], mask))
out.close()
if os.path.exists(outlierfile_all):
logger.info(
"All 'outliers' within field written to '{0}' based on catalog '{1}'."
.format(outlierfile_all,
os.path.split(cat)[1]))
else:
logger.error(
"Outlier file '{0}' doesn't exist, so sky model build went wrong. Will terminate process."
.format(outlierfile_all))
sys.exit(1)
#Write outlier file specific to this loop
if not os.path.exists(outlierfile_all) and os.path.exists(
'../{0}'.format(outlierfile_all)):
logger.warning(
'Using outliers from ../{0}'.format(outlierfile_all))
outlierfile_all = '../{0}'.format(outlierfile_all)
outliers = open(outlierfile_all).read()
out = open(outlierfile, 'w')
if os.path.exists(fitsname):
img = fits.open(fitsname)
head = img[0].header
img.close()
index = loop + 1
elif step == 'sky':
img = fits.PrimaryHDU()
head = img.header
head['CUNIT1'] = 'deg'
head['CUNIT2'] = 'deg'
head['CTYPE1'] = 'RA---SIN'
head['CTYPE2'] = 'DEC--SIN'
head['CRVAL1'] = ra
head['CRVAL2'] = dec
index = loop
#Update header to reflect image, and pop degenerate axes
imsize = imsize[index]
cell = cell[index]
if type(imsize) is not list:
imsize = [imsize, imsize]
if type(cell) is not list:
cell = [cell, cell]
head['NAXIS1'] = imsize[0]
head['NAXIS2'] = imsize[1]
head['CRPIX1'] = int(imsize[0] / 2 + 1)
head['CRPIX2'] = int(imsize[1] / 2 + 1)
xdelt = qa.convert(cell[0], 'deg')['value']
ydelt = qa.convert(cell[1], 'deg')['value']
if 'CDELT1' in head.keys() and head['CDELT1'] > 0:
head['CDELT1'] = xdelt
else:
head['CDELT1'] = -xdelt
if 'CDELT2' in head.keys() and head['CDELT2'] < 0:
head['CDELT2'] = -ydelt
else:
head['CDELT2'] = ydelt
head['NAXIS'] = 2
for axis in ['NAXIS3', 'NAXIS4']:
if axis in head.keys():
head.pop(head.index(axis))
w = WCS(head)
r = re.compile(r'phasecenter=J2000 (?P<ra>.*?) (?P<dec>.*?)\n')
positions = [m.groupdict() for m in r.finditer(outliers)]
outlier_bases = re.findall(r'imagename=(.*)\n', outliers)
num_outliers = 0
#Only write positions for this loop outside imaging area
for i, position in enumerate(positions):
pos = SkyCoord(**position)
if not w.footprint_contains(pos):
num_outliers += 1
mask = 'mask={0}'.format(pixmask)
phasecenter = 'J2000 {0}'.format(pos.to_string('hmsdms'))
if step == 'bdsf':
base = outlier_bases[i]
im = base + '.image'
outlier_cat = base + ".catalog.fits"
outlier_mask = '{0}.islmask'.format(base)
if nterms[loop] > 1 and deconvolver[loop] == 'mtmfs':
im += '.tt0'
if os.path.exists(im):
#Run PyBDSF on outlier and update mask
pybdsf(imbase,
rmsfile,
base,
im,
outlier_snr,
outlier_mask,
outlier_cat,
write_all=False)
outlier_pixmask = mask_image(**local,
outlier_base=base,
outlier_image=im)
else:
#Use main image, run PyBDSF on box around outlier, and update mask
ia.open(outimage)
pix = ia.topixel(pos.to_string('hmsdms'))['numeric']
x, y = pix[0], pix[1]
delta = outlier_imsize / 2
trim_box = (x - delta, x + delta, y - delta, y + delta)
ia.close()
pybdsf(imbase,
rmsfile,
imagename,
outimage,
outlier_snr,
outlier_mask,
outlier_cat,
trim_box=trim_box,
write_all=False)
outlier_pixmask = mask_image(**local,
outlier_base=base)
mask = 'mask={0}'.format(outlier_pixmask)
#If catalog written, take new PyBDSF position closest to previous position
if os.path.exists(outlier_cat):
tab = fits.open(outlier_cat)
data = tab[1].data
tab.close()
cat_positions = SkyCoord(ra=data['RA'],
dec=data['Dec'],
unit='deg,deg')
row, _, _ = pos.match_to_catalog_sky(cat_positions)
phasecenter = 'J2000 {0}'.format(
cat_positions[row].to_string('hmsdms'))
else:
logger.warning(
"PyBDSF catalogue '{0}' not created. Using old position and mask."
.format(outlier_cat))
mask = 'mask={0}'.format(pixmask)
out.write("""
imagename={0}_outlier{1}
imsize=[{2},{2}]
cell=[1.0arcsec,1.0arcsec]
phasecenter={3}
nterms={4}
gridder=standard
{5}\n""".format(imbase % (index), i, outlier_imsize,
phasecenter, nterms[loop], mask))
else:
logger.info(
'Excluding "{0}", as it lies within the image footprint.'.
format(outlier_bases[i]))
out.close()
if os.path.exists(outlierfile):
logger.info("Outlier file '{0}' written.".format(outlierfile))
else:
logger.error(
"Outlier file '{0}' doesn't exist, so self-calibration loop {1} failed. Will terminate selfcal process."
.format(outlierfile, loop))
sys.exit(1)
if num_outliers > 10:
logger.warning(
'The number of outliers written to "{0}" is > 10. As this will take some time to image, you may want to set a higher outlier_threshold than {1}, and/or increase your imsize beyond {2}.'
.format(outlierfile, orig_threshold, imsize))
return rmsfile, outlierfile
for i, item in enumerate(list_of_border_source_ids):
if i == len(list_of_border_source_ids) - 1:
continue
QUERIES.append(
query %
(list_of_border_source_ids[i], list_of_border_source_ids[i + 1] - 1))
t00 = time.time()
os.system("mkdir Gaia_Healpix_{:d}".format(hpx_query))
for i in range(11654, nhpx):
try:
t0 = time.time()
print(i, nhpx)
job = gaia.launch_job_async(QUERIES[i])
print(i, job)
r = job.get_results()
print(len(r))
np.save(
'./Gaia_Healpix_{:d}/lvl{:d}_{:06d}.npy'.format(
hpx_query, hpx_query, i), r)
lst = gaia.list_async_jobs(verbose=True)
for i2 in lst:
gaia.remove_jobs(i2.jobid)
t1 = time.time()
print(
"########################################################################################################################"
)
print(
"Finished query {:6d} of {:6d}. Duration: {:.2f} s Total passed time: {:.1f} s Estimate for completion: {:.1f}s "
def test_start_job(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
jobid = '12345'
# Phase POST response
responsePhase = DummyResponse()
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
responsePhase.set_data(method='POST',
context=None,
body=None,
headers=None)
req = "async/" + jobid + "/phase?PHASE=RUN"
connHandler.set_response(req, responsePhase)
# Launch response
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(303)
responseLaunchJob.set_message("OK")
# list of list (httplib implementation for headers in response)
launchResponseHeaders = [[
'location', 'http://test:1111/tap/async/' + jobid
]]
responseLaunchJob.set_data(method='POST',
context=None,
body=None,
headers=launchResponseHeaders)
query = 'query'
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"QUERY": str(query)
}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
req = "async?" + sortedKey
connHandler.set_response(req, responseLaunchJob)
# Phase response
responsePhase = DummyResponse()
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
responsePhase.set_data(method='GET',
context=None,
body="COMPLETED",
headers=None)
req = "async/" + jobid + "/phase"
connHandler.set_response(req, responsePhase)
# Results response
responseResultsJob = DummyResponse()
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseResultsJob.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = "async/" + jobid + "/results/result"
connHandler.set_response(req, responseResultsJob)
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
job = tap.launch_job_async(query, autorun=False)
assert job is not None, "Expected a valid job"
assert job.get_phase() == 'PENDING', \
"Wrong job phase. Expected: %s, found %s" % \
('PENDING', job.get_phase())
# start job
job.start()
assert job.get_phase() == 'QUEUED', \
"Wrong job phase. Expected: %s, found %s" % \
('QUEUED', job.get_phase())
# results
results = job.get_results()
assert len(results) == 3, \
"Wrong job results (num rows). Expected: %d, found %d" % \
(3, len(results))
assert job.get_phase() == 'COMPLETED', \
"Wrong job phase. Expected: %s, found %s" % \
('COMPLETED', job.get_phase())
# try to start again
with pytest.raises(Exception):
job.start()
def ir_query(self, ircat_name, view_adql=True):
"""
Performs a TAP+ query to irsa.ipac and returns the results.
Main reason for this function is to have basis for easily incorporating TAP queries in a general query function later.
-----------------------
Example catalogs:
"allwise_p3as_psd": AllWISE Source Catalog
"fp_psc": 2MASS Point Source Catalog
"glimpse_s07": GLIMPSE I Spring 07 Catalog (Spitzer)
"cosmos_phot": COSMOS Photometry Catalog
"iraspsc": IRAS Point Source Catalog
-----------------------
Arguments:
ircat_name [string]: name of the catalogue in the irsa.ipac TAP spec
view_adql [bool]: If True, prints the adql of the query generated.
Returns:
CatalogTable with query results.
"""
catalogs = {
"allwise": "allwise_p3as_psd",
"tmass": "fp_psc",
"glimpse": "glimpse_s07"
}
catalog = catalogs[ircat_name]
if ircat_name == "tmass":
cols = tmass_cols
cols_list = tmass_cols_list
if ircat_name == "allwise":
cols = allwise_cols
cols_list = allwise_cols_list
search_string = "SELECT {} FROM {} WHERE CONTAINS(POINT('ICRS',ra,dec),CIRCLE('ICRS',{},{},{}))=1 ".format(
cols, catalog, str(self.skycoord.ra.degree),
str(self.skycoord.dec.degree), self.radius)
if view_adql:
out(search_string)
ipac = TapPlus(url="https://irsa.ipac.caltech.edu/TAP/")
out("Creating query...")
job = ipac.launch_job_async(search_string)
out("Retrieving results...")
query_results = job.get_results()
for old, new in zip(query_results.colnames, cols_list):
query_results.rename_column(old, new)
out("Results retrieved.")
print(query_results.colnames)
#info_out(str(len(query_results['designation'])) + " sources detected.")
# Write query results to file.
fname = tpath + "/" + ircat_name + "_query.dat"
query_results.write(fname, format='ascii.ecsv')
return (CatalogTable([ircat_name], query_results))
def fs_table_conequery(args):
import numpy as np
import pandas as pd
import math
import time
import datetime
from astroquery.utils.tap.core import TapPlus
if args.verbose > 0:
print
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print 'fs_table_conequery'
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print
print datetime.datetime.now()
# Extract info from argument list...
radeg0 = args.radeg
decdeg0 = args.decdeg
radiusdeg = args.radiusdeg
outputFile = args.outputFile
# Read in nobs_fs_file...
nobs_fs_file = 'nobs_fs_table.csv'
df_nobs_fs = pd.read_csv(nobs_fs_file)
# Read in images_file...
images_file = 'images_table.csv'
df_images = pd.read_csv(images_file)
# Merge df_nobs_fs and df_images on image_id...
df_images_nobs = df_nobs_fs.merge(df_images, on=['image_id']).sort_values(
by='image_id', ascending=True).reset_index(drop=True)
# Calculate separation between center of cone
# and ra,dec of each image in df_images_nobs
rarad0 = math.radians(radeg0)
decrad0 = math.radians(decdeg0)
df_images_nobs.loc[:, 'rarad'] = np.radians(df_images_nobs.loc[:, 'ra'])
df_images_nobs.loc[:, 'declrad'] = np.radians(df_images_nobs.loc[:,
'decl'])
df_images_nobs.loc[:,'cosSep'] = math.sin(decrad0)*np.sin(df_images_nobs.loc[:,'declrad']) + \
math.cos(decrad0)*np.cos(df_images_nobs.loc[:,'declrad'])*np.cos(rarad0 - df_images_nobs.loc[:,'rarad'])
df_images_nobs.loc[:, 'sepDeg'] = np.degrees(
np.arccos(df_images_nobs.loc[:, 'cosSep']))
# Find which image_id's lie within the cone search radius...
mask = (df_images_nobs.loc[:, 'sepDeg'] < radiusdeg)
if args.verbose > 0:
print df_images_nobs.loc[
mask, ['image_id', 'filter', 'ra', 'decl', 'sepDeg']]
# Establish TAP connection to SkyMapper database...
skymapper = TapPlus(url="http://skymappertap.asvo.nci.org.au/ncitap/tap")
# Loop over image_id's...
i = 0
for image_id in df_images_nobs.loc[mask, 'image_id']:
# Formulate the query...
query = """SELECT * FROM dr1.fs_photometry where image_id=%d""" % (
image_id)
if args.verbose > 0:
print "Query: ", query
# Submit the query as an asynchronous job...
if args.verbose > 0:
print "Query start: ", datetime.datetime.now()
job = skymapper.launch_job_async(query)
if args.verbose > 0:
print "Query completed: ", datetime.datetime.now()
# Retrieve the results as an astropy Table...
if args.verbose > 0:
print "Retrieval start: ", datetime.datetime.now()
fs_table = job.get_results()
if args.verbose > 0:
print "Retrieval completed: ", datetime.datetime.now()
# Sort table by ra_img
# fs_table.sort('ra_img')
# "Append" does not seem to work for stropy table.write.
# For now, save each file individually....
# Save table to outputFile...
#if args.verbose>0:
# print "File output start: ", datetime.datetime.now()
#if i == 0:
# fs_table.write(outputFile)
#else:
# fs_table.write(outputFile, append=True)
#if args.verbose>0:
# print "File output completed: ", datetime.datetime.now()
tmpOutputFile = """%s.%d.csv""" % (outputFile, image_id)
if args.verbose > 0:
print "File output start: ", datetime.datetime.now()
fs_table.write(tmpOutputFile)
if args.verbose > 0:
print "File output completed: ", datetime.datetime.now()
i = i + 1
if args.verbose > 0: print
return 0
def test_launch_async_job():
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
jobid = '12345'
# Launch response
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(500)
responseLaunchJob.set_message("ERROR")
# list of list (httplib implementation for headers in response)
launchResponseHeaders = [[
'location', f'http://test:1111/tap/async/{jobid}'
]]
responseLaunchJob.set_data(method='POST',
context=None,
body=None,
headers=launchResponseHeaders)
query = 'query'
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"PHASE": "RUN",
"QUERY": str(query)
}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
req = f"async?{sortedKey}"
connHandler.set_response(req, responseLaunchJob)
# Phase response
responsePhase = DummyResponse()
responsePhase.set_status_code(500)
responsePhase.set_message("ERROR")
responsePhase.set_data(method='GET',
context=None,
body="COMPLETED",
headers=None)
req = f"async/{jobid}/phase"
connHandler.set_response(req, responsePhase)
# Results response
responseResultsJob = DummyResponse()
responseResultsJob.set_status_code(500)
responseResultsJob.set_message("ERROR")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseResultsJob.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = f"async/{jobid}/results/result"
connHandler.set_response(req, responseResultsJob)
with pytest.raises(Exception):
tap.launch_job_async(query)
responseLaunchJob.set_status_code(303)
responseLaunchJob.set_message("OK")
with pytest.raises(Exception):
tap.launch_job_async(query)
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
with pytest.raises(Exception):
tap.launch_job_async(query)
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
job = tap.launch_job_async(query)
assert job is not None
assert job.async_ is True
assert job.get_phase() == 'COMPLETED'
assert job.failed is False
# results
results = job.get_results()
assert len(results) == 3
__check_results_column(results, 'alpha', 'alpha', None, np.float64)
__check_results_column(results, 'delta', 'delta', None, np.float64)
__check_results_column(results, 'source_id', 'source_id', None, object)
__check_results_column(results, 'table1_oid', 'table1_oid', None, np.int32)
def test_job_parameters():
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
jobid = '12345'
# Launch response
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(303)
responseLaunchJob.set_message("OK")
# list of list (httplib implementation for headers in response)
launchResponseHeaders = [[
'location', f'http://test:1111/tap/async/{jobid}'
]]
responseLaunchJob.set_data(method='POST',
context=None,
body=None,
headers=launchResponseHeaders)
query = 'query'
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"QUERY": str(query)
}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
req = f"async?{sortedKey}"
connHandler.set_response(req, responseLaunchJob)
# Phase response
responsePhase = DummyResponse()
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
responsePhase.set_data(method='GET',
context=None,
body="COMPLETED",
headers=None)
req = f"async/{jobid}/phase"
connHandler.set_response(req, responsePhase)
# Results response
responseResultsJob = DummyResponse()
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseResultsJob.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = f"async/{jobid}/results/result"
connHandler.set_response(req, responseResultsJob)
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
job = tap.launch_job_async(query, autorun=False)
assert job is not None
assert job.get_phase() == 'PENDING'
# parameter response
responseParameters = DummyResponse()
responseParameters.set_status_code(200)
responseParameters.set_message("OK")
responseParameters.set_data(method='GET',
context=None,
body=None,
headers=None)
req = f"async/{jobid}?param1=value1"
connHandler.set_response(req, responseParameters)
# Phase POST response
responsePhase = DummyResponse()
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
responsePhase.set_data(method='POST',
context=None,
body=None,
headers=None)
req = f"async/{jobid}/phase?PHASE=RUN"
connHandler.set_response(req, responsePhase)
# send parameter OK
job.send_parameter("param1", "value1")
# start job
job.start()
assert job.get_phase() == 'QUEUED'
# try to send a parameter after execution
with pytest.raises(Exception):
job.send_parameter("param2", "value2")
def test_launc_async_job(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
jobid = '12345'
# Launch response
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(500)
responseLaunchJob.set_message("ERROR")
# list of list (httplib implementation for headers in response)
launchResponseHeaders = [
['location', 'http://test:1111/tap/async/' + jobid]
]
responseLaunchJob.set_data(method='POST',
context=None,
body=None,
headers=launchResponseHeaders)
query = 'query'
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"PHASE": "RUN",
"QUERY": str(query)}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
req = "async?" + sortedKey
connHandler.set_response(req, responseLaunchJob)
# Phase response
responsePhase = DummyResponse()
responsePhase.set_status_code(500)
responsePhase.set_message("ERROR")
responsePhase.set_data(method='GET',
context=None,
body="COMPLETED",
headers=None)
req = "async/" + jobid + "/phase"
connHandler.set_response(req, responsePhase)
# Results response
responseResultsJob = DummyResponse()
responseResultsJob.set_status_code(500)
responseResultsJob.set_message("ERROR")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseResultsJob.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = "async/" + jobid + "/results/result"
connHandler.set_response(req, responseResultsJob)
with pytest.raises(Exception):
tap.launch_job_async(query)
responseLaunchJob.set_status_code(303)
responseLaunchJob.set_message("OK")
with pytest.raises(Exception):
tap.launch_job_async(query)
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
with pytest.raises(Exception):
tap.launch_job_async(query)
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
job = tap.launch_job_async(query)
assert job is not None, "Expected a valid job"
assert job.is_sync() is False, "Expected an asynchronous job"
assert job.get_phase() == 'COMPLETED', \
"Wrong job phase. Expected: %s, found %s" % \
('COMPLETED', job.get_phase())
assert job.is_failed() is False, "Wrong job status (set Failed = True)"
# results
results = job.get_results()
assert len(results) == 3, \
"Wrong job results (num rows). Expected: %d, found %d" % \
(3, len(results))
self.__check_results_column(results,
'alpha',
'alpha',
None,
np.float64)
self.__check_results_column(results,
'delta',
'delta',
None,
np.float64)
self.__check_results_column(results,
'source_id',
'source_id',
None,
np.object)
self.__check_results_column(results,
'table1_oid',
'table1_oid',
None,
np.int32)
tables = tap.load_tables()
# We have to find the name of the Gaia DR2 table in the Vizier database.
# In[14]:
for t in tables:
if 'gaia' in t.name:
print(t.name)
# We now cross-match to LAMOST find Gaia stars within 1 arcsec (1/3600=0.00027777) of each LAMOST star.
# In[21]:
job = tap.launch_job_async(
'select top 10 * from "I/345/gaia2" as gaia, ' + '"V/153/dr4" as lamost ' +
"where 1=CONTAINS(POINT('ICRS',gaia.ra,gaia.dec)," +
"CIRCLE('ICRS',lamost.raj2000,lamost.dej2000,0.00027777))")
tableV = job.get_results()
print(tableV)
# In[22]:
tableV[['ObsID', 'RAJ2000', 'ra']]
# ## Gaia RVS selection function
# One other unique function provided in the Gaia archive is gaia_healpix_index which gives the healpix number of each Gaia source. Healpix are a method of dividing up the sphere into equal area pixels. This is useful as we can bin the catalogue by Healpix. In the following we count all sources in bins of G and healpix for a small region on the sky.
# In[27]:
from astroquery.gaia import Gaia
