def __init__(self,name,od,bn,ds=None,pol=None,polp=None,choice='directionless',
s_i=0.,s_v=0.,cri=None,**kwargs):
self.nm = name
self.od = od
self.bn = bn
self.ch = choice
self.cri = cri
self.__dict__.update(kwargs)
if ds:
with get_readable_fileobj(ds, cache=True) as f:
self.fitsfile = fits.open(f)
self.ds = self.fitsfile[0].data
self.dshd = self.fitsfile[0].header
else: self.ds = np.zeros([0])
self.ds[self.ds==0]=np.nan # remove irregular points
if pol:
self.pol = pol # fits file
with get_readable_fileobj(pol, cache=True) as e:
self.fitsfile = fits.open(e)
self.po = self.fitsfile[0].data
hdd = fits.open(pol)
self.header = hdd[0].header
else: self.po = np.zeros([0])
# default ds and po to zero arrays with size 0 if not given
self.po[self.po==0]=np.nan
if ds:
self.s_i = min(np.sqrt(np.nanmean(self.ds[0]**2)),
np.sqrt(np.nanmean(self.ds[-1]**2))) # set intensity rms to be the min rms
self.s_v = self.dshd["cdelt3"] / 1000. # channel width, in km/s
self.ds[self.ds < 2. * self.s_i]=np.nan # blank low SN points
def getObjectByCoord(ra, dec, service="ned"):
if service not in ["simbad", "ned"]:
raise ValueError("service must be one of 'ned' or 'simbad'")
data = {}
if service == "ned":
try:
with get_readable_fileobj(NED_COORD_QUERY % (ra, dec, "xml_main")) as f:
table = Table.read(f, format="votable")
except:
return None
data["RA"] = float(table["main_col3"][0])
data["DEC"] = float(table["main_col4"][0])
data["objtype"] = table["main_col5"][0]
data["descr"] = "Results of NED query"
data["name"] = table["main_col2"][0]
data["Mv"] = table["main_col9"][0]
data["rating"] = 0
with get_readable_fileobj(NED_NAME_QUERY % (urllib.quote(data["name"]), "xml_basic")) as f:
table = Table.read(f, format="votable")
try:
data["size"] = "%.1f arc-min" % (float(table["basic_col10"]))
except:
data["size"] = "..."
elif service == "simbad":
coord = "query coo %f %f radius=1m" % (ra, dec)
try:
with get_readable_fileobj(SIMBAD_URL + urllib.quote(SIMBAD_QUERY % coord)) as f:
table = Table.read(f, format="votable")
except:
return None
data["RA"] = table["RA_d_ICRS_2000_0_2000_0"][0]
data["DEC"] = table["DEC_d_ICRS_2000_0_2000_0"][0]
data["objtype"] = table["OTYPE_S"][0] or "--"
data["objtype"] = table["OTYPE_S"][0] or "--"
data["Mv"] = str((table["FLUX_V"][0] or "--"))
data["descr"] = "Results from SIMBAD query"
data["name"] = table["MAIN_ID"][0]
data["rating"] = 0
if table["GALDIM_MAJAXIS"][0]:
data["size"] = "%.1f arc-min" % table["GALDIM_MAJAXIS"][0]
else:
data["size"] = "--"
data["distance"] = -1.0
data["dark"] = False
data["comments"] = "None"
return data
def read_remote_spec(filename, encoding='binary', cache=True,
show_progress=True, **kwargs):
"""Read FITS or ASCII spectrum from a remote location.
Parameters
----------
filename : str
Spectrum filename.
encoding, cache, show_progress
See :func:`~astropy.utils.data.get_readable_fileobj`.
kwargs : dict
Keywords acceptable by :func:`read_fits_spec` (if FITS) or
:func:`read_ascii_spec` (if ASCII).
Returns
-------
header : dict
Metadata.
wavelengths, fluxes : `~astropy.units.quantity.Quantity`
Wavelength and flux of the spectrum.
"""
with get_readable_fileobj(filename, encoding=encoding, cache=cache,
show_progress=show_progress) as fd:
header, wavelengths, fluxes = read_spec(fd, fname=filename, **kwargs)
return header, wavelengths, fluxes
def get_remote_catalog_db(dbname, cache=True):
"""
Get a database of VO services (which is a JSON file) from a remote
location.
Parameters
----------
dbname : str
Prefix of JSON file to download from `astropy.vo.client.vos_baseurl`.
cache : bool
Use caching for VO Service database. Access to actual VO
websites referenced by the database still needs internet
connection.
Returns
-------
obj : `VOSDatabase` object
"""
with get_readable_fileobj(BASEURL() + dbname + '.json',
encoding='utf8', cache=cache) as fd:
tree = json.load(fd)
return VOSDatabase(tree)
def get_spectra_async(self, matches, plate=None, fiberID=None, mjd=None):
"""
Download spectrum from SDSS.
Parameters
----------
matches : astropy.table.Table instance (result of query_region).
Returns
-------
A list of context-managers that yield readable file-like objects
"""
if not isinstance(matches, Table):
raise ValueError
results = []
for row in matches:
plate = str(row['plate']).zfill(4)
fiber = str(row['fiberID']).zfill(3)
mjd = str(row['mjd'])
link = '%s/%s/1d/spSpec-%s-%s-%s.fit' % (SDSS.SPECTRO_1D, plate,
mjd, plate, fiber)
results.append(aud.get_readable_fileobj(link))
return results
def getfile(url,fn=None):
if fn is None:
fn = os.path.split(url)[-1]
if not os.path.exists(fn):
with aud.get_readable_fileobj(url) as f:
with open(fn,'w') as of:
of.write(f.read())
def test_compressed_stream():
import base64
gzipped_data = (b"H4sICIxwG1AAA2xvY2FsLmRhdAALycgsVkjLzElVANKlxakpCpl5CiUZqQ"
b"olqcUl8Tn5yYk58SmJJYnxWmCRzLx0hbTSvOSSzPy8Yi5nf78QV78QLgAlLytnRQAAAA==")
gzipped_data = base64.b64decode(gzipped_data)
assert isinstance(gzipped_data, bytes)
class FakeStream:
"""
A fake stream that has `read`, but no `seek`.
"""
def __init__(self, data):
self.data = data
def read(self, nbytes=None):
if nbytes is None:
result = self.data
self.data = b''
else:
result = self.data[:nbytes]
self.data = self.data[nbytes:]
return result
stream = FakeStream(gzipped_data)
with get_readable_fileobj(stream, encoding='binary') as f:
f.readline()
assert f.read().rstrip() == b'CONTENT'
def parse(cls, filename, date=None, _parser=_LCHParser, **kwargs):
"""Parse a closure file or stream into regions.
Closure files contain starlists lines followed by listings
of opening blocks when laser propogation is permitted. This
object parses these into individual :class:`Region` objects
which contain a list of :class:`Opening` periods.
Parameters
----------
filename : str or fileobj
Filename or readable file object for the parser.
date : :class:`~astropy.time.Time`
String, or Time value which will be used to set the
date for the closure file.
Returns
-------
regions : :class:`Regions`
The parsed regions.
"""
date = Time.now() if date is None else date
with get_readable_fileobj(filename) as stream:
parser = _parser(stream.name, date, regioncls=cls)
return parser(stream, **kwargs)
def _parse_result(self, response, verbose=False, retrieve_file=True):
"""
retrieve_file : bool
If True, will try to retrieve the file every 30s until it shows up.
Otherwise, just returns the filename (the job is still executed on
the remote server, though)
"""
if verbose:
print("Loading request from Besancon server ...")
# keep the text stored for possible later use
with aud.get_readable_fileobj(response.raw) as f:
text = f.read()
try:
filename = self.result_re.search(text).group()
except AttributeError: # if there are no matches
errors = parse_errors(text)
raise ValueError("Errors: "+"\n".join(errors))
if verbose:
print("File is %s and can be aquired from %s" % (filename, self.url_download+'/'+filename))
if retrieve_file:
return self.get_besancon_model_file(filename)
else:
return filename
def _convert_to_fd_or_read_function(fd):
"""
Returns a function suitable for streaming input, or a file object.
This function is only useful if passing off to C code where:
- If it's a real file object, we want to use it as a real
C file object to avoid the Python overhead.
- If it's not a real file object, it's much handier to just
have a Python function to call.
This is somewhat quirky behavior, of course, which is why it is
private. For a more useful version of similar behavior, see
`astropy.utils.misc.get_readable_fileobj`.
Parameters
----------
fd : object
May be:
- a file object. If the file is uncompressed, this raw
file object is returned verbatim. Otherwise, the read
method is returned.
- a function that reads from a stream, in which case it is
returned verbatim.
- a file path, in which case it is opened. Again, like a
file object, if it's uncompressed, a raw file object is
returned, otherwise its read method.
- an object with a :meth:`read` method, in which case that
method is returned.
Returns
-------
fd : context-dependent
See above.
"""
if is_callable(fd):
yield fd
return
from astropy.utils.data import get_readable_fileobj
with get_readable_fileobj(fd, encoding='binary') as new_fd:
if sys.platform.startswith('win'):
yield new_fd.read
else:
if IS_PY3K:
if isinstance(new_fd, io.FileIO):
yield new_fd
else:
yield new_fd.read
else:
if isinstance(new_fd, file):
yield new_fd
else:
yield new_fd.read
def get_extinction_table_async(
self, coordinate, radius=None, timeout=TIMEOUT):
"""
A query function similar to `astroquery.irsa_dust.IrsaDust.get_extinction_table`
but returns a file-handler to the remote files rather than downloading it.
Useful for asynchronous queries so that the actual download may be performed later.
Parameters
----------
coordinate : str
Can be either the name of an object or a coordinate string
If a name, must be resolveable by NED, SIMBAD, 2MASS, or SWAS.
Examples of acceptable coordinate strings, can be found here:
http://irsa.ipac.caltech.edu/applications/DUST/docs/coordinate.html
radius : str, optional
The size of the region to include in the dust query, in radian, degree
or hour as per format specified by `astropy.coordinates.Angle`. Defaults
to 5 degrees.
timeout : int, optional
Time limit for establishing successful connection with remote server.
Defaults to `astroquery.irsa_dust.IrsaDust.TIMEOUT`
Returns
-------
A context manager that yields a file like readable object
"""
url = IrsaDust.DUST_SERVICE_URL
request_payload = self._args_to_payload(coordinate, radius=radius)
response = commons.send_request(url, request_payload, timeout)
xml_tree = utils.xml(response.text)
result = SingleDustResult(xml_tree, coordinate)
return aud.get_readable_fileobj(result.ext_detail_table())
def fetch_json(url, verbose=False):
"""Fetch JSON data from a remote URL
Parameters
----------
url : `str`
the remote URL to fetch
verbose : `bool`, optional
display verbose download progress, default: `False`
Returns
------
json : `object`
the data fetched from ``url`` as parsed by :func:`json.loads`
See also
--------
json.loads
for details of the JSON parsing
Examples
--------
>>> from gwpy.io.losc import fetch_json
>>> fetch_json('https://losc.ligo.org/archive/1126257414/1126261510/json/')
"""
with get_readable_fileobj(url, show_progress=verbose) as response:
data = response.read()
try:
return json.loads(data)
except ValueError as exc:
exc.args = ("Failed to parse LOSC JSON from %r: %s"
% (url, str(exc)),)
raise
def search(cls, gravityspy_id, howmany=10,
era='ALL', ifos='H1L1', remote_timeout=20):
"""perform restful API version of search available here:
https://gravityspytools.ciera.northwestern.edu/search/
Parameters
----------
gravityspy_id : `str`,
This is the unique 10 character hash that identifies
a Gravity Spy Image
howmany : `int`, optional, default: 10
number of similar images you would like
Returns
-------
`GravitySpyTable` containing similar events based on
an evaluation of the Euclidean distance of the input image
to all other images in some Feature Space
"""
from astropy.utils.data import get_readable_fileobj
import json
from six.moves.urllib.error import HTTPError
from six.moves import urllib
# Need to build the url call for the restful API
base = 'https://gravityspytools.ciera.northwestern.edu' + \
'/search/similarity_search_restful_API'
map_era_to_url = {
'ALL': "event_time BETWEEN 1126400000 AND 1229176818",
'O1': "event_time BETWEEN 1126400000 AND 1137250000",
'ER10': "event_time BETWEEN 1161907217 AND 1164499217",
'O2a': "event_time BETWEEN 1164499217 AND 1219276818",
'ER13': "event_time BETWEEN 1228838418 AND 1229176818",
}
parts = {
'howmany': howmany,
'imageid': gravityspy_id,
'era': map_era_to_url[era],
'ifo': "{}".format(", ".join(
map(repr, [ifos[i:i+2] for i in range(0, len(ifos), 2)]),
)),
'database': 'updated_similarity_index_v2d0',
}
search = urllib.parse.urlencode(parts)
url = '{}/?{}'.format(base, search)
try:
with get_readable_fileobj(url, remote_timeout=remote_timeout) as f:
return GravitySpyTable(json.load(f))
except HTTPError as exc:
if exc.code == 500:
exc.msg += ', confirm the gravityspy_id is valid'
raise
def parse_cs(ivoid, cap_index=1):
"""Return test query pars as dict for given IVO ID and capability index."""
if isinstance(ivoid, bytes): # pragma: py3
ivoid = ivoid.decode('ascii')
# Production server.
url = ("http://vao.stsci.edu/regtap/tapservice.aspx/sync?lang=adql&"
"query=select%20detail_xpath%2Cdetail_value%20from%20"
"rr.res_detail%20where%20"
"ivoid%3D%27{0}%27%20and%20cap_index={1}%20and%20"
"detail_xpath%20in%20%28%27/capability/testQuery/ra%27%2C"
"%27/capability/testQuery/dec%27%2C%27/capability/testQuery/sr%27"
"%29".format(ivoid, cap_index))
urls_failed = False
default_sr = 0.1
try:
with get_readable_fileobj(url, encoding='binary',
show_progress=False) as fd:
t_query = Table.read(fd, format='votable')
except Exception as e: # pragma: no cover
urls_failed = True
urls_errmsg = '{0} raised {1}, using default'.format(
url, str(e))
if not urls_failed:
try:
xpath = t_query['detail_xpath']
ra = float(
t_query[xpath == b'/capability/testQuery/ra']['detail_value'])
dec = float(
t_query[xpath == b'/capability/testQuery/dec']['detail_value'])
sr = float(
t_query[xpath == b'/capability/testQuery/sr']['detail_value'])
# Handle big SR returning too big a table for some queries, causing
# tests to fail due to timeout.
if sr > default_sr:
warnings.warn(
'SR={0} is too large, using SR={1} for {2},{3}'.format(
sr, default_sr, ivoid, cap_index), AstropyUserWarning)
sr = default_sr
d = OrderedDict({'RA': ra, 'DEC': dec, 'SR': sr})
except Exception as e: # pragma: no cover
urls_failed = True
urls_errmsg = ('Failed to retrieve test query parameters for '
'{0},{1}, using default'.format(ivoid, cap_index))
# If no test query found, use default
if urls_failed: # pragma: no cover
d = OrderedDict({'RA': 0, 'DEC': 0, 'SR': default_sr})
warnings.warn(urls_errmsg, AstropyUserWarning)
return d
def test_web_profile(self):
# Check some additional queries to the server
with get_readable_fileobj('http://localhost:{0}/crossdomain.xml'.format(self.hub._web_port)) as f:
assert f.read() == CROSS_DOMAIN
with get_readable_fileobj('http://localhost:{0}/clientaccesspolicy.xml'.format(self.hub._web_port)) as f:
assert f.read() == CLIENT_ACCESS_POLICY
# Check headers
req = Request('http://localhost:{0}/crossdomain.xml'.format(self.hub._web_port))
req.add_header('Origin', 'test_web_profile')
resp = urlopen(req)
assert resp.getheader('Access-Control-Allow-Origin') == 'test_web_profile'
assert resp.getheader('Access-Control-Allow-Headers') == 'Content-Type'
assert resp.getheader('Access-Control-Allow-Credentials') == 'true'
def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True):
"""
Download a Besancon model from the website
Parameters
----------
filename : string
The besancon filename, with format ##########.######.resu
verbose : bool
Print details about the download process
save : bool
Save the table after acquiring it?
savename : None or string
If not specified, defaults to the .resu table name
overwrite : bool
Overwrite the file if it exists? Defaults to True because the .resu
tables should have unique names by default, so there's little risk of
accidentally overwriting important information
"""
url = url_download+filename
elapsed_time = 0
t0 = time.time()
sys.stdout.write("\n")
while 1:
sys.stdout.write(u"\r")
try:
U = urllib2.urlopen(url,timeout=5)
with aud.get_readable_fileobj(U, cache=True) as f:
results = f.read()
break
except urllib2.URLError:
sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0))
time.sleep(30)
elapsed_time += 30
continue
except socket.timeout:
sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0))
time.sleep(30)
elapsed_time += 30
continue
if save:
if savename is None:
savename = filename
if not overwrite and os.path.exists(savename):
raise IOError("File %s already exists." % savename)
outf = open(savename,'w')
print >>outf,results
outf.close()
return parse_besancon_model_string(results)
def test_local_data_obj_invalid(bad_compressed):
is_bz2 = bad_compressed.endswith('.bz2')
is_xz = bad_compressed.endswith('.xz')
# Note, since these invalid files are created on the fly in order to avoid
# problems with detection by antivirus software
# (see https://github.com/astropy/astropy/issues/6520), it is no longer
# possible to use ``get_pkg_data_fileobj`` to read the files. Technically,
# they're not local anymore: they just live in a temporary directory
# created by pytest. However, we can still use get_readable_fileobj for the
# test.
if (not HAS_BZ2 and is_bz2) or (not HAS_XZ and is_xz):
with pytest.raises(ValueError) as e:
with get_readable_fileobj(bad_compressed, encoding='binary') as f:
f.read()
assert ' format files are not supported' in str(e)
else:
with get_readable_fileobj(bad_compressed, encoding='binary') as f:
assert f.read().rstrip().endswith(b'invalid')
def get_images_async(self, coordinates, waveband='all', frame_type='stack',
image_width=1* u.arcmin, image_height=None, radius=None,
database='UKIDSSDR7PLUS', programme_id='all',
verbose=True, get_query_payload=False):
"""
Serves the same purpose as :meth:`~astroquery.ukidss.core.Ukidss.get_images` but
returns a list of file handlers to remote files
Parameters
----------
coordinates : str or `astropy.coordinates` object
The target around which to search. It may be specified as a string
in which case it is resolved using online services or as the appropriate
`astropy.coordinates` object. ICRS coordinates may also be entered as strings
as specified in the `astropy.coordinates` module.
waveband : str
The color filter to download. Must be one of ['all','J','H','K','H2','Z','Y','Br'].
frame_type : str
The type of image. Must be one of
['stack','normal','interleave','deep_stack','confidence','difference', 'leavstack', 'all']
image_width : str or `astropy.units.Quantity` object, optional
The image size (along X). Cannot exceed 15 arcmin. If missing, defaults to 1 arcmin.
image_height : str or `astropy.units.Quantity` object, optional
The image size (along Y). Cannot exceed 90 arcmin. If missing, same as image_width.
radius : str or `astropy.units.Quantity` object, optional
The string must be parsable by `astropy.coordinates.Angle`. The appropriate
`Quantity` object from `astropy.units` may also be used. When missing only image
around the given position rather than multi-frames are retrieved.
programme_id : str
The survey or programme in which to search for.
database : str
The UKIDSS database to use.
verbose : bool
Defaults to `True`. When `True` prints additional messages.
get_query_payload : bool, optional
if set to `True` then returns the dictionary sent as the HTTP request.
Defaults to `False`
Returns
-------
A list of context-managers that yield readable file-like objects
"""
image_urls = self.get_image_list(coordinates, waveband=waveband, frame_type=frame_type,
image_width=image_width, image_height=image_height,
database=database, programme_id=programme_id,
radius=radius, get_query_payload=get_query_payload)
if get_query_payload:
return image_urls
if verbose:
print("Found {num} targets".format(num=len(image_urls)))
return [aud.get_readable_fileobj(U) for U in image_urls]
def read(cls, *args, format=None, **kwargs):
"""
Read in data.
The arguments passed to this method depend on the format.
"""
ctx = None
try:
if format is None:
path = None
fileobj = None
if len(args):
if isinstance(args[0], PATH_TYPES):
from astropy.utils.data import get_readable_fileobj
# path might be a pathlib.Path object
if isinstance(args[0], pathlib.Path):
args = (str(args[0]),) + args[1:]
path = args[0]
try:
ctx = get_readable_fileobj(args[0], encoding='binary')
fileobj = ctx.__enter__()
except OSError:
raise
except Exception:
fileobj = None
else:
args = [fileobj] + list(args[1:])
elif hasattr(args[0], 'read'):
path = None
fileobj = args[0]
format = _get_valid_format(
'read', cls, path, fileobj, args, kwargs)
reader = get_reader(format, cls)
data = reader(*args, **kwargs)
if not isinstance(data, cls):
# User has read with a subclass where only the parent class is
# registered. This returns the parent class, so try coercing
# to desired subclass.
try:
data = cls(data)
except Exception:
raise TypeError('could not convert reader output to {0} '
'class.'.format(cls.__name__))
finally:
if ctx is not None:
ctx.__exit__(*sys.exc_info())
return data
def __fet(self):
from astropy.utils.data import get_readable_fileobj
from astropy.io import fits
with get_readable_fileobj(self.dcube_path, cache=True) as f:
fitsfile = fits.open(f)
gd = fitsfile[0].data[0][0]
dshd = fitsfile[0].header
with get_readable_fileobj(self.poldt_path, cache=True) as e:
fitsfile = fits.open(e)
po = fitsfile[0].data[0][0] + 90. # to B-field
pshd = fitsfile[0].header
tx = 'gd-pol'
fig = Figure()
self.fig_dict[tx] = fig
self.fg_dict[tx] = [gd,po]
self.mplfigs.addItem(tx)
axf = fig.add_subplot(111)
self.__quiver(gd,po,axf,tx)
def read_lockfile(lockfilename):
"""
Read in the lockfile given by ``lockfilename`` into a dictionary.
"""
# lockfilename may be a local file or a remote URL, but
# get_readable_fileobj takes care of this.
lockfiledict = {}
with get_readable_fileobj(lockfilename) as f:
for line in f:
if not line.startswith("#"):
kw, val = line.split("=")
lockfiledict[kw.strip()] = val.strip()
return lockfiledict
def _query_gator(options, debug=False):
# Construct query URL
url = GATOR_URL + "?" + \
string.join(["%s=%s" % (x, urllib.quote_plus(str(options[x]))) for x in options], "&")
if debug:
print(url)
# Request page
req = urllib2.Request(url)
response = urllib2.urlopen(req)
with aud.get_readable_fileobj(response, cache=True) as f:
result = f.read()
# Check if results were returned
if 'The catalog is not on the list' in result:
raise Exception("Catalog not found")
# Check that object name was not malformed
if 'Either wrong or missing coordinate/object name' in result:
raise Exception("Malformed coordinate/object name")
# Check that the results are not of length zero
if len(result) == 0:
raise Exception("The IRSA server sent back an empty reply")
# Write table to temporary file
output = tempfile.NamedTemporaryFile()
output.write(result)
output.flush()
# Read it in using the astropy VO table reader
try:
firsttable = votable.parse(output.name, pedantic=False).get_first_table()
array = firsttable.array
except Exception as ex:
print("Failed to parse votable! Returning output file instead.")
print(ex)
return open(output.name,'r')
# Convert to astropy.table.Table instance
table = array.to_table()
# Check if table is empty
if len(table) == 0:
warnings.warn("Query returned no results, so the table will be empty")
# Remove temporary file
output.close()
return table
def _vo_service_request(url, pedantic, kwargs):
if len(kwargs) and not (url.endswith('?') or url.endswith('&')):
raise VOSError("url should already end with '?' or '&'")
query = []
for key, value in kwargs.iteritems():
query.append('{}={}'.format(
urllib.quote(key), urllib.quote_plus(str(value))))
parsed_url = url + '&'.join(query)
with get_readable_fileobj(parsed_url) as req:
tab = table.parse(req, filename=parsed_url, pedantic=pedantic)
return vo_tab_parse(tab, url, kwargs)
def get_besancon_model_file(self, filename, verbose=True, timeout=5.0):
"""
Download a Besancon model from the website
Parameters
----------
filename : string
The besancon filename, with format ##########.######.resu
verbose : bool
Print details about the download process
timeout : float
Amount of time to wait after pinging the server to see if a file is
present. Default 5s, which is probably reasonable.
"""
url = self.url_download+"/"+filename
elapsed_time = 0
t0 = time.time()
if verbose:
sys.stdout.write("Awaiting Besancon file...\n")
while True:
if verbose:
sys.stdout.write(u"\r")
sys.stdout.flush()
try:
# U = requests.get(url,timeout=timeout,stream=True)
# TODO: add timeout= keyword to get_readable_fileobj (when PR https://github.com/astropy/astropy/pull/1258 is merged)
with aud.get_readable_fileobj(url, cache=True) as f:
results = f.read()
break
except urllib2.URLError:
if verbose:
sys.stdout.write(u"Waiting %0.1fs for model to finish (elapsed wait time %0.1fs, total wait time %0.1f)\r" % (self.ping_delay,elapsed_time,time.time()-t0))
sys.stdout.flush()
time.sleep(self.ping_delay)
elapsed_time += self.ping_delay
continue
except socket.timeout:
if verbose:
sys.stdout.write(u"Waiting %0.1fs for model to finish (elapsed wait time %0.1fs, total wait time %0.1f)\r" % (self.ping_delay,elapsed_time,time.time()-t0))
sys.stdout.flush()
time.sleep(self.ping_delay)
elapsed_time += self.ping_delay
continue
return parse_besancon_model_string(results)
def test_find_by_hash():
from astropy.utils.data import clear_download_cache
with get_readable_fileobj(TESTURL, encoding="binary", cache=True) as page:
hash = hashlib.md5(page.read())
hashstr = 'hash/' + hash.hexdigest()
fnout = get_pkg_data_filename(hashstr)
assert os.path.isfile(fnout)
clear_download_cache(hashstr[5:])
assert not os.path.isfile(fnout)
lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')
assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def __init__(self,dataset,center=[0.,0.],pa=0.,length=10.,
vrange=None,rms=None,gray=True,level=[1.,10.]):
self.ds = dataset
self.cr = center
self.pa = pa
self.ln = length
self.vr = vrange
self.rms = rms
self.gs = gray
self.lv = level
with get_readable_fileobj(dataset, cache=True) as f:
self.fitsfile = fits.open(f)
self.data = self.fitsfile[0].data
self.header = self.fitsfile[0].header
w = wcs.WCS(self.header)
def get_images_async(self, coordinates, radius=0.25 * u.arcmin, max_rms=10000,
band="all", get_uvfits=False, verbose=True, get_query_payload=False):
"""
Serves the same purpose as :meth:`~astroquery.nvas.core.Nvas.get_images` but
returns a list of file handlers to remote files
Parameters
----------
coordinates : str or `astropy.coordinates` object
The target around which to search. It may be specified as a string
in which case it is resolved using online services or as the appropriate
`astropy.coordinates` object. ICRS coordinates may also be entered as strings
as specified in the `astropy.coordinates` module.
radius : str or `astropy.units.Quantity` object, optional
The string must be parsable by `astropy.coordinates.Angle`. The appropriate
`Quantity` object from `astropy.units` may also be used. Defaults to 0.25 arcmin.
max_rms : float, optional
Maximum allowable noise level in the image (mJy). Defaults to 10000 mJy.
band : str, optional
The band of the image to fetch. Valid bands must be from
["all","L","C","X","U","K","Q"]. Defaults to 'all'
get_uvfits : bool, optional
Gets the UVfits files instead of the IMfits files when set to `True`.
Defaults to `False`.
verbose : bool, optional
When `True` print out additional messgages. Defaults to `True`.
get_query_payload : bool, optional
if set to `True` then returns the dictionary sent as the HTTP request.
Defaults to `False`.
Returns
-------
A list of context-managers that yield readable file-like objects
"""
image_urls = self.get_image_list(coordinates, radius=radius, max_rms=max_rms,
band=band, get_uvfits=get_uvfits,
get_query_payload=get_query_payload)
if get_query_payload:
return image_urls
if verbose:
print("{num} images found.".format(num=len(image_urls)))
return [aud.get_readable_fileobj(U) for U in image_urls]
def search(cls, uniqueID, howmany=10):
"""perform restful API version of search available here:
https://gravityspytools.ciera.northwestern.edu/search/
Parameters
----------
uniqueID : `str`,
This is the unique 10 character hash that identifies
a Gravity Spy Image
howmany : `int`, optional, default: 10
number of similar images you would like
Returns
-------
`GravitySpyTable` containing similar events based on
an evaluation of the Euclidean distance of the input image
to all other images in some Feature Space
"""
from astropy.utils.data import get_readable_fileobj
import json
from six.moves.urllib.error import HTTPError
# Need to build the url call for the restful API
base = 'https://gravityspytools.ciera.northwestern.edu' + \
'/search/similarity_search_restful_API'
parts = {
'howmany': howmany,
'imageid': uniqueID,
}
search = '&'.join('{}={}'.format(key, value) for
key, value in parts.items())
url = '{}/?{}'.format(base, search)
try:
with get_readable_fileobj(url) as f:
return GravitySpyTable(json.load(f))
except HTTPError as exc:
if exc.code == 500:
exc.msg = exc.msg + ', please confirm the uniqueID is valid'
raise
def get_spectral_template_async(self, kind='qso'):
"""
Download spectral templates from SDSS DR-2, which are located here:
http://www.sdss.org/dr5/algorithms/spectemplates/
There 32 spectral templates available from DR-2, from stellar spectra,
to galaxies, to quasars. To see the available templates, do:
from astroquery.sdss import SDSS
print sdss.AVAILABLE_TEMPLATES
Parameters
----------
kind : str, list
Which spectral template to download? Options are stored in the
dictionary astroquery.sdss.SDSS.AVAILABLE_TEMPLATES
Examples
--------
>>> qso = SDSS.get_spectral_template(kind='qso')
>>> Astar = SDSS.get_spectral_template(kind='star_A')
>>> Fstar = SDSS.get_spectral_template(kind='star_F')
Returns
-------
List of PyFITS HDUList objects.
"""
if kind == 'all':
indices = list(np.arange(33))
else:
indices = spec_templates[kind]
if type(indices) is not list:
indices = [indices]
results = []
for index in indices:
name = str(index).zfill(3)
link = '%s-%s.fit' % (SDSS.TEMPLATES, name)
results.append(aud.get_readable_fileobj(link))
return results
def _parse_result(self, response, verbose=False):
"""
Parses the raw HTTP response and returns it as an `astropy.table.Table`.
Parameters
----------
response : `requests.Response`
The HTTP response object
verbose : bool, optional
Defaults to false. When true it will display warnings whenever the VOtable
returned from the service doesn't conform to the standard.
Returns
-------
table : `astropy.table.Table`
"""
table_links = self.extract_urls(response.content)
# keep only one link that is not a webstart
if len(table_links) == 0:
raise Exception("No VOTable found on returned webpage!")
table_link = [link for link in table_links if "8080" not in link][0]
with aud.get_readable_fileobj(table_link) as f:
content = f.read()
if not verbose:
commons.suppress_vo_warnings()
try:
tf = tempfile.NamedTemporaryFile()
tf.write(content.encode("utf-8"))
tf.flush()
first_table = votable.parse(tf.name, pedantic=False).get_first_table()
table = first_table.to_table()
if len(table) == 0:
warnings.warn("Query returned no results, so the table will be empty")
return table
except Exception as ex:
self.response = content
self.table_parse_error = ex
raise TableParseError(
"Failed to parse UKIDSS votable! The raw response can be found "
"in self.response, and the error in self.table_parse_error."
)
def from_json(cls, filename, **kwargs):
"""
Create a database of VO services from a JSON file.
Example JSON format for Cone Search::
{
"__version__": 1,
"catalogs" : {
"My Cone Search": {
"capabilityClass": "ConeSearch",
"title": "My Cone Search",
"url": "http://foo/cgi-bin/search?CAT=bar&",
...
},
"Another Cone Search": {
...
}
}
}
Parameters
----------
filename : str
JSON file.
kwargs : dict
Keywords accepted by
:func:`~astropy.utils.data.get_readable_fileobj`.
Returns
-------
db : `VOSDatabase`
Database from given file.
"""
with get_readable_fileobj(filename, **kwargs) as fd:
tree = json.load(fd)
return cls(tree)
def vospace_readable_fileobj(name_or_obj, token=None, **kwargs):
"""Read data from VOSpace or some other place.
Notes
-----
Most of the heavy lifting is done with
:func:`~astropy.io.data.get_readable_fileobj`. Any additional keywords
passed to this function will get passed directly to that function.
Parameters
----------
name_or_obj : :class:`str` or file-like object
The filename of the file to access (if given as a string), or
the file-like object to access.
If a file-like object, it must be opened in binary mode.
token : :class:`str`
A token granting access to VOSpace.
Returns
-------
file
A readable file-like object.
"""
fileobj = name_or_obj
close_fileobj = False
if (isinstance(name_or_obj, str) and name_or_obj.find('://') > 0):
uri = name_or_obj[:name_or_obj.find('://')]
if authClient.isValidUser(uri):
# VOSpace call
fileobj = BytesIO(storeClient.get(name_or_obj, mode='binary'))
close_fileobj = True
with get_readable_fileobj(fileobj, **kwargs) as f:
try:
yield f
finally:
if close_fileobj:
fileobj.close()
def xml_readlines(source):
"""
Get the lines from a given XML file. Correctly determines the
encoding and always returns unicode.
Parameters
----------
source : path-like, readable file-like, or callable
Handle that contains the data or function that reads it.
If a function or callable object, it must directly read from a stream.
Non-callable objects must define a ``read`` method.
Returns
-------
lines : list of unicode
"""
encoding = get_xml_encoding(source)
with data.get_readable_fileobj(source, encoding=encoding) as input:
input.seek(0)
xml_lines = input.readlines()
return xml_lines
def request_besancon(email,
glon,
glat,
smallfield=True,
extinction=0.7,
area=0.0001,
verbose=True,
clouds=None,
absmag_limits=(-7, 15),
mag_limits=copy.copy(mag_limits),
colors_limits=copy.copy(colors_limits),
retrieve_file=True,
**kwargs):
"""
Perform a query on the Besancon model of the galaxy
http://model.obs-besancon.fr/
Parameters
----------
email : string
A valid e-mail address to send the report of completion to
glon : float
glat : float
Galactic latitude and longitude at the center
smallfield : bool
Small field (True) or Large Field (False)
LARGE FIELD NOT SUPPORTED YET
extinction : float
Extinction per kpc in A_V
area : float
Area in square degrees
absmag_limits : (float,float)
Absolute magnitude lower,upper limits
colors_limits : dict of (float,float)
Should contain 4 elements listing color differences in the valid bands, e.g.:
{"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)}
mag_limits = dict of (float,float)
Lower and Upper magnitude difference limits for each magnitude band
U B V R I J H K L
clouds : list of 2-tuples
Up to 25 line-of-sight clouds can be specified in pairs of (A_V,
distance in pc)
verbose : bool
Print out extra error messages?
retrieve_file : bool
If True, will try to retrieve the file every 30s until it shows up.
Otherwise, just returns the filename (the job is still executed on
the remote server, though)
kwargs : dict
Can override any argument in the request if you know the name of the
POST keyword.
Returns
-------
Either the filename or the table depending on whether 'retrieve file' is
specified
"""
# create a new keyword dict based on inputs + defaults
kwd = copy.copy(keyword_defaults)
for key, val in kwargs.iteritems():
if key in keyword_defaults:
kwd[key] = val
elif verbose:
print "Skipped invalid key %s" % key
kwd['kleg'] = 1 if smallfield else 2
if not smallfield:
raise NotImplementedError
kwd['adif'] = extinction
kwd['soli'] = area
kwd['oo'][0] = absmag_limits[0]
kwd['ff'][0] = absmag_limits[1]
for ii, (key, val) in enumerate(colors_limits.items()):
if key[0] in mag_order and key[1] == '-' and key[2] in mag_order:
kwd['colind'][ii] = key
kwd['oo'][ii + 9] = val[0]
kwd['ff'][ii + 9] = val[1]
else:
raise ValueError('Invalid color %s' % key)
for (key, val) in mag_limits.iteritems():
if key in mag_order:
kwd['band0'][mag_order.index(key)] = val[0]
kwd['bandf'][mag_order.index(key)] = val[1]
else:
raise ValueError('Invalid band %s' % key)
if clouds is not None:
for ii, (AV, di) in enumerate(clouds):
kwd[AV][ii] = AV
kwd[di][ii] = di
# parse the default dictionary
request = parse_besancon_dict(keyword_defaults)
# an e-mail address is required
request.append(('email', email))
request = urllib.urlencode(request)
# load the URL as text
U = urllib.urlopen(url_request, request)
# keep the text stored for possible later use
with aud.get_readable_fileobj(U) as f:
text = f.read()
try:
filename = result_re.search(text).group()
except AttributeError: # if there are no matches
errors = parse_errors(text)
raise ValueError("Errors: " + "\n".join(errors))
if verbose:
print "File is %s and can be aquired from %s" % (
filename, url_download + filename)
if retrieve_file:
return get_besancon_model_file(filename)
else:
return filename
def test_url_nocache():
with get_readable_fileobj(TESTURL, cache=False, encoding='utf-8') as page:
assert page.read().find('Astropy') > -1
def validate(source, output=None, xmllint=False, filename=None):
"""
Prints a validation report for the given file.
Parameters
----------
source : str or readable file-like object
Path to a VOTABLE_ xml file or pathlib.path
object having Path to a VOTABLE_ xml file.
output : writable file-like object, optional
Where to output the report. Defaults to ``sys.stdout``.
If `None`, the output will be returned as a string.
xmllint : bool, optional
When `True`, also send the file to ``xmllint`` for schema and
DTD validation. Requires that ``xmllint`` is installed. The
default is `False`. ``source`` must be a file on the local
filesystem in order for ``xmllint`` to work.
filename : str, optional
A filename to use in the error messages. If not provided, one
will be automatically determined from ``source``.
Returns
-------
is_valid : bool or str
Returns `True` if no warnings were found. If ``output`` is
`None`, the return value will be a string.
"""
from astropy.utils.console import print_code_line, color_print
if output is None:
output = sys.stdout
return_as_str = False
if output is None:
output = io.StringIO()
lines = []
votable = None
reset_vo_warnings()
with data.get_readable_fileobj(source, encoding='binary') as fd:
content = fd.read()
content_buffer = io.BytesIO(content)
content_buffer.seek(0)
if filename is None:
if isinstance(source, str):
filename = source
elif hasattr(source, 'name'):
filename = source.name
elif hasattr(source, 'url'):
filename = source.url
else:
filename = "<unknown>"
with warnings.catch_warnings(record=True) as warning_lines:
warnings.resetwarnings()
warnings.simplefilter("always", exceptions.VOWarning, append=True)
try:
votable = parse(content_buffer, verify='warn', filename=filename)
except ValueError as e:
lines.append(str(e))
lines = [
str(x.message)
for x in warning_lines if issubclass(x.category, exceptions.VOWarning)
] + lines
content_buffer.seek(0)
output.write("Validation report for {0}\n\n".format(filename))
if len(lines):
xml_lines = iterparser.xml_readlines(content_buffer)
for warning in lines:
w = exceptions.parse_vowarning(warning)
if not w['is_something']:
output.write(w['message'])
output.write('\n\n')
else:
line = xml_lines[w['nline'] - 1]
warning = w['warning']
if w['is_warning']:
color = 'yellow'
else:
color = 'red'
color_print('{0:d}: '.format(w['nline']),
'',
warning or 'EXC',
color,
': ',
'',
textwrap.fill(w['message'],
initial_indent=' ',
subsequent_indent=' ').lstrip(),
file=output)
print_code_line(line, w['nchar'], file=output)
output.write('\n')
else:
output.write('astropy.io.votable found no violations.\n\n')
success = 0
if xmllint and os.path.exists(filename):
from . import xmlutil
if votable is None:
version = "1.1"
else:
version = votable.version
success, stdout, stderr = xmlutil.validate_schema(filename, version)
if success != 0:
output.write('xmllint schema violations:\n\n')
output.write(stderr.decode('utf-8'))
else:
output.write('xmllint passed\n')
if return_as_str:
return output.getvalue()
return len(lines) == 0 and success == 0
def read(table, guess=None, **kwargs):
# Docstring defined below
del _read_trace[:]
# Downstream readers might munge kwargs
kwargs = copy.deepcopy(kwargs)
# Convert 'fast_reader' key in kwargs into a dict if not already and make sure
# 'enable' key is available.
fast_reader = _get_fast_reader_dict(kwargs)
kwargs['fast_reader'] = fast_reader
if fast_reader['enable'] and fast_reader.get('chunk_size'):
return _read_in_chunks(table, **kwargs)
if 'fill_values' not in kwargs:
kwargs['fill_values'] = [('', '0')]
# If an Outputter is supplied in kwargs that will take precedence.
if 'Outputter' in kwargs: # user specified Outputter, not supported for fast reading
fast_reader['enable'] = False
format = kwargs.get('format')
# Dictionary arguments are passed by reference per default and thus need
# special protection:
new_kwargs = copy.deepcopy(kwargs)
kwargs['fast_reader'] = copy.deepcopy(fast_reader)
# Get the Reader class based on possible format and Reader kwarg inputs.
Reader = _get_format_class(format, kwargs.get('Reader'), 'Reader')
if Reader is not None:
new_kwargs['Reader'] = Reader
format = Reader._format_name
# Remove format keyword if there, this is only allowed in read() not get_reader()
if 'format' in new_kwargs:
del new_kwargs['format']
if guess is None:
guess = _GUESS
if guess:
# If ``table`` is probably an HTML file then tell guess function to add
# the HTML reader at the top of the guess list. This is in response to
# issue #3691 (and others) where libxml can segfault on a long non-HTML
# file, thus prompting removal of the HTML reader from the default
# guess list.
new_kwargs['guess_html'] = _probably_html(table)
# If `table` is a filename or readable file object then read in the
# file now. This prevents problems in Python 3 with the file object
# getting closed or left at the file end. See #3132, #3013, #3109,
# #2001. If a `readme` arg was passed that implies CDS format, in
# which case the original `table` as the data filename must be left
# intact.
if 'readme' not in new_kwargs:
encoding = kwargs.get('encoding')
try:
with get_readable_fileobj(table, encoding=encoding) as fileobj:
table = fileobj.read()
except ValueError: # unreadable or invalid binary file
raise
except Exception:
pass
else:
# Ensure that `table` has at least one \r or \n in it
# so that the core.BaseInputter test of
# ('\n' not in table and '\r' not in table)
# will fail and so `table` cannot be interpreted there
# as a filename. See #4160.
if not re.search(r'[\r\n]', table):
table = table + os.linesep
# If the table got successfully read then look at the content
# to see if is probably HTML, but only if it wasn't already
# identified as HTML based on the filename.
if not new_kwargs['guess_html']:
new_kwargs['guess_html'] = _probably_html(table)
# Get the table from guess in ``dat``. If ``dat`` comes back as None
# then there was just one set of kwargs in the guess list so fall
# through below to the non-guess way so that any problems result in a
# more useful traceback.
dat = _guess(table, new_kwargs, format, fast_reader)
if dat is None:
guess = False
if not guess:
if format is None:
reader = get_reader(**new_kwargs)
format = reader._format_name
# Try the fast reader version of `format` first if applicable. Note that
# if user specified a fast format (e.g. format='fast_basic') this test
# will fail and the else-clause below will be used.
if fast_reader['enable'] and 'fast_{0}'.format(
format) in core.FAST_CLASSES:
fast_kwargs = copy.deepcopy(new_kwargs)
fast_kwargs['Reader'] = core.FAST_CLASSES['fast_{0}'.format(
format)]
fast_reader_rdr = get_reader(**fast_kwargs)
try:
dat = fast_reader_rdr.read(table)
_read_trace.append({
'kwargs':
copy.deepcopy(fast_kwargs),
'Reader':
fast_reader_rdr.__class__,
'status':
'Success with fast reader (no guessing)'
})
except (core.ParameterError, cparser.CParserError,
UnicodeEncodeError) as err:
# special testing value to avoid falling back on the slow reader
if fast_reader['enable'] == 'force':
raise core.InconsistentTableError(
'fast reader {} exception: {}'.format(
fast_reader_rdr.__class__, err))
# If the fast reader doesn't work, try the slow version
reader = get_reader(**new_kwargs)
dat = reader.read(table)
_read_trace.append({
'kwargs':
copy.deepcopy(new_kwargs),
'Reader':
reader.__class__,
'status':
'Success with slow reader after failing'
' with fast (no guessing)'
})
else:
reader = get_reader(**new_kwargs)
dat = reader.read(table)
_read_trace.append({
'kwargs':
copy.deepcopy(new_kwargs),
'Reader':
reader.__class__,
'status':
'Success with specified Reader class '
'(no guessing)'
})
return dat
def read_multi(flatten, cls, source, *args, **kwargs):
"""Read sources into a `cls` with multiprocessing
This method should be called by `cls.read` and uses the `nproc`
keyword to enable and handle pool-based multiprocessing of
multiple source files, using `flatten` to combine the
chunked data into a single object of the correct type.
Parameters
----------
flatten : `callable`
a method to take a list of ``cls`` instances, and combine them
into a single ``cls`` instance
cls : `type`
the object type to read
source : `str`, `list` of `str`, ...
the input data source, can be of in many different forms
*args
positional arguments to pass to the reader
**kwargs
keyword arguments to pass to the reader
"""
# parse input as a list of files
try: # try and map to a list of file-like objects
files = file_list(source)
except ValueError: # otherwise treat as single file
files = [source]
# determine input format (so we don't have to do it multiple times)
# -- this is basically harvested from astropy.io.registry.read()
if kwargs.get('format', None) is None:
ctx = None
if isinstance(source, FILE_LIKE):
fileobj = source
elif isinstance(source, string_types):
try:
ctx = get_readable_fileobj(files[0], encoding='binary')
fileobj = ctx.__enter__()
except IOError:
raise
except Exception:
fileobj = None
kwargs['format'] = get_format('read', cls, files[0], fileobj, args,
kwargs)
if ctx is not None:
ctx.__exit__(*sys.exc_info())
# calculate maximum number of processes
nproc = min(kwargs.pop('nproc', 1), len(files))
# define multiprocessing method
def _read_single_file(f):
try:
return f, io_read(cls, f, *args, **kwargs)
except Exception as e:
if nproc == 1:
raise
elif isinstance(e, SAXException): # SAXExceptions don't pickle
return f, e.getException()
else:
return f, e
# read files
output = mp_utils.multiprocess_with_queues(nproc,
_read_single_file,
files,
raise_exceptions=False)
# raise exceptions (from multiprocessing, single process raises inline)
for f, x in output:
if isinstance(x, Exception):
x.args = ('Failed to read %s: %s' % (f, str(x)), )
raise x
# return combined object
_, out = zip(*output)
return flatten(out)
__author__ = "Alex Urban <*****@*****.**>"
__currentmodule__ = 'gwpy.timeseries'
# First, we prepare one second of Gaussian noise:
from numpy import random
from gwpy.timeseries import TimeSeries
noise = TimeSeries(random.normal(scale=.1, size=16384), sample_rate=16384)
# Then we can download a simulation of the GW150914 signal from LOSC:
from astropy.utils.data import get_readable_fileobj
source = 'https://losc.ligo.org/s/events/GW150914/P150914/'
url = '%s/fig2-unfiltered-waveform-H.txt' % source
with get_readable_fileobj(url) as f:
signal = TimeSeries.read(f, format='txt')
signal.t0 = .5 # make sure this intersects with noise time samples
# Note, since this simulation cuts off before a certain time, it is
# important to taper its ends to zero to avoid ringing artifacts.
# We can accomplish this using the
# :meth:`~gwpy.timeseries.TimeSeries.taper` method.
signal = signal.taper()
# Since the time samples overlap, we can inject this into our noise data
# using :meth:`~gwpy.types.series.Series.inject`:
data = noise.inject(signal)
def from_veto_definer_file(cls, fp, start=None, end=None, ifo=None,
format='ligolw'):
"""Read a `DataQualityDict` from a LIGO_LW XML VetoDefinerTable.
Parameters
----------
fp : `str`
path of veto definer file to read
start : `~gwpy.time.LIGOTimeGPS`, `int`, optional
GPS start time at which to restrict returned flags
end : `~gwpy.time.LIGOTimeGPS`, `int`, optional
GPS end time at which to restrict returned flags
ifo : `str`, optional
interferometer prefix whose flags you want to read
format : `str`, optional
format of file to read, currently only 'ligolw' is supported
Returns
-------
flags : `DataQualityDict`
a `DataQualityDict` of flags parsed from the `veto_def_table`
of the input file.
Notes
-----
This method does not automatically `~DataQualityDict.populate`
the `active` segment list of any flags, a separate call should
be made for that as follows
>>> flags = DataQualityDict.from_veto_definer_file('/path/to/file.xml')
>>> flags.populate()
"""
from ..io.ligolw import table_from_file
if format != 'ligolw':
raise NotImplementedError("Reading veto definer from non-ligolw "
"format file is not currently "
"supported")
if start is not None:
start = to_gps(start)
if end is not None:
end = to_gps(end)
# read veto definer file
with get_readable_fileobj(fp, show_progress=False) as f:
veto_def_table = table_from_file(f, 'veto_definer')
# parse flag definitions
out = cls()
for row in veto_def_table:
if ifo and row.ifo != ifo:
continue
if start and 0 < row.end_time <= start:
continue
elif start:
row.start_time = max(row.start_time, start)
if end and row.start_time >= end:
continue
elif end and not row.end_time:
row.end_time = end
elif end:
row.end_time = min(row.end_time, end)
flag = DataQualityFlag.from_veto_def(row)
if flag.name in out:
out[flag.name].known.extend(flag.known)
out[flag.name].known.coalesce()
else:
out[flag.name] = flag
return out
def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False,
verbose=True, savename=None, overwrite=False):
"""
Get all sources in the catalog within some radius
Parameters
----------
glon : float
glat : float
Galactic latitude and longitude at the center
directory : None or string
Directory to download files into. Defaults to self.directory
radius : float
Radius in which to search for catalog entries in arcminutes
savename : string or None
The file name to save the catalog to. If unspecified, will save as
UKIDSS_catalog_G###.###-###.###_r###.fits.gz, where the #'s indicate
galactic lon/lat and radius
Returns
-------
List of fits.primaryHDU instances containing FITS tables
Example
-------
>>> R = UKIDSSQuery()
>>> data = R.get_catalog_gal(10.625,-0.38,radius=0.1)
>>> bintable = data[0][1]
"""
# Construct request
self.request = {}
self.request['database'] = self.database
self.request['programmeID'] = verify_programme_id(self.programmeID,querytype='catalog')
self.request['from'] = 'source'
self.request['formaction'] = 'region'
self.request['ra'] = glon
self.request['dec'] = glat
self.request['sys'] = 'G'
self.request['radius'] = radius
self.request['xSize'] = ''
self.request['ySize'] = ''
self.request['boxAlignment'] = 'RADec'
self.request['emailAddress'] = ''
self.request['format'] = 'FITS'
self.request['compress'] = 'GZIP'
self.request['rows'] = 1
self.request['select'] = '*'
self.request['where'] = ''
self.query_str = url_getcatalog +"?"+ urlencode(self.request)
if directory is None:
directory = self.directory
# Retrieve page
page = self.opener.open(url_getcatalog, urlencode(self.request))
with aud.get_readable_fileobj(page) as f:
results = f.read()
# Parse results for links
format = formatter.NullFormatter() # create default formatter
htmlparser = LinksExtractor(format) # create new parser object
htmlparser.feed(results)
htmlparser.close()
links = list(set(htmlparser.get_links()))
# Loop through links and retrieve FITS tables
c = 0
data = []
for link in links:
if not "8080" in link:
c = c + 1
if not os.path.exists(directory):
os.mkdir(directory)
if save:
if savename is None:
savename = "UKIDSS_catalog_G%07.3f%+08.3f_r%03i.fits.gz" % (glon,glat,radius)
filename = directory + "/" + savename
U = self.opener.open(link)
with aud.get_readable_fileobj(U, cache=True) as f:
results = f.read()
S = BytesIO(results)
try:
fitsfile = fits.open(S,ignore_missing_end=True)
except IOError:
S.seek(0)
G = gzip.GzipFile(fileobj=S)
fitsfile = fits.open(G,ignore_missing_end=True)
data.append(fitsfile)
if save:
fitsfile.writeto(filename.rstrip(".gz"), clobber=overwrite)
return data
def search(cls,
gravityspy_id,
howmany=10,
era='ALL',
ifos='H1L1',
remote_timeout=20):
"""perform restful API version of search available here:
https://gravityspytools.ciera.northwestern.edu/search/
Parameters
----------
gravityspy_id : `str`,
This is the unique 10 character hash that identifies
a Gravity Spy Image
howmany : `int`, optional, default: 10
number of similar images you would like
Returns
-------
`GravitySpyTable` containing similar events based on
an evaluation of the Euclidean distance of the input image
to all other images in some Feature Space
"""
# Need to build the url call for the restful API
base = 'https://gravityspytools.ciera.northwestern.edu' + \
'/search/similarity_search_restful_API'
map_era_to_url = {
'ALL': "event_time BETWEEN 1126400000 AND 1584057618",
'O1': "event_time BETWEEN 1126400000 AND 1137250000",
'ER10': "event_time BETWEEN 1161907217 AND 1164499217",
'O2a': "event_time BETWEEN 1164499217 AND 1219276818",
'ER13': "event_time BETWEEN 1228838418 AND 1229176818",
}
parts = {
'howmany':
howmany,
'imageid':
gravityspy_id,
'era':
map_era_to_url[era],
'ifo':
"{}".format(", ".join(
map(repr, [ifos[i:i + 2] for i in range(0, len(ifos), 2)]), )),
'database':
'similarity_index_o3',
}
search = urlencode(parts)
url = '{}/?{}'.format(base, search)
try:
with get_readable_fileobj(url, remote_timeout=remote_timeout) as f:
return GravitySpyTable(json.load(f))
except HTTPError as exc:
if exc.code == 500:
exc.msg += ', confirm the gravityspy_id is valid'
raise
def download_list_of_fitsfiles(linklist,
output_directory=None,
output_prefix=None,
save=False,
overwrite=False,
verbose=False,
output_coord_format=None,
filename_header_keywords=None,
include_input_filename=True):
"""
Given a list of file URLs, download them and (optionally) rename them
example:
download_list_of_fitsfiles(
['http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH00.fits',
'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH01.fits',
'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_SC00.fits'],
output_directory='fermi_m31',
output_prefix='FermiLAT',
save=True,
overwrite=False,
verbose=True,
output_coord_format=None, # FITS tables don't have crval/crpix, good one is: "%08.3g%+08.3g",
filename_header_keywords=None, # couldn't find any useful ones
include_input_filename=True)
"""
# Loop through links and retrieve FITS images
images = {}
for link in linklist:
if output_directory is None:
output_directory = ""
elif output_directory[-1] != "/":
output_directory += "/"
if not os.path.exists(output_directory):
os.mkdir(output_directory)
with aud.get_readable_fileobj(link, cache=True) as f:
results = f.read()
S = StringIO.StringIO(results)
try:
# try to open as a fits file
fitsfile = fits.open(S, ignore_missing_end=True)
except IOError:
# if that fails, try to open as a gzip'd fits file
# have to rewind to the start
S.seek(0)
G = gzip.GzipFile(fileobj=S)
fitsfile = fits.open(G, ignore_missing_end=True)
# Get Multiframe ID from the header
images[link] = fitsfile
if save:
h0 = fitsfile[0].header
if filename_header_keywords: # is not None or empty
nametxt = "_".join([
validify_filename(str(h0[key]))
for key in filename_header_keywords
])
else:
nametxt = ""
if output_coord_format:
lon = h0['CRVAL1']
lat = h0['CRVAL2']
# this part will eventually be handled by astropy.coordinates directly
# ctype = h0['CTYPE1']
# if 'RA' in ctype:
# coordinate = coord.ICRSCoordinates(lon,lat,unit=('deg','deg'))
# elif 'GLON' in ctype:
# coordinate = coord.GalacticCoordinates(lon,lat,unit=('deg','deg'))
# else:
# raise TypeError("Don't recognize ctype %s" % ctype)
# coordstr = coordinate.format(output_coord_format)
try:
coordstr = output_coord_format.format(lon, lat)
except TypeError:
coordstr = output_coord_format % (lon, lat)
nametxt += "_" + coordstr
if include_input_filename:
filename_root = os.path.split(link)[1]
else:
filename_root = ""
savename = output_prefix if output_prefix else ""
savename += nametxt
savename += "_" + filename_root
# Set final directory and file names
final_file = output_directory + savename
if verbose:
print "Saving file %s" % final_file
try:
fitsfile.writeto(final_file, clobber=overwrite)
except IOError:
print "Skipped writing file %s because it exists and overwrite=False" % final_file
return images
def get_nrao_image(lon,
lat,
system='galactic',
epoch='J2000',
size=1.0,
max_rms=1e4,
band="",
verbose=True,
savename=None,
save=True,
overwrite=False,
directory='./',
get_uvfits=False):
"""
Search for and download
Parameters
----------
lon : float
lat : float
Right ascension and declination or glon/glat
system : ['celestial','galactic']
System of lon/lat. Can be any valid coordinate system supported by the
astropy.coordinates package
epoch : string
Epoch of the coordinate system (e.g., B1950, J2000)
savename : None or string
filename to save fits file as. If None, will become G###.###p###.###_(survey).fits
size : float
Size of search radius (arcminutes)
max_rms : float
Maximum allowable noise level in the image (mJy)
verbose : bool
Print out extra error messages?
save : bool
Save FITS file?
overwrite : bool
Overwrite if file already exists?
directory : string
Directory to store file in. Defaults to './'.
get_uvfits : bool
Get the UVfits files instead of the IMfits files?
Examples
--------
>>> fitsfile = get_nrao_image(49.489,-0.37)
"""
if band not in valid_bands:
raise ValueError("Invalid band. Valid bands are: %s" % valid_bands)
if system == 'celestial':
radec = coord.FK5Coordinates(lon, lat, unit=('deg', 'deg'))
galactic = radec.galactic
elif system == 'galactic':
galactic = coord.GalacticCoordinates(lon, lat, unit=('deg', 'deg'))
radec = galactic.fk5
radecstr = radec.ra.format(sep=' ') + ' ' + radec.dec.format(sep=' ')
glon, glat = galactic.lonangle.degrees, galactic.latangle.degrees
# Construct request
request = {}
request["nvas_pos"] = radecstr
request["nvas_rad"] = size
request["nvas_rms"] = max_rms
request["nvas_scl"] = size
request["submit"] = "Search"
request["nvas_bnd"] = band
# create the request header data
request = urllib.urlencode(request)
# load the URL as text
U = urllib.urlopen(request_URL, request)
# read results with progressbar
results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report)
if get_uvfits:
links = uvfits_re.findall(results)
else:
links = imfits_re.findall(results)
configurations = config_re.findall(results)
if len(links) == 0:
if verbose:
print "No matches found at ra,dec = %s." % (radecstr)
return []
if verbose > 1:
print "Configurations: "
print "\n".join(
["%40s: %20s" % (L, C) for L, C in zip(links, configurations)])
if save and not os.path.exists(directory):
os.mkdir(directory)
if save:
opener = urllib2.build_opener()
if verbose:
print "Found %i imfits files" % len(links)
images = []
for link, config in zip(links, configurations):
# Get the file
U = opener.open(link)
with aud.get_readable_fileobj(U) as f:
results = f.read()
S = StringIO.StringIO(results)
try:
fitsfile = fits.open(S, ignore_missing_end=True)
except IOError:
S.seek(0)
G = gzip.GzipFile(fileobj=S)
fitsfile = fits.open(G, ignore_missing_end=True)
# Get Multiframe ID from the header
images.append(fitsfile)
if save:
h0 = fitsfile[0].header
freq_ghz = h0['CRVAL3'] / 1e9
for bn, bandlimits in band_freqs.iteritems():
if freq_ghz < bandlimits[1] and freq_ghz > bandlimits[0]:
bandname = bn
obj = str(h0['OBJECT']).strip()
program = h0['OBSERVER'].strip()
h0['CONFIG'] = config
if savename is None:
if get_uvfits:
filename = "VLA_%s_G%07.3f%+08.3f_%s_%s.uvfits" % (
bandname, glon, glat, obj, program)
else:
filename = "VLA_%s_G%07.3f%+08.3f_%s_%s.fits" % (
bandname, glon, glat, obj, program)
else:
filename = savename
# Set final directory and file names
final_file = directory + '/' + filename
if verbose:
print "Saving file %s" % final_file
fitsfile.writeto(final_file, clobber=overwrite)
return images
def parse_cs(rid):
"""Return ``<testQuery>`` pars as dict for given Resource ID."""
if isinstance(rid, bytes): # pragma: py3
rid = rid.decode('ascii')
# Production server.
url = ('http://vao.stsci.edu/directory/getRecord.aspx?'
'id={0}&format=xml'.format(rid))
# Test server (in case production server fails).
backup_url = ('http://vaotest.stsci.edu/directory/getRecord.aspx?'
'id={0}&format=xml'.format(rid))
tqp = ['ra', 'dec', 'sr']
d = OrderedDict()
urls_failed = False
urls_errmsg = ''
try:
with get_readable_fileobj(url, encoding='binary',
show_progress=False) as fd:
dom = minidom.parse(fd)
except Exception as e: # pragma: no cover
try:
warnings.warn(
'{0} raised {1}, trying {2}'.format(url, str(e), backup_url),
AstropyUserWarning)
with get_readable_fileobj(backup_url,
encoding='binary',
show_progress=False) as fd:
dom = minidom.parse(fd)
except Exception as e:
urls_failed = True
urls_errmsg = '{0} raised {1}, using default'.format(
backup_url, str(e))
if not urls_failed:
tq = dom.getElementsByTagName('testQuery')
if tq:
for key in tqp:
try:
d[key.upper()] = tq[0].getElementsByTagName(
key)[0].firstChild.nodeValue.strip()
except Exception: # pragma: no cover
urls_failed = True
urls_errmsg = ('Incomplete testQuery for {0}, '
'using default'.format(rid))
else: # pragma: no cover
urls_failed = True
urls_errmsg = 'No testQuery found for {0}, using default'.format(
rid)
# Handle big SR returning too big a table for some queries, causing
# tests to fail due to timeout.
default_sr = '0.1'
# If no testQuery found, use default
if urls_failed: # pragma: no cover
d = OrderedDict({'RA': '0', 'DEC': '0', 'SR': default_sr})
warnings.warn(urls_errmsg, AstropyUserWarning)
# Force SR to be reasonably small
elif d['SR'] > default_sr:
warnings.warn(
'SR={0} is too large, using SR={1} for {2}'.format(
d['SR'], default_sr, rid), AstropyUserWarning)
d['SR'] = default_sr
return d
def get_image_gal(self, glon, glat, filter='all', frametype='stack',
directory=None, size=1.0, verbose=True, save=True, savename=None,
overwrite=False):
"""
Get an image at a specified glon/glat. Size can be specified
Parameters
----------
glon : float
glat : float
Galactic latitude and longitude at the center
filter : ['all','J','H','K','H2','Z','Y','Br']
The color filter to download.
frametype : ['stack','normal','interleave','deep%stack','confidence','difference','leavstack','all']
The type of image
directory : None or string
Directory to download files into. Defaults to self.directory
size : float
Size of cutout (symmetric) in arcminutes
verbose : bool
Print out extra error messages?
save : bool
Save FITS file?
savename : string or None
The file name to save the catalog to. If unspecified, will save as
UKIDSS_[band]_G###.###-###.###_[obj].fits.gz, where the #'s
indicate galactic lon/lat and [band] and [obj] refer to the filter
and the object name
overwrite : bool
Overwrite if file already exists?
Examples
--------
>>> R = UKIDSSQuery()
>>> fitsfile = R.get_image_gal(10.5,0.0)
# get UWISH2 data (as per http://astro.kent.ac.uk/uwish2/main.html)
>>> R.database='U09B8v20120403'
>>> R.login(username='******',password='******',community='nonSurvey')
>>> R.get_image_gal(49.489,-0.27,frametype='leavstack',size=20,filter='H2')
"""
# Check for validity of requested frame_type
if frametype not in frame_types:
raise ValueError("Invalide frame type. Valid frame types are: %s"
% frame_types)
if filter not in self.filters:
raise ValueError("Invalide filter. Valid filters are: %s"
% self.filters.keys())
# Construct request
self.request = {}
self.request['database'] = self.database
self.request['programmeID'] = verify_programme_id(self.programmeID,querytype='image')
self.request['ra'] = glon
self.request['dec'] = glat
self.request['sys'] = 'G'
self.request['filterID'] = self.filters[filter]
self.request['xsize'] = size
self.request['ysize'] = size
self.request['obsType'] = 'object'
self.request['frameType'] = frametype
self.request['mfid'] = ''
self.query_str = url_getimage +"?"+ urlencode(self.request)
if directory is None:
directory = self.directory
# Retrieve page
page = self.opener.open(url_getimage, urlencode(self.request))
with aud.get_readable_fileobj(page) as f:
results = f.read()
# Parse results for links
format = formatter.NullFormatter()
htmlparser = LinksExtractor(format)
htmlparser.feed(results)
htmlparser.close()
links = htmlparser.get_links()
if verbose:
print("Found %i targets" % (len(links)))
# Loop through links and retrieve FITS images
images = []
for link in links:
if not os.path.exists(directory):
os.mkdir(directory)
# Get the file
U = self.opener.open(link.replace("getImage", "getFImage"))
with aud.get_readable_fileobj(U, cache=True) as f:
results = f.read()
S = BytesIO(results)
try:
# try to open as a fits file
fitsfile = fits.open(S,ignore_missing_end=True)
except IOError:
# if that fails, try to open as a gzip'd fits file
# have to rewind to the start
S.seek(0)
G = gzip.GzipFile(fileobj=S)
fitsfile = fits.open(G,ignore_missing_end=True)
# Get Multiframe ID from the header
images.append(fitsfile)
if save:
h0 = fitsfile[0].header
filt = str(h0['FILTER']).strip()
obj = filt + "_" + str(h0['OBJECT']).strip().replace(":", ".")
if savename is None:
filename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt,glon,glat,obj)
else:
filename = savename
# Set final directory and file names
final_file = directory + '/' + filename
if verbose:
print("Saving file %s" % final_file)
fitsfile.writeto(final_file, clobber=overwrite)
return images
def test_path_objects_get_readable_fileobj():
fpath = pathlib.Path(TESTLOCAL)
with get_readable_fileobj(fpath) as f:
assert f.read().rstrip() == ('This file is used in the test_local_data_* '
'testing functions\nCONTENT')
def read(self, cls, *args, format=None, cache=False, **kwargs):
"""
Read in data.
Parameters
----------
cls : class
*args
The arguments passed to this method depend on the format.
format : str or None
cache : bool
Whether to cache the results of reading in the data.
**kwargs
The arguments passed to this method depend on the format.
Returns
-------
object or None
The output of the registered reader.
"""
ctx = None
try:
# Expand a tilde-prefixed path if present in args[0]
args = _expand_user_in_args(args)
if format is None:
path = None
fileobj = None
if len(args):
if isinstance(args[0],
PATH_TYPES) and not os.path.isdir(args[0]):
from astropy.utils.data import get_readable_fileobj
# path might be a os.PathLike object
if isinstance(args[0], os.PathLike):
args = (os.fspath(args[0]), ) + args[1:]
path = args[0]
try:
ctx = get_readable_fileobj(args[0],
encoding='binary',
cache=cache)
fileobj = ctx.__enter__()
except OSError:
raise
except Exception:
fileobj = None
else:
args = [fileobj] + list(args[1:])
elif hasattr(args[0], 'read'):
path = None
fileobj = args[0]
format = self._get_valid_format('read', cls, path, fileobj,
args, kwargs)
reader = self.get_reader(format, cls)
data = reader(*args, **kwargs)
if not isinstance(data, cls):
# User has read with a subclass where only the parent class is
# registered. This returns the parent class, so try coercing
# to desired subclass.
try:
data = cls(data)
except Exception:
raise TypeError('could not convert reader output to {} '
'class.'.format(cls.__name__))
finally:
if ctx is not None:
ctx.__exit__(*sys.exc_info())
return data
def find_api_page(obj, version=None, openinbrowser=True, timeout=None):
"""
Determines the URL of the API page for the specified object, and
optionally open that page in a web browser.
.. note::
You must be connected to the internet for this to function even if
``openinbrowser`` is `False`, unless you provide a local version of
the documentation to ``version`` (e.g., ``file:///path/to/docs``).
Parameters
----------
obj
The object to open the docs for or its fully-qualified name
(as a str).
version : str
The doc version - either a version number like '0.1', 'dev' for
the development/latest docs, or a URL to point to a specific
location that should be the *base* of the documentation. Defaults to
latest if you are on aren't on a release, otherwise, the version you
are on.
openinbrowser : bool
If `True`, the `webbrowser` package will be used to open the doc
page in a new web browser window.
timeout : number, optional
The number of seconds to wait before timing-out the query to
the astropy documentation. If not given, the default python
stdlib timeout will be used.
Returns
-------
url : str
The loaded URL
Raises
------
ValueError
If the documentation can't be found
"""
import webbrowser
from zlib import decompress
from astropy.utils.data import get_readable_fileobj
if (not isinstance(obj, str) and hasattr(obj, '__module__')
and hasattr(obj, '__name__')):
obj = obj.__module__ + '.' + obj.__name__
elif inspect.ismodule(obj):
obj = obj.__name__
if version is None:
from astropy import version
if version.release:
version = 'v' + version.version
else:
version = 'dev'
if '://' in version:
if version.endswith('index.html'):
baseurl = version[:-10]
elif version.endswith('/'):
baseurl = version
else:
baseurl = version + '/'
elif version == 'dev' or version == 'latest':
baseurl = 'http://devdocs.astropy.org/'
else:
baseurl = f'https://docs.astropy.org/en/{version}/'
# Custom request headers; see
# https://github.com/astropy/astropy/issues/8990
url = baseurl + 'objects.inv'
headers = {'User-Agent': f'Astropy/{version}'}
with get_readable_fileobj(url,
encoding='binary',
remote_timeout=timeout,
http_headers=headers) as uf:
oiread = uf.read()
# need to first read/remove the first four lines, which have info before
# the compressed section with the actual object inventory
idx = -1
headerlines = []
for _ in range(4):
oldidx = idx
idx = oiread.index(b'\n', oldidx + 1)
headerlines.append(oiread[(oldidx + 1):idx].decode('utf-8'))
# intersphinx version line, project name, and project version
ivers, proj, vers, compr = headerlines
if 'The remainder of this file is compressed using zlib' not in compr:
raise ValueError('The file downloaded from {} does not seem to be'
'the usual Sphinx objects.inv format. Maybe it '
'has changed?'.format(baseurl + 'objects.inv'))
compressed = oiread[(idx + 1):]
decompressed = decompress(compressed).decode('utf-8')
resurl = None
for l in decompressed.strip().splitlines():
ls = l.split()
name = ls[0]
loc = ls[3]
if loc.endswith('$'):
loc = loc[:-1] + name
if name == obj:
resurl = baseurl + loc
break
if resurl is None:
raise ValueError(f'Could not find the docs for the object {obj}')
elif openinbrowser:
webbrowser.open(resurl)
return resurl
def get_images_radius(self, ra, dec, radius, filter='all',
frametype='stack', directory=None, n_concurrent=1, save=True,
verbose=True, overwrite=False):
"""
Get all images within some radius of a specified RA/Dec
Parameters
----------
ra : float
dec : float
ra/dec center to search around
radius : float
Radius of circle to search within
filter : ['all','J','H','K']
The color filter to download.
frametype : ['stack', 'normal', 'interleave', 'deep%stack',
'confidence', 'difference', 'leavstack', 'all']
The type of image
directory : None or string
Directory to download files into. Defaults to self.directory
verbose : bool
Print out extra error messages?
save : bool
Save FITS file?
overwrite : bool
Overwrite if file already exists?
n_concurrent : int
Number of concurrent download threads to start
Examples
--------
>>> R = UKIDSSQuery()
>>> fitsfile = R.get_image_gal(10.5,0.0)
"""
# Check for validity of requested frame_type
if frametype not in frame_types:
raise ValueError("Invalide frame type. Valid frame types are: %s"
% frame_types)
if filter not in self.filters:
raise ValueError("Invalide filter. Valid filters are: %s"
% self.filters.keys())
if directory is None:
directory = self.directory
# Construct self.request
self.request = {}
self.request['database'] = self.database
self.request['programmeID'] = verify_programme_id(self.programmeID,querytype='image')
self.request['userSelect'] = 'default'
self.request['obsType'] = 'object'
self.request['frameType'] = frametype
self.request['filterID'] = self.filters[filter]
self.request['minRA'] = str(round(ra - radius / cos(radians(dec)),2))
self.request['maxRA'] = str(round(ra + radius / cos(radians(dec)),2))
self.request['formatRA'] = 'degrees'
self.request['minDec'] = str(dec - radius)
self.request['maxDec'] = str(dec + radius)
self.request['formatDec'] = 'degrees'
self.request['startDay'] = 0
self.request['startMonth'] = 0
self.request['startYear'] = 0
self.request['endDay'] = 0
self.request['endMonth'] = 0
self.request['endYear'] = 0
self.request['dep'] = 0
self.request['mfid'] = ''
self.request['lmfid'] = ''
self.request['fsid'] = ''
self.request['rows'] = 1000
self.query_str = url_getimages + "?" + urlencode(self.request)
# Retrieve page
page = self.opener.open(url_getimages, urlencode(self.request))
with aud.get_readable_fileobj(page) as f:
results = f.read()
# Parse results for links
format = formatter.NullFormatter()
htmlparser = LinksExtractor(format)
htmlparser.feed(results)
htmlparser.close()
links = htmlparser.get_links()
# Loop through links and retrieve FITS images
for link in links:
if not os.path.exists(directory):
os.mkdir(directory)
if not os.path.exists(directory + '/' + frametype):
os.mkdir(directory + '/' + frametype)
if ('fits_download' in link and '_cat.fits' not in link and
'_two.fit' not in link):
# Get image filename
basename = os.path.basename(link.split("&")[0])
temp_file = directory + '/' + frametype + '/' + basename
if verbose:
print("Downloading %s..." % basename)
p = mp.Process(
target=progressbar.retrieve,
args=(link, temp_file, self.opener))
else:
p = mp.Process(
target=urllib.urlretrieve, args=(link, temp_file))
p.start()
while True:
if len(mp.active_children()) < n_concurrent:
break
time.sleep(0.1)
def parse_cs(ivoid, cap_index=1):
"""Return test query pars as dict for given IVO ID and capability index."""
if isinstance(ivoid, bytes): # ASTROPY_LT_4_1
ivoid = ivoid.decode('ascii')
# Production server.
url = ("https://vao.stsci.edu/regtap/tapservice.aspx/sync?lang=adql&"
"query=select%20detail_xpath%2Cdetail_value%20from%20"
"rr.res_detail%20where%20"
"ivoid%3D%27{0}%27%20and%20cap_index={1}%20and%20"
"detail_xpath%20in%20%28%27/capability/testQuery/ra%27%2C"
"%27/capability/testQuery/dec%27%2C%27/capability/testQuery/sr%27"
"%29".format(ivoid, cap_index))
urls_failed = False
default_sr = 0.1
try:
with get_readable_fileobj(url, encoding='binary',
show_progress=False) as fd:
t_query = Table.read(fd, format='votable')
except Exception as e: # pragma: no cover
urls_failed = True
urls_errmsg = '{0} raised {1}, using default'.format(
url, str(e))
if not urls_failed:
try:
xpath = t_query['detail_xpath']
if ASTROPY_LT_4_1:
ra = float(
t_query[xpath == b'/capability/testQuery/ra']['detail_value'])
dec = float(
t_query[xpath == b'/capability/testQuery/dec']['detail_value'])
sr = float(
t_query[xpath == b'/capability/testQuery/sr']['detail_value'])
else:
ra = float(
t_query[xpath == '/capability/testQuery/ra']['detail_value'])
dec = float(
t_query[xpath == '/capability/testQuery/dec']['detail_value'])
sr = float(
t_query[xpath == '/capability/testQuery/sr']['detail_value'])
# Handle big SR returning too big a table for some queries, causing
# tests to fail due to timeout.
if sr > default_sr:
warnings.warn(
'SR={0} is too large, using SR={1} for {2},{3}'.format(
sr, default_sr, ivoid, cap_index), AstropyUserWarning)
sr = default_sr
d = OrderedDict({'RA': ra, 'DEC': dec, 'SR': sr})
except Exception as e: # pragma: no cover
urls_failed = True
urls_errmsg = ('Failed to retrieve test query parameters for '
'{0},{1}, using default: {2}'.format(ivoid, cap_index, str(e)))
# If no test query found, use default
if urls_failed: # pragma: no cover
d = OrderedDict({'RA': 0, 'DEC': 0, 'SR': default_sr})
warnings.warn(urls_errmsg, AstropyUserWarning)
return d
def _download_file(url, cache=None, verbose=False):
if cache is None:
cache = bool_env('GWPY_CACHE', False)
return get_readable_fileobj(url, cache=cache, show_progress=verbose)
def get_readable_fileobj(*args, **kwargs):
"""
Overload astropy's get_readable_fileobj so that we can safely monkeypatch
it in astroquery without affecting astropy core functionality
"""
return aud.get_readable_fileobj(*args, **kwargs)
def get_besancon_model_file(filename,
verbose=True,
save=True,
savename=None,
overwrite=True):
"""
Download a Besancon model from the website
Parameters
----------
filename : string
The besancon filename, with format ##########.######.resu
verbose : bool
Print details about the download process
save : bool
Save the table after acquiring it?
savename : None or string
If not specified, defaults to the .resu table name
overwrite : bool
Overwrite the file if it exists? Defaults to True because the .resu
tables should have unique names by default, so there's little risk of
accidentally overwriting important information
"""
url = url_download + filename
elapsed_time = 0
t0 = time.time()
sys.stdout.write("\n")
while 1:
sys.stdout.write(u"\r")
try:
U = urllib2.urlopen(url, timeout=5)
with aud.get_readable_fileobj(U, cache=True) as f:
results = f.read()
break
except urllib2.URLError:
sys.stdout.write(
u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r"
% (elapsed_time, time.time() - t0))
time.sleep(30)
elapsed_time += 30
continue
except socket.timeout:
sys.stdout.write(
u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r"
% (elapsed_time, time.time() - t0))
time.sleep(30)
elapsed_time += 30
continue
if save:
if savename is None:
savename = filename
if not overwrite and os.path.exists(savename):
raise IOError("File %s already exists." % savename)
outf = open(savename, 'w')
print >> outf, results
outf.close()
return parse_besancon_model_string(results)
def from_registry(cls, registry_url, timeout=60, **kwargs):
"""
Create a database of VO services from VO registry URL.
This is described in detail in :ref:`vo-sec-validator-build-db`,
except for the ``validate_xxx`` keys that are added by the
validator itself.
Parameters
----------
registry_url : str
URL of VO registry that returns a VO Table.
For example, see
``astroquery.vo_conesearch.validator.conf.cs_mstr_list``.
Pedantic is automatically set to `False` for parsing.
timeout : number
Temporarily set ``astropy.utils.data.conf.remote_timeout``
to this value to avoid time out error while reading the
entire registry.
kwargs : dict
Keywords accepted by
:func:`~astropy.utils.data.get_readable_fileobj`.
Returns
-------
db : `VOSDatabase`
Database from given registry.
Raises
------
VOSError
Invalid VO registry.
"""
# Download registry as VO table
with data_conf.set_temp('remote_timeout', timeout):
with get_readable_fileobj(registry_url, **kwargs) as fd:
tab_all = parse_single_table(fd, pedantic=False)
# Registry must have these fields
compulsory_fields = ['res_title', 'access_url']
cat_fields = tab_all.array.dtype.names
for field in compulsory_fields:
if field not in cat_fields: # pragma: no cover
raise VOSError('"{0}" is missing from registry.'.format(field))
title_counter = defaultdict(int)
title_fmt = '{0} {1}'
db = cls.create_empty()
# Each row in the table becomes a catalog
for arr in tab_all.array.data:
cur_cat = {}
cur_key = ''
# Process each field and build the catalog.
# Catalog is completely built before being thrown out
# because codes need less changes should we decide to
# allow duplicate URLs in the future.
for field in cat_fields:
# For primary key, a number needs to be appended to the title
# because registry can have multiple entries with the same
# title but different URLs.
if field == 'res_title':
cur_title = arr['res_title']
title_counter[cur_title] += 1 # Starts with 1
if isinstance(cur_title, bytes): # pragma: py3
cur_key = title_fmt.format(cur_title.decode('utf-8'),
title_counter[cur_title])
else: # pragma: py2
cur_key = title_fmt.format(cur_title,
title_counter[cur_title])
# Special handling of title and access URL,
# otherwise no change.
if field == 'access_url':
s = unescape_all(arr['access_url'])
if isinstance(s, six.binary_type):
s = s.decode('utf-8')
cur_cat['url'] = s
elif field == 'res_title':
cur_cat['title'] = arr[field]
else:
cur_cat[field] = arr[field]
# New field to track duplicate access URLs.
cur_cat['duplicatesIgnored'] = 0
# Add catalog to database, unless duplicate access URL exists.
# In that case, the entry is thrown out and the associated
# counter is updated.
dup_keys = db._url_keys[cur_cat['url']]
if len(dup_keys) < 1:
db.add_catalog(cur_key,
VOSCatalog(cur_cat),
allow_duplicate_url=False)
else:
db._catalogs[dup_keys[0]]['duplicatesIgnored'] += 1
warnings.warn(
'{0} is thrown out because it has same access URL as '
'{1}.'.format(cur_key, dup_keys[0]), AstropyUserWarning)
return db
def read(table, guess=None, **kwargs):
"""
Read the input ``table`` and return the table. Most of
the default behavior for various parameters is determined by the Reader
class.
See also:
- http://docs.astropy.org/en/stable/io/ascii/
- http://docs.astropy.org/en/stable/io/ascii/read.html
Parameters
----------
table : str, file-like, list, pathlib.Path object
Input table as a file name, file-like object, list of strings,
single newline-separated string or pathlib.Path object .
guess : bool
Try to guess the table format. Defaults to None.
format : str, `~astropy.io.ascii.BaseReader`
Input table format
Inputter : `~astropy.io.ascii.BaseInputter`
Inputter class
Outputter : `~astropy.io.ascii.BaseOutputter`
Outputter class
delimiter : str
Column delimiter string
comment : str
Regular expression defining a comment line in table
quotechar : str
One-character string to quote fields containing special characters
header_start : int
Line index for the header line not counting comment or blank lines.
A line with only whitespace is considered blank.
data_start : int
Line index for the start of data not counting comment or blank lines.
A line with only whitespace is considered blank.
data_end : int
Line index for the end of data not counting comment or blank lines.
This value can be negative to count from the end.
converters : dict
Dictionary of converters
data_Splitter : `~astropy.io.ascii.BaseSplitter`
Splitter class to split data columns
header_Splitter : `~astropy.io.ascii.BaseSplitter`
Splitter class to split header columns
names : list
List of names corresponding to each data column
include_names : list
List of names to include in output.
exclude_names : list
List of names to exclude from output (applied after ``include_names``)
fill_values : dict
specification of fill values for bad or missing table values
fill_include_names : list
List of names to include in fill_values.
fill_exclude_names : list
List of names to exclude from fill_values (applied after ``fill_include_names``)
fast_reader : bool or dict
Whether to use the C engine, can also be a dict with options which
defaults to `False`; parameters for options dict:
use_fast_converter: bool
enable faster but slightly imprecise floating point conversion method
parallel: bool or int
multiprocessing conversion using ``cpu_count()`` or ``'number'`` processes
exponent_style: str
One-character string defining the exponent or ``'Fortran'`` to auto-detect
Fortran-style scientific notation like ``'3.14159D+00'`` (``'E'``, ``'D'``, ``'Q'``),
all case-insensitive; default ``'E'``, all other imply ``use_fast_converter``
chunk_size : int
If supplied with a value > 0 then read the table in chunks of
approximately ``chunk_size`` bytes. Default is reading table in one pass.
chunk_generator : bool
If True and ``chunk_size > 0`` then return an iterator that returns a
table for each chunk. The default is to return a single stacked table
for all the chunks.
Reader : `~astropy.io.ascii.BaseReader`
Reader class (DEPRECATED)
encoding: str
Allow to specify encoding to read the file (default= ``None``).
Returns
-------
dat : `~astropy.table.Table` OR <generator>
Output table
"""
del _read_trace[:]
# Downstream readers might munge kwargs
kwargs = copy.deepcopy(kwargs)
# Convert 'fast_reader' key in kwargs into a dict if not already and make sure
# 'enable' key is available.
fast_reader = _get_fast_reader_dict(kwargs)
kwargs['fast_reader'] = fast_reader
if fast_reader['enable'] and fast_reader.get('chunk_size'):
return _read_in_chunks(table, **kwargs)
if 'fill_values' not in kwargs:
kwargs['fill_values'] = [('', '0')]
# If an Outputter is supplied in kwargs that will take precedence.
if 'Outputter' in kwargs: # user specified Outputter, not supported for fast reading
fast_reader['enable'] = False
format = kwargs.get('format')
# Dictionary arguments are passed by reference per default and thus need
# special protection:
new_kwargs = copy.deepcopy(kwargs)
kwargs['fast_reader'] = copy.deepcopy(fast_reader)
# Get the Reader class based on possible format and Reader kwarg inputs.
Reader = _get_format_class(format, kwargs.get('Reader'), 'Reader')
if Reader is not None:
new_kwargs['Reader'] = Reader
format = Reader._format_name
# Remove format keyword if there, this is only allowed in read() not get_reader()
if 'format' in new_kwargs:
del new_kwargs['format']
if guess is None:
guess = _GUESS
if guess:
# If ``table`` is probably an HTML file then tell guess function to add
# the HTML reader at the top of the guess list. This is in response to
# issue #3691 (and others) where libxml can segfault on a long non-HTML
# file, thus prompting removal of the HTML reader from the default
# guess list.
new_kwargs['guess_html'] = _probably_html(table)
# If `table` is a filename or readable file object then read in the
# file now. This prevents problems in Python 3 with the file object
# getting closed or left at the file end. See #3132, #3013, #3109,
# #2001. If a `readme` arg was passed that implies CDS format, in
# which case the original `table` as the data filename must be left
# intact.
if 'readme' not in new_kwargs:
encoding = kwargs.get('encoding')
try:
with get_readable_fileobj(table, encoding=encoding) as fileobj:
table = fileobj.read()
except ValueError: # unreadable or invalid binary file
raise
except Exception:
pass
else:
# Ensure that `table` has at least one \r or \n in it
# so that the core.BaseInputter test of
# ('\n' not in table and '\r' not in table)
# will fail and so `table` cannot be interpreted there
# as a filename. See #4160.
if not re.search(r'[\r\n]', table):
table = table + os.linesep
# If the table got successfully read then look at the content
# to see if is probably HTML, but only if it wasn't already
# identified as HTML based on the filename.
if not new_kwargs['guess_html']:
new_kwargs['guess_html'] = _probably_html(table)
# Get the table from guess in ``dat``. If ``dat`` comes back as None
# then there was just one set of kwargs in the guess list so fall
# through below to the non-guess way so that any problems result in a
# more useful traceback.
dat = _guess(table, new_kwargs, format, fast_reader)
if dat is None:
guess = False
if not guess:
if format is None:
reader = get_reader(**new_kwargs)
format = reader._format_name
# Try the fast reader version of `format` first if applicable. Note that
# if user specified a fast format (e.g. format='fast_basic') this test
# will fail and the else-clause below will be used.
if fast_reader['enable'] and 'fast_{0}'.format(
format) in core.FAST_CLASSES:
fast_kwargs = copy.deepcopy(new_kwargs)
fast_kwargs['Reader'] = core.FAST_CLASSES['fast_{0}'.format(
format)]
fast_reader_rdr = get_reader(**fast_kwargs)
try:
dat = fast_reader_rdr.read(table)
_read_trace.append({
'kwargs':
copy.deepcopy(fast_kwargs),
'Reader':
fast_reader_rdr.__class__,
'status':
'Success with fast reader (no guessing)'
})
except (core.ParameterError, cparser.CParserError,
UnicodeEncodeError) as err:
# special testing value to avoid falling back on the slow reader
if fast_reader['enable'] == 'force':
raise core.InconsistentTableError(
'fast reader {} exception: {}'.format(
fast_reader_rdr.__class__, err))
# If the fast reader doesn't work, try the slow version
reader = get_reader(**new_kwargs)
dat = reader.read(table)
_read_trace.append({
'kwargs':
copy.deepcopy(new_kwargs),
'Reader':
reader.__class__,
'status':
'Success with slow reader after failing'
' with fast (no guessing)'
})
else:
reader = get_reader(**new_kwargs)
dat = reader.read(table)
_read_trace.append({
'kwargs':
copy.deepcopy(new_kwargs),
'Reader':
reader.__class__,
'status':
'Success with specified Reader class '
'(no guessing)'
})
return dat