def get_VLASS_images(position, radius, showgrid=False, savefile=''):
cadc = Cadc()
result = cadc.query_region(position, collection='VLASS')
if len(result) == 0:
print('No Data at CADC at position: {}'.format(
position.to_string('hmsdms')))
else:
urls = cadc.get_data_urls(result)
for url in urls:
cutout_url = construct_cadc_url(url, position, radius)
plot_http_fits(cutout_url, showgrid, savefile)
def test_query_region(self):
cadc = Cadc()
result = cadc.query_region('08h45m07.5s +54d18m00s', collection='CFHT')
# do some manipulation of the results. Below it's filtering out based
# on target name but other manipulations are possible.
assert len(result) > 0
urls = cadc.get_data_urls(result[result['target_name'] == 'Nr3491_1'])
assert len(urls) > 0
# urls are a subset of the results that match target_name==Nr3491_1
assert len(result) >= len(urls)
urls_data_only = len(urls)
# now get the auxilary files too
urls = cadc.get_data_urls(result[result['target_name'] == 'Nr3491_1'],
include_auxiliaries=True)
assert urls_data_only <= len(urls)
# the same result should be obtained by querying the entire region
# and filtering out on the CFHT collection
result2 = cadc.query_region('08h45m07.5s +54d18m00s')
assert len(result) == len(result2[result2['collection'] == 'CFHT'])
# search for a target
results = cadc.query_region(SkyCoord.from_name('M31'), radius=0.016)
assert len(results) > 20
def test_query_region(self):
cadc = Cadc()
result = cadc.query_region('08h45m07.5s +54d18m00s', collection='CFHT')
# do some manipulation of the results. Below it's filtering out based
# on target name but other manipulations are possible.
assert len(result) > 0
urls = cadc.get_data_urls(result[result['target_name'] == 'Nr3491_1'])
assert len(urls) > 0
# urls are a subset of the results that match target_name==Nr3491_1
assert len(result) >= len(urls)
urls_data_only = len(urls)
# now get the auxilary files too
urls = cadc.get_data_urls(result[result['target_name'] == 'Nr3491_1'],
include_auxiliaries=True)
assert urls_data_only <= len(urls)
# the same result should be obtained by querying the entire region
# and filtering out on the CFHT collection
result2 = cadc.query_region('08h45m07.5s +54d18m00s')
assert len(result) == len(result2[result2['collection'] == 'CFHT'])
# search for a target
results = cadc.query_region(SkyCoord.from_name('M31'))
assert len(results) > 20
def test_get_data_urls():
def get(*args, **kwargs):
class CapsResponse:
def __init__(self):
self.status_code = 200
self.content = b''
def raise_for_status(self):
pass
return CapsResponse()
class Result:
pass
file1 = Mock()
file1.semantics = '#this'
file1.access_url = 'https://get.your.data/path'
file2 = Mock()
file2.semantics = '#aux'
file2.access_url = 'https://get.your.data/auxpath'
file3 = Mock()
file3.semantics = '#preview'
file3.access_url = 'https://get.your.data/previewpath'
# add the package file that should be filtered out
package_file_old = Mock()
package_file_old.semantics = 'http://www.opencadc.org/caom2#pkg'
package_file = Mock()
package_file.semantics = '#package'
result = [file1, file2, file3, package_file_old, package_file]
with patch('pyvo.dal.adhoc.DatalinkResults.from_result_url') as \
dl_results_mock:
dl_results_mock.return_value = result
cadc = Cadc()
cadc._request = get # mock the request
assert [file1.access_url] == \
cadc.get_data_urls({'publisherID': ['ivo://cadc.nrc.ca/foo']})
assert [file1.access_url, file2.access_url, file3.access_url] == \
cadc.get_data_urls({'publisherID': ['ivo://cadc.nrc.ca/foo']},
include_auxiliaries=True)
with pytest.raises(AttributeError):
cadc.get_data_urls(None)
with pytest.raises(AttributeError):
cadc.get_data_urls({'noPublisherID': 'test'})
def test_get_data_urls(monkeypatch):
monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock)
def get(*args, **kwargs):
class CapsResponse(object):
def __init__(self):
self.status_code = 200
self.content = b''
def raise_for_status(self):
pass
return CapsResponse()
class Result(object):
pass
vot_result = Result()
vot_result.array = [{'semantics': b'#this',
'access_url': b'https://get.your.data/path'},
{'semantics': b'#aux',
'access_url': b'https://get.your.data/auxpath'}]
monkeypatch.setattr(cadc_core, 'parse_single_table', lambda x: vot_result)
dummyTapHandler = DummyTapHandler()
cadc = Cadc(tap_plus_handler=dummyTapHandler)
cadc._request = get # mock the request
assert [vot_result.array[0]['access_url'].decode('ascii')] == \
cadc.get_data_urls({'caomPublisherID': ['ivo://cadc.nrc.ca/foo']})
assert [vot_result.array[0]['access_url'].decode('ascii'),
vot_result.array[1]['access_url'].decode('ascii')] == \
cadc.get_data_urls({'caomPublisherID': ['ivo://cadc.nrc.ca/foo']},
include_auxiliaries=True)
with pytest.raises(AttributeError):
cadc.get_data_urls(None)
with pytest.raises(AttributeError):
cadc.get_data_urls({'noPublisherID': 'test'})
sys.stderr.write("Nothing to do, next object!\n")
continue
if (ntodo > 0):
useful = useful[:ntodo]
# Download new images in chunks (URL-fetch is slow):
#sys.stderr.write("making URLs ... ")
nchunks = int(np.ceil(len(useful) / float(chunksize)))
#sys.stderr.write("nchunks: %d\n" % nchunks)
uidx = np.arange(len(useful))
chunkidx = np.array_split(np.arange(len(useful)), nchunks)
for ii, cidx in enumerate(chunkidx, 1):
sys.stderr.write("Chunk %d of %d ...\n" % (ii, nchunks))
snag = useful[cidx]
imurls = cadc.get_data_urls(snag)
dlspec = [(uu, ss, tmp_dl_path)
for uu, ss in zip(imurls, snag['isave'])]
#fdl.smart_fetch_bulk(dlspec)
download_from_cadc(dlspec)
#for uu,ss in zip(imurls, snag['isave']):
# success = download_from_cadc(uu, ss, tmp_dl_path)
######################################################################
# CHANGELOG (fetch_hsa_data.py):
#---------------------------------------------------------------------
#
# 2020-03-02:
# -- Increased __version__ to 0.1.0.
# -- First created fetch_CFHT_data.py.
#
