def test_query_name(self): cadc = Cadc() result1 = cadc.query_name('M31') assert len(result1) > 20 # test case insensitive result2 = cadc.query_name('m31') assert len(result1) == len(result2)
def test_async(self): # test async calls cadc = Cadc() # run the query in sync mode first expected = cadc.exec_sync( "select top 3 observationID from caom2.Observation where " "collection='IRIS' order by observationID") # now run the query in async mode job = cadc.create_async( "select top 3 observationID from caom2.Observation where " "collection='IRIS' order by observationID") job = job.run().wait() job.raise_if_error() result = job.fetch_result().to_table() assert len(expected) == len(result) for ii in range(0, 2): assert expected['observationID'][ii] == result['observationID'][ii] # load job again loaded_job = cadc.load_async_job(job.job_id) result = loaded_job.fetch_result().to_table() assert len(expected) == len(result) for ii in range(0, 2): assert expected['observationID'][ii] == result['observationID'][ii]
def test_misc(): cadc = Cadc() coords = '08h45m07.5s +54d18m00s' coords_ra = parse_coordinates(coords).fk5.ra.degree coords_dec = parse_coordinates(coords).fk5.dec.degree assert "SELECT * from caom2.Observation o join caom2.Plane p ON " \ "o.obsID=p.obsID WHERE INTERSECTS( CIRCLE('ICRS', " \ "{}, {}, 0.3), position_bounds) = 1 " \ "AND (quality_flag IS NULL OR quality_flag != 'junk') " \ "AND collection='CFHT' AND dataProductType='image'".\ format(coords_ra, coords_dec) == \ cadc._args_to_payload(**{'coordinates': coords, 'radius': 0.3 * u.deg, 'collection': 'CFHT', 'data_product_type': 'image'})['query'] # no collection or data_product_type assert "SELECT * from caom2.Observation o join caom2.Plane p ON " \ "o.obsID=p.obsID WHERE INTERSECTS( CIRCLE('ICRS', " \ "{}, {}, 0.3), position_bounds) = 1 AND (quality_flag IS NULL OR " \ "quality_flag != 'junk')".format(coords_ra, coords_dec) == \ cadc._args_to_payload(**{'coordinates': coords, 'radius': '0.3 deg'})['query']
def test_query_name(self): cadc = Cadc() result1 = cadc.query_name('M31-B14') assert len(result1) > 20 # test case insensitive result2 = cadc.query_name('m31-b14') assert len(result1) == len(result2)
def test_exec_sync(): # save results in a file # create the VOTable result # example from http://docs.astropy.org/en/stable/io/votable/ votable = VOTableFile() resource = Resource() votable.resources.append(resource) table = Table(votable) resource.tables.append(table) table.fields.extend([ Field(votable, name="filename", datatype="char", arraysize="*"), Field(votable, name="matrix", datatype="double", arraysize="2x2")]) table.create_arrays(2) table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) buffer = BytesIO() votable.to_xml(buffer) cadc = Cadc(auth_session=requests.Session()) response = Mock() response.to_table.return_value = buffer.getvalue() cadc.cadctap.search = Mock(return_value=response) output_file = '{}/test_vooutput.xml'.format(tempfile.tempdir) cadc.exec_sync('some query', output_file=output_file) actual = parse(output_file) assert len(votable.resources) == len(actual.resources) == 1 assert len(votable.resources[0].tables) ==\ len(actual.resources[0].tables) == 1 actual_table = actual.resources[0].tables[0] try: # TODO remove when astropy LTS upgraded from astropy.utils.diff import report_diff_values assert report_diff_values(table, actual_table, fileobj=sys.stdout) except ImportError: pass
def test_login(self): cadc = Cadc() now = datetime.utcnow() cadc.login(os.environ['CADC_USER'], os.environ['CADC_PASSWD']) query = "select top 1 * from caom2.Plane where " \ "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.query(query) assert len(result) == 1
def test_get_images(self): cadc = Cadc() coords = '08h45m07.5s +54d18m00s' radius = 0.005 * u.deg images = cadc.get_images(coords, radius, collection='CFHT') assert images is not None for image in images: assert isinstance(image, fits.HDUList)
def test_get_images_async(self): cadc = Cadc() coords = '01h45m07.5s +23d18m00s' radius = '0.05 deg' readable_objs = cadc.get_images_async(coords, radius, collection="CFHT") assert readable_objs is not None for obj in readable_objs: assert isinstance(obj, FileContainer)
def test_get_tables(): # default parameters table_set = PropertyMock() table_set.keys.return_value = ['table1', 'table2'] table_set.values.return_value = ['tab1val', 'tab2val', 'tab3val'] with patch('astroquery.cadc.core.pyvo.dal.TAPService', autospec=True) as tapservice_mock: tapservice_mock.return_value.tables = table_set cadc = Cadc() assert len(cadc.get_tables(only_names=True)) == 2 assert len(cadc.get_tables()) == 3
def test_get_images(): with patch('astroquery.utils.commons.get_readable_fileobj', autospec=True) as readable_fobj_mock: readable_fobj_mock.return_value = open(data_path('query_images.fits'), 'rb') cadc = Cadc() fits_images = cadc.get_images('08h45m07.5s +54d18m00s', 0.01*u.deg, get_url_list=True) assert fits_images == ['https://some.url'] fits_images = cadc.get_images('08h45m07.5s +54d18m00s', '0.01 deg') assert fits_images is not None assert isinstance(fits_images[0], HDUList)
def test_query(self): cadc = Cadc() result = cadc.query( "select count(*) from caom2.Observation where target_name='M31'") assert 1000 < result[0][0] # test that no proprietary results are returned when not logged in now = datetime.utcnow() query = "select top 1 * from caom2.Plane where " \ "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.query(query) assert len(result) == 0
def test_query(self): cadc = Cadc() result = cadc.exec_sync( "select count(*) from caom2.Observation where target_name='M31'") assert 1000 < result[0][0] # test that no proprietary results are returned when not logged in now = datetime.utcnow() query = "select top 1 * from caom2.Plane where " \ "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 0
def test_load_async_job(): with patch('astroquery.cadc.core.pyvo.dal.TAPService', autospec=True) as tapservice_mock: with patch('astroquery.cadc.core.pyvo.dal.AsyncTAPJob', autospec=True) as tapjob_mock: tapservice_mock.return_value.baseurl.return_value = 'https://www.example.com/tap' mock_job = Mock() mock_job.job_id = '123' tapjob_mock.return_value = mock_job cadc = Cadc() jobid = '123' job = cadc.load_async_job(jobid) assert job.job_id == '123'
def get_VLASS_images(position, radius, showgrid=False, savefile=''): cadc = Cadc() result = cadc.query_region(position, collection='VLASS') if len(result) == 0: print('No Data at CADC at position: {}'.format( position.to_string('hmsdms'))) else: urls = cadc.get_data_urls(result) for url in urls: cutout_url = construct_cadc_url(url, position, radius) plot_http_fits(cutout_url, showgrid, savefile)
def test_get_table(): table_set = PropertyMock() tables_result = [Mock() for _ in range(3)] tables_result[0].name = 'tab1' tables_result[1].name = 'tab2' tables_result[2].name = 'tab3' table_set.values.return_value = tables_result with patch('astroquery.cadc.core.pyvo.dal.TAPService', autospec=True) as tapservice_mock: tapservice_mock.return_value.tables = table_set cadc = Cadc() assert cadc.get_table('tab2').name == 'tab2' assert cadc.get_table('foo') is None
def test_get_images_async(): with patch('astroquery.utils.commons.get_readable_fileobj', autospec=True) as readable_fobj_mock: readable_fobj_mock.return_value = open(data_path('query_images.fits'), 'rb') cadc = Cadc() readable_objs = cadc.get_images_async('08h45m07.5s +54d18m00s', 0.01*u.arcmin, get_url_list=True) assert readable_objs == ['https://some.url'] readable_objs = cadc.get_images_async('08h45m07.5s +54d18m00s', '0.01 arcsec') assert readable_objs is not None assert isinstance(readable_objs[0], FileContainer)
def test_list_tables(self): cadc = Cadc() table_names = cadc.get_tables(only_names=True) assert len(table_names) > 20 assert 'caom2.Observation' in table_names assert 'ivoa.ObsCore' in table_names assert 'tap_schema.tables' in table_names tables = cadc.get_tables() assert len(table_names) == len(tables) for table in tables: assert table.name in table_names table = cadc.get_table('caom2.Observation') assert 'caom2.Observation' == table.name
def test_misc(monkeypatch): dummyTapHandler = DummyTapHandler() monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) cadc = Cadc(tap_plus_handler=dummyTapHandler) class Result(object): pass result = Result() result._phase = 'RUN' with pytest.raises(RuntimeError): cadc._parse_result(result) result._phase = 'COMPLETED' result.results = 'WELL DONE' assert result.results == cadc._parse_result(result) coords = '08h45m07.5s +54d18m00s' coords_ra = parse_coordinates(coords).ra.degree coords_dec = parse_coordinates(coords).dec.degree assert "SELECT * from caom2.Observation o join caom2.Plane p ON " \ "o.obsID=p.obsID WHERE INTERSECTS( CIRCLE('ICRS', " \ "{}, {}, 0.3), position_bounds) = 1 " \ "AND (quality_flag IS NULL OR quality_flag != 'junk') " \ "AND collection='CFHT'".format(coords_ra, coords_dec) == \ cadc._args_to_payload(**{'coordinates': coords, 'radius': 0.3, 'collection': 'CFHT'})['query'] # no collection assert "SELECT * from caom2.Observation o join caom2.Plane p ON " \ "o.obsID=p.obsID WHERE INTERSECTS( CIRCLE('ICRS', " \ "{}, {}, 0.3), position_bounds) = 1 AND (quality_flag IS NULL OR " \ "quality_flag != 'junk')".format(coords_ra, coords_dec) == \ cadc._args_to_payload(**{'coordinates': coords, 'radius': 0.3})['query']
def test_get_collections(self): cadc = Cadc() result = cadc.get_collections() assert len(result) > 10 # test for the presence of a few known collections assert 'CFHT' in result assert 'Infrared' in result['CFHT']['Bands'] assert 'Optical' in result['CFHT']['Bands'] assert 'MOST' in result assert 'Optical' in result['MOST']['Bands'] assert 'GEMINI' in result assert 'Infrared' in result['GEMINI']['Bands'] assert 'Optical' in result['GEMINI']['Bands'] assert 'JCMT' in result assert 'Millimeter' in result['JCMT']['Bands'] assert 'DAO' in result assert 'Infrared' in result['DAO']['Bands'] assert 'Optical' in result['DAO']['Bands']
def test_run_query(monkeypatch): dummyTapHandler = DummyTapHandler() monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) tap = Cadc(tap_plus_handler=dummyTapHandler) query = "query" operation = 'sync' # default parameters parameters = {} parameters['query'] = query parameters['name'] = None parameters['output_file'] = None parameters['output_format'] = 'votable' parameters['verbose'] = False parameters['dump_to_file'] = False parameters['upload_resource'] = None parameters['upload_table_name'] = None tap.run_query(query, operation) dummyTapHandler.check_call('run_query', parameters) # test with parameters dummyTapHandler.reset() output_file = 'output' output_format = 'format' verbose = True upload_resource = 'upload_res' upload_table_name = 'upload_table' parameters['query'] = query parameters['name'] = None parameters['output_file'] = output_file parameters['output_format'] = output_format parameters['verbose'] = verbose parameters['dump_to_file'] = True parameters['upload_resource'] = upload_resource parameters['upload_table_name'] = upload_table_name tap.run_query(query, operation, output_file=output_file, output_format=output_format, verbose=verbose, upload_resource=upload_resource, upload_table_name=upload_table_name) dummyTapHandler.check_call('run_query', parameters)
def test_get_collections(): cadc = Cadc() def mock_run_query(query, output_format=None, maxrec=None, output_file=None): assert query == \ 'select distinct collection, energy_emBand from caom2.EnumField' assert output_format is None assert maxrec is None assert output_file is None table = AstroTable(rows=[('CFHT', 'Optical'), ('CFHT', 'Infrared'), ('JCMT', 'Millimeter'), ('DAO', 'Optical'), ('DAO', 'Infrared')], names=('collection', 'energy_emBand')) return table cadc.exec_sync = mock_run_query result = cadc.get_collections() assert len(result) == 3 assert 'CFHT' in result assert 'JCMT' in result assert 'DAO' in result
def test_login(monkeypatch): def get_access_url_mock(arg1, arg2): return "some.url" monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) dummyTapHandler = DummyTapHandler() tap = Cadc(tap_plus_handler=dummyTapHandler) user = '******' password = '******' cert = 'cert' # default parameters parameters = {} parameters['cookie_prefix'] = cadc_core.CADC_COOKIE_PREFIX parameters['login_url'] = "some.url" parameters['verbose'] = False parameters['user'] = None parameters['password'] = None parameters['certificate_file'] = None tap.login(None, None, None) dummyTapHandler.check_call('login', parameters) # test with parameters dummyTapHandler.reset() parameters['user'] = user parameters['password'] = password parameters['certificate_file'] = cert tap.login(user, password, cert) dummyTapHandler.check_call('login', parameters)
def test_get_data_urls(): def get(*args, **kwargs): class CapsResponse: def __init__(self): self.status_code = 200 self.content = b'' def raise_for_status(self): pass return CapsResponse() class Result: pass file1 = Mock() file1.semantics = '#this' file1.access_url = 'https://get.your.data/path' file2 = Mock() file2.semantics = '#aux' file2.access_url = 'https://get.your.data/auxpath' file3 = Mock() file3.semantics = '#preview' file3.access_url = 'https://get.your.data/previewpath' # add the package file that should be filtered out package_file_old = Mock() package_file_old.semantics = 'http://www.opencadc.org/caom2#pkg' package_file = Mock() package_file.semantics = '#package' result = [file1, file2, file3, package_file_old, package_file] with patch('pyvo.dal.adhoc.DatalinkResults.from_result_url') as \ dl_results_mock: dl_results_mock.return_value = result cadc = Cadc() cadc._request = get # mock the request assert [file1.access_url] == \ cadc.get_data_urls({'publisherID': ['ivo://cadc.nrc.ca/foo']}) assert [file1.access_url, file2.access_url, file3.access_url] == \ cadc.get_data_urls({'publisherID': ['ivo://cadc.nrc.ca/foo']}, include_auxiliaries=True) with pytest.raises(AttributeError): cadc.get_data_urls(None) with pytest.raises(AttributeError): cadc.get_data_urls({'noPublisherID': 'test'})
def test_get_images_against_AS(self): cadc = Cadc() coords = '08h45m07.5s +54d18m00s' radius = 0.05 * u.deg # Compare results from cadc advanced search to get_images query = cadc._args_to_payload(**{ 'coordinates': coords, 'radius': radius, 'data_product_type': 'image' })['query'] result = cadc.exec_sync(query) uri_list = [uri.decode('ascii') for uri in result['publisherID']] access_url = 'https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/download' icrs_coords = parse_coordinates(coords).icrs data = { 'uris': ' '.join(uri_list), 'params': 'cutout=Circle ICRS {} {} {}'.format(icrs_coords.ra.degree, icrs_coords.dec.degree, radius.value), 'method': 'URL List' } resp_urls = requests.post(access_url, data).text.split('\r\n') # Filter out the errors and empty strings filtered_resp_urls = list( filter(lambda url: not url.startswith('ERROR') and url != '', resp_urls)) # This function should return nearly the same urls image_urls = cadc.get_images(coords, radius, get_url_list=True) assert len(filtered_resp_urls) == len(image_urls)
def test_list_jobs(self): cadc = Cadc() cadc.login(certificate_file=os.environ['CADC_CERT']) job = cadc.create_async( "select top 3 observationID from caom2.Observation where " "collection='IRIS' order by observationID") job = job.run().wait() job.raise_if_error() job.fetch_result().to_table() jobs = cadc.list_async_jobs() assert len(jobs) > 0 if len(jobs) > 5: jobs_subset = cadc.list_async_jobs(last=5) assert len(jobs_subset) == 5
def test_login_with_cert(self): for auth_session in [requests.Session()]: cadc = Cadc(auth_session=auth_session) now = datetime.utcnow() query = \ "select top 1 * from caom2.Plane where metaRelease>'{}'".\ format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 0 # following query is to test login with certificates when an # anonymous query is executed first. cadc.login(certificate_file=os.environ['CADC_CERT']) query = \ "select top 1 * from caom2.Plane where metaRelease>'{}'".\ format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 1 cadc.logout() query = \ "select top 1 * from caom2.Plane where metaRelease>'{}'".\ format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 0
def test_get_data_urls(monkeypatch): monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) def get(*args, **kwargs): class CapsResponse(object): def __init__(self): self.status_code = 200 self.content = b'' def raise_for_status(self): pass return CapsResponse() class Result(object): pass vot_result = Result() vot_result.array = [{'semantics': b'#this', 'access_url': b'https://get.your.data/path'}, {'semantics': b'#aux', 'access_url': b'https://get.your.data/auxpath'}] monkeypatch.setattr(cadc_core, 'parse_single_table', lambda x: vot_result) dummyTapHandler = DummyTapHandler() cadc = Cadc(tap_plus_handler=dummyTapHandler) cadc._request = get # mock the request assert [vot_result.array[0]['access_url'].decode('ascii')] == \ cadc.get_data_urls({'caomPublisherID': ['ivo://cadc.nrc.ca/foo']}) assert [vot_result.array[0]['access_url'].decode('ascii'), vot_result.array[1]['access_url'].decode('ascii')] == \ cadc.get_data_urls({'caomPublisherID': ['ivo://cadc.nrc.ca/foo']}, include_auxiliaries=True) with pytest.raises(AttributeError): cadc.get_data_urls(None) with pytest.raises(AttributeError): cadc.get_data_urls({'noPublisherID': 'test'})
def test_login_with_user_password(self): for auth_session in [ None, authsession.AuthSession(), requests.Session() ]: cadc = Cadc(auth_session=auth_session) now = datetime.utcnow() query = \ "select top 1 * from caom2.Plane where metaRelease>'{}'".\ format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 0 cadc.login(os.environ['CADC_USER'], os.environ['CADC_PASSWD']) query = "select top 1 * from caom2.Plane where metaRelease>'{}'".\ format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 1 # repeat after logout cadc.logout() query = \ "select top 1 * from caom2.Plane where metaRelease>'{}'".\ format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 0
def test_logout(monkeypatch): dummyTapHandler = DummyTapHandler() monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) tap = Cadc(tap_plus_handler=dummyTapHandler) # default parameters parameters = {} parameters['verbose'] = False tap.logout(False) dummyTapHandler.check_call('logout', parameters) # test with parameters dummyTapHandler.reset() parameters['verbose'] = True tap.logout(True) dummyTapHandler.check_call('logout', parameters)
def test_get_tables(monkeypatch): dummyTapHandler = DummyTapHandler() monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) tap = Cadc(tap_plus_handler=dummyTapHandler) # default parameters parameters = {} parameters['only_names'] = False parameters['verbose'] = False tap.get_tables() dummyTapHandler.check_call('get_tables', parameters) # test with parameters dummyTapHandler.reset() parameters = {} parameters['only_names'] = True parameters['verbose'] = True tap.get_tables(True, True) dummyTapHandler.check_call('get_tables', parameters)
def test_load_async_job(monkeypatch): dummyTapHandler = DummyTapHandler() monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) tap = Cadc(tap_plus_handler=dummyTapHandler) jobid = '123' # default parameters parameters = {} parameters['jobid'] = jobid parameters['verbose'] = False tap.load_async_job(jobid) dummyTapHandler.check_call('load_async_job', parameters) # test with parameters dummyTapHandler.reset() parameters['jobid'] = jobid parameters['verbose'] = True tap.load_async_job(jobid, verbose=True) dummyTapHandler.check_call('load_async_job', parameters)
def test_authsession(self): # repeat previous test auth_session = requests.Session() auth_session.cert = os.environ['CADC_CERT'] cadc = Cadc(auth_session=auth_session) now = datetime.utcnow() query = "select top 1 * from caom2.Plane where " \ "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 1 annon_session = requests.Session() cadc = Cadc(auth_session=annon_session) now = datetime.utcnow() query = "select top 1 * from caom2.Plane where " \ "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f')) result = cadc.exec_sync(query) assert len(result) == 0
def test_save_results(monkeypatch): dummyTapHandler = DummyTapHandler() monkeypatch.setattr(cadc_core, 'get_access_url', get_access_url_mock) tap = Cadc(tap_plus_handler=dummyTapHandler) job = '123' # default parameters parameters = {} parameters['job'] = job parameters['filename'] = 'file.txt' parameters['verbose'] = False tap.save_results(job, 'file.txt') dummyTapHandler.check_call('save_results', parameters) # test with parameters dummyTapHandler.reset() parameters['job'] = job parameters['filename'] = 'file.txt' parameters['verbose'] = True tap.save_results(job, 'file.txt', verbose=True) dummyTapHandler.check_call('save_results', parameters)
def test_query_region(self): cadc = Cadc() result = cadc.query_region('08h45m07.5s +54d18m00s', collection='CFHT') # do some manipulation of the results. Below it's filtering out based # on target name but other manipulations are possible. assert len(result) > 0 urls = cadc.get_data_urls(result[result['target_name'] == 'Nr3491_1']) assert len(urls) > 0 # urls are a subset of the results that match target_name==Nr3491_1 assert len(result) >= len(urls) urls_data_only = len(urls) # now get the auxilary files too urls = cadc.get_data_urls(result[result['target_name'] == 'Nr3491_1'], include_auxiliaries=True) assert urls_data_only <= len(urls) # the same result should be obtained by querying the entire region # and filtering out on the CFHT collection result2 = cadc.query_region('08h45m07.5s +54d18m00s') assert len(result) == len(result2[result2['collection'] == 'CFHT']) # search for a target results = cadc.query_region(SkyCoord.from_name('M31')) assert len(results) > 20