def test_oneimage(cdciplatform, *a, **aa):
print("running test one image at ", cdciplatform)
from oda_api.api import DispatcherAPI
from oda_api.plot_tools import OdaImage, OdaLightCurve
from oda_api.data_products import BinaryData
import os
from astropy.io import fits
import numpy as np
from numpy import sqrt
endpoint = platform_endpoint(cdciplatform)
disp = DispatcherAPI(host=endpoint)
print(disp)
onescw = aa.get("scw", "186500220010.001")
disp.set_custom_progress_formatter(custom_progress_formatter)
data = disp.get_product(instrument='isgri',
product='isgri_image',
scw_list=[onescw],
E1_keV=25,
E2_keV=80,
osa_version='OSA11.0',
RA=0,
DEC=0,
detection_threshold=15,
product_type='Real')
print(data)
def test_product_description_not_null():
from oda_api.api import DispatcherAPI
disp = DispatcherAPI(
host=get_platform_dispatcher(),
instrument="mock",
)
assert disp.get_product_description('isgri', 'isgri_image') is not None
def test_instruments():
from oda_api.api import DispatcherAPI
disp = DispatcherAPI(
host=get_platform_dispatcher(),
instrument="mock",
)
assert disp.get_instruments_list() == ['isgri', 'jemx', 'polar', 'spi_acs']
def test_n_recentscw(cdciplatform, timestamp=None, n_scw=3, *a, **aa):
import requests
import time
if timestamp is None:
timestamp = time.time()
else:
timestamp = float(time.time())
t1 = timestamp - 24 * 3600 * 580
t2 = timestamp - 24 * 3600 * 570
s = "https://www.astro.unige.ch/cdci/astrooda/dispatch-data/gw/timesystem/api/v1.0/scwlist/cons/{}/{}?&ra=83&dec=22&radius=200.0&min_good_isgri=1000".format(
time.strftime("%Y-%m-%dT%H:00:00", time.gmtime(t1)),
time.strftime("%Y-%m-%dT%H:00:00", time.gmtime(t2)),
)
print(s)
r = requests.get(s)
print(r.json())
scwpick = r.json()[:n_scw]
print("picked")
assert len(scwpick) > 0
print("running test image at ", cdciplatform)
from oda_api.api import DispatcherAPI
from oda_api.plot_tools import OdaImage, OdaLightCurve
from oda_api.data_products import BinaryData
import os
from astropy.io import fits
import numpy as np
from numpy import sqrt
endpoint = platform_endpoint(cdciplatform)
disp = DispatcherAPI(host=endpoint)
print(disp)
data = disp.get_product(instrument='isgri',
product='isgri_lc',
scw_list=[str(s) + ".001" for s in scwpick],
E1_keV=20,
E2_keV=80,
osa_version='OSA10.2',
RA=0,
DEC=0,
detection_threshold=15,
product_type='Real')
print(data)
def test_jemx_lc_source_name_formatting(cdciplatform, *a, **aa):
"""
feature: source names
feature: attribute source name
issue: https://github.com/oda-hub/dispatcher-plugin-integral/issues/40
"""
import pkg_resources
from packaging.version import parse
assert parse(pkg_resources.get_distribution("oda_api").version) >= parse("1.1.14")
from oda_api.api import DispatcherAPI
disp = DispatcherAPI(url=platform_endpoint(cdciplatform))
par_dict = {
"src_name": "4U 1700-377",
"RA": "270.80",
"DEC": "-29.80",
"T1": "2019-04-01",
"T2": "2019-06-05",
"T_format": "isot",
"instrument": "jemx",
"osa_version": "OSA11.0",
"radius": "4",
"max_pointings": "50",
"integral_data_rights": "public",
"jemx_num": "1",
"E1_keV": "3",
"E2_keV": "20",
"product_type": "Real",
"detection_threshold": "5",
"product": "jemx_lc",
"time_bin": "4",
"time_bin_format": "sec",
"catalog_selected_objects": "1,2,3",
"selected_catalog": '{"cat_frame": "fk5", "cat_coord_units": "deg", "cat_column_list": [[0, 1, 2], ["GX 5-1", "MAXI SRC", "H 1820-303"], [96.1907958984375, 74.80066680908203, 66.31670379638672], [270.2771301269531, 270.7560729980469, 275.914794921875], [-25.088342666625977, -29.84027099609375, -30.366628646850586], [0, 1, 0], [0.05000000074505806, 0.05000000074505806, 0.05000000074505806]], "cat_column_names": ["meta_ID", "src_names", "significance", "ra", "dec", "FLAG", "ERR_RAD"], "cat_column_descr": [["meta_ID", "<i8"], ["src_names", "<U10"], ["significance", "<f8"], ["ra", "<f8"], ["dec", "<f8"], ["FLAG", "<i8"], ["ERR_RAD", "<f8"]], "cat_lat_name": "dec", "cat_lon_name": "ra"}',
}
data_collection_lc = disp.get_product(**par_dict)
data_collection_lc.show()
l_output = data_collection_lc.as_list()
print('len(l_output): \n', len(l_output))
assert len(l_output) == 2
assert l_output[0]['prod_name'] == 'jemx_lc_0_H1820m303'
assert l_output[0]['meta_data:']['src_name'] == 'H 1820-303'
assert l_output[1]['prod_name'] == 'jemx_lc_1_MAXISRC'
assert l_output[1]['meta_data:']['src_name'] == 'MAXI SRC'
def get_disp(wait=True, platform="staging-1-2"):
from oda_api.api import DispatcherAPI
return DispatcherAPI(
host=get_platform_dispatcher(platform),
instrument="isgri",
wait=wait,
)
def test_one(cdciplatform, *a, **aa):
print("running test one lc at ", cdciplatform)
from oda_api.api import DispatcherAPI
from oda_api.plot_tools import OdaImage, OdaLightCurve
from oda_api.data_products import BinaryData
import os
from astropy.io import fits
import numpy as np
from numpy import sqrt
endpoint = platform_endpoint(cdciplatform)
disp = DispatcherAPI(host=endpoint)
print(disp)
data = disp.get_product(instrument='isgri',
product='isgri_lc',
scw_list=["066500220010.001"],
E1_keV=20,
E2_keV=80,
osa_version='OSA10.2',
RA=0,
DEC=0,
detection_threshold=15,
product_type='Real')
print(data)
print(data.show())
print(data._p_list)
print(data.isgri_lc_0_Crab)
print(data.isgri_lc_0_Crab.data_unit[1].data)
assert 'XAX_E' in data.isgri_lc_0_Crab.data_unit[1].data
return data
def test_instruments(cdciplatform, *a, **aa):
print("running test ", cdciplatform)
from oda_api.api import DispatcherAPI
from oda_api.plot_tools import OdaImage, OdaLightCurve
from oda_api.data_products import BinaryData
import os
from astropy.io import fits
import numpy as np
from numpy import sqrt
import json
endpoint = platform_endpoint(cdciplatform)
disp = DispatcherAPI(host=endpoint)
print(disp)
r = disp.get_instrument_description('isgri')
print("\033[34m", json.dumps(r, indent=4, sort_keys=True), "\033[0m")
def disp_for_platform(cdciplatform):
from oda_api.api import DispatcherAPI
from oda_api.plot_tools import OdaImage, OdaLightCurve
from oda_api.data_products import BinaryData
import os
from astropy.io import fits
import numpy as np
from numpy import sqrt
endpoint = platform_endpoint(cdciplatform)
return DispatcherAPI(host=endpoint)
def test_get_spectrum(instrument,product,product_type='Reale'):
disp = DispatcherAPI.build_from_envs()
data = disp.get_product(instrument=instrument,
product=product,
T1='2003-03-15T23:27:40.0',
T2='2003-03-16T00:03:12.0',
osa_version='OSA10.2',
RA=255.986542,
DEC=-37.844167,
detection_threshold=5.0,
radius=15.,
E1_keV=3.,
E2_keV=35.,
product_type=product_type)
assert (data != None)
return data
def test_n_recentscw(cdciplatform,
timestamp=None,
osaversion="osa10.2",
n_scw=2,
e_offset=0,
source=None,
*a,
**aa):
import requests
import time
logging.getLogger('oda_api').setLevel(logging.DEBUG)
logging.getLogger('oda_api').addHandler(logging.StreamHandler())
from logging_tree import printout
printout()
if timestamp is None:
timestamp = time.time()
else:
timestamp = float(time.time())
if source is None:
ra = 0
dec = 0
radius = 180
t1 = timestamp - 24 * 3600 * 580
t2 = timestamp - 24 * 3600 * 530
elif source == "Crab":
ra = 83
dec = 22
radius = 5
t1 = timestamp - 24 * 3600 * 980
t2 = timestamp - 24 * 3600 * 530
elif source in ["Sco X-1", "Cyg X-1"]:
c = get_named_source_coord(source)
ra = c.ra.deg
dec = c.dec.deg
print(c)
radius = 5
t1 = timestamp - 24 * 3600 * 365 * 15
t2 = timestamp - 24 * 3600 * 365 * 1.5
else:
raise NotImplementedError
r = get_scw_list(ra_obj=ra,
dec_obj=dec,
radius=radius,
start_date=time.strftime("%Y-%m-%dT%H:00:00",
time.gmtime(t1)),
end_date=time.strftime("%Y-%m-%dT%H:00:00",
time.gmtime(t2)))
random.seed(0)
scwpick = random.sample(r, n_scw)
print("picked:", scwpick)
assert len(scwpick) > 0
print("running test image at ", cdciplatform)
from oda_api.api import DispatcherAPI
from oda_api.plot_tools import OdaImage, OdaLightCurve
from oda_api.data_products import BinaryData
import os
from astropy.io import fits
import numpy as np
from numpy import sqrt
endpoint = platform_endpoint(cdciplatform)
disp = DispatcherAPI(host=endpoint)
disp.set_custom_progress_formatter(custom_progress_formatter)
print(disp)
if '10.2' in osaversion:
osa_version = 'OSA10.2'
elif '11.0' in osaversion:
osa_version = 'OSA11.0'
else:
osa_version = 'OSA10.2' # default
data = disp.get_product(instrument='isgri',
product='isgri_image',
scw_list=[str(s) for s in scwpick],
E1_keV=25 + e_offset,
E2_keV=80 + e_offset,
osa_version=osa_version,
RA=0,
DEC=0,
detection_threshold=15,
product_type='Real')
printout()
print(data)
print(dir(data))
catalog_table = data.dispatcher_catalog_1.table
m = catalog_table['significance'] >= 0.0
print(catalog_table[m])
if source is not None:
print(f"\033[31m source check requested for {source}\033[0m")
print(f"found sources:", catalog_table['src_names'])
t = catalog_table[catalog_table['src_names'] == source]
print(t)
assert len(t) == 1
def integration_test():
disp = DispatcherAPI(host='10.194.169.161', port=32784, instrument='mock')
# In[4]:
instr_list = disp.get_instruments_list()
for i in instr_list:
print(i)
# In[ ]:
# ### get the description of the instrument
# In[5]:
disp.get_instrument_description('isgri')
# ### get the description of the product
#
# In[6]:
disp.get_product_description(instrument='isgri',
product_name='isgri_image')
# ## Get ODA products
# now we skip the dry_run to actually get the products
# In[8]:
data = disp.get_product(instrument='isgri',
product='isgri_image',
T1='2003-03-15T23:27:40.0',
T2='2003-03-16T00:03:15.0',
E1_keV=20.0,
E2_keV=40.0,
osa_version='OSA10.2',
RA=255.986542,
DEC=-37.844167,
detection_threshold=5.0,
radius=15.,
product_type='Real')
# ### the ODA data structure
# In[9]:
data.show()
# you can acess memeber by name:
# In[10]:
data.mosaic_image_0
# or by position in the data list
# In[11]:
data._p_list[0]
# ### the ODA catalog
# In[12]:
data.dispatcher_catalog_1.table
# you can use astropy.table commands to modify the table of the catatlog http://docs.astropy.org/en/stable/table/modify_table.html
# to generate a catalog to pass to the dispatcher api
# In[13]:
api_cat = data.dispatcher_catalog_1.get_api_dictionary()
# In[14]:
api_cat
# In[15]:
data = disp.get_product(instrument='isgri',
product='isgri_image',
T1='2003-03-15T23:27:40.0',
T2='2003-03-16T00:03:15.0',
E1_keV=20.0,
E2_keV=40.0,
osa_version='OSA10.2',
RA=255.986542,
DEC=-37.844167,
detection_threshold=5.0,
radius=15.,
product_type='Real',
selected_catalog=api_cat)
# you can explore the image with the following command
# In[16]:
data.mosaic_image_0.show()
# In[17]:
data.mosaic_image_0.show_meta()
# In[23]:
data.mosaic_image_0.data_unit[1].data
# In[24]:
hdu = data.mosaic_image_0.to_fits_hdu_list()
# In[25]:
data.mosaic_image_0.data_unit[1].data.shape
# In[26]:
data.mosaic_image_0.write_fits_file('test.fits', overwrite=True)
# ### the ODA Image plotting tool
# In[34]:
#interactive
#%matplotlib notebook
im = OdaImage(data.mosaic_image_0)
# In[35]:
im.show(unit_ID=1)
# In[33]:
data.mosaic_image_0.data_unit[1].header
# ### the ODA LC plotting tool
# In[36]:
data = disp.get_product(instrument='isgri',
product='isgri_lc',
T1='2003-03-15T23:27:40.0',
T2='2003-03-16T00:03:12.0',
time_bin=70,
osa_version='OSA10.2',
RA=255.986542,
DEC=-37.844167,
detection_threshold=5.0,
radius=15.,
product_type='Real')
# ### explore LC
# In[37]:
data.show()
# In[38]:
data.isgri_lc_0.show_meta()
# In[39]:
for ID, s in enumerate(data._p_list):
print(ID, s.meta_data['src_name'])
# In[40]:
lc = data._p_list[0]
lc.data_unit[1].data
# In[41]:
lc.show()
# In[42]:
lc.meta_data
# In[43]:
#get_ipython().run_line_magic('matplotlib', 'inline')
#OdaLightCurve(lc).show(unit_ID=1)
# In[44]:
lc.data_unit[0].header
# ### Polar LC
# In[45]:
#conda create --name=polar_root root=5 python=3 -c nlesc
#source activate poloar_root
#conda install astropy future -c nlesc
#conda install -c conda-forge json_tricks
#from oda_api.api import DispatcherAPI
#from oda_api.data_products import BinaryData
#from oda_api.plot_tools import OdaImage,OdaLightCurve
#disp=DispatcherAPI(host='10.194.169.161',port=32784,instrument='mock',protocol='http')
data = disp.get_product(instrument='polar',
product='polar_lc',
T1='2016-12-18T08:32:21.000',
T2='2016-12-18T08:34:01.000',
time_bin=0.5,
verbose=True,
dry_run=False)
# In[46]:
data.show()
# In[47]:
data._p_list[0]
# In[48]:
lc = data._p_list[0]
root = data._p_list[1]
open('lc.root', "wb").write(root)
# In[49]:
open('lc.root', "wb").write(root)
# In[51]:
#get_ipython().run_line_magic('matplotlib', 'inline')
#OdaLightCurve(lc).show(unit_ID=1)
# ### SPIACS LC
# In[52]:
disp.get_instrument_description('spi_acs')
# In[53]:
data = disp.get_product(instrument='spi_acs',
product='spi_acs_lc',
T1='2003-03-15T23:27:40.0',
T2='2003-03-15T23:57:12.0',
time_bin=2,
osa_version='OSA10.2',
RA=255.986542,
DEC=-37.844167,
detection_threshold=5.0,
radius=15.,
product_type='Real')
# In[54]:
data.show()
# In[55]:
lc = data._p_list[0]
# In[56]:
lc.show()
# In[58]:
lc.data_unit[1].header
# In[59]:
lc.data_unit[1].data[0:10]
# In[60]:
#OdaLightCurve(lc).show(unit_ID=1)
# ### the ODA and spectra
# In[61]:
data = disp.get_product(instrument='isgri',
product='isgri_spectrum',
T1='2003-03-15T23:27:40.0',
T2='2003-03-16T00:03:12.0',
time_bin=50,
osa_version='OSA10.2',
RA=255.986542,
DEC=-37.844167,
detection_threshold=5.0,
radius=15.,
product_type='Real')
# ### explore spectra
# In[62]:
for ID, s in enumerate(data._p_list):
print(ID, s.meta_data)
# In[63]:
data._p_list[87].write_fits_file('spec.fits')
data._p_list[88].write_fits_file('arf.fits')
data._p_list[89].write_fits_file('rmf.fits')
# In[64]:
s.show()
# In[65]:
d = data._p_list[3]
# In[ ]:
# In[66]:
d.data_unit[1].header
# ### JEM-X test
# In[67]:
disp.get_instrument_description('jemx')
# In[68]:
data = disp.get_product(instrument='jemx',
jemx_num='2',
product='jemx_image',
scw_list=['010200230010.001'],
osa_version='OSA10.2',
detection_threshold=5.0,
radius=15.,
product_type='dummy')
# In[69]:
data = disp.get_product(instrument='jemx',
jemx_nume='2',
product='jemx_lc',
scw_list=['010200230010.001'],
osa_version='OSA10.2',
detection_threshold=5.0,
radius=15.,
product_type='Real')
# In[70]:
data = disp.get_product(instrument='jemx',
jemx_num='2',
product='jemx_spectrum',
scw_list=['010200230010.001'],
osa_version='OSA10.2',
detection_threshold=5.0,
radius=15.,
product_type='Real')
def test_instruments():
from oda_api.api import DispatcherAPI
disp = DispatcherAPI(host="cdcihn/staging-1.2/dispatcher",
instrument="mock")
print(disp.get_instruments_list())
def set_api_code(self, query_dict, url=None):
self.prod_dictionary['api_code'] = DispatcherAPI.set_api_code(
query_dict, url=url)