def test_image_cached():
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ii_skyimage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
# now get cached
fb = ddosa.ii_skyimage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fb.produce_disabled = True
import time
t0 = time.time()
fb.get()
assert time.time() - t0 < 3
def test_catforspec_tolc():
import ddosa
import gencat
cat = gencat.SourceCatalog()
cat.catalog = [dict(
NAME="TEST_SOURCE",
RA=83,
DEC=22,
)]
cat.promote()
ddosa.ScWData(input_scwid="066500220010.001").get().promote()
gcat = gencat.CatForSpectra().get()
d = pyfits.open(gcat.cat.get_path())[1].data
assert len(d) == 1
assert d[0]['NAME'] == cat.catalog[0]['NAME']
assert d[0]['RA_FIN'] == cat.catalog[0]['RA']
assert d[0]['DEC_FIN'] == cat.catalog[0]['DEC']
lc = ddosa.ii_lc_extract(input_cat=gencat.CatForSpectra,
use_read_caches=[]).get()
assert hasattr(lc, 'lightcurve')
d = pyfits.open(lc.lightcurve.get_path())
print d[1].data
assert len(d[1].data) == 1
assert len(d[2:]) == 1
assert d[2].header['NAME'] == cat.catalog[0]['NAME']
def test_cat_add_postimage():
import ddosa
import dataanalysis.core as da
da.debug_output()
ddosa.ScWData(input_scwid="066500230010.001").promote()
sl=ddosa.SourceList(use_sources=[dict(name="NEW_added",ra=83,dec=22)])
a = ddosa.CatExtract(
input_extra_sources=sl,
)
a.promote()
b = ddosa.CatForSpectraFromImaging(
input_extra_sources=sl,
use_minsig=10,
)
b.promote()
a.cached=False
b.cached=False
b.get()
print b.cat.get_path()
f=fits.open(b.cat.get_path())[1]
print f.data['NAME']
assert "NEW_added" in f.data['NAME']
def test_cat_add():
import ddosa
import dataanalysis.core as da
da.debug_output()
ddosa.ScWData(input_scwid="066500230010.001").promote()
print "was before",ddosa.CatExtract()
a = ddosa.CatExtract(
input_extra_sources=ddosa.SourceList(use_sources=[dict(name="NEW_added",ra=83,dec=22)]),
)
print "got back",a
assert hasattr(a,'input_extra_sources')
a.cached=False
a.get()
print a.cat.get_path()
f=fits.open(a.cat.get_path())[1]
print f.data['NAME']
assert "NEW_added" in f.data['NAME']
def test_osa_blob():
import dataanalysis.core as da
from sdsc import SDSCCache, blob_store
blob_store.purge()
import ddosa
cache = SDSCCache()
im = ddosa.ii_skyimage(
assume=[ddosa.ScWData(input_scwid="066500220010.001")])
im.get()
im.write_caches.append(cache.__class__)
da.debug_output()
cache.store(im._da_locally_complete, im)
assert len(blob_store.blobs) == 1
blob = blob_store.blobs.items()[0][1]
import tarfile
import StringIO
tar = tarfile.open(fileobj=StringIO.StringIO(blob))
print tar.getnames()
assert './blob/isgri_sky_res.fits.gz' in tar.getnames()
def test_store_restore_in_analysis():
import dataanalysis.core as da
import sdsc
import ddosa
import ddosadm
import subprocess
scw = ddosadm.ScWData(input_scwid="066500230010.001")
obj = ddosa.BinEventsImage(
assume=[ddosa.ScWData(input_scwid="066500220010.001")])
c = obj.cache
found_sdsc = False
while c is not None:
print("cache:", c)
c = c.parent
if c == sdsc.cache: found_sdsc = True
assert found_sdsc
assert sdsc.cache.__class__ in obj.write_caches
assert obj.cached
try:
obj.get()
except subprocess.CalledProcessError:
pass
key = sdsc.cache.blob_store.keys[0]
print "key:", key
blob = sdsc.cache.blob_store.get_blob(key)
tar = tarfile.open(fileobj=StringIO.StringIO(blob))
print tar.getnames()
def test_mosaic_ii_skyimage():
import ddosa
import gencat
cat = gencat.SourceCatalog()
cat.catalog = [dict(
NAME="TEST_SOURCE",
RA=83,
DEC=22,
)]
cat.promote()
assert cat.virtual
assert hasattr(cat, 'catalog')
grcat = gencat.GRcat()
grcat.serialize()
sc = ddosa.da.AnalysisFactory.byname("SourceCatalog")
assert cat == sc
print sc.get_version()
#ips=ddosa.ImageProcessingSummary().get()
#print "HASH:",ips.hash
##
#da.debug_output()
mf = gencat.GRcat()
c = mf.process(output_required=False, run_if_haveto=False)[0]
print("mf:", c)
mf = gencat.GRcat(assume=ddosa.ScWData(input_scwid="any"))
c = mf.process(output_required=False, run_if_haveto=False)[0]
print("mf:", c)
assert c[1][2].startswith("SourceCatalog.v0.1.")
##
mosaic = ddosa.mosaic_ii_skyimage(assume=[
ddosa.ii_skyimage(input_cat=gencat.CatForImage),
ddosa.ImageGroups(input_scwlist=ddosa.IDScWList(
use_scwid_list=["066500330010.001", "066500340010.001"])),
ddosa.ImageBins(use_ebins=[(25, 40)], use_version="onebin_25_40"),
ddosa.ImagingConfig(use_SouFit=0, use_version="soufit0")
])
mosaic.read_caches = []
mosaic.ii_NegModels = 1
mosaic.get()
d = pyfits.open(mosaic.srclres.get_path())[1].data
print d['NAME']
assert len(d) == 2
assert d[0]['NAME'] == cat.catalog[0]['NAME']
assert d[0]['RA_FIN'] == cat.catalog[0]['RA']
assert d[0]['DEC_FIN'] == cat.catalog[0]['DEC']
def test_lightcurves():
da.debug_output()
fa = ddosa.ii_lc_extract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
assert os.path.exists(fa.lightcurve.get_path())
def test_catextract():
da.reset()
import ddosa
reload(ddosa)
da.debug_output()
fa = ddosa.CatExtract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
assert hasattr(fa, 'cat')
da.reset()
fa = ddosa.CatExtract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.produce_disabled = True
def test_scw():
#da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ScWData(input_scwid="066500230010.001")
fa.get()
print fa.scwpath
def test_gti_cached():
da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ibis_gti(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.get()
# now get cached
fb = ddosa.ibis_gti(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fb.produce_disabled = True
fb.get()
def test_gti():
# da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ibis_gti(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
def test_catextract_grcat():
import ddosa
import gencat
cat = gencat.SourceCatalog()
cat.catalog = [dict(
NAME="TEST_SOURCE",
RA=83,
DEC=22,
)]
cat.promote()
scw = ddosa.ScWData(input_scwid="066500220010.001")
scw.get().promote()
igrcat = gencat.ISGRIRefCat(use_read_caches=[]).get()
assert os.path.exists(igrcat.cat)
d = pyfits.open(igrcat.cat)[1].data
assert len(d) == 1
assert d[0]['NAME'] == cat.catalog[0]['NAME']
assert d[0]['RA_OBJ'] == cat.catalog[0]['RA']
assert d[0]['DEC_OBJ'] == cat.catalog[0]['DEC']
assert d[0]['ISGRI_FLAG'] == 1
assert d[0]['ISGRI_FLAG2'] == 0
ecat = ddosa.CatExtract(use_read_caches=[]).get()
assert isinstance(ecat.input_cat, gencat.ISGRIRefCat)
assert os.path.exists(ecat.input_cat.cat)
d = pyfits.open(ecat.input_cat.cat)[1].data
assert len(d) == 1
assert d[0]['NAME'] == cat.catalog[0]['NAME']
assert d[0]['RA_OBJ'] == cat.catalog[0]['RA']
assert d[0]['DEC_OBJ'] == cat.catalog[0]['DEC']
assert d[0]['ISGRI_FLAG'] == 1
assert d[0]['ISGRI_FLAG2'] == 0
d = pyfits.open(ecat.cat.get_path())[1].data
assert len(d) == 1
assert d[0]['NAME'] == cat.catalog[0]['NAME']
assert d[0]['RA_OBJ'] == cat.catalog[0]['RA']
assert d[0]['DEC_OBJ'] == cat.catalog[0]['DEC']
im = ddosa.ii_skyimage(use_read_caches=[]).get()
d = pyfits.open(im.skyres.get_path())[2].data
assert len(d['NAME']) >= 1
assert d[0]['NAME'] == cat.catalog[0]['NAME']
assert d[0]['RA_OBJ'] == cat.catalog[0]['RA']
assert d[0]['DEC_OBJ'] == cat.catalog[0]['DEC']
def test_spectra():
da.reset()
import ddosa
reload(ddosa)
#da.debug_output()
fa = ddosa.ii_spectra_extract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
ddosa.SpectraBins(use_rmfpath="/data/resources/rmf_62bands.fits"),
])
fa.read_caches = []
fa.get()
assert os.path.exists(fa.spectrum.get_path())
def test_imagebins():
#
# da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.BinEventsImage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
ddosa.ImageBins(use_autoversion=True, use_ebins=[(25, 45)])
])
fa.read_caches = []
fa.get()
print fa.shadow_detector
def test_image():
#
da.reset()
import ddosa
reload(ddosa)
da.debug_output()
fa = ddosa.ii_skyimage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
print fa.skyima
def extract_from_index(self,scw_index,rep_base_prod=os.environ['INTEGRAL_DATA'],scwversion="001"):
scx=SkyCoord(scw_index['RA_SCX'],scw_index['DEC_SCX'],unit="deg")
target=SkyCoord(self.coordinates['RA'],self.coordinates['DEC'],unit="deg")
m_avail=scw_index['SW_TYPE']=="POINTING"
m_coord=scx.separation(target).deg<self.coordinates['radius']
print("for coordinates",target,"found",sum(m_coord & m_avail),"within",self.coordinates['radius'],"degrees")
t1_ijd=float(converttime("UTC",self.timespan['T1'],"IJD"))
t2_ijd=float(converttime("UTC",self.timespan['T2'],"IJD"))
t1=scw_index['TSTART']
t2=scw_index['TSTOP']
m_time=(t1>t1_ijd) & (t2<t2_ijd)
print("for time",t1_ijd,t2_ijd,"found",sum(m_coord & m_avail & m_time), "total range was",t1.min(),t2.max(),"(",(t2.max()-t1.min())/365,"years)")
pre_selection=scw_index['SWID'][m_coord & m_avail & m_time]
selection=[]
if not self.randomize_pick:
for scwid in pre_selection:
evtsfn=rep_base_prod+"/scw/%s/%s.%s/isgri_events.fits.gz"%(scwid[:4],scwid,scwversion)
print("searching for",evtsfn)
if not os.path.exists(evtsfn):
print("skipping",scwid)
continue
if not isfile(evtsfn) or not access(evtsfn, R_OK):
print("File {} doesn't exist or isn't readable".format(evtsfn))
continue
selection.append(scwid)
if self.max_pointings is not None and len(selection)>=self.max_pointings:
print("choosing only first",self.max_pointings)
break
else:
pick_size=min(self.max_pointings,len(pre_selection))
print("choosing only random",pick_size)
random.seed(0)
pre_selection=sorted(random.sample(pre_selection,pick_size))
for scwid in pre_selection:
evtsfn=rep_base_prod+"/scw/%s/%s.%s/isgri_events.fits.gz"%(scwid[:4],scwid,scwversion)
print("searching for",evtsfn)
if not os.path.exists(evtsfn):
print("skipping",scwid)
continue
if not isfile(evtsfn) or not access(evtsfn, R_OK):
print("File {} doesn't exist or isn't readable".format(evtsfn))
continue
else:
print("File {} exist and readable".format(evtsfn))
selection.append(scwid)
print("selection:",selection)
return [ddosa.ScWData(input_scwid=scwid+"."+scwversion) for scwid in selection]
def test_find():
ur = useresponse.FindResponse(assume=ddosa.ScWData(
input_scwid="066500220010.001"))
ur.get()
assert ur.arf_path == "/home/isdc/savchenk/osa11_deployment/deployment/ic/ic/ibis/rsp/isgr_arf_rsp_0037.fits"