def test_mosaic_ii_exception_empty():
remote = ddaclient.AutoRemoteDDA()
try:
product = remote.query(
target="mosaic_ii_skyimage",
modules=[
"git://ddosa", "git://osahk", "git://mosaic",
'git://rangequery'
],
assume=[
'ddosa.ImageGroups(input_scwlist=rangequery.TimeDirectionScWList)',
'rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2010-04-12T11:11:11",T2="2009-04-12T11:11:11"),\
use_max_pointings=50 \
)\
',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
])
assert os.path.exists(product.skyima)
except ddaclient.AnalysisException as e:
print(e)
assert hasattr(e, 'exceptions')
assert len(e.exceptions) == 1
print((e.exceptions))
assert e.exceptions[0]['exception'] == 'EmptyScWList()'
def test_cat_injection_image_empty_cat():
remote = ddaclient.AutoRemoteDDA()
cat = [
'SourceCatalog', {
"catalog": [],
"version": "v0_empty",
""
"cached": False,
}
]
product = remote.query(
target="ii_skyimage",
modules=ddosa_modules,
assume=[
scwsource_module + '.ScWData(input_scwid="' + test_scw + '")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")',
],
inject=[])
print(("product:", product))
d = fits.open(product.skyres)[2].data
def test_lc_delegation():
remote = ddaclient.AutoRemoteDDA()
random_ra = 83 + (random.random() - 0.5) * 5
with pytest.raises(ddaclient.AnalysisDelegatedException) as excinfo:
product = remote.query(
target="lc_pick",
modules=["git://ddosa", "git://ddosadm", 'git://rangequery'],
assume=[
'ddosa.ImageGroups(\
input_scwlist=\
rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=%.5lg,DEC=22,radius=5),\
use_timespan=dict(T1="2014-04-12T11:11:11",T2="2015-04-12T11:11:11"),\
use_max_pointings=1 \
)\
)\
)\
',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
],
prompt_delegate=True,
callback=default_callback,
)
def test_analysis_exception():
remote = ddaclient.AutoRemoteDDA()
print(remote)
while True:
try:
product = remote.query(
target="ibis_gti",
modules=[
"git://ddosa/staging-1-3", 'git://findic/staging-1-3',
"git://ddosa11/staging-1-3"
],
assume=[
'ddosa.ScWData(input_scwid="335200230010.001")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
])
break
except ddaclient.AnalysisException as e:
print("found as expected exception", e)
return
except ddaclient.AnalysisDelegatedException as e:
print("waiting", repr(e))
time.sleep(2)
raise RuntimeError("should have raised exception!")
def test_cat_injection():
remote = ddaclient.AutoRemoteDDA()
cat = [
'SourceCatalog', {
"catalog": [{
"DEC": 23,
"NAME": "TEST_SOURCE",
"RA": 83
}, {
"DEC": 13,
"NAME": "TEST_SOURCE2",
"RA": 83
}],
"version":
"v1"
}
]
product = remote.query(
target="CatForImage",
modules=ddosa_modules + ["git://gencat"],
assume=[
scwsource_module + '.ScWData(input_scwid="' + test_scw + '")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
],
inject=[cat])
print(("product:", product))
d = fits.open(product.cat)[1].data
assert cat[1]['catalog'][0]['NAME'] in d['NAME']
def test_sum_spectrum_extract_all():
remote = ddaclient.AutoRemoteDDA()
try:
product = remote.query(
target="ISGRISpectraSum",
modules=[
"ddosa", "git://ddosadm", "git://useresponse/cd7855bf7",
"git://process_isgri_spectra/2200bfd", "git://rangequery"
],
assume=[
'process_isgri_spectra.ScWSpectraList(\
input_scwlist=\
rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2008-04-12T11:11:11",T2="2009-04-12T11:11:11"),\
use_max_pointings=5 \
)\
)\
',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")',
'process_isgri_spectra.ISGRISpectraSum(use_extract_all=True)'
])
except ddaclient.WorkerException as e:
if len(e.args) > 2:
print()
e[2]
raise
import astropy.io.fits as fits
assert fits.open(product.isgri_sum_Crab)[1].header['EXPOSURE'] > 3000
def test_report_scwlist():
remote = ddaclient.AutoRemoteDDA()
try:
product = remote.query(target="ReportScWList",
modules=["git://ddosa", "git://rangequery"],
assume=[
'rangequery.ReportScWList(\
input_scwlist=\
rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83.,DEC=22.,radius=5.),\
use_timespan=dict(T1="2008-04-12T11:11:11",T2="2009-04-12T11:11:11"),\
use_max_pointings=3 \
)\
)'
])
except ddaclient.WorkerException as e:
if len(e.args) > 2:
print()
e[2]
raise
assert hasattr(product, 'scwidlist')
print((product.scwidlist))
assert len(product.scwidlist) == 3
assert product.scwidlist == [
'072700080010.001', '072700090010.001', '072700100010.001'
]
def test_mosaic_exception_failed():
remote = ddaclient.AutoRemoteDDA()
job_id = time.strftime("%y%m%d_%H%M%S")
encoded = urllib.parse.urlencode(
dict(job_id=job_id, session_id="test_mosaic"))
custom_version = "imgbins_for_" + job_id
kwargs = dict(
target="mosaic_ii_skyimage",
modules=["git://ddosa"],
assume=[
'ddosa.ImageGroups(input_scwlist=ddosa.IDScWList)',
'ddosa.IDScWList(use_scwid_list=["058900660010.001"])',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="%s")' %
custom_version,
],
prompt_delegate=True,
callback="http://mock-dispatcher?" + encoded,
)
with pytest.raises(ddaclient.AnalysisDelegatedException) as excinfo:
product = remote.query(**kwargs)
e = excinfo.value
print(e)
assert hasattr(e, 'exceptions')
assert len(e.exceptions) == 1
print((e.exceptions))
assert e.exceptions[0]['exception'] == 'NoValidScW()'
def test_single_image():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(
target="jemx_image",
modules=["git://ddosa", "git://ddosadm", "git://ddjemx"],
assume=[scwsource_module + '.ScWData(input_scwid="' + test_scw + '")'])
assert os.path.exists(product.skyima)
def test_image():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(
target="ii_skyimage",
modules=ddosa_modules,
assume=[
scwsource_module + '.ScWData(input_scwid="' + test_scw + '")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
])
def test_graph_exception():
remote = ddaclient.AutoRemoteDDA()
with pytest.raises(ddaclient.AnalysisException):
product = remote.query(
target="CatExtract",
modules=["ddosa", "git://ddosadm"],
assume=[
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
])
def test_mosaic_delegation_cat():
remote = ddaclient.AutoRemoteDDA()
random_ra = 83 + (random.random() - 0.5) * 5
cat = [
'SourceCatalog', {
"catalog": [{
"DEC": 23,
"NAME": "TEST_SOURCE1",
"RA": 83
}, {
"DEC": 13,
"NAME": "TEST_SOURCE2",
"RA": random_ra
}],
"version":
"v1"
}
]
job_id = time.strftime("%y%m%d_%H%M%S")
encoded = urllib.parse.urlencode(
dict(job_id=job_id, session_id="test_mosaic"))
print(("encoded:", encoded))
with pytest.raises(ddaclient.AnalysisDelegatedException) as excinfo:
product = remote.query(
target="mosaic_ii_skyimage",
modules=[
"git://ddosa", "git://ddosadm", 'git://rangequery',
'git://gencat'
],
assume=[
'ddosa.ImageGroups(\
input_scwlist=\
rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2014-04-12T11:11:11",T2="2015-04-12T11:11:11"),\
use_max_pointings=100 \
)\
)\
',
'ddosa.ImageBins(use_ebins=[(20,80)],use_autoversion=True)',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
],
callback=default_callback + "?" + encoded,
prompt_delegate=True,
inject=[cat],
)
# callback="http://intggcn01:5000/callback?job_id=1&asdsd=2",
assert excinfo.value.delegation_state == "submitted"
def test_mosaic_delegation_failing():
remote = ddaclient.AutoRemoteDDA()
job_id = time.strftime("%y%m%d_%H%M%S")
encoded = urllib.parse.urlencode(
dict(job_id=job_id, session_id="test_mosaic"))
print(("encoded:", encoded))
custom_version = "imgbins_for_" + job_id
kwargs = dict(
target="mosaic_ii_skyimage",
modules=["git://ddosa", 'git://rangequery', 'git://ddosa_delegate'],
assume=[
'ddosa.ImageGroups(input_scwlist=rangequery.TimeDirectionScWList)',
'rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2014-04-12T11:11:11",T2="2015-04-12T11:11:11"),\
use_max_pointings=%i \
)' %
int(os.environ.get("TEST_N_POINTINGS", 3)),
'ddosa.ghost_bustersImage(input_fail=ddosa.FailingMedia)',
'ddosa.ImageBins(use_ebins=[(20,80)],use_autoversion=False, use_version="%s")'
% custom_version,
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
],
callback=default_callback + "?" + encoded,
prompt_delegate=True,
)
with pytest.raises(ddaclient.AnalysisDelegatedException) as excinfo:
product = remote.query(**kwargs)
# callback="http://intggcn01:5000/callback?job_id=1&asdsd=2",
assert excinfo.value.delegation_state == "submitted"
while True:
time.sleep(5)
try:
product = remote.query(**kwargs)
except ddaclient.AnalysisDelegatedException as e:
print(("state:", e.delegation_state))
except ddaclient.WorkerException:
print(("worker exception:", e.__class__))
break
except Exception as e:
print(("undefined failure:", e.__class__, e))
raise
else:
print(("DONE:", product))
break
def test_bad_request():
remote = ddaclient.AutoRemoteDDA()
#with pytest.raises(requests.ConnectionError):
with pytest.raises(ddaclient.WorkerException):
product = remote.query(
target="Undefined",
modules=["ddosa", "git://ddosadm"],
assume=[
scwsource_module + '.ScWData(input_scwid="035200250010.001")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
])
def test_spectrum():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(
target="ii_spectra_extract",
modules=ddosa_modules,
assume=[
scwsource_module + '.ScWData(input_scwid="' + test_scw + '")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2")',
'ddosa.CatForLC(use_minsig=3)',
])
assert os.path.exists(product.spectrum)
def test_very_large_request():
remote = ddaclient.AutoRemoteDDA()
with pytest.raises(requests.ConnectionError):
product = remote.query(
target="ii_spectra_extract",
modules=["ddosa", "git://ddosadm"],
assume=[
'ddosa.ScWData(input_scwid=[{}])'.format('035200230010.001' *
2000),
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
])
assert product
def test_spectrum_show_standard_catalog():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(
target="CatForSpectraFromImaging",
modules=["git://ddosa"] + ddosa_data_modules,
assume=[
scwsource_module + '.ScWData(input_scwid="' + test_scw + '")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2")',
'ddosa.CatForLC(use_minsig=3)',
'ddosa.CatForSpectraFromImaging(use_cached=True)'
])
assert os.path.exists(product.cat)
def test_spectrum_sum():
remote = ddaclient.AutoRemoteDDA()
custom_version = time.strftime("%Y%m%d_%H%M%S")
kwargs = dict(
target="spe_pick",
modules=["git://ddosa", "git://ddjemx", 'git://rangequery'],
assume=[
'ddjemx.JMXImageSpectraGroups(input_scwlist=rangequery.TimeDirectionScWList)',
'rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2008-04-12T11:11:11",T2="2009-04-12T11:11:11"),\
use_max_pointings=1 \
)',
'ddjemx.JEnergyBins(use_version="%s",use_bins=[(5,20)])' %
custom_version
],
callback='http://mock-dispatcher?' + urllib.parse.urlencode(
dict(job_id=custom_version, session_id="test_jemx")),
prompt_delegate=True,
)
with pytest.raises(ddaclient.AnalysisDelegatedException) as excinfo:
product = remote.query(**kwargs)
assert excinfo.value.delegation_state == "submitted"
while True:
time.sleep(5)
try:
product = remote.query(**kwargs)
except ddaclient.AnalysisDelegatedException as e:
print(("state:", e.delegation_state))
except ddaclient.WorkerException:
print(("worker exception:", e.__class__))
break
except Exception as e:
print(("undefined failure:", e.__class__, e))
raise
else:
print(("DONE:", product))
break
product = remote.query(**kwargs)
assert os.path.exists(product.spectrum_Crab)
def test_lc():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(
target="lc_pick",
modules=ddosa_modules + ["git://rangequery"],
assume=[
'ddosa.LCGroups(input_scwlist=rangequery.TimeDirectionScWList)',
'rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2014-04-12T11:11:11",T2="2015-04-12T11:11:11"),\
use_max_pointings=1 \
)',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
],
)
def test_mosaic():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(target="mosaic_jemx",
modules=[
"git://ddosa", "git://ddosadm", "git://ddjemx",
'git://rangequery'
],
assume=[
'ddjemx.JMXScWImageList(\
input_scwlist=\
ddosa.IDScWList(\
use_scwid_list=%s\
)\
)' % test_scw_list_str
])
assert os.path.exists(product.skyima)
def test_summary_injection():
remote = ddaclient.AutoRemoteDDA()
cat = [
'SourceCatalog', {
"catalog": [{
"DEC": 23,
"NAME": "TEST_SOURCE",
"RA": 83
}, {
"DEC": 13,
"NAME": "TEST_SOURCE2",
"RA": 83
}],
"version":
"v1"
}
]
def test_handled_exception():
remote = ddaclient.AutoRemoteDDA()
try:
product = remote.query(
target="CatExtract",
modules=["git://ddosa", "git://ddosadm"],
assume=[
scwsource_module + '.ScWData(input_scwid="935200230010.001")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
])
except ddaclient.AnalysisException as e:
print(e)
assert hasattr(e, 'exceptions')
assert len(e.exceptions) == 1
print((e.exceptions))
assert e.exceptions[0]['node'] == "ScWData"
def test_fit_one():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(
target="FitSourcePowerlaw",
modules=[
"ddosa", "git://ddosadm", "git://useresponse/cd7855bf7",
"git://process_isgri_spectra/2200bfd", "git://rangequery",
"git://ddjemx", "git://fit"
],
assume=[
scwsource_module + '.ScWData(input_scwid="066500330010.001")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2")',
'ddosa.CatForLC(use_minsig=3)',
'fit.FitSourcePowerlaw(use_sname="Crab")'
])
print(("product:", product))
def test_poke_sleeping():
remote = ddaclient.AutoRemoteDDA()
import threading
def worker():
remote.query("sleep:10")
t = threading.Thread(target=worker)
t.start()
for i in range(15):
time.sleep(1)
try:
r = remote.poke()
print(i, r)
break
except ddaclient.WorkerException as e:
print(e)
def test_delegation():
remote = ddaclient.AutoRemoteDDA()
random_rev = random.randint(50, 1800)
with pytest.raises(ddaclient.AnalysisDelegatedException) as excinfo:
product = remote.query(
target="ii_skyimage",
modules=["ddosa", "git://ddosadm"],
assume=[
'ddosa.ScWData(input_scwid="%.4i00430010.001")' % random_rev,
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
],
prompt_delegate=True,
callback=default_callback,
)
assert excinfo.value.delegation_state == "submitted"
def test_mosaic_rangequery():
remote = ddaclient.AutoRemoteDDA()
product = remote.query(target="mosaic_jemx",
modules=[
"git://ddosa", "git://ddosadm", "git://ddjemx",
'git://rangequery'
],
assume=[
'ddjemx.JMXScWImageList(\
input_scwlist=\
rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2008-04-12T11:11:11",T2="2009-04-12T11:11:11"),\
use_max_pointings=2 \
)\
)'
])
assert os.path.exists(product.skyima)
def test_cat_injection_image():
remote = ddaclient.AutoRemoteDDA()
cat = [
'SourceCatalog', {
"catalog": [{
"DEC": 23,
"NAME": "TEST_SOURCE",
"RA": 83
}, {
"DEC": 13,
"NAME": "TEST_SOURCE2",
"RA": 83
}],
"version":
"v0",
"cached":
False,
}
]
product = remote.query(
target="ii_skyimage",
modules=ddosa_modules + ["git://gencat"],
assume=[
scwsource_module + '.ScWData(input_scwid="' + test_scw + '")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")',
'ddosa.ii_skyimage(input_cat=gencat.CatForImage)'
],
inject=[cat])
print(("product:", product))
print(("product:", product.skyres))
os.path.exists(product.skyres)
os.system("ls -lotr " + product.skyres)
d = fits.open(product.skyres)[2].data
assert cat[1]['catalog'][0]['NAME'] in d['NAME']
def test_mosaic_injection():
remote = ddaclient.AutoRemoteDDA()
cat = [
'SourceCatalog', {
"catalog": [{
"DEC": 23,
"NAME": "TEST_SOURCE",
"RA": 83
}, {
"DEC": 13,
"NAME": "TEST_SOURCE2",
"RA": 83
}],
"version":
"v1"
}
]
product = remote.query(
target="Mosaic",
modules=[
"git://ddosa", "git://ddosadm", "git://osahk", "git://mosaic",
'git://rangequery', 'git://gencat'
],
assume=[
'mosaic.ScWImageList(\
input_scwlist=\
rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=83,DEC=22,radius=5),\
use_timespan=dict(T1="2008-04-12T11:11:11",T2="2009-04-12T11:11:11"),\
use_max_pointings=2 \
)\
)\
', 'mosaic.Mosaic(use_pixdivide=4)',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")'
],
inject=[cat])
assert os.path.exists(product.skyima)
def test_cat_injection_lc():
remote = ddaclient.AutoRemoteDDA()
cat = [
'SourceCatalog', {
"catalog": [{
"DEC": 23,
"NAME": "TEST_SOURCE",
"RA": 83
}, {
"DEC": 13,
"NAME": "TEST_SOURCE2",
"RA": 83
}],
"version":
"v1"
}
]
product = remote.query(
target="ii_lc_extract",
modules=ddosa_modules + ["git://gencat"],
assume=[
scwsource_module + '.ScWData(input_scwid="' + test_scw + '")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2")',
'ddosa.ii_lc_extract(input_cat=gencat.CatForSpectra)',
],
inject=[cat])
assert os.path.exists(product.lightcurve)
print(("product:", product))
assert hasattr(product, 'lightcurve')
d = fits.open(product.lightcurve)
assert len(d[1].data) == 2
assert len(d[2:]) == 2
assert d[2].header['NAME'] == cat[1]['catalog'][0]['NAME']
def test_delegated_failing_request():
remote = ddaclient.AutoRemoteDDA()
#with pytest.raises(requests.ConnectionError):
try:
product = remote.query(
target="FailingMedia",
modules=["ddosa", "git://ddosadm"],
assume=[
scwsource_module + '.ScWData(input_scwid="035200250010.001")',
'ddosa.ImageBins(use_ebins=[(20,40)],use_version="onebin_20_40")',
'ddosa.ImagingConfig(use_SouFit=0,use_version="soufit0")',
'ddosa.FailingMedia(use_version="%i")' %
random.randint(0, 1000)
],
prompt_delegate=True,
callback="file:///data/ddcache/test_callback",
)
except ddaclient.AnalysisDelegatedException:
pass
