def test_osa_blob():
import dataanalysis.core as da
from sdsc import SDSCCache, blob_store
blob_store.purge()
import ddosa
cache = SDSCCache()
im = ddosa.ii_skyimage(
assume=[ddosa.ScWData(input_scwid="066500220010.001")])
im.get()
im.write_caches.append(cache.__class__)
da.debug_output()
cache.store(im._da_locally_complete, im)
assert len(blob_store.blobs) == 1
blob = blob_store.blobs.items()[0][1]
import tarfile
import StringIO
tar = tarfile.open(fileobj=StringIO.StringIO(blob))
print tar.getnames()
assert './blob/isgri_sky_res.fits.gz' in tar.getnames()
def test_scwid_list():
import ddosa
import dataanalysis.core as da
da.debug_output()
fa = ddosa.IDScWList(use_scwid_list=["066500230010.001"])
print(fa.get())
print(fa.get_version())
assert len(fa.scwlistdata) == 1
fa = ddosa.IDScWList(
use_scwid_list=["066500230010.001", "066500250010.001"])
print(fa.get())
print(fa.get_version())
assert len(fa.scwlistdata) == 2
fa = ddosa.IDScWList(
use_scwid_list=["066500230010.001", "066600250010.001"])
print(fa.get())
print(fa.get_version())
assert len(fa.scwlistdata) == 2
def test_live_multiple_resource_delegation(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append(
"ClientDelegatableAnalysis.*")
A = ddmoduletest.TwoCDInputAnalysis()
with pytest.raises(
dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources) == 2
assert isinstance(excinfo.value.resources[0],
dataanalysis.caches.resources.WebResource)
assert isinstance(excinfo.value.resources[1],
dataanalysis.caches.resources.WebResource)
print(excinfo.value.resources[0].identity.get_modules_loadable())
print(excinfo.value.resources)
def test_passing_assumptions(ddservice, app):
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
ddmoduletest.cache.delegation_mode = "interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice
#ddmoduletest.cache.resource_factory.getter=getter
A = ddmoduletest.ChainedServerProducer(
assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")])
A.produce_disabled = True
a = A.get()
print a.data
assert a.data == {'a': 'dataAfromclient', 'b': 'dataBdataAfromclient'}
assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b[
'main_executed_on']['pid']
def test_base_cache_check_location():
from dataanalysis import core as da
from dataanalysis import caches
da.reset()
da.debug_output()
cache = caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached = True
read_caches = []
store_preview_yaml = True
def main(self):
self.data = "test123"
A = Analysis()
A.get()
cached_path = cache.construct_cached_file_path(A._da_locally_complete, A)
import os
assert os.path.exists(cached_path)
assert os.path.exists(cached_path + "/hash.txt")
assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
def test_live_resource_server_cachable(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"]
A = ddmoduletest.ServerCachableAnalysis()
A.cache=ddmoduletest.cache
with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources)==1
getter=lambda x:client.get(x).json
fr = excinfo.value.resources[0].get(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
fr = excinfo.value.resources[0].fetch(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
def test_passing_unmanagable_assumptions(ddservice, app):
import dataanalysis.core as da
import dataanalysis.caches.resources
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
ddmoduletest.cache.delegation_mode = "interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice
#ddmoduletest.cache.resource_factory.getter=getter
A = ddmoduletest.ChainedServerProducer(assume=[
ddmoduletest.AAnalysis(input_x=ddmoduletest.TwoCDInputAnalysis())
])
A.produce_disabled = True
with pytest.raises(
dataanalysis.caches.resources.GenericResourceException) as excinfo:
a = A.get()
assert 'mismatch' in excinfo.value.args[2]['comment']
def test_cat_add_postimage():
import ddosa
import dataanalysis.core as da
da.debug_output()
ddosa.ScWData(input_scwid="066500230010.001").promote()
sl=ddosa.SourceList(use_sources=[dict(name="NEW_added",ra=83,dec=22)])
a = ddosa.CatExtract(
input_extra_sources=sl,
)
a.promote()
b = ddosa.CatForSpectraFromImaging(
input_extra_sources=sl,
use_minsig=10,
)
b.promote()
a.cached=False
b.cached=False
b.get()
print b.cat.get_path()
f=fits.open(b.cat.get_path())[1]
print f.data['NAME']
assert "NEW_added" in f.data['NAME']
def test_obscure_assumptions():
from dataanalysis import core as da
da.reset()
da.debug_output()
class BAnalysis(da.DataAnalysis):
data="1"
class AAnalysis(da.DataAnalysis):
input_b=BAnalysis
cached = True
def main(self):
self.data="a."+self.input_b.data
A=AAnalysis().get()
data=A.export_data()
hashe=A._da_locally_complete
cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A)
import os
assert os.path.exists(cached_path)
print(cached_path)
da.reset()
nA=da.DataAnalysis.from_hashe_and_data(hashe, data).get()
print(nA)
assert nA.export_data()['data'] == data['data']
assert nA._da_locally_complete == hashe
def test_base_cache_check_location():
from dataanalysis import core as da
from dataanalysis import caches
da.reset()
da.debug_output()
cache=caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached=True
read_caches=[]
store_preview_yaml=True
def main(self):
self.data="test123"
A=Analysis()
A.get()
cached_path=cache.construct_cached_file_path(A._da_locally_complete,A)
import os
assert os.path.exists(cached_path)
assert os.path.exists(cached_path+"/hash.txt")
assert os.path.exists(cached_path+"/object_identity.yaml.gz")
assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
def test_selective_delegation():
import dataanalysis.caches.delegating
da.debug_output()
class TCache(dataanalysis.caches.delegating.SelectivelyDelegatingCache):
delegating_analysis=['AAnalysis','BAnalysis$']
#cache=Cache()
class AAnalysis(da.DataAnalysis):
#read_caches = [q_cache.__class__]
cached = True
cache=TCache()
def main(self):
#self.data = "datadata1"
raise Exception("this should have been delegated")
class BAnalysis(da.DataAnalysis):
#read_caches = [q_cache.__class__]
cached = True
cache=TCache()
def main(self):
self.data = "datadata2"
A=AAnalysis()
with pytest.raises(da.AnalysisDelegatedException):
A.get()
B=BAnalysis()
B.get()
def test_generate_list():
da.debug_output()
da.reset()
da.AnalysisFactory.reset()
class BAnalysis(da.DataAnalysis):
c=None
_da_settings=["c"]
def main(self):
print("test",self.c)
self.data="data "+repr(self.c)
def __repr__(self):
return "[A %s]"%repr(self.c)
class AAnalysis(da.DataAnalysis):
run_for_hashe=True
def main(self):
print("test",self.__class__)
r=[BAnalysis(use_c=c) for c in range(3)]
return r
A=AAnalysis()
r=A.get()
print(r[1].c,r[1].data)
assert r[0].data == 'data 0'
assert r[1].data == 'data 1'
def test_queue_reconstruct_env():
from dataanalysis import caches
import dataanalysis.caches.queue
da.debug_output()
da.reset()
q_cache=caches.queue.QueueCache()
q_cache.wipe_queue(["waiting", "done", "running"])
define_analysis()
A=dataanalysis.core.AnalysisFactory['AAnalysis']
A.produce_disabled=True
A.cache = q_cache
A.cached = True
with pytest.raises(da.AnalysisDelegatedException):
A.get()
# worker part
f_cache=caches.cache_core.CacheNoIndex()
#f_cache.parent=q_cache
da.reset()
#A = dataanalysis.core.AnalysisFactory['Analysis']
#A.produce_disabled = False
#A.cache = f_cache
#A.cached = True
worker=caches.queue.QueueCacheWorker()
print(worker.run_all())
def test_cat_add():
import ddosa
import dataanalysis.core as da
da.debug_output()
ddosa.ScWData(input_scwid="066500230010.001").promote()
print "was before",ddosa.CatExtract()
a = ddosa.CatExtract(
input_extra_sources=ddosa.SourceList(use_sources=[dict(name="NEW_added",ra=83,dec=22)]),
)
print "got back",a
assert hasattr(a,'input_extra_sources')
a.cached=False
a.get()
print a.cat.get_path()
f=fits.open(a.cat.get_path())[1]
print f.data['NAME']
assert "NEW_added" in f.data['NAME']
def test_obscure_node_cached():
from dataanalysis import core as da
da.reset()
da.debug_output()
class BAnalysis(da.DataAnalysis):
data="1"
class AAnalysis(da.DataAnalysis):
input_b=BAnalysis
cached = True
def main(self):
self.data="a."+self.input_b.data
A=AAnalysis().get()
data=A.export_data()
hashe=A._da_locally_complete
cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A)
import os
assert os.path.exists(cached_path)
print(cached_path)
da.reset()
nA=da.DataAnalysis.from_hashe(hashe).get()
print(nA)
assert nA.export_data()['data'] == data['data']
assert nA._da_locally_complete == hashe
def test_generate_list():
da.debug_output()
da.reset()
da.AnalysisFactory.reset()
class BAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[A %s]" % repr(self.c)
class AAnalysis(da.DataAnalysis):
run_for_hashe = True
def main(self):
print("test", self.__class__)
r = [BAnalysis(use_c=c) for c in range(3)]
return r
A = AAnalysis()
r = A.get()
print(r[1].c, r[1].data)
assert r[0].data == 'data 0'
assert r[1].data == 'data 1'
def test_passing_assumptions(ddservice_fixture, app):
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
ddmoduletest.cache.delegation_mode="interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture
#ddmoduletest.cache.resource_factory.getter=getter
A=ddmoduletest.ChainedServerProducer(assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")])
A.produce_disabled=True
a=A.get()
print a.data
assert a.data=={'a': 'dataAassumed:fromclient', 'b': 'dataB_A:dataAassumed:fromclient'}
assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b['main_executed_on']['pid']
def test_live_resource_server_cachable(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"]
A = ddmoduletest.ServerCachableAnalysis()
A.cache = ddmoduletest.cache
with pytest.raises(
dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources) == 1
getter = lambda x: client.get(x).json
fr = excinfo.value.resources[0].get(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
fr = excinfo.value.resources[0].fetch(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
def test_lightcurves():
da.debug_output()
fa = ddosa.ii_lc_extract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
assert os.path.exists(fa.lightcurve.get_path())
def test_serialize():
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get()
ed=S.export_data()
assert '_da_stored_string_input_s' in ed
def test_serialize():
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get()
ed=S.export_data()
assert '_da_stored_string_input_s' in ed
def test_queue_cache_generative():
from dataanalysis import caches
import dataanalysis.caches.queue
da.debug_output()
da.reset()
q_cache=caches.queue.QueueCache("./queue_test")
q_cache.queue.purge()
assert len(q_cache.queue.list())==0
define_analysis()
A=dataanalysis.core.AnalysisFactory['GenerativeAnalysis']
for name in 'AAnalysis',:
aA=dataanalysis.core.AnalysisFactory[name].__class__
aA.cache = q_cache
aA.produce_disabled=True
aA.cached = True
with pytest.raises(da.AnalysisDelegatedException) as e:
A.get()
print('AnalysisDelegatedException',e.value,e.value.origin,e.value.hashe)
assert len(e.value.hashe[1:])==2
assert len(e.value.source_exceptions)==2
for se in e.value.source_exceptions:
print('found multiple AnalysisDelegatedException',se,se.origin)
print(q_cache.queue.list())
# assert len(q_cache.queue.list())==2
assert len(q_cache.queue.list())==1
# worker part
f_cache=caches.cache_core.CacheNoIndex()
#f_cache.parent=q_cache
define_analysis()
A = dataanalysis.core.AnalysisFactory['AAnalysis']
A.produce_disabled = False
A.cache = f_cache
A.cached = True
worker=caches.queue.QueueCacheWorker("./queue_test")
print(worker.run_once())
def test_self_assumption():
from dataanalysis import core as da
da.debug_output()
da.reset()
class AAnalysis(da.DataAnalysis):
pass
aa = AAnalysis(assume=AAnalysis(use_arg="arg2", use_nonearg=None)).get()
assert aa._da_locally_complete
assert not hasattr(aa,'arg') # current behavoir and it is bad
def test_constructor_input():
from dataanalysis import analysisfactory
from dataanalysis import core as da
da.reset()
class BAnalysis(da.DataAnalysis):
def main(self):
print("test")
self.data="data"
class Analysis(da.DataAnalysis):
#input_b=BAnalysis
def main(self):
print("test")
if hasattr(self,'input_b'):
self.data = self.input_b.data
else:
self.data = 'otherdata'
A=Analysis()
A.get()
assert A.data == 'otherdata'
A=Analysis(input_b=BAnalysis())
assert hasattr(A,'input_b')
da.debug_output()
print("*" * 80)
A.promote()
print("/" * 80)
A_fn=analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get()
print(A, A_fn)
assert A == A_fn
A_f = analysisfactory.AnalysisFactory.get(A).get()
print(A,A_f)
assert A == A_f
A.get()
assert A.data == 'data'
inputs=A.list_inputs()
assert isinstance(inputs,list)
assert len(inputs)==1
def test_constructor_input():
from dataanalysis import analysisfactory
from dataanalysis import core as da
da.reset()
class BAnalysis(da.DataAnalysis):
def main(self):
print("test")
self.data = "data"
class Analysis(da.DataAnalysis):
#input_b=BAnalysis
def main(self):
print("test")
if hasattr(self, 'input_b'):
self.data = self.input_b.data
else:
self.data = 'otherdata'
A = Analysis()
A.get()
assert A.data == 'otherdata'
A = Analysis(input_b=BAnalysis())
assert hasattr(A, 'input_b')
da.debug_output()
print("*" * 80)
A.promote()
print("/" * 80)
A_fn = analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get()
print(A, A_fn)
assert A == A_fn
A_f = analysisfactory.AnalysisFactory.get(A).get()
print(A, A_f)
assert A == A_f
A.get()
assert A.data == 'data'
inputs = A.list_inputs()
assert isinstance(inputs, list)
assert len(inputs) == 1
def test_numpy_to_datafile_recursive():
from dataanalysis import core as da
import os
da.reset()
import numpy as np
import pandas as pd
class Analysis(da.DataAnalysis):
read_caches=[]
cached=True
def main(self):
print("test")
self.data = {'a':np.linspace(0,1,100),'b/c!':pd.DataFrame(np.linspace(0,2,200))}
A = Analysis()
A.get()
content=A.export_data()
cc=A.cache.adopt_datafiles(content)
assert cc['data']['a'] is None
#da.TransientCacheInstance.reset()
da.reset()
da.debug_output()
B = Analysis()
B.produce_disabled=True
B.read_caches = [A.cache.__class__]
B.get()
print(B.data['a'])
#B.cache.restore(A._da_locally_complete,A)
assert hasattr(B,'_datafile_data_a')
assert hasattr(B._datafile_data_a,'adopted_format')
assert B._datafile_data_a.adopted_format == "numpy"
print B._datafile_data_a.get_path()
assert os.path.exists(B._datafile_data_a.get_path())
assert hasattr(B, 'data')
assert all(A.data['a'] == B.data['a'])
def test_object_injection_reset():
from dataanalysis import core as da
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
# arg=None
pass
A1 = AAnalysis(use_arg="arg1")
assert A1.arg == "arg1"
d1 = A1.export_data()
print("has data:", d1)
assert d1['arg'] == 'arg1'
a1 = A1.serialize()
print("serilization:",a1)
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
pass
da.AnalysisFactory.inject_serialization(a1)
print("factory has",da.AnalysisFactory.cache['AAnalysis'])
aanalysis=da.AnalysisFactory['AAnalysis']
assert aanalysis.arg == "arg1"
class Analysis(da.DataAnalysis):
#cached = True
input_arg=AAnalysis
def main(self):
print("test")
# time.sleep(3)
self.data = "data_"+self.input_arg.arg
A=Analysis()
A.get()
print(A.data)
assert A.data == "data_arg1"
def test_object_injection_reset():
from dataanalysis import core as da
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
# arg=None
pass
A1 = AAnalysis(use_arg="arg1")
assert A1.arg == "arg1"
d1 = A1.export_data()
print("has data:", d1)
assert d1['arg'] == 'arg1'
a1 = A1.serialize()
print("serilization:", a1)
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
pass
da.AnalysisFactory.inject_serialization(a1)
print("factory has", da.AnalysisFactory.cache['AAnalysis'])
aanalysis = da.AnalysisFactory['AAnalysis']
assert aanalysis.arg == "arg1"
class Analysis(da.DataAnalysis):
#cached = True
input_arg = AAnalysis
def main(self):
print("test")
# time.sleep(3)
self.data = "data_" + self.input_arg.arg
A = Analysis()
A.get()
print(A.data)
assert A.data == "data_arg1"
def test_image():
#
da.reset()
import ddosa
reload(ddosa)
da.debug_output()
fa = ddosa.ii_skyimage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
print fa.skyima
def test_recursive_datafile():
from dataanalysis import core as da
import os
da.reset()
da.debug_output()
class Analysis(da.DataAnalysis):
cached=True
def main(self):
print("test")
fn="file.txt"
with open(fn,"w") as f:
f.write("test")
fn1="file1.txt"
with open(fn1,"w") as f:
f.write("test")
self.data = [da.DataFile(fn)]
self.data1 = da.DataFile(fn1)
A = Analysis()
A.read_caches = []
A.get()
da.reset()
os.remove("file.txt")
os.remove("file1.txt")
B = Analysis()
B.produce_disabled=True
B.get()
#B.cache.restore(A._da_locally_complete,A)
assert hasattr(B,'data')
print B.data
#assert hasattr(B._datafile_data,'adopted_format')
#assert B._datafile_data.adopted_format == "numpy"
#print B._datafile_data.get_path()
assert os.path.exists(B.data1.get_path())
assert os.path.exists(B.data[0].get_path())
def test_live_delegation_assumptions(ddservice_fixture,app):
import requests
import dataanalysis.core as da
da.reset()
da.debug_output()
r=requests.get(ddservice_fixture+url_for('produce', target="BAnalysis", modules="ddmoduletest", assume="AAnalysis.assumed_data=\"clients\""))
print(r.content)
factory_r=r.json()
print(factory_r)
assert factory_r['data']['data']=='dataB_A:dataAassumed:clients'
assert len(factory_r)>0
def test_pandas_to_datafile():
from dataanalysis import core as da
import os
da.reset()
import pandas as pd
import numpy as np
class Analysis(da.DataAnalysis):
read_caches=[]
cached=True
def main(self):
print("test")
self.data = pd.DataFrame()
self.data['data']=np.linspace(0,1,100)
A = Analysis()
A.get()
#da.TransientCacheInstance.reset()
da.reset()
da.debug_output()
B = Analysis()
B.produce_disabled=True
B.read_caches = [A.cache.__class__]
B.get()
#B.cache.restore(A._da_locally_complete,A)
assert hasattr(B,'_datafile_data')
assert hasattr(B._datafile_data,'adopted_format')
assert B._datafile_data.adopted_format == "pandas"
print B._datafile_data.get_path()
assert os.path.exists(B._datafile_data.get_path())
assert hasattr(B, 'data')
assert all(abs(A.data.data - B.data.data)<1e-5)
def test_object_serialization_with_use():
from dataanalysis import core as da
da.debug_output()
da.reset()
class AAnalysis(da.DataAnalysis):
class_arg=1
A=AAnalysis(use_usearg=2)
A.promote()
AA=AAnalysis()
aac=AA.serialize()
assert aac[1]['usearg'] == 2
assert aac[1]['class_arg'] == 1
assert 'version' in aac[1]
def test_live_chained_delegation(ddservice_fixture, app):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*")
A=ddmoduletest.ChainedDelegator()
with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources)==1
assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource)
def test_live_delegation(ddservice, app):
import requests
import dataanalysis.core as da
da.reset()
da.debug_output()
r = requests.get(
ddservice +
url_for('produce', target="BAnalysis", modules="ddmoduletest"))
print(r.content)
factory_r = r.json()
print(factory_r)
assert factory_r['data']['data'] == 'dataBdataA'
assert len(factory_r) > 0
def test_object_serialization_with_use():
from dataanalysis import core as da
da.debug_output()
da.reset()
class AAnalysis(da.DataAnalysis):
class_arg = 1
A = AAnalysis(use_usearg=2)
A.promote()
AA = AAnalysis()
aac = AA.serialize()
assert aac[1]['usearg'] == 2
assert aac[1]['class_arg'] == 1
assert 'version' in aac[1]
def test_queue_cache():
from dataanalysis import caches
import dataanalysis.caches.queue
da.debug_output()
da.reset()
q_cache=caches.queue.QueueCache()
q_cache.queue.purge()
define_analysis()
assert len(da.AnalysisFactory.dda_modules_used)==1
A=dataanalysis.core.AnalysisFactory['AAnalysis']
A.produce_disabled=True
A.cache = q_cache
A.cached = True
with pytest.raises(da.AnalysisDelegatedException):
A.get()
print(q_cache.queue.info)
assert q_cache
# worker part
f_cache=caches.cache_core.CacheNoIndex()
#f_cache.parent=q_cache
define_analysis()
A = dataanalysis.core.AnalysisFactory['AAnalysis']
A.produce_disabled = False
A.cache = f_cache
A.cached = True
worker=caches.queue.QueueCacheWorker("/tmp/queue")
print(q_cache.queue.info)
print("with worker",worker)
print(worker.run_all())
def test_live_resource_delegation(client):
import os
import threading
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis = ["ClientDelegatableAnalysisA"]
A = ddmoduletest.ClientDelegatableAnalysisA1()
with pytest.raises(
dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources) == 1
getter = lambda x: client.get(x).json
fr = excinfo.value.resources[0].fetch(getter=getter)
print fr
print fr.data
assert fr.status == 'not allowed to produce'
R = excinfo.value.resources[0].get(getter=getter)
print(R)
assert R.data['data'] == 'dataA1'
print(R.data.keys())
print(R.data['resource_stats']['main_executed_on'])
assert os.getpid() == R.data['resource_stats']['main_executed_on']['pid']
assert threading.current_thread(
).ident == R.data['resource_stats']['main_executed_on']['thread_id']
def test_catextract():
da.reset()
import ddosa
reload(ddosa)
da.debug_output()
fa = ddosa.CatExtract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
assert hasattr(fa, 'cat')
da.reset()
fa = ddosa.CatExtract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.produce_disabled = True
def test_gti_cached():
da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ibis_gti(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.get()
# now get cached
fb = ddosa.ibis_gti(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fb.produce_disabled = True
fb.get()
def test_live_chained_delegation(ddservice, app):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*")
A = ddmoduletest.ChainedDelegator()
with pytest.raises(
dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources) == 1
assert isinstance(excinfo.value.resources[0],
dataanalysis.caches.resources.WebResource)
def test_passing_unmanagable_assumptions(ddservice_fixture, app):
import dataanalysis.core as da
import dataanalysis.caches.resources
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
ddmoduletest.cache.delegation_mode="interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture
#ddmoduletest.cache.resource_factory.getter=getter
A=ddmoduletest.ChainedServerProducer(assume=[ddmoduletest.AAnalysis(input_x=ddmoduletest.TwoCDInputAnalysis())])
A.produce_disabled=True
#with pytest.raises(dataanalysis.caches.resources.GenericResourceException) as excinfo:
a=A.get()
def test_chained_waiting(ddservice, app):
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*")
ddmoduletest.cache.delegation_mode = "interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice
#ddmoduletest.cache.resource_factory.getter=getter
A = ddmoduletest.ChainedDelegator()
with pytest.raises(da.AnalysisDelegatedException) as excinfo:
a = A.get()
assert len(excinfo.value.resources) == 1
assert excinfo.value.resources[0].hashe[-1] == "ChainedDelegator.v0"
def test_chained_waiting(ddservice_fixture, app):
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*")
ddmoduletest.cache.delegation_mode="interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture
#ddmoduletest.cache.resource_factory.getter=getter
A=ddmoduletest.ChainedDelegator()
with pytest.raises(da.AnalysisDelegatedException) as excinfo:
a=A.get()
assert len(excinfo.value.resources)==1
assert excinfo.value.resources[0].hashe[-1] == "ChainedDelegator.v0"
def test_resource_delegation(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
ddmoduletest.cache.delegating_analysis="AAnalysis"
A=ddmoduletest.AAnalysis()
with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources)==1
assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource)
print(excinfo.value.resources[0].identity.get_modules_loadable())
print(excinfo.value.resources[0])
def test_osa_scw_blob():
import dataanalysis.core as da
import subprocess
from sdsc import SDSCCache, blob_store
blob_store.purge()
assert len(blob_store.blobs) == 0
import ddosa
import ddosadm
cache = SDSCCache()
scw = ddosadm.ScWData(input_scwid="066500220010.001")
scw.read_caches = []
scw.write_caches.append(cache.__class__)
try:
scw.get()
except subprocess.CalledProcessError:
pass
da.debug_output()
cache.store(scw._da_locally_complete, scw)
assert len(blob_store.blobs) == 1
print blob_store.blobs.keys()
blob = blob_store.blobs.items()[0][1]
import tarfile
import StringIO
tar = tarfile.open(fileobj=StringIO.StringIO(blob))
print tar.getnames()
def test_live_resource_delegation(client):
import dataanalysis.core as da
import dataanalysis
#print(dir(client))
#raise Exception(client.__module__)
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis = ["ClientDelegatableAnalysisA"]
A = ddmoduletest.ClientDelegatableAnalysisA1()
with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources)==1
getter=lambda x:client.get(x).json
fr=excinfo.value.resources[0].fetch(getter=getter)
print(fr)
print(fr.data)
assert fr.status == 'not allowed to produce'
R=excinfo.value.resources[0].get(getter=getter)
print(R)
assert R.data['data'] == 'dataA1'
print(R.data.keys())
def test_cache_blob():
from dataanalysis import core as da
from dataanalysis import caches
import glob
cache = caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached = True
read_caches = []
def main(self):
self.data = "datax"
open("file.txt", "w").write("filedata")
self.datafile = da.DataFile("file.txt")
A = Analysis()
A.get()
A._da_locally_complete
assert A._da_locally_complete == A._da_expected_full_hashe
print(glob.glob(A._da_cached_path + "/*"))
assert len(glob.glob(A._da_cached_path + "/*")) > 2
da.debug_output()
cache.store_to_directory(A._da_locally_complete, A, "/tmp/blob/")
print(glob.glob("/tmp/blob/*"))
assert len(glob.glob("/tmp/blob/*")) > 2
blob = cache.assemble_blob(A._da_locally_complete, A).read()
print(blob)
assert len(blob) > 50
def test_resource_delegation(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
ddmoduletest.cache.delegating_analysis = "AAnalysis"
A = ddmoduletest.AAnalysis()
with pytest.raises(
dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources) == 1
assert isinstance(excinfo.value.resources[0],
dataanalysis.caches.resources.WebResource)
print(excinfo.value.resources[0].identity.get_modules_loadable())
print(excinfo.value.resources[0])
def test_cache_blob():
from dataanalysis import core as da
from dataanalysis import caches
import glob
cache = caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached=True
read_caches=[]
def main(self):
self.data="datax"
open("file.txt","w").write("filedata")
self.datafile=da.DataFile("file.txt")
A = Analysis()
A.get()
A._da_locally_complete
assert A._da_locally_complete == A._da_expected_full_hashe
print(glob.glob(A._da_cached_path+"/*"))
assert len(glob.glob(A._da_cached_path+"/*"))>2
da.debug_output()
cache.store_to_directory(A._da_locally_complete,A,"/tmp/blob/")
print(glob.glob("/tmp/blob/*"))
assert len(glob.glob("/tmp/blob/*")) > 2
blob=cache.assemble_blob(A._da_locally_complete, A).read()
print(blob)
assert len(blob)>50
def test_live_multiple_resource_delegation(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ClientDelegatableAnalysis.*")
A=ddmoduletest.TwoCDInputAnalysis()
with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources)==2
assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource)
assert isinstance(excinfo.value.resources[1], dataanalysis.caches.resources.WebResource)
print(excinfo.value.resources[0].identity.get_modules_loadable())
print(excinfo.value.resources)
data=dict(
resources=emerged_object.guess_main_resources(),
dependencies=[r.jsonify() for r in e.resources],
),
).jsonify()
class Callback(Resource):
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('target', type=str, help='')
args = parser.parse_args()
def create_app():
app = Flask(__name__)
api = Api(app)
api_version="v0"
api.add_resource(List, '/api/%s/list'%api_version)
api.add_resource(Produce, '/api/%s/produce'%api_version)
api.add_resource(Status, '/api/%s/status'%api_version)
api.add_resource(Callback, '/api/%s/callback' % api_version)
return app
if __name__ == '__main__':
import dataanalysis as da
da.debug_output()
create_app().run(debug=False,port=6767)
def test_generate_aliased():
da.reset()
da.debug_output()
my_cache=cache_core.Cache("./local-test")
class BAnalysis(da.DataAnalysis):
c = None
cache=my_cache
cached=True
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[A %s]" % repr(self.c)
b=BAnalysis(use_c=1).get()
assert b.cached
assert b._da_locally_complete
da.reset()
class BAnalysis(da.DataAnalysis):
c = None
cached=True
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[B %s]" % repr(self.c)
class CAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[C %s]" % repr(self.c)
class AAnalysis(da.DataAnalysis):
run_for_hashe=True
allow_alias=True
def main(self):
print("test", self.__class__)
r = [[BAnalysis(use_c=c),CAnalysis(use_c=c)] for c in range(3)]
return r
A = AAnalysis()
r = A.get()
print r
print r.output
for b,c in r.output:
print b,b.data,b._da_locally_complete
print c, c.data,c._da_locally_complete
print
print(r.output[1][0].c, r.output[1][0].data)
assert isinstance(r.output[0][0], BAnalysis)
assert r.output[0][0].cached
assert r.output[0][0]._da_locally_complete
assert r.output[0][0].data == 'data 0'
assert isinstance(r.output[0][1], CAnalysis)
assert not r.output[0][1].cached
assert r.output[0][1]._da_locally_complete
assert r.output[1][1].data == 'data 1'
def test_delegation():
from dataanalysis.caches.queue import QueueCacheWorker
queue_name="/tmp/queue"
qw=QueueCacheWorker(queue_name)
qw.queue.purge()
print("cache worker:",qw)
randomized_version="v%i"%random.randint(1,10000)
callback_file = "./callback"
cmd=[
'dda-run',
'Mosaic',
'-m','ddmoduletest',
'-a','ddmoduletest.RandomModifier(use_version="%s")'%randomized_version,
# '-Q',queue_name,
'--callback','file://'+callback_file,
'--delegate-target',
]
if os.path.exists(callback_file):
os.remove(callback_file)
exception_report="exception.yaml"
if os.path.exists(exception_report):
os.remove(exception_report)
assert not os.path.exists(callback_file)
qw.queue.wipe(["waiting","locked","done","failed","running"])
# run it
print("CMD:"," ".join(cmd))
try:
subprocess.check_call(cmd,stderr=subprocess.STDOUT,env=env)
except subprocess.CalledProcessError as e:
pass
else:
raise Exception("expected AnalysisDelegatedException")
#p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
#p.wait()
#print(p.stdout.read())
assert os.path.exists(exception_report)
recovered_exception = yaml.load(open(exception_report))
print(recovered_exception)
print(qw.queue.info)
assert qw.queue.info['waiting'] == 1, qw.queue.info
assert qw.queue.info['locked'] == 0
assert qw.queue.info['done'] == 0
print("\n\nWORKER")
qw.run_once()
print(qw.queue.info)
assert qw.queue.info['waiting'] == 2
assert qw.queue.info['locked'] == 1
assert qw.queue.info['done'] == 0
assert os.path.exists(callback_file)
callback_info = open(callback_file).readlines()
print("".join(callback_info))
assert len(callback_info) == 1
print("\n\nWORKER")
qw.run_once()
assert qw.queue.info['waiting'] == 1
assert qw.queue.info['locked'] == 1
assert qw.queue.info['done'] == 1
print("\n\nWORKER")
qw.run_once()
assert qw.queue.info['waiting'] == 0
assert qw.queue.info['locked'] == 1
assert qw.queue.info['done'] == 2
print("\n\nWORKER run to unlock")
qw.run_once()
assert qw.queue.info['waiting'] == 1
assert qw.queue.info['locked'] == 0
assert qw.queue.info['done'] == 2
qw.run_once()
assert qw.queue.info['waiting'] == 0
assert qw.queue.info['locked'] == 0
assert qw.queue.info['done'] == 3
# run again, expecting from cache
exception_report = "exception.yaml"
if os.path.exists(exception_report):
os.remove(exception_report)
print("\n\nAGAIN")
print("cmd:"," ".join(cmd+["-V",'--delegate-target']))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,env=env)
p.wait()
print(p.stdout.read())
assert not os.path.exists(exception_report)
import ddmoduletest
da.debug_output()
da.AnalysisFactory.WhatIfCopy("test",ddmoduletest.RandomModifier(use_version=randomized_version))
A=ddmoduletest.Mosaic()
A.write_caches=[]
A.produce_disabled=True
A.run_if_haveto=False
A.get()
assert A._da_output_origin=="cache"
print(A.resource_stats)
