def test_osa_blob(): import dataanalysis.core as da from sdsc import SDSCCache, blob_store blob_store.purge() import ddosa cache = SDSCCache() im = ddosa.ii_skyimage( assume=[ddosa.ScWData(input_scwid="066500220010.001")]) im.get() im.write_caches.append(cache.__class__) da.debug_output() cache.store(im._da_locally_complete, im) assert len(blob_store.blobs) == 1 blob = blob_store.blobs.items()[0][1] import tarfile import StringIO tar = tarfile.open(fileobj=StringIO.StringIO(blob)) print tar.getnames() assert './blob/isgri_sky_res.fits.gz' in tar.getnames()
def test_scwid_list(): import ddosa import dataanalysis.core as da da.debug_output() fa = ddosa.IDScWList(use_scwid_list=["066500230010.001"]) print(fa.get()) print(fa.get_version()) assert len(fa.scwlistdata) == 1 fa = ddosa.IDScWList( use_scwid_list=["066500230010.001", "066500250010.001"]) print(fa.get()) print(fa.get_version()) assert len(fa.scwlistdata) == 2 fa = ddosa.IDScWList( use_scwid_list=["066500230010.001", "066600250010.001"]) print(fa.get()) print(fa.get_version()) assert len(fa.scwlistdata) == 2
def test_live_multiple_resource_delegation(client): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append( "ClientDelegatableAnalysis.*") A = ddmoduletest.TwoCDInputAnalysis() with pytest.raises( dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources) == 2 assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource) assert isinstance(excinfo.value.resources[1], dataanalysis.caches.resources.WebResource) print(excinfo.value.resources[0].identity.get_modules_loadable()) print(excinfo.value.resources)
def test_passing_assumptions(ddservice, app): import dataanalysis.core as da da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*") ddmoduletest.cache.delegation_mode = "interactive" ddmoduletest.cache.resource_factory.endpoint = ddservice #ddmoduletest.cache.resource_factory.getter=getter A = ddmoduletest.ChainedServerProducer( assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")]) A.produce_disabled = True a = A.get() print a.data assert a.data == {'a': 'dataAfromclient', 'b': 'dataBdataAfromclient'} assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid() assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid() assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b[ 'main_executed_on']['pid']
def test_base_cache_check_location(): from dataanalysis import core as da from dataanalysis import caches da.reset() da.debug_output() cache = caches.cache_core.Cache() class Analysis(da.DataAnalysis): cached = True read_caches = [] store_preview_yaml = True def main(self): self.data = "test123" A = Analysis() A.get() cached_path = cache.construct_cached_file_path(A._da_locally_complete, A) import os assert os.path.exists(cached_path) assert os.path.exists(cached_path + "/hash.txt") assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
def test_live_resource_server_cachable(client): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"] A = ddmoduletest.ServerCachableAnalysis() A.cache=ddmoduletest.cache with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources)==1 getter=lambda x:client.get(x).json fr = excinfo.value.resources[0].get(getter=getter) assert fr.status == "result" print fr.data assert fr.data['data'] == 'dataAdataB' fr = excinfo.value.resources[0].fetch(getter=getter) assert fr.status == "result" print fr.data assert fr.data['data'] == 'dataAdataB'
def test_passing_unmanagable_assumptions(ddservice, app): import dataanalysis.core as da import dataanalysis.caches.resources da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*") ddmoduletest.cache.delegation_mode = "interactive" ddmoduletest.cache.resource_factory.endpoint = ddservice #ddmoduletest.cache.resource_factory.getter=getter A = ddmoduletest.ChainedServerProducer(assume=[ ddmoduletest.AAnalysis(input_x=ddmoduletest.TwoCDInputAnalysis()) ]) A.produce_disabled = True with pytest.raises( dataanalysis.caches.resources.GenericResourceException) as excinfo: a = A.get() assert 'mismatch' in excinfo.value.args[2]['comment']
def test_cat_add_postimage(): import ddosa import dataanalysis.core as da da.debug_output() ddosa.ScWData(input_scwid="066500230010.001").promote() sl=ddosa.SourceList(use_sources=[dict(name="NEW_added",ra=83,dec=22)]) a = ddosa.CatExtract( input_extra_sources=sl, ) a.promote() b = ddosa.CatForSpectraFromImaging( input_extra_sources=sl, use_minsig=10, ) b.promote() a.cached=False b.cached=False b.get() print b.cat.get_path() f=fits.open(b.cat.get_path())[1] print f.data['NAME'] assert "NEW_added" in f.data['NAME']
def test_obscure_assumptions(): from dataanalysis import core as da da.reset() da.debug_output() class BAnalysis(da.DataAnalysis): data="1" class AAnalysis(da.DataAnalysis): input_b=BAnalysis cached = True def main(self): self.data="a."+self.input_b.data A=AAnalysis().get() data=A.export_data() hashe=A._da_locally_complete cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A) import os assert os.path.exists(cached_path) print(cached_path) da.reset() nA=da.DataAnalysis.from_hashe_and_data(hashe, data).get() print(nA) assert nA.export_data()['data'] == data['data'] assert nA._da_locally_complete == hashe
def test_base_cache_check_location(): from dataanalysis import core as da from dataanalysis import caches da.reset() da.debug_output() cache=caches.cache_core.Cache() class Analysis(da.DataAnalysis): cached=True read_caches=[] store_preview_yaml=True def main(self): self.data="test123" A=Analysis() A.get() cached_path=cache.construct_cached_file_path(A._da_locally_complete,A) import os assert os.path.exists(cached_path) assert os.path.exists(cached_path+"/hash.txt") assert os.path.exists(cached_path+"/object_identity.yaml.gz") assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
def test_selective_delegation(): import dataanalysis.caches.delegating da.debug_output() class TCache(dataanalysis.caches.delegating.SelectivelyDelegatingCache): delegating_analysis=['AAnalysis','BAnalysis$'] #cache=Cache() class AAnalysis(da.DataAnalysis): #read_caches = [q_cache.__class__] cached = True cache=TCache() def main(self): #self.data = "datadata1" raise Exception("this should have been delegated") class BAnalysis(da.DataAnalysis): #read_caches = [q_cache.__class__] cached = True cache=TCache() def main(self): self.data = "datadata2" A=AAnalysis() with pytest.raises(da.AnalysisDelegatedException): A.get() B=BAnalysis() B.get()
def test_generate_list(): da.debug_output() da.reset() da.AnalysisFactory.reset() class BAnalysis(da.DataAnalysis): c=None _da_settings=["c"] def main(self): print("test",self.c) self.data="data "+repr(self.c) def __repr__(self): return "[A %s]"%repr(self.c) class AAnalysis(da.DataAnalysis): run_for_hashe=True def main(self): print("test",self.__class__) r=[BAnalysis(use_c=c) for c in range(3)] return r A=AAnalysis() r=A.get() print(r[1].c,r[1].data) assert r[0].data == 'data 0' assert r[1].data == 'data 1'
def test_queue_reconstruct_env(): from dataanalysis import caches import dataanalysis.caches.queue da.debug_output() da.reset() q_cache=caches.queue.QueueCache() q_cache.wipe_queue(["waiting", "done", "running"]) define_analysis() A=dataanalysis.core.AnalysisFactory['AAnalysis'] A.produce_disabled=True A.cache = q_cache A.cached = True with pytest.raises(da.AnalysisDelegatedException): A.get() # worker part f_cache=caches.cache_core.CacheNoIndex() #f_cache.parent=q_cache da.reset() #A = dataanalysis.core.AnalysisFactory['Analysis'] #A.produce_disabled = False #A.cache = f_cache #A.cached = True worker=caches.queue.QueueCacheWorker() print(worker.run_all())
def test_cat_add(): import ddosa import dataanalysis.core as da da.debug_output() ddosa.ScWData(input_scwid="066500230010.001").promote() print "was before",ddosa.CatExtract() a = ddosa.CatExtract( input_extra_sources=ddosa.SourceList(use_sources=[dict(name="NEW_added",ra=83,dec=22)]), ) print "got back",a assert hasattr(a,'input_extra_sources') a.cached=False a.get() print a.cat.get_path() f=fits.open(a.cat.get_path())[1] print f.data['NAME'] assert "NEW_added" in f.data['NAME']
def test_obscure_node_cached(): from dataanalysis import core as da da.reset() da.debug_output() class BAnalysis(da.DataAnalysis): data="1" class AAnalysis(da.DataAnalysis): input_b=BAnalysis cached = True def main(self): self.data="a."+self.input_b.data A=AAnalysis().get() data=A.export_data() hashe=A._da_locally_complete cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A) import os assert os.path.exists(cached_path) print(cached_path) da.reset() nA=da.DataAnalysis.from_hashe(hashe).get() print(nA) assert nA.export_data()['data'] == data['data'] assert nA._da_locally_complete == hashe
def test_generate_list(): da.debug_output() da.reset() da.AnalysisFactory.reset() class BAnalysis(da.DataAnalysis): c = None _da_settings = ["c"] def main(self): print("test", self.c) self.data = "data " + repr(self.c) def __repr__(self): return "[A %s]" % repr(self.c) class AAnalysis(da.DataAnalysis): run_for_hashe = True def main(self): print("test", self.__class__) r = [BAnalysis(use_c=c) for c in range(3)] return r A = AAnalysis() r = A.get() print(r[1].c, r[1].data) assert r[0].data == 'data 0' assert r[1].data == 'data 1'
def test_passing_assumptions(ddservice_fixture, app): import dataanalysis.core as da da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*") ddmoduletest.cache.delegation_mode="interactive" ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture #ddmoduletest.cache.resource_factory.getter=getter A=ddmoduletest.ChainedServerProducer(assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")]) A.produce_disabled=True a=A.get() print a.data assert a.data=={'a': 'dataAassumed:fromclient', 'b': 'dataB_A:dataAassumed:fromclient'} assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid() assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid() assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b['main_executed_on']['pid']
def test_live_resource_server_cachable(client): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"] A = ddmoduletest.ServerCachableAnalysis() A.cache = ddmoduletest.cache with pytest.raises( dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources) == 1 getter = lambda x: client.get(x).json fr = excinfo.value.resources[0].get(getter=getter) assert fr.status == "result" print fr.data assert fr.data['data'] == 'dataAdataB' fr = excinfo.value.resources[0].fetch(getter=getter) assert fr.status == "result" print fr.data assert fr.data['data'] == 'dataAdataB'
def test_lightcurves(): da.debug_output() fa = ddosa.ii_lc_extract(assume=[ ddosa.ScWData(input_scwid="066500230010.001"), ]) fa.read_caches = [] fa.get() assert os.path.exists(fa.lightcurve.get_path())
def test_serialize(): import dataanalysis.core as da da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get() ed=S.export_data() assert '_da_stored_string_input_s' in ed
def test_serialize(): import dataanalysis.core as da da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get() ed=S.export_data() assert '_da_stored_string_input_s' in ed
def test_queue_cache_generative(): from dataanalysis import caches import dataanalysis.caches.queue da.debug_output() da.reset() q_cache=caches.queue.QueueCache("./queue_test") q_cache.queue.purge() assert len(q_cache.queue.list())==0 define_analysis() A=dataanalysis.core.AnalysisFactory['GenerativeAnalysis'] for name in 'AAnalysis',: aA=dataanalysis.core.AnalysisFactory[name].__class__ aA.cache = q_cache aA.produce_disabled=True aA.cached = True with pytest.raises(da.AnalysisDelegatedException) as e: A.get() print('AnalysisDelegatedException',e.value,e.value.origin,e.value.hashe) assert len(e.value.hashe[1:])==2 assert len(e.value.source_exceptions)==2 for se in e.value.source_exceptions: print('found multiple AnalysisDelegatedException',se,se.origin) print(q_cache.queue.list()) # assert len(q_cache.queue.list())==2 assert len(q_cache.queue.list())==1 # worker part f_cache=caches.cache_core.CacheNoIndex() #f_cache.parent=q_cache define_analysis() A = dataanalysis.core.AnalysisFactory['AAnalysis'] A.produce_disabled = False A.cache = f_cache A.cached = True worker=caches.queue.QueueCacheWorker("./queue_test") print(worker.run_once())
def test_self_assumption(): from dataanalysis import core as da da.debug_output() da.reset() class AAnalysis(da.DataAnalysis): pass aa = AAnalysis(assume=AAnalysis(use_arg="arg2", use_nonearg=None)).get() assert aa._da_locally_complete assert not hasattr(aa,'arg') # current behavoir and it is bad
def test_constructor_input(): from dataanalysis import analysisfactory from dataanalysis import core as da da.reset() class BAnalysis(da.DataAnalysis): def main(self): print("test") self.data="data" class Analysis(da.DataAnalysis): #input_b=BAnalysis def main(self): print("test") if hasattr(self,'input_b'): self.data = self.input_b.data else: self.data = 'otherdata' A=Analysis() A.get() assert A.data == 'otherdata' A=Analysis(input_b=BAnalysis()) assert hasattr(A,'input_b') da.debug_output() print("*" * 80) A.promote() print("/" * 80) A_fn=analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get() print(A, A_fn) assert A == A_fn A_f = analysisfactory.AnalysisFactory.get(A).get() print(A,A_f) assert A == A_f A.get() assert A.data == 'data' inputs=A.list_inputs() assert isinstance(inputs,list) assert len(inputs)==1
def test_constructor_input(): from dataanalysis import analysisfactory from dataanalysis import core as da da.reset() class BAnalysis(da.DataAnalysis): def main(self): print("test") self.data = "data" class Analysis(da.DataAnalysis): #input_b=BAnalysis def main(self): print("test") if hasattr(self, 'input_b'): self.data = self.input_b.data else: self.data = 'otherdata' A = Analysis() A.get() assert A.data == 'otherdata' A = Analysis(input_b=BAnalysis()) assert hasattr(A, 'input_b') da.debug_output() print("*" * 80) A.promote() print("/" * 80) A_fn = analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get() print(A, A_fn) assert A == A_fn A_f = analysisfactory.AnalysisFactory.get(A).get() print(A, A_f) assert A == A_f A.get() assert A.data == 'data' inputs = A.list_inputs() assert isinstance(inputs, list) assert len(inputs) == 1
def test_numpy_to_datafile_recursive(): from dataanalysis import core as da import os da.reset() import numpy as np import pandas as pd class Analysis(da.DataAnalysis): read_caches=[] cached=True def main(self): print("test") self.data = {'a':np.linspace(0,1,100),'b/c!':pd.DataFrame(np.linspace(0,2,200))} A = Analysis() A.get() content=A.export_data() cc=A.cache.adopt_datafiles(content) assert cc['data']['a'] is None #da.TransientCacheInstance.reset() da.reset() da.debug_output() B = Analysis() B.produce_disabled=True B.read_caches = [A.cache.__class__] B.get() print(B.data['a']) #B.cache.restore(A._da_locally_complete,A) assert hasattr(B,'_datafile_data_a') assert hasattr(B._datafile_data_a,'adopted_format') assert B._datafile_data_a.adopted_format == "numpy" print B._datafile_data_a.get_path() assert os.path.exists(B._datafile_data_a.get_path()) assert hasattr(B, 'data') assert all(A.data['a'] == B.data['a'])
def test_object_injection_reset(): from dataanalysis import core as da da.reset() da.debug_output() class AAnalysis(da.DataAnalysis): # arg=None pass A1 = AAnalysis(use_arg="arg1") assert A1.arg == "arg1" d1 = A1.export_data() print("has data:", d1) assert d1['arg'] == 'arg1' a1 = A1.serialize() print("serilization:",a1) da.reset() da.debug_output() class AAnalysis(da.DataAnalysis): pass da.AnalysisFactory.inject_serialization(a1) print("factory has",da.AnalysisFactory.cache['AAnalysis']) aanalysis=da.AnalysisFactory['AAnalysis'] assert aanalysis.arg == "arg1" class Analysis(da.DataAnalysis): #cached = True input_arg=AAnalysis def main(self): print("test") # time.sleep(3) self.data = "data_"+self.input_arg.arg A=Analysis() A.get() print(A.data) assert A.data == "data_arg1"
def test_object_injection_reset(): from dataanalysis import core as da da.reset() da.debug_output() class AAnalysis(da.DataAnalysis): # arg=None pass A1 = AAnalysis(use_arg="arg1") assert A1.arg == "arg1" d1 = A1.export_data() print("has data:", d1) assert d1['arg'] == 'arg1' a1 = A1.serialize() print("serilization:", a1) da.reset() da.debug_output() class AAnalysis(da.DataAnalysis): pass da.AnalysisFactory.inject_serialization(a1) print("factory has", da.AnalysisFactory.cache['AAnalysis']) aanalysis = da.AnalysisFactory['AAnalysis'] assert aanalysis.arg == "arg1" class Analysis(da.DataAnalysis): #cached = True input_arg = AAnalysis def main(self): print("test") # time.sleep(3) self.data = "data_" + self.input_arg.arg A = Analysis() A.get() print(A.data) assert A.data == "data_arg1"
def test_image(): # da.reset() import ddosa reload(ddosa) da.debug_output() fa = ddosa.ii_skyimage(assume=[ ddosa.ScWData(input_scwid="066500230010.001"), ]) fa.read_caches = [] fa.get() print fa.skyima
def test_recursive_datafile(): from dataanalysis import core as da import os da.reset() da.debug_output() class Analysis(da.DataAnalysis): cached=True def main(self): print("test") fn="file.txt" with open(fn,"w") as f: f.write("test") fn1="file1.txt" with open(fn1,"w") as f: f.write("test") self.data = [da.DataFile(fn)] self.data1 = da.DataFile(fn1) A = Analysis() A.read_caches = [] A.get() da.reset() os.remove("file.txt") os.remove("file1.txt") B = Analysis() B.produce_disabled=True B.get() #B.cache.restore(A._da_locally_complete,A) assert hasattr(B,'data') print B.data #assert hasattr(B._datafile_data,'adopted_format') #assert B._datafile_data.adopted_format == "numpy" #print B._datafile_data.get_path() assert os.path.exists(B.data1.get_path()) assert os.path.exists(B.data[0].get_path())
def test_live_delegation_assumptions(ddservice_fixture,app): import requests import dataanalysis.core as da da.reset() da.debug_output() r=requests.get(ddservice_fixture+url_for('produce', target="BAnalysis", modules="ddmoduletest", assume="AAnalysis.assumed_data=\"clients\"")) print(r.content) factory_r=r.json() print(factory_r) assert factory_r['data']['data']=='dataB_A:dataAassumed:clients' assert len(factory_r)>0
def test_pandas_to_datafile(): from dataanalysis import core as da import os da.reset() import pandas as pd import numpy as np class Analysis(da.DataAnalysis): read_caches=[] cached=True def main(self): print("test") self.data = pd.DataFrame() self.data['data']=np.linspace(0,1,100) A = Analysis() A.get() #da.TransientCacheInstance.reset() da.reset() da.debug_output() B = Analysis() B.produce_disabled=True B.read_caches = [A.cache.__class__] B.get() #B.cache.restore(A._da_locally_complete,A) assert hasattr(B,'_datafile_data') assert hasattr(B._datafile_data,'adopted_format') assert B._datafile_data.adopted_format == "pandas" print B._datafile_data.get_path() assert os.path.exists(B._datafile_data.get_path()) assert hasattr(B, 'data') assert all(abs(A.data.data - B.data.data)<1e-5)
def test_object_serialization_with_use(): from dataanalysis import core as da da.debug_output() da.reset() class AAnalysis(da.DataAnalysis): class_arg=1 A=AAnalysis(use_usearg=2) A.promote() AA=AAnalysis() aac=AA.serialize() assert aac[1]['usearg'] == 2 assert aac[1]['class_arg'] == 1 assert 'version' in aac[1]
def test_live_chained_delegation(ddservice_fixture, app): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*") A=ddmoduletest.ChainedDelegator() with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources)==1 assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource)
def test_live_delegation(ddservice, app): import requests import dataanalysis.core as da da.reset() da.debug_output() r = requests.get( ddservice + url_for('produce', target="BAnalysis", modules="ddmoduletest")) print(r.content) factory_r = r.json() print(factory_r) assert factory_r['data']['data'] == 'dataBdataA' assert len(factory_r) > 0
def test_object_serialization_with_use(): from dataanalysis import core as da da.debug_output() da.reset() class AAnalysis(da.DataAnalysis): class_arg = 1 A = AAnalysis(use_usearg=2) A.promote() AA = AAnalysis() aac = AA.serialize() assert aac[1]['usearg'] == 2 assert aac[1]['class_arg'] == 1 assert 'version' in aac[1]
def test_queue_cache(): from dataanalysis import caches import dataanalysis.caches.queue da.debug_output() da.reset() q_cache=caches.queue.QueueCache() q_cache.queue.purge() define_analysis() assert len(da.AnalysisFactory.dda_modules_used)==1 A=dataanalysis.core.AnalysisFactory['AAnalysis'] A.produce_disabled=True A.cache = q_cache A.cached = True with pytest.raises(da.AnalysisDelegatedException): A.get() print(q_cache.queue.info) assert q_cache # worker part f_cache=caches.cache_core.CacheNoIndex() #f_cache.parent=q_cache define_analysis() A = dataanalysis.core.AnalysisFactory['AAnalysis'] A.produce_disabled = False A.cache = f_cache A.cached = True worker=caches.queue.QueueCacheWorker("/tmp/queue") print(q_cache.queue.info) print("with worker",worker) print(worker.run_all())
def test_live_resource_delegation(client): import os import threading import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis = ["ClientDelegatableAnalysisA"] A = ddmoduletest.ClientDelegatableAnalysisA1() with pytest.raises( dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources) == 1 getter = lambda x: client.get(x).json fr = excinfo.value.resources[0].fetch(getter=getter) print fr print fr.data assert fr.status == 'not allowed to produce' R = excinfo.value.resources[0].get(getter=getter) print(R) assert R.data['data'] == 'dataA1' print(R.data.keys()) print(R.data['resource_stats']['main_executed_on']) assert os.getpid() == R.data['resource_stats']['main_executed_on']['pid'] assert threading.current_thread( ).ident == R.data['resource_stats']['main_executed_on']['thread_id']
def test_catextract(): da.reset() import ddosa reload(ddosa) da.debug_output() fa = ddosa.CatExtract(assume=[ ddosa.ScWData(input_scwid="066500230010.001"), ]) fa.read_caches = [] fa.get() assert hasattr(fa, 'cat') da.reset() fa = ddosa.CatExtract(assume=[ ddosa.ScWData(input_scwid="066500230010.001"), ]) fa.produce_disabled = True
def test_gti_cached(): da.debug_output() da.reset() import ddosa reload(ddosa) fa = ddosa.ibis_gti(assume=[ ddosa.ScWData(input_scwid="066500230010.001"), ]) fa.get() # now get cached fb = ddosa.ibis_gti(assume=[ ddosa.ScWData(input_scwid="066500230010.001"), ]) fb.produce_disabled = True fb.get()
def test_live_chained_delegation(ddservice, app): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*") A = ddmoduletest.ChainedDelegator() with pytest.raises( dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources) == 1 assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource)
def test_passing_unmanagable_assumptions(ddservice_fixture, app): import dataanalysis.core as da import dataanalysis.caches.resources da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*") ddmoduletest.cache.delegation_mode="interactive" ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture #ddmoduletest.cache.resource_factory.getter=getter A=ddmoduletest.ChainedServerProducer(assume=[ddmoduletest.AAnalysis(input_x=ddmoduletest.TwoCDInputAnalysis())]) A.produce_disabled=True #with pytest.raises(dataanalysis.caches.resources.GenericResourceException) as excinfo: a=A.get()
def test_chained_waiting(ddservice, app): import dataanalysis.core as da da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*") ddmoduletest.cache.delegation_mode = "interactive" ddmoduletest.cache.resource_factory.endpoint = ddservice #ddmoduletest.cache.resource_factory.getter=getter A = ddmoduletest.ChainedDelegator() with pytest.raises(da.AnalysisDelegatedException) as excinfo: a = A.get() assert len(excinfo.value.resources) == 1 assert excinfo.value.resources[0].hashe[-1] == "ChainedDelegator.v0"
def test_chained_waiting(ddservice_fixture, app): import dataanalysis.core as da da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ChainedDelegator.*") ddmoduletest.cache.delegation_mode="interactive" ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture #ddmoduletest.cache.resource_factory.getter=getter A=ddmoduletest.ChainedDelegator() with pytest.raises(da.AnalysisDelegatedException) as excinfo: a=A.get() assert len(excinfo.value.resources)==1 assert excinfo.value.resources[0].hashe[-1] == "ChainedDelegator.v0"
def test_resource_delegation(client): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest ddmoduletest.cache.delegating_analysis="AAnalysis" A=ddmoduletest.AAnalysis() with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources)==1 assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource) print(excinfo.value.resources[0].identity.get_modules_loadable()) print(excinfo.value.resources[0])
def test_osa_scw_blob(): import dataanalysis.core as da import subprocess from sdsc import SDSCCache, blob_store blob_store.purge() assert len(blob_store.blobs) == 0 import ddosa import ddosadm cache = SDSCCache() scw = ddosadm.ScWData(input_scwid="066500220010.001") scw.read_caches = [] scw.write_caches.append(cache.__class__) try: scw.get() except subprocess.CalledProcessError: pass da.debug_output() cache.store(scw._da_locally_complete, scw) assert len(blob_store.blobs) == 1 print blob_store.blobs.keys() blob = blob_store.blobs.items()[0][1] import tarfile import StringIO tar = tarfile.open(fileobj=StringIO.StringIO(blob)) print tar.getnames()
def test_live_resource_delegation(client): import dataanalysis.core as da import dataanalysis #print(dir(client)) #raise Exception(client.__module__) da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis = ["ClientDelegatableAnalysisA"] A = ddmoduletest.ClientDelegatableAnalysisA1() with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources)==1 getter=lambda x:client.get(x).json fr=excinfo.value.resources[0].fetch(getter=getter) print(fr) print(fr.data) assert fr.status == 'not allowed to produce' R=excinfo.value.resources[0].get(getter=getter) print(R) assert R.data['data'] == 'dataA1' print(R.data.keys())
def test_cache_blob(): from dataanalysis import core as da from dataanalysis import caches import glob cache = caches.cache_core.Cache() class Analysis(da.DataAnalysis): cached = True read_caches = [] def main(self): self.data = "datax" open("file.txt", "w").write("filedata") self.datafile = da.DataFile("file.txt") A = Analysis() A.get() A._da_locally_complete assert A._da_locally_complete == A._da_expected_full_hashe print(glob.glob(A._da_cached_path + "/*")) assert len(glob.glob(A._da_cached_path + "/*")) > 2 da.debug_output() cache.store_to_directory(A._da_locally_complete, A, "/tmp/blob/") print(glob.glob("/tmp/blob/*")) assert len(glob.glob("/tmp/blob/*")) > 2 blob = cache.assemble_blob(A._da_locally_complete, A).read() print(blob) assert len(blob) > 50
def test_resource_delegation(client): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest ddmoduletest.cache.delegating_analysis = "AAnalysis" A = ddmoduletest.AAnalysis() with pytest.raises( dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources) == 1 assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource) print(excinfo.value.resources[0].identity.get_modules_loadable()) print(excinfo.value.resources[0])
def test_cache_blob(): from dataanalysis import core as da from dataanalysis import caches import glob cache = caches.cache_core.Cache() class Analysis(da.DataAnalysis): cached=True read_caches=[] def main(self): self.data="datax" open("file.txt","w").write("filedata") self.datafile=da.DataFile("file.txt") A = Analysis() A.get() A._da_locally_complete assert A._da_locally_complete == A._da_expected_full_hashe print(glob.glob(A._da_cached_path+"/*")) assert len(glob.glob(A._da_cached_path+"/*"))>2 da.debug_output() cache.store_to_directory(A._da_locally_complete,A,"/tmp/blob/") print(glob.glob("/tmp/blob/*")) assert len(glob.glob("/tmp/blob/*")) > 2 blob=cache.assemble_blob(A._da_locally_complete, A).read() print(blob) assert len(blob)>50
def test_live_multiple_resource_delegation(client): import dataanalysis.core as da import dataanalysis da.reset() da.debug_output() import ddmoduletest reload(ddmoduletest) ddmoduletest.cache.delegating_analysis.append("ClientDelegatableAnalysis.*") A=ddmoduletest.TwoCDInputAnalysis() with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo: A.get() assert len(excinfo.value.resources)==2 assert isinstance(excinfo.value.resources[0], dataanalysis.caches.resources.WebResource) assert isinstance(excinfo.value.resources[1], dataanalysis.caches.resources.WebResource) print(excinfo.value.resources[0].identity.get_modules_loadable()) print(excinfo.value.resources)
data=dict( resources=emerged_object.guess_main_resources(), dependencies=[r.jsonify() for r in e.resources], ), ).jsonify() class Callback(Resource): def get(self): parser = reqparse.RequestParser() parser.add_argument('target', type=str, help='') args = parser.parse_args() def create_app(): app = Flask(__name__) api = Api(app) api_version="v0" api.add_resource(List, '/api/%s/list'%api_version) api.add_resource(Produce, '/api/%s/produce'%api_version) api.add_resource(Status, '/api/%s/status'%api_version) api.add_resource(Callback, '/api/%s/callback' % api_version) return app if __name__ == '__main__': import dataanalysis as da da.debug_output() create_app().run(debug=False,port=6767)
def test_generate_aliased(): da.reset() da.debug_output() my_cache=cache_core.Cache("./local-test") class BAnalysis(da.DataAnalysis): c = None cache=my_cache cached=True _da_settings = ["c"] def main(self): print("test", self.c) self.data = "data " + repr(self.c) def __repr__(self): return "[A %s]" % repr(self.c) b=BAnalysis(use_c=1).get() assert b.cached assert b._da_locally_complete da.reset() class BAnalysis(da.DataAnalysis): c = None cached=True _da_settings = ["c"] def main(self): print("test", self.c) self.data = "data " + repr(self.c) def __repr__(self): return "[B %s]" % repr(self.c) class CAnalysis(da.DataAnalysis): c = None _da_settings = ["c"] def main(self): print("test", self.c) self.data = "data " + repr(self.c) def __repr__(self): return "[C %s]" % repr(self.c) class AAnalysis(da.DataAnalysis): run_for_hashe=True allow_alias=True def main(self): print("test", self.__class__) r = [[BAnalysis(use_c=c),CAnalysis(use_c=c)] for c in range(3)] return r A = AAnalysis() r = A.get() print r print r.output for b,c in r.output: print b,b.data,b._da_locally_complete print c, c.data,c._da_locally_complete print print(r.output[1][0].c, r.output[1][0].data) assert isinstance(r.output[0][0], BAnalysis) assert r.output[0][0].cached assert r.output[0][0]._da_locally_complete assert r.output[0][0].data == 'data 0' assert isinstance(r.output[0][1], CAnalysis) assert not r.output[0][1].cached assert r.output[0][1]._da_locally_complete assert r.output[1][1].data == 'data 1'
def test_delegation(): from dataanalysis.caches.queue import QueueCacheWorker queue_name="/tmp/queue" qw=QueueCacheWorker(queue_name) qw.queue.purge() print("cache worker:",qw) randomized_version="v%i"%random.randint(1,10000) callback_file = "./callback" cmd=[ 'dda-run', 'Mosaic', '-m','ddmoduletest', '-a','ddmoduletest.RandomModifier(use_version="%s")'%randomized_version, # '-Q',queue_name, '--callback','file://'+callback_file, '--delegate-target', ] if os.path.exists(callback_file): os.remove(callback_file) exception_report="exception.yaml" if os.path.exists(exception_report): os.remove(exception_report) assert not os.path.exists(callback_file) qw.queue.wipe(["waiting","locked","done","failed","running"]) # run it print("CMD:"," ".join(cmd)) try: subprocess.check_call(cmd,stderr=subprocess.STDOUT,env=env) except subprocess.CalledProcessError as e: pass else: raise Exception("expected AnalysisDelegatedException") #p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) #p.wait() #print(p.stdout.read()) assert os.path.exists(exception_report) recovered_exception = yaml.load(open(exception_report)) print(recovered_exception) print(qw.queue.info) assert qw.queue.info['waiting'] == 1, qw.queue.info assert qw.queue.info['locked'] == 0 assert qw.queue.info['done'] == 0 print("\n\nWORKER") qw.run_once() print(qw.queue.info) assert qw.queue.info['waiting'] == 2 assert qw.queue.info['locked'] == 1 assert qw.queue.info['done'] == 0 assert os.path.exists(callback_file) callback_info = open(callback_file).readlines() print("".join(callback_info)) assert len(callback_info) == 1 print("\n\nWORKER") qw.run_once() assert qw.queue.info['waiting'] == 1 assert qw.queue.info['locked'] == 1 assert qw.queue.info['done'] == 1 print("\n\nWORKER") qw.run_once() assert qw.queue.info['waiting'] == 0 assert qw.queue.info['locked'] == 1 assert qw.queue.info['done'] == 2 print("\n\nWORKER run to unlock") qw.run_once() assert qw.queue.info['waiting'] == 1 assert qw.queue.info['locked'] == 0 assert qw.queue.info['done'] == 2 qw.run_once() assert qw.queue.info['waiting'] == 0 assert qw.queue.info['locked'] == 0 assert qw.queue.info['done'] == 3 # run again, expecting from cache exception_report = "exception.yaml" if os.path.exists(exception_report): os.remove(exception_report) print("\n\nAGAIN") print("cmd:"," ".join(cmd+["-V",'--delegate-target'])) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,env=env) p.wait() print(p.stdout.read()) assert not os.path.exists(exception_report) import ddmoduletest da.debug_output() da.AnalysisFactory.WhatIfCopy("test",ddmoduletest.RandomModifier(use_version=randomized_version)) A=ddmoduletest.Mosaic() A.write_caches=[] A.produce_disabled=True A.run_if_haveto=False A.get() assert A._da_output_origin=="cache" print(A.resource_stats)