def test_base_cache_check_location():
from dataanalysis import core as da
from dataanalysis import caches
da.reset()
da.debug_output()
cache=caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached=True
read_caches=[]
store_preview_yaml=True
def main(self):
self.data="test123"
A=Analysis()
A.get()
cached_path=cache.construct_cached_file_path(A._da_locally_complete,A)
import os
assert os.path.exists(cached_path)
assert os.path.exists(cached_path+"/hash.txt")
assert os.path.exists(cached_path+"/object_identity.yaml.gz")
assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
def test_runtimenamed():
debug_output()
da.reset()
class BAnalysis(da.DataAnalysis):
c = 1
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[B %s]" % repr(self.c)
class AAnalysis(da.DataAnalysis):
input_b = da.NamedAnalysis("BAnalysis")
# run_for_hashe=True
def main(self):
print("test", self.__class__)
self.data = "A,b:" + self.input_b.data
def __repr__(self):
return "[A %s]" % repr(self.input_b)
class CAnalysis(da.DataAnalysis):
input_a = AAnalysis
C = CAnalysis(assume=BAnalysis(use_c=1))
C.get()
assert C._da_locally_complete
assert C.input_a.data == "A,b:data 1"
def test_base_cache_check_location():
from dataanalysis import core as da
from dataanalysis import caches
da.reset()
da.debug_output()
cache = caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached = True
read_caches = []
store_preview_yaml = True
def main(self):
self.data = "test123"
A = Analysis()
A.get()
cached_path = cache.construct_cached_file_path(A._da_locally_complete, A)
import os
assert os.path.exists(cached_path)
assert os.path.exists(cached_path + "/hash.txt")
assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
def test_base_cache_object():
from dataanalysis import core as da
from dataanalysis import caches
da.reset()
cache = caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached = True
def main(self):
self.data = "test123"
A = Analysis()
A.get()
da.reset()
class Analysis(da.DataAnalysis):
cached = True
produce_disabled = True
B = Analysis()
B.get()
assert B.data == A.data
def test_image_cached():
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ii_skyimage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
# now get cached
fb = ddosa.ii_skyimage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fb.produce_disabled = True
import time
t0 = time.time()
fb.get()
assert time.time() - t0 < 3
def test_base_cache_object():
from dataanalysis import core as da
from dataanalysis import caches
da.reset()
cache=caches.cache_core.Cache()
class Analysis(da.DataAnalysis):
cached=True
def main(self):
self.data="test123"
A=Analysis()
A.get()
da.reset()
class Analysis(da.DataAnalysis):
cached = True
produce_disabled=True
B = Analysis()
B.get()
assert B.data == A.data
def test_obscure_node_cached():
from dataanalysis import core as da
da.reset()
da.debug_output()
class BAnalysis(da.DataAnalysis):
data="1"
class AAnalysis(da.DataAnalysis):
input_b=BAnalysis
cached = True
def main(self):
self.data="a."+self.input_b.data
A=AAnalysis().get()
data=A.export_data()
hashe=A._da_locally_complete
cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A)
import os
assert os.path.exists(cached_path)
print(cached_path)
da.reset()
nA=da.DataAnalysis.from_hashe(hashe).get()
print(nA)
assert nA.export_data()['data'] == data['data']
assert nA._da_locally_complete == hashe
def test_object_export_import():
from dataanalysis import core as da
#da.debug_output()
da.reset()
class Analysis(da.DataAnalysis):
#cached = True
def main(self):
print("test")
# time.sleep(3)
self.data = "data1"
A = Analysis()
A.get()
print(A.data)
assert A.data == "data1"
object_data = A.export_data()
da.reset()
B = Analysis()
with pytest.raises(AttributeError):
print(B.data)
B.import_data(object_data)
assert B.data == "data1"
def test_live_resource_server_cachable(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"]
A = ddmoduletest.ServerCachableAnalysis()
A.cache=ddmoduletest.cache
with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources)==1
getter=lambda x:client.get(x).json
fr = excinfo.value.resources[0].get(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
fr = excinfo.value.resources[0].fetch(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
def test_factorize_run_for_hashe_analysis():
import dataanalysis.core as da
import dataanalysis.graphtools as gt
da.reset()
reload(gt)
class AAnalysis(da.DataAnalysis):
pass
class BAnalysis(da.DataAnalysis):
run_for_hashe=True
input_a=AAnalysis
class CAnalysis(da.DataAnalysis):
input_b=BAnalysis
class DAnalysis(da.DataAnalysis):
input_c=CAnalysis
c=DAnalysis().get()
print("got:",c._da_locally_complete)
fct=gt.Factorize(use_root='DAnalysis',use_leaves=['BAnalysis'])
fct=fct.get()
print(fct)
assert isinstance(fct, da.DataHandle)
assert fct.str() == "Factorize.v0.Factor_DAnalysis.By_BAnalysis.processing_definition.1fa218f1"
def test_obscure_assumptions():
from dataanalysis import core as da
da.reset()
da.debug_output()
class BAnalysis(da.DataAnalysis):
data="1"
class AAnalysis(da.DataAnalysis):
input_b=BAnalysis
cached = True
def main(self):
self.data="a."+self.input_b.data
A=AAnalysis().get()
data=A.export_data()
hashe=A._da_locally_complete
cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A)
import os
assert os.path.exists(cached_path)
print(cached_path)
da.reset()
nA=da.DataAnalysis.from_hashe_and_data(hashe, data).get()
print(nA)
assert nA.export_data()['data'] == data['data']
assert nA._da_locally_complete == hashe
def test_queue_reconstruct_env():
from dataanalysis import caches
import dataanalysis.caches.queue
da.debug_output()
da.reset()
q_cache=caches.queue.QueueCache()
q_cache.wipe_queue(["waiting", "done", "running"])
define_analysis()
A=dataanalysis.core.AnalysisFactory['AAnalysis']
A.produce_disabled=True
A.cache = q_cache
A.cached = True
with pytest.raises(da.AnalysisDelegatedException):
A.get()
# worker part
f_cache=caches.cache_core.CacheNoIndex()
#f_cache.parent=q_cache
da.reset()
#A = dataanalysis.core.AnalysisFactory['Analysis']
#A.produce_disabled = False
#A.cache = f_cache
#A.cached = True
worker=caches.queue.QueueCacheWorker()
print(worker.run_all())
def test_factorize_run_for_hashe_analysis():
import dataanalysis.core as da
import dataanalysis.graphtools as gt
da.reset()
reload(gt)
class AAnalysis(da.DataAnalysis):
pass
class BAnalysis(da.DataAnalysis):
run_for_hashe = True
input_a = AAnalysis
class CAnalysis(da.DataAnalysis):
input_b = BAnalysis
class DAnalysis(da.DataAnalysis):
input_c = CAnalysis
c = DAnalysis().get()
print("got:", c._da_locally_complete)
fct = gt.Factorize(use_root='DAnalysis', use_leaves=['BAnalysis'])
fct = fct.get()
print(fct)
assert isinstance(fct, da.DataHandle)
assert fct.str(
) == "Factorize.v0.Factor_DAnalysis.By_BAnalysis.processing_definition.1fa218f1"
def test_input_assumption_version():
import dataanalysis.core as da
da.reset()
import ddmoduletest
reload(ddmoduletest)
C=ddmoduletest.BAnalysis(assume=[ddmoduletest.AAnalysis(use_version="xxx_version")])
C.get()
print C.data
ident=C.get_identity()
assert len(ident.assumptions)==1
print "assumption",ident.assumptions[0]
print "assumption data",ident.assumptions[0][1]['assumed_data'],type(ident.assumptions[0][1]['assumed_data'])
import dataanalysis.emerge as emerge
da.reset()
#print ident.assumptions[]
eC=emerge.emerge_from_identity(ident)
eC.get()
assert C.data == eC.data
print C._da_locally_complete
assert C._da_locally_complete[1][2]=='AAnalysis.xxx_version'
def test_generate_list():
da.debug_output()
da.reset()
da.AnalysisFactory.reset()
class BAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[A %s]" % repr(self.c)
class AAnalysis(da.DataAnalysis):
run_for_hashe = True
def main(self):
print("test", self.__class__)
r = [BAnalysis(use_c=c) for c in range(3)]
return r
A = AAnalysis()
r = A.get()
print(r[1].c, r[1].data)
assert r[0].data == 'data 0'
assert r[1].data == 'data 1'
def test_passing_assumptions(ddservice_fixture, app):
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
ddmoduletest.cache.delegation_mode="interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture
#ddmoduletest.cache.resource_factory.getter=getter
A=ddmoduletest.ChainedServerProducer(assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")])
A.produce_disabled=True
a=A.get()
print a.data
assert a.data=={'a': 'dataAassumed:fromclient', 'b': 'dataB_A:dataAassumed:fromclient'}
assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b['main_executed_on']['pid']
def test_used_modules_stacking():
import dataanalysis.core as da
da.reset()
import ddmoduletest
reload(ddmoduletest)
import ddmoduletest2
reload(ddmoduletest)
reload(ddmoduletest2)
reload(ddmoduletest)
reload(ddmoduletest2)
print()
for i,m in enumerate(da.AnalysisFactory.dda_modules_used):
print(i,m)
assert len(da.AnalysisFactory.dda_modules_used) == 2
C=ddmoduletest.AAnalysis()
C.get()
ident=C.get_identity()
print("identity",ident)
import dataanalysis.emerge as emerge
da.reset()
eC=emerge.emerge_from_identity(ident)
eC.get()
assert C.data == eC.data
def test_object_injection():
from dataanalysis import core as da
#da.debug_output()
da.reset()
class Analysis(da.DataAnalysis):
#cached = True
def main(self):
print("test")
# time.sleep(3)
self.data = "data2"
A = Analysis()
A.get()
assert A.data == "data2"
serialization = A.serialize()
da.reset()
class Analysis(da.DataAnalysis):
pass
da.AnalysisFactory.inject_serialization(serialization)
B = Analysis()
assert B.data == "data2"
def test_object_injection():
from dataanalysis import core as da
#da.debug_output()
da.reset()
class Analysis(da.DataAnalysis):
#cached = True
def main(self):
print("test")
# time.sleep(3)
self.data = "data2"
A=Analysis()
A.get()
assert A.data == "data2"
serialization=A.serialize()
da.reset()
class Analysis(da.DataAnalysis):
pass
da.AnalysisFactory.inject_serialization(serialization)
B=Analysis()
assert B.data == "data2"
def test_object_export_import():
from dataanalysis import core as da
#da.debug_output()
da.reset()
class Analysis(da.DataAnalysis):
#cached = True
def main(self):
print("test")
# time.sleep(3)
self.data = "data1"
A=Analysis()
A.get()
print(A.data)
assert A.data == "data1"
object_data=A.export_data()
da.reset()
B=Analysis()
with pytest.raises(AttributeError):
print(B.data)
B.import_data(object_data)
assert B.data == "data1"
def test_object_injection_identity():
from dataanalysis import core as da
from dataanalysis.emerge import emerge_from_identity
da.reset()
import ddmoduletest
reload(ddmoduletest)
da.AnalysisFactory.assume_serialization(("AAnalysis",{'assumed_data':"bdata1"}))
B=ddmoduletest.BAnalysis()
B.get()
assert B.data=="dataB_A:dataAassumed:bdata1"
ident=B.get_identity()
assert len(ident.assumptions)==1
print(ident.assumptions)
assert ident.assumptions[0][1]['assumed_data']=="bdata1"
da.reset()
C=emerge_from_identity(ident)
C.get()
assert C.data == "dataB_A:dataAassumed:bdata1"
def test_generate_list():
da.debug_output()
da.reset()
da.AnalysisFactory.reset()
class BAnalysis(da.DataAnalysis):
c=None
_da_settings=["c"]
def main(self):
print("test",self.c)
self.data="data "+repr(self.c)
def __repr__(self):
return "[A %s]"%repr(self.c)
class AAnalysis(da.DataAnalysis):
run_for_hashe=True
def main(self):
print("test",self.__class__)
r=[BAnalysis(use_c=c) for c in range(3)]
return r
A=AAnalysis()
r=A.get()
print(r[1].c,r[1].data)
assert r[0].data == 'data 0'
assert r[1].data == 'data 1'
def test_passing_unmanagable_assumptions(ddservice, app):
import dataanalysis.core as da
import dataanalysis.caches.resources
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
ddmoduletest.cache.delegation_mode = "interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice
#ddmoduletest.cache.resource_factory.getter=getter
A = ddmoduletest.ChainedServerProducer(assume=[
ddmoduletest.AAnalysis(input_x=ddmoduletest.TwoCDInputAnalysis())
])
A.produce_disabled = True
with pytest.raises(
dataanalysis.caches.resources.GenericResourceException) as excinfo:
a = A.get()
assert 'mismatch' in excinfo.value.args[2]['comment']
def test_passing_assumptions(ddservice, app):
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
ddmoduletest.cache.delegation_mode = "interactive"
ddmoduletest.cache.resource_factory.endpoint = ddservice
#ddmoduletest.cache.resource_factory.getter=getter
A = ddmoduletest.ChainedServerProducer(
assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")])
A.produce_disabled = True
a = A.get()
print a.data
assert a.data == {'a': 'dataAfromclient', 'b': 'dataBdataAfromclient'}
assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid()
assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b[
'main_executed_on']['pid']
def test_live_multiple_resource_delegation(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis.append(
"ClientDelegatableAnalysis.*")
A = ddmoduletest.TwoCDInputAnalysis()
with pytest.raises(
dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources) == 2
assert isinstance(excinfo.value.resources[0],
dataanalysis.caches.resources.WebResource)
assert isinstance(excinfo.value.resources[1],
dataanalysis.caches.resources.WebResource)
print(excinfo.value.resources[0].identity.get_modules_loadable())
print(excinfo.value.resources)
def test_live_resource_server_cachable(client):
import dataanalysis.core as da
import dataanalysis
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"]
A = ddmoduletest.ServerCachableAnalysis()
A.cache = ddmoduletest.cache
with pytest.raises(
dataanalysis.core.AnalysisDelegatedException) as excinfo:
A.get()
assert len(excinfo.value.resources) == 1
getter = lambda x: client.get(x).json
fr = excinfo.value.resources[0].get(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
fr = excinfo.value.resources[0].fetch(getter=getter)
assert fr.status == "result"
print fr.data
assert fr.data['data'] == 'dataAdataB'
def emerge_from_identity(identity):
da.reset()
reload(dataanalysis.graphtools)
print("fresh factory knows",da.AnalysisFactory.cache)
import_ddmodules(identity.modules)
log("assumptions:",identity.assumptions)
#A = da.AnalysisFactory.byname(identity.factory_name)
for i,assumption in enumerate(identity.assumptions):
log("requested assumption:", assumption)
if assumption[0] == '':
log("ATTENTION: dangerous eval from string",assumption[1])
da.AnalysisFactory.WhatIfCopy('queue emergence %i'%i, eval(assumption[1]))
else:
a = da.AnalysisFactory.byname(assumption[0])
a.import_data(assumption[1])
da.AnalysisFactory.WhatIfCopy('queue emergence %i'%i, a)
print(a, "from", assumption)
A = da.AnalysisFactory.byname(identity.factory_name)
producable_hashe=A.get_hashe()
producable_hashe_jsonified=jsonify(hashtools.hashe_replace_object(producable_hashe, None, "None"))
expected_hashe_jsonified=jsonify(identity.expected_hashe)
if identity.expected_hashe is None or identity.expected_hashe == "None":
log("expected hashe verification skipped")
elif producable_hashe_jsonified != expected_hashe_jsonified:
log("producable:\n",jsonify(producable_hashe_jsonified),"\n",level="top")
log("requested:\n",jsonify(expected_hashe_jsonified),level="top")
try:
from dataanalysis import displaygraph
displaygraph.plot_hashe(producable_hashe_jsonified,"producable.png")
displaygraph.plot_hashe(expected_hashe_jsonified,"expected.png")
except Exception as e:
pass
log_logstash("emergence exception",note="inconsistent_emergence",producable=producable_hashe_jsonified,expected_hashe=expected_hashe_jsonified)
raise InconsitentEmergence(
"unable to produce\n"+repr(producable_hashe_jsonified)+"\n while can produce"+repr(expected_hashe_jsonified),
producable_hashe_jsonified,
expected_hashe_jsonified,
)
log("successfully emerged")
return A
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('modules', type=str, help='')
parser.add_argument('assume', type=str, help='')
args = parser.parse_args()
da.reset()
emerge.import_ddmodules(args['modules'].split(","))
return da.AnalysisFactory.cache.keys()
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('modules', type=str, help='')
parser.add_argument('assume', type=str, help='')
args = parser.parse_args()
da.reset()
emerge.import_ddmodules(args['modules'].split(","))
return da.AnalysisFactory.cache.keys()
def test_scw():
#da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ScWData(input_scwid="066500230010.001")
fa.get()
print fa.scwpath
def test_basic():
import dataanalysis.core as da
import dataanalysis.importing as importing
da.reset()
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
m=importing.load_by_name(['filesystem','moduletest',dir_path+"/ddmoduletest.py"])
assert not hasattr(m[0], '__dda_module_global_name__')
assert not hasattr(m[0], '__dda_module_origin__')
def test_serialize():
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get()
ed=S.export_data()
assert '_da_stored_string_input_s' in ed
def test_serialize():
import dataanalysis.core as da
da.reset()
da.debug_output()
import ddmoduletest
reload(ddmoduletest)
S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get()
ed=S.export_data()
assert '_da_stored_string_input_s' in ed
def test_gti():
# da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.ibis_gti(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
def test_delegating_analysis():
#import dataanalysis
from dataanalysis import caches
import dataanalysis.caches.delegating
# da.debug_output()
da.reset()
q_cache=caches.delegating.DelegatingCache()
da.DataAnalysis.cache.tail_parent(q_cache)
class A1Analysis(da.DataAnalysis):
read_caches = [q_cache.__class__]
cached = True
def main(self):
self.data="datadata1"
class A2Analysis(da.DataAnalysis):
read_caches = [q_cache.__class__]
cached = True
def main(self):
self.data="datadata2"
class A3Analysis(da.DataAnalysis):
def main(self):
self.data="datadata3"
class BAnalysis(da.DataAnalysis):
input_a1 = A1Analysis
input_a2 = A2Analysis
input_a3 = A3Analysis
def main(self):
self.data="datadata"
A=BAnalysis()
print(A.cache.list_parent_stack())
with pytest.raises(da.AnalysisDelegatedException):
A.get()
try:
A.get()
except da.AnalysisDelegatedException as e:
print("resources:",e.resources)
del da.DataAnalysis.cache.parent
def test_queue_cache_generative():
from dataanalysis import caches
import dataanalysis.caches.queue
da.debug_output()
da.reset()
q_cache=caches.queue.QueueCache("./queue_test")
q_cache.queue.purge()
assert len(q_cache.queue.list())==0
define_analysis()
A=dataanalysis.core.AnalysisFactory['GenerativeAnalysis']
for name in 'AAnalysis',:
aA=dataanalysis.core.AnalysisFactory[name].__class__
aA.cache = q_cache
aA.produce_disabled=True
aA.cached = True
with pytest.raises(da.AnalysisDelegatedException) as e:
A.get()
print('AnalysisDelegatedException',e.value,e.value.origin,e.value.hashe)
assert len(e.value.hashe[1:])==2
assert len(e.value.source_exceptions)==2
for se in e.value.source_exceptions:
print('found multiple AnalysisDelegatedException',se,se.origin)
print(q_cache.queue.list())
# assert len(q_cache.queue.list())==2
assert len(q_cache.queue.list())==1
# worker part
f_cache=caches.cache_core.CacheNoIndex()
#f_cache.parent=q_cache
define_analysis()
A = dataanalysis.core.AnalysisFactory['AAnalysis']
A.produce_disabled = False
A.cache = f_cache
A.cached = True
worker=caches.queue.QueueCacheWorker("./queue_test")
print(worker.run_once())
def test_self_assumption():
from dataanalysis import core as da
da.debug_output()
da.reset()
class AAnalysis(da.DataAnalysis):
pass
aa = AAnalysis(assume=AAnalysis(use_arg="arg2", use_nonearg=None)).get()
assert aa._da_locally_complete
assert not hasattr(aa,'arg') # current behavoir and it is bad
def run_task(self,task):
task_data=task.task_data
log("emerging from object_identity",task_data['object_identity'])
object_identity=da.DataAnalysisIdentity.from_dict(task_data['object_identity'])
da.reset()
reload(dataanalysis.graphtools)
log("fresh factory knows",da.AnalysisFactory.cache)
log(object_identity)
A=emerge.emerge_from_identity(object_identity)
A._da_delegation_allowed=False
dataanalysis.callback.Callback.set_callback_accepted_classes([da.byname(object_identity.factory_name).__class__])
for url in task.submission_info['callbacks']:
log("setting object callback",A,url)
A.set_callback(url)
log("emerged object:",A)
request_root_node=getattr(A, 'request_root_node', False)
if request_root_node:
final_state = "done"
else:
final_state = "task_done"
try:
result=A.get(requested_by=[repr(self)],isolated_directory_key=task.key,isolated_directory_cleanup=True)
A.raise_stored_exceptions()
except da.AnalysisException as e:
A.process_hooks("top", A, message="task complete", state=final_state, task_comment="completed with failure "+repr(e))
A.process_hooks("top", A, message="analysis exception", exception=repr(e),state="node_analysis_exception")
except da.AnalysisDelegatedException as delegation_exception:
final_state = "task_done"
log("delegated dependencies:",delegation_exception)
A.process_hooks("top",A,message="task dependencies delegated",state=final_state, task_comment="task dependencies delegated",delegation_exception=repr(delegation_exception))
raise
except dqueue.TaskStolen:
raise
except Exception as e:
A.process_hooks("top", A, message="task complete", state=final_state, task_comment="completed with unexpected failure "+repr(e))
client=get_sentry_client()
if client is not None:
client.captureException()
raise
else:
A.process_hooks("top",A,message="task complete",state=final_state, task_comment="completed with success")
return result
def test_constructor_input():
from dataanalysis import analysisfactory
from dataanalysis import core as da
da.reset()
class BAnalysis(da.DataAnalysis):
def main(self):
print("test")
self.data="data"
class Analysis(da.DataAnalysis):
#input_b=BAnalysis
def main(self):
print("test")
if hasattr(self,'input_b'):
self.data = self.input_b.data
else:
self.data = 'otherdata'
A=Analysis()
A.get()
assert A.data == 'otherdata'
A=Analysis(input_b=BAnalysis())
assert hasattr(A,'input_b')
da.debug_output()
print("*" * 80)
A.promote()
print("/" * 80)
A_fn=analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get()
print(A, A_fn)
assert A == A_fn
A_f = analysisfactory.AnalysisFactory.get(A).get()
print(A,A_f)
assert A == A_f
A.get()
assert A.data == 'data'
inputs=A.list_inputs()
assert isinstance(inputs,list)
assert len(inputs)==1
def test_generate_structures():
debug_output()
da.reset()
class BAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[A %s]" % repr(self.c)
class CAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[C %s]" % repr(self.c)
class AAnalysis(da.DataAnalysis):
# run_for_hashe=True
def main(self):
print("test", self.__class__)
r = [[BAnalysis(use_c=c),CAnalysis(use_c=c)] for c in range(3)]
return r
A = AAnalysis()
r = A.get()
print r
print(r[1].c, r[1].data)
#assert r[0]
#assert isinstance(r[0][0],BAnalysis)
assert r[0].data == 'data 0'
assert r[2].data == 'data 1'
assert r[1]._da_locally_complete
def test_git_list():
import os
import dataanalysis.core as da
import dataanalysis.importing as importing
da.reset()
m=importing.load_by_name("git://test",local_gitroot=os.environ.get("TMPDIR","/tmp")+"/git/",remote_git_root='volodymyrss-public')
assert hasattr(m[0], '__dda_module_global_name__')
assert hasattr(m[0], '__dda_module_origin__')
assert m[0].__dda_module_origin__=="git"
assert m[0].__dda_module_global_name__ == "git://test"
def test_constructor_input():
from dataanalysis import analysisfactory
from dataanalysis import core as da
da.reset()
class BAnalysis(da.DataAnalysis):
def main(self):
print("test")
self.data = "data"
class Analysis(da.DataAnalysis):
#input_b=BAnalysis
def main(self):
print("test")
if hasattr(self, 'input_b'):
self.data = self.input_b.data
else:
self.data = 'otherdata'
A = Analysis()
A.get()
assert A.data == 'otherdata'
A = Analysis(input_b=BAnalysis())
assert hasattr(A, 'input_b')
da.debug_output()
print("*" * 80)
A.promote()
print("/" * 80)
A_fn = analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get()
print(A, A_fn)
assert A == A_fn
A_f = analysisfactory.AnalysisFactory.get(A).get()
print(A, A_f)
assert A == A_f
A.get()
assert A.data == 'data'
inputs = A.list_inputs()
assert isinstance(inputs, list)
assert len(inputs) == 1
def test_numpy_to_datafile_recursive():
from dataanalysis import core as da
import os
da.reset()
import numpy as np
import pandas as pd
class Analysis(da.DataAnalysis):
read_caches=[]
cached=True
def main(self):
print("test")
self.data = {'a':np.linspace(0,1,100),'b/c!':pd.DataFrame(np.linspace(0,2,200))}
A = Analysis()
A.get()
content=A.export_data()
cc=A.cache.adopt_datafiles(content)
assert cc['data']['a'] is None
#da.TransientCacheInstance.reset()
da.reset()
da.debug_output()
B = Analysis()
B.produce_disabled=True
B.read_caches = [A.cache.__class__]
B.get()
print(B.data['a'])
#B.cache.restore(A._da_locally_complete,A)
assert hasattr(B,'_datafile_data_a')
assert hasattr(B._datafile_data_a,'adopted_format')
assert B._datafile_data_a.adopted_format == "numpy"
print B._datafile_data_a.get_path()
assert os.path.exists(B._datafile_data_a.get_path())
assert hasattr(B, 'data')
assert all(A.data['a'] == B.data['a'])
def test_object_injection_reset():
from dataanalysis import core as da
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
# arg=None
pass
A1 = AAnalysis(use_arg="arg1")
assert A1.arg == "arg1"
d1 = A1.export_data()
print("has data:", d1)
assert d1['arg'] == 'arg1'
a1 = A1.serialize()
print("serilization:",a1)
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
pass
da.AnalysisFactory.inject_serialization(a1)
print("factory has",da.AnalysisFactory.cache['AAnalysis'])
aanalysis=da.AnalysisFactory['AAnalysis']
assert aanalysis.arg == "arg1"
class Analysis(da.DataAnalysis):
#cached = True
input_arg=AAnalysis
def main(self):
print("test")
# time.sleep(3)
self.data = "data_"+self.input_arg.arg
A=Analysis()
A.get()
print(A.data)
assert A.data == "data_arg1"
def test_generate_structures():
debug_output()
da.reset()
class BAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[A %s]" % repr(self.c)
class CAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[C %s]" % repr(self.c)
class AAnalysis(da.DataAnalysis):
# run_for_hashe=True
def main(self):
print("test", self.__class__)
r = [[BAnalysis(use_c=c), CAnalysis(use_c=c)] for c in range(3)]
return r
A = AAnalysis()
r = A.get()
print r
print(r[1].c, r[1].data)
#assert r[0]
#assert isinstance(r[0][0],BAnalysis)
assert r[0].data == 'data 0'
assert r[2].data == 'data 1'
assert r[1]._da_locally_complete
def test_spectra():
da.reset()
import ddosa
reload(ddosa)
#da.debug_output()
fa = ddosa.ii_spectra_extract(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
ddosa.SpectraBins(use_rmfpath="/data/resources/rmf_62bands.fits"),
])
fa.read_caches = []
fa.get()
assert os.path.exists(fa.spectrum.get_path())
def test_object_injection_reset():
from dataanalysis import core as da
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
# arg=None
pass
A1 = AAnalysis(use_arg="arg1")
assert A1.arg == "arg1"
d1 = A1.export_data()
print("has data:", d1)
assert d1['arg'] == 'arg1'
a1 = A1.serialize()
print("serilization:", a1)
da.reset()
da.debug_output()
class AAnalysis(da.DataAnalysis):
pass
da.AnalysisFactory.inject_serialization(a1)
print("factory has", da.AnalysisFactory.cache['AAnalysis'])
aanalysis = da.AnalysisFactory['AAnalysis']
assert aanalysis.arg == "arg1"
class Analysis(da.DataAnalysis):
#cached = True
input_arg = AAnalysis
def main(self):
print("test")
# time.sleep(3)
self.data = "data_" + self.input_arg.arg
A = Analysis()
A.get()
print(A.data)
assert A.data == "data_arg1"
def test_image():
#
da.reset()
import ddosa
reload(ddosa)
da.debug_output()
fa = ddosa.ii_skyimage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
])
fa.read_caches = []
fa.get()
print fa.skyima
def test_imagebins():
#
# da.debug_output()
da.reset()
import ddosa
reload(ddosa)
fa = ddosa.BinEventsImage(assume=[
ddosa.ScWData(input_scwid="066500230010.001"),
ddosa.ImageBins(use_autoversion=True, use_ebins=[(25, 45)])
])
fa.read_caches = []
fa.get()
print fa.shadow_detector
def test_one_object():
from dataanalysis import core as da
da.reset()
class Analysis(da.DataAnalysis):
def main(self):
print("test")
self.data = "datacontent"
A = Analysis()
A.get()
assert A.data == 'datacontent'
j = A.jsonify()
assert j['data'] == 'datacontent'
def test_factorize_note():
import dataanalysis.core as da
da.reset()
da.AnalysisFactory.note_factorization(dict(a=1,b=2,c=[1,2,{1:2}]))
class AAnalysis(da.DataAnalysis):
read_caches=[]
cached=True
A=AAnalysis().get()
fn=A._da_cached_path+"/factorizations.txt"
assert os.path.exists(fn)
print(">>>>\n"+open(fn).read()+"<<<<")
def test_live_delegation_assumptions(ddservice_fixture,app):
import requests
import dataanalysis.core as da
da.reset()
da.debug_output()
r=requests.get(ddservice_fixture+url_for('produce', target="BAnalysis", modules="ddmoduletest", assume="AAnalysis.assumed_data=\"clients\""))
print(r.content)
factory_r=r.json()
print(factory_r)
assert factory_r['data']['data']=='dataB_A:dataAassumed:clients'
assert len(factory_r)>0
def test_recursive_datafile():
from dataanalysis import core as da
import os
da.reset()
da.debug_output()
class Analysis(da.DataAnalysis):
cached=True
def main(self):
print("test")
fn="file.txt"
with open(fn,"w") as f:
f.write("test")
fn1="file1.txt"
with open(fn1,"w") as f:
f.write("test")
self.data = [da.DataFile(fn)]
self.data1 = da.DataFile(fn1)
A = Analysis()
A.read_caches = []
A.get()
da.reset()
os.remove("file.txt")
os.remove("file1.txt")
B = Analysis()
B.produce_disabled=True
B.get()
#B.cache.restore(A._da_locally_complete,A)
assert hasattr(B,'data')
print B.data
#assert hasattr(B._datafile_data,'adopted_format')
#assert B._datafile_data.adopted_format == "numpy"
#print B._datafile_data.get_path()
assert os.path.exists(B.data1.get_path())
assert os.path.exists(B.data[0].get_path())
def test_factorize_note():
import dataanalysis.core as da
da.reset()
da.AnalysisFactory.note_factorization(dict(a=1, b=2, c=[1, 2, {1: 2}]))
class AAnalysis(da.DataAnalysis):
read_caches = []
cached = True
A = AAnalysis().get()
fn = A._da_cached_path + "/factorizations.txt"
assert os.path.exists(fn)
print(">>>>\n" + open(fn).read() + "<<<<")
def test_generate_factory_assumptions_references():
debug_output()
da.reset()
class BAnalysis(da.DataAnalysis):
c = None
_da_settings = ["c"]
def main(self):
print("test", self.c)
self.data = "data " + repr(self.c)
def __repr__(self):
return "[B %s]" % repr(self.c)
class AAnalysis(da.DataAnalysis):
input_b=None
# run_for_hashe=True
def main(self):
print("test", self.__class__)
self.data="A,b:"+self.input_b.data
def __repr__(self):
return "[A %s]" % repr(self.input_b)
class CAnalysis(da.DataAnalysis):
input_a=AAnalysis
da.AnalysisFactory.WhatIfCopy("test",AAnalysis(input_b=BAnalysis,use_version="v2"))
print(da.AnalysisFactory.cache_assumptions)
assert len(da.AnalysisFactory.cache_assumptions)==1
C = CAnalysis(assume=BAnalysis(use_c=1))
C.get()
assert C._da_locally_complete
assert C.input_a.data=="A,b:data 1"
C = CAnalysis(assume=BAnalysis(use_c=2))
C.get()
assert C._da_locally_complete
assert C.input_a.data=="A,b:data 2"
def test_pandas_to_datafile():
from dataanalysis import core as da
import os
da.reset()
import pandas as pd
import numpy as np
class Analysis(da.DataAnalysis):
read_caches=[]
cached=True
def main(self):
print("test")
self.data = pd.DataFrame()
self.data['data']=np.linspace(0,1,100)
A = Analysis()
A.get()
#da.TransientCacheInstance.reset()
da.reset()
da.debug_output()
B = Analysis()
B.produce_disabled=True
B.read_caches = [A.cache.__class__]
B.get()
#B.cache.restore(A._da_locally_complete,A)
assert hasattr(B,'_datafile_data')
assert hasattr(B._datafile_data,'adopted_format')
assert B._datafile_data.adopted_format == "pandas"
print B._datafile_data.get_path()
assert os.path.exists(B._datafile_data.get_path())
assert hasattr(B, 'data')
assert all(abs(A.data.data - B.data.data)<1e-5)
