def test_base_cache_check_location():
    from dataanalysis import core as da
    from dataanalysis import caches
    da.reset()
    da.debug_output()

    cache=caches.cache_core.Cache()

    class Analysis(da.DataAnalysis):
        cached=True
        read_caches=[]
        store_preview_yaml=True

        def main(self):
            self.data="test123"

    A=Analysis()
    A.get()

    cached_path=cache.construct_cached_file_path(A._da_locally_complete,A)

    import os
    assert os.path.exists(cached_path)
    assert os.path.exists(cached_path+"/hash.txt")
    assert os.path.exists(cached_path+"/object_identity.yaml.gz")
    assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
def test_runtimenamed():
    debug_output()
    da.reset()

    class BAnalysis(da.DataAnalysis):
        c = 1

        _da_settings = ["c"]

        def main(self):
            print("test", self.c)
            self.data = "data " + repr(self.c)

        def __repr__(self):
            return "[B %s]" % repr(self.c)

    class AAnalysis(da.DataAnalysis):
        input_b = da.NamedAnalysis("BAnalysis")

        # run_for_hashe=True
        def main(self):
            print("test", self.__class__)
            self.data = "A,b:" + self.input_b.data

        def __repr__(self):
            return "[A %s]" % repr(self.input_b)

    class CAnalysis(da.DataAnalysis):
        input_a = AAnalysis

    C = CAnalysis(assume=BAnalysis(use_c=1))
    C.get()
    assert C._da_locally_complete
    assert C.input_a.data == "A,b:data 1"
Example #3
0
def test_base_cache_check_location():
    from dataanalysis import core as da
    from dataanalysis import caches
    da.reset()
    da.debug_output()

    cache = caches.cache_core.Cache()

    class Analysis(da.DataAnalysis):
        cached = True
        read_caches = []
        store_preview_yaml = True

        def main(self):
            self.data = "test123"

    A = Analysis()
    A.get()

    cached_path = cache.construct_cached_file_path(A._da_locally_complete, A)

    import os
    assert os.path.exists(cached_path)
    assert os.path.exists(cached_path + "/hash.txt")
    assert os.path.exists(cached_path + "/cache_preview.yaml.gz")
Example #4
0
def test_base_cache_object():
    from dataanalysis import core as da
    from dataanalysis import caches
    da.reset()

    cache = caches.cache_core.Cache()

    class Analysis(da.DataAnalysis):
        cached = True

        def main(self):
            self.data = "test123"

    A = Analysis()
    A.get()

    da.reset()

    class Analysis(da.DataAnalysis):
        cached = True
        produce_disabled = True

    B = Analysis()
    B.get()

    assert B.data == A.data
Example #5
0
def test_image_cached():
    da.reset()
    import ddosa
    reload(ddosa)

    fa = ddosa.ii_skyimage(assume=[
        ddosa.ScWData(input_scwid="066500230010.001"),
    ])
    fa.read_caches = []

    fa.get()

    # now get cached

    fb = ddosa.ii_skyimage(assume=[
        ddosa.ScWData(input_scwid="066500230010.001"),
    ])

    fb.produce_disabled = True

    import time
    t0 = time.time()

    fb.get()

    assert time.time() - t0 < 3
def test_base_cache_object():
    from dataanalysis import core as da
    from dataanalysis import caches
    da.reset()

    cache=caches.cache_core.Cache()

    class Analysis(da.DataAnalysis):
        cached=True

        def main(self):
            self.data="test123"

    A=Analysis()
    A.get()

    da.reset()

    class Analysis(da.DataAnalysis):
        cached = True
        produce_disabled=True

    B = Analysis()
    B.get()


    assert B.data == A.data
def test_obscure_node_cached():
    from dataanalysis import core as da
    da.reset()
    da.debug_output()

    class BAnalysis(da.DataAnalysis):
        data="1"

    class AAnalysis(da.DataAnalysis):
        input_b=BAnalysis

        cached = True

        def main(self):
            self.data="a."+self.input_b.data

    A=AAnalysis().get()

    data=A.export_data()
    hashe=A._da_locally_complete

    cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A)

    import os
    assert os.path.exists(cached_path)
    print(cached_path)

    da.reset()

    nA=da.DataAnalysis.from_hashe(hashe).get()
    print(nA)

    assert nA.export_data()['data'] == data['data']
    assert nA._da_locally_complete == hashe
def test_object_export_import():
    from dataanalysis import core as da

    #da.debug_output()
    da.reset()

    class Analysis(da.DataAnalysis):
        #cached = True

        def main(self):
            print("test")
            # time.sleep(3)
            self.data = "data1"

    A = Analysis()
    A.get()

    print(A.data)

    assert A.data == "data1"

    object_data = A.export_data()

    da.reset()

    B = Analysis()

    with pytest.raises(AttributeError):
        print(B.data)

    B.import_data(object_data)
    assert B.data == "data1"
Example #9
0
def test_live_resource_server_cachable(client):
    import dataanalysis.core as da
    import dataanalysis

    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)
    ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"]

    A = ddmoduletest.ServerCachableAnalysis()
    A.cache=ddmoduletest.cache

    with pytest.raises(dataanalysis.core.AnalysisDelegatedException) as excinfo:
        A.get()

    assert len(excinfo.value.resources)==1

    getter=lambda x:client.get(x).json

    fr = excinfo.value.resources[0].get(getter=getter)

    assert fr.status == "result"
    print fr.data
    assert fr.data['data'] == 'dataAdataB'

    fr = excinfo.value.resources[0].fetch(getter=getter)

    assert fr.status == "result"

    print fr.data

    assert fr.data['data'] == 'dataAdataB'
def test_factorize_run_for_hashe_analysis():
    import dataanalysis.core as da
    import dataanalysis.graphtools as gt
    da.reset()
    reload(gt)

    class AAnalysis(da.DataAnalysis):
        pass

    class BAnalysis(da.DataAnalysis):
        run_for_hashe=True
        input_a=AAnalysis

    class CAnalysis(da.DataAnalysis):
        input_b=BAnalysis

    class DAnalysis(da.DataAnalysis):
        input_c=CAnalysis

    c=DAnalysis().get()
    print("got:",c._da_locally_complete)

    fct=gt.Factorize(use_root='DAnalysis',use_leaves=['BAnalysis'])

    fct=fct.get()

    print(fct)

    assert isinstance(fct, da.DataHandle)
    assert fct.str() == "Factorize.v0.Factor_DAnalysis.By_BAnalysis.processing_definition.1fa218f1"
def test_obscure_assumptions():
    from dataanalysis import core as da
    da.reset()
    da.debug_output()

    class BAnalysis(da.DataAnalysis):
        data="1"

    class AAnalysis(da.DataAnalysis):
        input_b=BAnalysis

        cached = True

        def main(self):
            self.data="a."+self.input_b.data

    A=AAnalysis().get()

    data=A.export_data()
    hashe=A._da_locally_complete

    cached_path=A.cache.construct_cached_file_path(A._da_locally_complete, A)

    import os
    assert os.path.exists(cached_path)
    print(cached_path)

    da.reset()

    nA=da.DataAnalysis.from_hashe_and_data(hashe, data).get()
    print(nA)

    assert nA.export_data()['data'] == data['data']
    assert nA._da_locally_complete == hashe
def test_queue_reconstruct_env():
    from dataanalysis import caches
    import dataanalysis.caches.queue

    da.debug_output()
    da.reset()

    q_cache=caches.queue.QueueCache()
    q_cache.wipe_queue(["waiting", "done", "running"])

    define_analysis()

    A=dataanalysis.core.AnalysisFactory['AAnalysis']
    A.produce_disabled=True
    A.cache = q_cache
    A.cached = True

    with pytest.raises(da.AnalysisDelegatedException):
        A.get()

    # worker part

    f_cache=caches.cache_core.CacheNoIndex()
    #f_cache.parent=q_cache

    da.reset()

    #A = dataanalysis.core.AnalysisFactory['Analysis']
    #A.produce_disabled = False
    #A.cache = f_cache
    #A.cached = True

    worker=caches.queue.QueueCacheWorker()

    print(worker.run_all())
def test_factorize_run_for_hashe_analysis():
    import dataanalysis.core as da
    import dataanalysis.graphtools as gt
    da.reset()
    reload(gt)

    class AAnalysis(da.DataAnalysis):
        pass

    class BAnalysis(da.DataAnalysis):
        run_for_hashe = True
        input_a = AAnalysis

    class CAnalysis(da.DataAnalysis):
        input_b = BAnalysis

    class DAnalysis(da.DataAnalysis):
        input_c = CAnalysis

    c = DAnalysis().get()
    print("got:", c._da_locally_complete)

    fct = gt.Factorize(use_root='DAnalysis', use_leaves=['BAnalysis'])

    fct = fct.get()

    print(fct)

    assert isinstance(fct, da.DataHandle)
    assert fct.str(
    ) == "Factorize.v0.Factor_DAnalysis.By_BAnalysis.processing_definition.1fa218f1"
Example #14
0
def test_input_assumption_version():
    import dataanalysis.core as da
    da.reset()

    import ddmoduletest
    reload(ddmoduletest)
    C=ddmoduletest.BAnalysis(assume=[ddmoduletest.AAnalysis(use_version="xxx_version")])
    C.get()

    print C.data

    ident=C.get_identity()

    assert len(ident.assumptions)==1

    print "assumption",ident.assumptions[0]
    print "assumption data",ident.assumptions[0][1]['assumed_data'],type(ident.assumptions[0][1]['assumed_data'])

    import dataanalysis.emerge as emerge
    da.reset()

    #print ident.assumptions[]

    eC=emerge.emerge_from_identity(ident)
    eC.get()

    assert C.data == eC.data

    print C._da_locally_complete
    assert C._da_locally_complete[1][2]=='AAnalysis.xxx_version'
Example #15
0
def test_generate_list():
    da.debug_output()
    da.reset()
    da.AnalysisFactory.reset()

    class BAnalysis(da.DataAnalysis):
        c = None

        _da_settings = ["c"]

        def main(self):
            print("test", self.c)
            self.data = "data " + repr(self.c)

        def __repr__(self):
            return "[A %s]" % repr(self.c)

    class AAnalysis(da.DataAnalysis):
        run_for_hashe = True

        def main(self):
            print("test", self.__class__)
            r = [BAnalysis(use_c=c) for c in range(3)]
            return r

    A = AAnalysis()
    r = A.get()

    print(r[1].c, r[1].data)
    assert r[0].data == 'data 0'
    assert r[1].data == 'data 1'
Example #16
0
def test_passing_assumptions(ddservice_fixture, app):
    import dataanalysis.core as da

    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)

    ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
    ddmoduletest.cache.delegation_mode="interactive"

    ddmoduletest.cache.resource_factory.endpoint = ddservice_fixture
    #ddmoduletest.cache.resource_factory.getter=getter

    A=ddmoduletest.ChainedServerProducer(assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")])
    A.produce_disabled=True

    a=A.get()


    print a.data

    assert a.data=={'a': 'dataAassumed:fromclient', 'b': 'dataB_A:dataAassumed:fromclient'}
    assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid()
    assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid()
    assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b['main_executed_on']['pid']
Example #17
0
def test_used_modules_stacking():
    import dataanalysis.core as da
    da.reset()

    import ddmoduletest
    reload(ddmoduletest)
    import ddmoduletest2
    reload(ddmoduletest)
    reload(ddmoduletest2)
    reload(ddmoduletest)
    reload(ddmoduletest2)

    print()
    for i,m in enumerate(da.AnalysisFactory.dda_modules_used):
        print(i,m)

    assert len(da.AnalysisFactory.dda_modules_used) == 2

    C=ddmoduletest.AAnalysis()
    C.get()


    ident=C.get_identity()

    print("identity",ident)

    import dataanalysis.emerge as emerge
    da.reset()

    eC=emerge.emerge_from_identity(ident)
    eC.get()

    assert C.data == eC.data
def test_object_injection():
    from dataanalysis import core as da

    #da.debug_output()
    da.reset()

    class Analysis(da.DataAnalysis):
        #cached = True

        def main(self):
            print("test")
            # time.sleep(3)
            self.data = "data2"

    A = Analysis()
    A.get()

    assert A.data == "data2"

    serialization = A.serialize()

    da.reset()

    class Analysis(da.DataAnalysis):
        pass

    da.AnalysisFactory.inject_serialization(serialization)
    B = Analysis()

    assert B.data == "data2"
def test_object_injection():
    from dataanalysis import core as da

    #da.debug_output()
    da.reset()

    class Analysis(da.DataAnalysis):
        #cached = True

        def main(self):
            print("test")
            # time.sleep(3)
            self.data = "data2"

    A=Analysis()
    A.get()

    assert A.data == "data2"

    serialization=A.serialize()

    da.reset()

    class Analysis(da.DataAnalysis):
        pass

    da.AnalysisFactory.inject_serialization(serialization)
    B=Analysis()

    assert B.data == "data2"
def test_object_export_import():
    from dataanalysis import core as da

    #da.debug_output()
    da.reset()

    class Analysis(da.DataAnalysis):
        #cached = True


        def main(self):
            print("test")
            # time.sleep(3)
            self.data = "data1"

    A=Analysis()
    A.get()

    print(A.data)

    assert A.data == "data1"

    object_data=A.export_data()

    da.reset()

    B=Analysis()

    with pytest.raises(AttributeError):
        print(B.data)

    B.import_data(object_data)
    assert B.data == "data1"
def test_object_injection_identity():
    from dataanalysis import core as da
    from dataanalysis.emerge import emerge_from_identity

    da.reset()

    import ddmoduletest
    reload(ddmoduletest)

    da.AnalysisFactory.assume_serialization(("AAnalysis",{'assumed_data':"bdata1"}))

    B=ddmoduletest.BAnalysis()
    B.get()

    assert B.data=="dataB_A:dataAassumed:bdata1"

    ident=B.get_identity()

    assert len(ident.assumptions)==1
    print(ident.assumptions)
    assert ident.assumptions[0][1]['assumed_data']=="bdata1"

    da.reset()

    C=emerge_from_identity(ident)
    C.get()

    assert C.data == "dataB_A:dataAassumed:bdata1"
def test_generate_list():
    da.debug_output()
    da.reset()
    da.AnalysisFactory.reset()

    class BAnalysis(da.DataAnalysis):
        c=None

        _da_settings=["c"]

        def main(self):
            print("test",self.c)
            self.data="data "+repr(self.c)

        def __repr__(self):
            return "[A %s]"%repr(self.c)

    
    class AAnalysis(da.DataAnalysis):
        run_for_hashe=True
        def main(self):
            print("test",self.__class__)
            r=[BAnalysis(use_c=c) for c in range(3)]
            return r

    A=AAnalysis()
    r=A.get()

    print(r[1].c,r[1].data)
    assert r[0].data == 'data 0'
    assert r[1].data == 'data 1'
Example #23
0
def test_passing_unmanagable_assumptions(ddservice, app):
    import dataanalysis.core as da
    import dataanalysis.caches.resources

    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)

    ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
    ddmoduletest.cache.delegation_mode = "interactive"

    ddmoduletest.cache.resource_factory.endpoint = ddservice
    #ddmoduletest.cache.resource_factory.getter=getter

    A = ddmoduletest.ChainedServerProducer(assume=[
        ddmoduletest.AAnalysis(input_x=ddmoduletest.TwoCDInputAnalysis())
    ])
    A.produce_disabled = True

    with pytest.raises(
            dataanalysis.caches.resources.GenericResourceException) as excinfo:
        a = A.get()

    assert 'mismatch' in excinfo.value.args[2]['comment']
Example #24
0
def test_passing_assumptions(ddservice, app):
    import dataanalysis.core as da

    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)

    ddmoduletest.cache.delegating_analysis.append("ChainedServerProducer.*")
    ddmoduletest.cache.delegation_mode = "interactive"

    ddmoduletest.cache.resource_factory.endpoint = ddservice
    #ddmoduletest.cache.resource_factory.getter=getter

    A = ddmoduletest.ChainedServerProducer(
        assume=[ddmoduletest.AAnalysis(use_assumed_data="fromclient")])
    A.produce_disabled = True

    a = A.get()

    print a.data

    assert a.data == {'a': 'dataAfromclient', 'b': 'dataBdataAfromclient'}
    assert a.resource_stats_a['main_executed_on']['pid'] != os.getpid()
    assert a.resource_stats_b['main_executed_on']['pid'] != os.getpid()
    assert a.resource_stats_a['main_executed_on']['pid'] == a.resource_stats_b[
        'main_executed_on']['pid']
Example #25
0
def test_live_multiple_resource_delegation(client):
    import dataanalysis.core as da
    import dataanalysis

    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)
    ddmoduletest.cache.delegating_analysis.append(
        "ClientDelegatableAnalysis.*")

    A = ddmoduletest.TwoCDInputAnalysis()

    with pytest.raises(
            dataanalysis.core.AnalysisDelegatedException) as excinfo:
        A.get()

    assert len(excinfo.value.resources) == 2

    assert isinstance(excinfo.value.resources[0],
                      dataanalysis.caches.resources.WebResource)
    assert isinstance(excinfo.value.resources[1],
                      dataanalysis.caches.resources.WebResource)

    print(excinfo.value.resources[0].identity.get_modules_loadable())

    print(excinfo.value.resources)
Example #26
0
def test_live_resource_server_cachable(client):
    import dataanalysis.core as da
    import dataanalysis

    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)
    ddmoduletest.cache.delegating_analysis = ["ServerCachableAnalysis"]

    A = ddmoduletest.ServerCachableAnalysis()
    A.cache = ddmoduletest.cache

    with pytest.raises(
            dataanalysis.core.AnalysisDelegatedException) as excinfo:
        A.get()

    assert len(excinfo.value.resources) == 1

    getter = lambda x: client.get(x).json

    fr = excinfo.value.resources[0].get(getter=getter)

    assert fr.status == "result"
    print fr.data
    assert fr.data['data'] == 'dataAdataB'

    fr = excinfo.value.resources[0].fetch(getter=getter)

    assert fr.status == "result"

    print fr.data

    assert fr.data['data'] == 'dataAdataB'
Example #27
0
def emerge_from_identity(identity):
    da.reset()

    reload(dataanalysis.graphtools)
    print("fresh factory knows",da.AnalysisFactory.cache)


    import_ddmodules(identity.modules)

    log("assumptions:",identity.assumptions)

    #A = da.AnalysisFactory.byname(identity.factory_name)


    for i,assumption in enumerate(identity.assumptions):
        log("requested assumption:", assumption)

        if assumption[0] == '':
            log("ATTENTION: dangerous eval from string",assumption[1])
            da.AnalysisFactory.WhatIfCopy('queue emergence %i'%i, eval(assumption[1]))
        else:
            a = da.AnalysisFactory.byname(assumption[0])
            a.import_data(assumption[1])
            da.AnalysisFactory.WhatIfCopy('queue emergence %i'%i, a)

            print(a, "from", assumption)

    A = da.AnalysisFactory.byname(identity.factory_name)
    producable_hashe=A.get_hashe()

    producable_hashe_jsonified=jsonify(hashtools.hashe_replace_object(producable_hashe, None, "None"))
    expected_hashe_jsonified=jsonify(identity.expected_hashe)

    if identity.expected_hashe is None or identity.expected_hashe == "None":
        log("expected hashe verification skipped")
    elif producable_hashe_jsonified != expected_hashe_jsonified:
        log("producable:\n",jsonify(producable_hashe_jsonified),"\n",level="top")
        log("requested:\n",jsonify(expected_hashe_jsonified),level="top")

        try:
            from dataanalysis import displaygraph
            displaygraph.plot_hashe(producable_hashe_jsonified,"producable.png")
            displaygraph.plot_hashe(expected_hashe_jsonified,"expected.png")
        except Exception as e:
            pass

        log_logstash("emergence exception",note="inconsistent_emergence",producable=producable_hashe_jsonified,expected_hashe=expected_hashe_jsonified)

        raise InconsitentEmergence(
            "unable to produce\n"+repr(producable_hashe_jsonified)+"\n while can produce"+repr(expected_hashe_jsonified),
            producable_hashe_jsonified,
            expected_hashe_jsonified,
        )

    log("successfully emerged")

    return A
Example #28
0
    def get(self):
        parser = reqparse.RequestParser()
        parser.add_argument('modules', type=str, help='')
        parser.add_argument('assume', type=str, help='')
        args = parser.parse_args()

        da.reset()
        emerge.import_ddmodules(args['modules'].split(","))

        return da.AnalysisFactory.cache.keys()
Example #29
0
    def get(self):
        parser = reqparse.RequestParser()
        parser.add_argument('modules', type=str, help='')
        parser.add_argument('assume', type=str, help='')
        args = parser.parse_args()

        da.reset()
        emerge.import_ddmodules(args['modules'].split(","))

        return da.AnalysisFactory.cache.keys()
Example #30
0
def test_scw():
    #da.debug_output()
    da.reset()
    import ddosa
    reload(ddosa)

    fa = ddosa.ScWData(input_scwid="066500230010.001")

    fa.get()

    print fa.scwpath
Example #31
0
def test_basic():
    import dataanalysis.core as da
    import dataanalysis.importing as importing

    da.reset()

    import os
    dir_path = os.path.dirname(os.path.realpath(__file__))
    m=importing.load_by_name(['filesystem','moduletest',dir_path+"/ddmoduletest.py"])

    assert not hasattr(m[0], '__dda_module_global_name__')
    assert not hasattr(m[0], '__dda_module_origin__')
Example #32
0
def test_serialize():
    import dataanalysis.core as da
    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)
    S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get()

    ed=S.export_data()

    assert '_da_stored_string_input_s' in ed
Example #33
0
def test_serialize():
    import dataanalysis.core as da
    da.reset()
    da.debug_output()

    import ddmoduletest
    reload(ddmoduletest)
    S = ddmoduletest.SAnalysis(input_s="inputassumedthis").get()

    ed=S.export_data()

    assert '_da_stored_string_input_s' in ed
Example #34
0
def test_gti():
    #    da.debug_output()
    da.reset()
    import ddosa
    reload(ddosa)

    fa = ddosa.ibis_gti(assume=[
        ddosa.ScWData(input_scwid="066500230010.001"),
    ])
    fa.read_caches = []

    fa.get()
def test_delegating_analysis():
    #import dataanalysis
    from dataanalysis import caches
    import dataanalysis.caches.delegating

   # da.debug_output()
    da.reset()

    q_cache=caches.delegating.DelegatingCache()

    da.DataAnalysis.cache.tail_parent(q_cache)

    class A1Analysis(da.DataAnalysis):
        read_caches = [q_cache.__class__]
        cached = True

        def main(self):
            self.data="datadata1"

    class A2Analysis(da.DataAnalysis):
        read_caches = [q_cache.__class__]
        cached = True

        def main(self):
            self.data="datadata2"

    class A3Analysis(da.DataAnalysis):

        def main(self):
            self.data="datadata3"

    class BAnalysis(da.DataAnalysis):
        input_a1 = A1Analysis
        input_a2 = A2Analysis
        input_a3 = A3Analysis

        def main(self):
            self.data="datadata"

    A=BAnalysis()

    print(A.cache.list_parent_stack())

    with pytest.raises(da.AnalysisDelegatedException):
        A.get()

    try:
        A.get()
    except da.AnalysisDelegatedException as e:
        print("resources:",e.resources)

    del da.DataAnalysis.cache.parent
def test_queue_cache_generative():
    from dataanalysis import caches
    import dataanalysis.caches.queue

    da.debug_output()
    da.reset()

    q_cache=caches.queue.QueueCache("./queue_test")
    q_cache.queue.purge()

    assert len(q_cache.queue.list())==0

    define_analysis()

    A=dataanalysis.core.AnalysisFactory['GenerativeAnalysis']

    for name in 'AAnalysis',:
        aA=dataanalysis.core.AnalysisFactory[name].__class__
        aA.cache = q_cache
        aA.produce_disabled=True
        aA.cached = True

    with pytest.raises(da.AnalysisDelegatedException) as e:
        A.get()

    print('AnalysisDelegatedException',e.value,e.value.origin,e.value.hashe)
    assert len(e.value.hashe[1:])==2

    assert len(e.value.source_exceptions)==2
    for se in e.value.source_exceptions:
        print('found multiple AnalysisDelegatedException',se,se.origin)


    print(q_cache.queue.list())
#    assert len(q_cache.queue.list())==2
    assert len(q_cache.queue.list())==1

    # worker part

    f_cache=caches.cache_core.CacheNoIndex()
    #f_cache.parent=q_cache

    define_analysis()

    A = dataanalysis.core.AnalysisFactory['AAnalysis']
    A.produce_disabled = False
    A.cache = f_cache
    A.cached = True

    worker=caches.queue.QueueCacheWorker("./queue_test")

    print(worker.run_once())
Example #37
0
def test_self_assumption():
    from dataanalysis import core as da

    da.debug_output()
    da.reset()

    class AAnalysis(da.DataAnalysis):
        pass

    aa = AAnalysis(assume=AAnalysis(use_arg="arg2", use_nonearg=None)).get()
    assert aa._da_locally_complete

    assert not hasattr(aa,'arg') # current behavoir and it is bad
Example #38
0
    def run_task(self,task):
        task_data=task.task_data
        log("emerging from object_identity",task_data['object_identity'])
        object_identity=da.DataAnalysisIdentity.from_dict(task_data['object_identity'])
        da.reset()

        reload(dataanalysis.graphtools)
        log("fresh factory knows",da.AnalysisFactory.cache)

        log(object_identity)
        A=emerge.emerge_from_identity(object_identity)
        A._da_delegation_allowed=False

        dataanalysis.callback.Callback.set_callback_accepted_classes([da.byname(object_identity.factory_name).__class__])

        for url in task.submission_info['callbacks']:
            log("setting object callback",A,url)
            A.set_callback(url)

        log("emerged object:",A)

        request_root_node=getattr(A, 'request_root_node', False)
        if request_root_node:
            final_state = "done"
        else:
            final_state = "task_done"

        try:
            result=A.get(requested_by=[repr(self)],isolated_directory_key=task.key,isolated_directory_cleanup=True)
            A.raise_stored_exceptions()
        except da.AnalysisException as e:
            A.process_hooks("top", A, message="task complete", state=final_state, task_comment="completed with failure "+repr(e))
            A.process_hooks("top", A, message="analysis exception", exception=repr(e),state="node_analysis_exception")
        except da.AnalysisDelegatedException as delegation_exception:
            final_state = "task_done"
            log("delegated dependencies:",delegation_exception)
            A.process_hooks("top",A,message="task dependencies delegated",state=final_state, task_comment="task dependencies delegated",delegation_exception=repr(delegation_exception))
            raise
        except dqueue.TaskStolen:
            raise
        except Exception as e:
            A.process_hooks("top", A, message="task complete", state=final_state, task_comment="completed with unexpected failure "+repr(e))

            client=get_sentry_client()
            if client is not None:
                client.captureException()

            raise
        else:
            A.process_hooks("top",A,message="task complete",state=final_state, task_comment="completed with success")
            return result
Example #39
0
def test_constructor_input():
    from dataanalysis import analysisfactory
    from dataanalysis import core as da
    da.reset()

    class BAnalysis(da.DataAnalysis):
        def main(self):
            print("test")
            self.data="data"

    class Analysis(da.DataAnalysis):
        #input_b=BAnalysis

        def main(self):
            print("test")
            if hasattr(self,'input_b'):
                self.data = self.input_b.data
            else:
                self.data = 'otherdata'

    A=Analysis()
    A.get()

    assert A.data == 'otherdata'

    A=Analysis(input_b=BAnalysis())

    assert hasattr(A,'input_b')

    da.debug_output()
    print("*" * 80)
    A.promote()
    print("/" * 80)

    A_fn=analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get()
    print(A, A_fn)
    assert A == A_fn

    A_f = analysisfactory.AnalysisFactory.get(A).get()

    print(A,A_f)
    assert A == A_f

    A.get()

    assert A.data == 'data'

    inputs=A.list_inputs()
    assert isinstance(inputs,list)
    assert len(inputs)==1
def test_generate_structures():
    debug_output()
    da.reset()

    class BAnalysis(da.DataAnalysis):
        c = None

        _da_settings = ["c"]

        def main(self):
            print("test", self.c)
            self.data = "data " + repr(self.c)

        def __repr__(self):
            return "[A %s]" % repr(self.c)

    class CAnalysis(da.DataAnalysis):
        c = None

        _da_settings = ["c"]

        def main(self):
            print("test", self.c)
            self.data = "data " + repr(self.c)

        def __repr__(self):
            return "[C %s]" % repr(self.c)


    class AAnalysis(da.DataAnalysis):
        # run_for_hashe=True
        def main(self):
            print("test", self.__class__)
            r = [[BAnalysis(use_c=c),CAnalysis(use_c=c)] for c in range(3)]
            return r

    A = AAnalysis()
    r = A.get()

    print r

    print(r[1].c, r[1].data)

    #assert r[0]

    #assert isinstance(r[0][0],BAnalysis)
    assert r[0].data == 'data 0'
    assert r[2].data == 'data 1'

    assert r[1]._da_locally_complete
Example #41
0
def test_git_list():
    import os
    import dataanalysis.core as da
    import dataanalysis.importing as importing

    da.reset()

    m=importing.load_by_name("git://test",local_gitroot=os.environ.get("TMPDIR","/tmp")+"/git/",remote_git_root='volodymyrss-public')

    assert hasattr(m[0], '__dda_module_global_name__')
    assert hasattr(m[0], '__dda_module_origin__')

    assert m[0].__dda_module_origin__=="git"
    assert m[0].__dda_module_global_name__ == "git://test"
Example #42
0
def test_constructor_input():
    from dataanalysis import analysisfactory
    from dataanalysis import core as da
    da.reset()

    class BAnalysis(da.DataAnalysis):
        def main(self):
            print("test")
            self.data = "data"

    class Analysis(da.DataAnalysis):
        #input_b=BAnalysis

        def main(self):
            print("test")
            if hasattr(self, 'input_b'):
                self.data = self.input_b.data
            else:
                self.data = 'otherdata'

    A = Analysis()
    A.get()

    assert A.data == 'otherdata'

    A = Analysis(input_b=BAnalysis())

    assert hasattr(A, 'input_b')

    da.debug_output()
    print("*" * 80)
    A.promote()
    print("/" * 80)

    A_fn = analysisfactory.AnalysisFactory.get_by_name(A.get_signature()).get()
    print(A, A_fn)
    assert A == A_fn

    A_f = analysisfactory.AnalysisFactory.get(A).get()

    print(A, A_f)
    assert A == A_f

    A.get()

    assert A.data == 'data'

    inputs = A.list_inputs()
    assert isinstance(inputs, list)
    assert len(inputs) == 1
Example #43
0
def test_numpy_to_datafile_recursive():
    from dataanalysis import core as da
    import os
    da.reset()

    import numpy as np
    import pandas as pd

    class Analysis(da.DataAnalysis):
        read_caches=[]

        cached=True

        def main(self):
            print("test")
            self.data = {'a':np.linspace(0,1,100),'b/c!':pd.DataFrame(np.linspace(0,2,200))}


    A = Analysis()
    A.get()

    content=A.export_data()
    cc=A.cache.adopt_datafiles(content)
    assert cc['data']['a'] is None

    #da.TransientCacheInstance.reset()
    da.reset()
    da.debug_output()

    B = Analysis()
    B.produce_disabled=True
    B.read_caches = [A.cache.__class__]
    B.get()

    print(B.data['a'])

    #B.cache.restore(A._da_locally_complete,A)

    assert hasattr(B,'_datafile_data_a')

    assert hasattr(B._datafile_data_a,'adopted_format')
    assert B._datafile_data_a.adopted_format == "numpy"

    print B._datafile_data_a.get_path()
    assert os.path.exists(B._datafile_data_a.get_path())

    assert hasattr(B, 'data')

    assert all(A.data['a'] == B.data['a'])
def test_object_injection_reset():
    from dataanalysis import core as da
    da.reset()
    da.debug_output()

    class AAnalysis(da.DataAnalysis):
#        arg=None
        pass

    A1 = AAnalysis(use_arg="arg1")

    assert  A1.arg == "arg1"
    d1 = A1.export_data()
    print("has data:", d1)
    assert d1['arg'] == 'arg1'

    a1 = A1.serialize()

    print("serilization:",a1)

    da.reset()
    da.debug_output()

    class AAnalysis(da.DataAnalysis):
        pass

    da.AnalysisFactory.inject_serialization(a1)


    print("factory has",da.AnalysisFactory.cache['AAnalysis'])

    aanalysis=da.AnalysisFactory['AAnalysis']
    assert aanalysis.arg == "arg1"

    class Analysis(da.DataAnalysis):
        #cached = True
        input_arg=AAnalysis

        def main(self):
            print("test")
            # time.sleep(3)
            self.data = "data_"+self.input_arg.arg

    A=Analysis()
    A.get()

    print(A.data)

    assert A.data == "data_arg1"
Example #45
0
def test_generate_structures():
    debug_output()
    da.reset()

    class BAnalysis(da.DataAnalysis):
        c = None

        _da_settings = ["c"]

        def main(self):
            print("test", self.c)
            self.data = "data " + repr(self.c)

        def __repr__(self):
            return "[A %s]" % repr(self.c)

    class CAnalysis(da.DataAnalysis):
        c = None

        _da_settings = ["c"]

        def main(self):
            print("test", self.c)
            self.data = "data " + repr(self.c)

        def __repr__(self):
            return "[C %s]" % repr(self.c)

    class AAnalysis(da.DataAnalysis):
        # run_for_hashe=True
        def main(self):
            print("test", self.__class__)
            r = [[BAnalysis(use_c=c), CAnalysis(use_c=c)] for c in range(3)]
            return r

    A = AAnalysis()
    r = A.get()

    print r

    print(r[1].c, r[1].data)

    #assert r[0]

    #assert isinstance(r[0][0],BAnalysis)
    assert r[0].data == 'data 0'
    assert r[2].data == 'data 1'

    assert r[1]._da_locally_complete
Example #46
0
def test_spectra():
    da.reset()
    import ddosa
    reload(ddosa)
    #da.debug_output()

    fa = ddosa.ii_spectra_extract(assume=[
        ddosa.ScWData(input_scwid="066500230010.001"),
        ddosa.SpectraBins(use_rmfpath="/data/resources/rmf_62bands.fits"),
    ])
    fa.read_caches = []

    fa.get()

    assert os.path.exists(fa.spectrum.get_path())
def test_object_injection_reset():
    from dataanalysis import core as da
    da.reset()
    da.debug_output()

    class AAnalysis(da.DataAnalysis):
        #        arg=None
        pass

    A1 = AAnalysis(use_arg="arg1")

    assert A1.arg == "arg1"
    d1 = A1.export_data()
    print("has data:", d1)
    assert d1['arg'] == 'arg1'

    a1 = A1.serialize()

    print("serilization:", a1)

    da.reset()
    da.debug_output()

    class AAnalysis(da.DataAnalysis):
        pass

    da.AnalysisFactory.inject_serialization(a1)

    print("factory has", da.AnalysisFactory.cache['AAnalysis'])

    aanalysis = da.AnalysisFactory['AAnalysis']
    assert aanalysis.arg == "arg1"

    class Analysis(da.DataAnalysis):
        #cached = True
        input_arg = AAnalysis

        def main(self):
            print("test")
            # time.sleep(3)
            self.data = "data_" + self.input_arg.arg

    A = Analysis()
    A.get()

    print(A.data)

    assert A.data == "data_arg1"
Example #48
0
def test_image():
    #
    da.reset()
    import ddosa
    reload(ddosa)

    da.debug_output()

    fa = ddosa.ii_skyimage(assume=[
        ddosa.ScWData(input_scwid="066500230010.001"),
    ])
    fa.read_caches = []

    fa.get()

    print fa.skyima
Example #49
0
def test_imagebins():
    #
    # da.debug_output()
    da.reset()
    import ddosa
    reload(ddosa)

    fa = ddosa.BinEventsImage(assume=[
        ddosa.ScWData(input_scwid="066500230010.001"),
        ddosa.ImageBins(use_autoversion=True, use_ebins=[(25, 45)])
    ])
    fa.read_caches = []

    fa.get()

    print fa.shadow_detector
Example #50
0
def test_one_object():
    from dataanalysis import core as da
    da.reset()

    class Analysis(da.DataAnalysis):
        def main(self):
            print("test")
            self.data = "datacontent"

    A = Analysis()
    A.get()

    assert A.data == 'datacontent'

    j = A.jsonify()

    assert j['data'] == 'datacontent'
def test_factorize_note():

    import dataanalysis.core as da
    da.reset()

    da.AnalysisFactory.note_factorization(dict(a=1,b=2,c=[1,2,{1:2}]))

    class AAnalysis(da.DataAnalysis):
        read_caches=[]
        cached=True

    A=AAnalysis().get()

    fn=A._da_cached_path+"/factorizations.txt"
    assert os.path.exists(fn)

    print(">>>>\n"+open(fn).read()+"<<<<")
Example #52
0
def test_live_delegation_assumptions(ddservice_fixture,app):
    import requests
    import dataanalysis.core as da
    da.reset()
    da.debug_output()

    r=requests.get(ddservice_fixture+url_for('produce', target="BAnalysis", modules="ddmoduletest", assume="AAnalysis.assumed_data=\"clients\""))

    print(r.content)

    factory_r=r.json()

    print(factory_r)

    assert factory_r['data']['data']=='dataB_A:dataAassumed:clients'

    assert len(factory_r)>0
Example #53
0
def test_recursive_datafile():
    from dataanalysis import core as da
    import os
    da.reset()
    da.debug_output()
    class Analysis(da.DataAnalysis):
        cached=True

        def main(self):
            print("test")
            fn="file.txt"
            with open(fn,"w") as f:
                f.write("test")

            fn1="file1.txt"
            with open(fn1,"w") as f:
                f.write("test")

            self.data = [da.DataFile(fn)]
            self.data1 = da.DataFile(fn1)


    A = Analysis()
    A.read_caches = []
    A.get()

    da.reset()

    os.remove("file.txt")
    os.remove("file1.txt")
    B = Analysis()
    B.produce_disabled=True
    B.get()

    #B.cache.restore(A._da_locally_complete,A)

    assert hasattr(B,'data')

    print B.data

    #assert hasattr(B._datafile_data,'adopted_format')
    #assert B._datafile_data.adopted_format == "numpy"

    #print B._datafile_data.get_path()
    assert os.path.exists(B.data1.get_path())
    assert os.path.exists(B.data[0].get_path())
def test_factorize_note():

    import dataanalysis.core as da
    da.reset()

    da.AnalysisFactory.note_factorization(dict(a=1, b=2, c=[1, 2, {1: 2}]))

    class AAnalysis(da.DataAnalysis):
        read_caches = []
        cached = True

    A = AAnalysis().get()

    fn = A._da_cached_path + "/factorizations.txt"
    assert os.path.exists(fn)

    print(">>>>\n" + open(fn).read() + "<<<<")
def test_generate_factory_assumptions_references():
    debug_output()
    da.reset()

    class BAnalysis(da.DataAnalysis):
        c = None

        _da_settings = ["c"]

        def main(self):
            print("test", self.c)
            self.data = "data " + repr(self.c)

        def __repr__(self):
            return "[B %s]" % repr(self.c)

    class AAnalysis(da.DataAnalysis):
        input_b=None

        # run_for_hashe=True
        def main(self):
            print("test", self.__class__)
            self.data="A,b:"+self.input_b.data

        def __repr__(self):
            return "[A %s]" % repr(self.input_b)

    class CAnalysis(da.DataAnalysis):
        input_a=AAnalysis

    da.AnalysisFactory.WhatIfCopy("test",AAnalysis(input_b=BAnalysis,use_version="v2"))

    print(da.AnalysisFactory.cache_assumptions)
    assert len(da.AnalysisFactory.cache_assumptions)==1

    C = CAnalysis(assume=BAnalysis(use_c=1))
    C.get()
    assert C._da_locally_complete
    assert C.input_a.data=="A,b:data 1"

    C = CAnalysis(assume=BAnalysis(use_c=2))
    C.get()
    assert C._da_locally_complete
    assert C.input_a.data=="A,b:data 2"
Example #56
0
def test_pandas_to_datafile():
    from dataanalysis import core as da
    import os
    da.reset()

    import pandas as pd
    import numpy as np

    class Analysis(da.DataAnalysis):
        read_caches=[]

        cached=True

        def main(self):
            print("test")
            self.data = pd.DataFrame()
            self.data['data']=np.linspace(0,1,100)


    A = Analysis()
    A.get()

    #da.TransientCacheInstance.reset()
    da.reset()
    da.debug_output()

    B = Analysis()
    B.produce_disabled=True
    B.read_caches = [A.cache.__class__]
    B.get()

    #B.cache.restore(A._da_locally_complete,A)

    assert hasattr(B,'_datafile_data')

    assert hasattr(B._datafile_data,'adopted_format')
    assert B._datafile_data.adopted_format == "pandas"

    print B._datafile_data.get_path()
    assert os.path.exists(B._datafile_data.get_path())

    assert hasattr(B, 'data')

    assert all(abs(A.data.data - B.data.data)<1e-5)