Exemple #1
0
        def wrapper_data(func, *args, **kwargs):
            start = time.time()
            # retreive intermediate data if it exist and execute the act otherwise
            id_task = get_task_id(func, *args, **kwargs)

            dname = hashlib.md5(id_task).hexdigest()
            #     cache_data_path = os.path.join(cp.cache_dir, "data", dname)

            # download data from refs:
            for data in args:
                if isinstance(data, Data):
                    if not data.value:
                        try:
                            path_to_data = index.is_in(data.id)
                            tmp_data = load_intermediate_data(
                                dname=None,
                                data_path=path_to_data,
                                method=cp.method,
                                sftp_client=cc.sftp_client)
                            data.value = tmp_data.value
                        except:
                            pass
            # Compute
            print("COMPUTE DATA")
            tmp_data = execute(func, *args, **kwargs)

            dtime = time.time()

            #     dsize=0
            dsize = getsize(tmp_data.value)
            tmp_data.size = dsize
            tmp_data.time = time.time() - start
            tmp_data.cpu_time = dtime - start
            tmp_data.dltime = time.time() - dtime
            return tmp_data
Exemple #2
0
        def wrapper_data(func, *args, **kwargs):
            start = time.time()
            # retreive intermediate data if it exist and execute the act otherwise
            id_task = get_task_id(func, *args, **kwargs)
            dname = hashlib.md5(id_task).hexdigest()
            p = re.compile('\(\d*\,')
            m = p.findall(id_task)
            workflow = m[0][1:-1]

            intermediate_data_path = index.is_in(id_task)
            if intermediate_data_path:
                # GIVE REF! Not data
                print("LOAD DATA", intermediate_data_path)
                dtime = time.time()
                fake_data = load_intermediate_data_local(
                    data_path=intermediate_data_path)
                tmp_data = Data(id=id_task, value=fake_data, workflow=workflow)
                end = time.time()
            else:
                # download data from refs:
                for data in args:
                    if isinstance(data, Data):
                        if not data.value:
                            try:
                                path_to_data = index.is_in(data.id)
                                tmp_data = load_intermediate_data_local(
                                    data_path=path_to_data)
                                data.value = tmp_data.value
                                data.workflow = workflow
                            except:
                                pass

                    # Compute
                    print("COMPUTE DATA")
                    tmp_data = execute(func, *args, **kwargs)
                    dtime = time.time()
                    end = time.time()

                # print('ADD to cache')
                # # get name of file:
                # pathcache = "/home/ubuntu/openalea/"
                # dname = hashlib.md5(str(id_task)).hexdigest()
                # pathcache = os.path.join(pathcache, dname)
                # write_intermediate_data_local(tmp_data, pathcache)
                # index.add_data(data_id=str(id_task), path=pathcache)
            # fake write:
            print('rewrite')
            pathcache = "/home/ubuntu/openalea/"
            dname = hashlib.md5(str(id_task)).hexdigest()
            pathcache = os.path.join(pathcache, dname)
            write_intermediate_data_local(tmp_data, pathcache)
            #     dsize=0
            dsize = getsize(tmp_data.value)
            tmp_data.size = dsize
            tmp_data.time = time.time() - start
            tmp_data.cpu_time = dtime - start
            tmp_data.dltime = end - dtime
            return tmp_data