def wrapper_data(func, *args, **kwargs): start = time.time() # retreive intermediate data if it exist and execute the act otherwise id_task = get_task_id(func, *args, **kwargs) dname = hashlib.md5(id_task).hexdigest() # cache_data_path = os.path.join(cp.cache_dir, "data", dname) # download data from refs: for data in args: if isinstance(data, Data): if not data.value: try: path_to_data = index.is_in(data.id) tmp_data = load_intermediate_data( dname=None, data_path=path_to_data, method=cp.method, sftp_client=cc.sftp_client) data.value = tmp_data.value except: pass # Compute print("COMPUTE DATA") tmp_data = execute(func, *args, **kwargs) dtime = time.time() # dsize=0 dsize = getsize(tmp_data.value) tmp_data.size = dsize tmp_data.time = time.time() - start tmp_data.cpu_time = dtime - start tmp_data.dltime = time.time() - dtime return tmp_data
def wrapper_data(func, *args, **kwargs): start = time.time() # retreive intermediate data if it exist and execute the act otherwise id_task = get_task_id(func, *args, **kwargs) dname = hashlib.md5(id_task).hexdigest() p = re.compile('\(\d*\,') m = p.findall(id_task) workflow = m[0][1:-1] intermediate_data_path = index.is_in(id_task) if intermediate_data_path: # GIVE REF! Not data print("LOAD DATA", intermediate_data_path) dtime = time.time() fake_data = load_intermediate_data_local( data_path=intermediate_data_path) tmp_data = Data(id=id_task, value=fake_data, workflow=workflow) end = time.time() else: # download data from refs: for data in args: if isinstance(data, Data): if not data.value: try: path_to_data = index.is_in(data.id) tmp_data = load_intermediate_data_local( data_path=path_to_data) data.value = tmp_data.value data.workflow = workflow except: pass # Compute print("COMPUTE DATA") tmp_data = execute(func, *args, **kwargs) dtime = time.time() end = time.time() # print('ADD to cache') # # get name of file: # pathcache = "/home/ubuntu/openalea/" # dname = hashlib.md5(str(id_task)).hexdigest() # pathcache = os.path.join(pathcache, dname) # write_intermediate_data_local(tmp_data, pathcache) # index.add_data(data_id=str(id_task), path=pathcache) # fake write: print('rewrite') pathcache = "/home/ubuntu/openalea/" dname = hashlib.md5(str(id_task)).hexdigest() pathcache = os.path.join(pathcache, dname) write_intermediate_data_local(tmp_data, pathcache) # dsize=0 dsize = getsize(tmp_data.value) tmp_data.size = dsize tmp_data.time = time.time() - start tmp_data.cpu_time = dtime - start tmp_data.dltime = end - dtime return tmp_data