def wrapper_data(func, *args, **kwargs):
start = time.time()
# retreive intermediate data if it exist and execute the act otherwise
id_task = get_task_id(func, *args, **kwargs)
dname = hashlib.md5(id_task).hexdigest()
# cache_data_path = os.path.join(cp.cache_dir, "data", dname)
# download data from refs:
for data in args:
if isinstance(data, Data):
if not data.value:
try:
path_to_data = index.is_in(data.id)
tmp_data = load_intermediate_data(
dname=None,
data_path=path_to_data,
method=cp.method,
sftp_client=cc.sftp_client)
data.value = tmp_data.value
except:
pass
# Compute
print("COMPUTE DATA")
tmp_data = execute(func, *args, **kwargs)
dtime = time.time()
# dsize=0
dsize = getsize(tmp_data.value)
tmp_data.size = dsize
tmp_data.time = time.time() - start
tmp_data.cpu_time = dtime - start
tmp_data.dltime = time.time() - dtime
return tmp_data
def wrapper_data(func, *args, **kwargs):
start = time.time()
# retreive intermediate data if it exist and execute the act otherwise
id_task = get_task_id(func, *args, **kwargs)
dname = hashlib.md5(id_task).hexdigest()
p = re.compile('\(\d*\,')
m = p.findall(id_task)
workflow = m[0][1:-1]
intermediate_data_path = index.is_in(id_task)
if intermediate_data_path:
# GIVE REF! Not data
print("LOAD DATA", intermediate_data_path)
dtime = time.time()
fake_data = load_intermediate_data_local(
data_path=intermediate_data_path)
tmp_data = Data(id=id_task, value=fake_data, workflow=workflow)
end = time.time()
else:
# download data from refs:
for data in args:
if isinstance(data, Data):
if not data.value:
try:
path_to_data = index.is_in(data.id)
tmp_data = load_intermediate_data_local(
data_path=path_to_data)
data.value = tmp_data.value
data.workflow = workflow
except:
pass
# Compute
print("COMPUTE DATA")
tmp_data = execute(func, *args, **kwargs)
dtime = time.time()
end = time.time()
# print('ADD to cache')
# # get name of file:
# pathcache = "/home/ubuntu/openalea/"
# dname = hashlib.md5(str(id_task)).hexdigest()
# pathcache = os.path.join(pathcache, dname)
# write_intermediate_data_local(tmp_data, pathcache)
# index.add_data(data_id=str(id_task), path=pathcache)
# fake write:
print('rewrite')
pathcache = "/home/ubuntu/openalea/"
dname = hashlib.md5(str(id_task)).hexdigest()
pathcache = os.path.join(pathcache, dname)
write_intermediate_data_local(tmp_data, pathcache)
# dsize=0
dsize = getsize(tmp_data.value)
tmp_data.size = dsize
tmp_data.time = time.time() - start
tmp_data.cpu_time = dtime - start
tmp_data.dltime = end - dtime
return tmp_data