def synchronize_db_up(context, targets):
""" Syncrhonizes the DB up """
db = context.get_compmake_db()
# first create the volume if it doesn't exist
vol = create_db_volume(db)
# now we need to put all files
keys = []
cq = CacheQueryDB(db)
jobs = set()
jobs.update(targets)
jobs.update(cq.tree(targets))
#print('%d targets, %d jobs' % (len(targets), len(jobs)))
# XXX: not all jobs
for job_id in jobs:
resources = [job2jobargskey, job2userobjectkey,
job2cachekey, job2key]
for r in resources:
key = r(job_id)
if key in db:
keys.append(key)
#print('Found %s files to upload' % len(keys))
# Shadow storage
db2 = StorageFilesystem(basepath=vol.mount_path)
already = set([os.path.basename(x['path']) for x in vol.ls('.')])
filename2contents = {}
#print('obtained: %r' % already)
for key in keys:
f = db.filename_for_key(key)
f2 = db2.filename_for_key(key)
local_path = f
remote_path = os.path.relpath(f2, db2.basepath)
if remote_path in already:
#print('skipping %r' % local_path)
continue
size = os.stat(local_path).st_size
use_compact = size < 6*1024
if use_compact:
with open(local_path) as f:
filename2contents[f2] = f.read()
else:
#print('%s -> %s' % (local_path, remote_path))
assert os.path.join(db2.basepath, remote_path) == f2
vol.put_file(local_path, remote_path, target_mode=None)
import multyvac
multyvac_job_id = multyvac.submit(copy_files, filename2contents,
_vol=[vol.name])
multyvac_job = multyvac.get(multyvac_job_id)
multyvac_job.get_result()
return vol, db2
