def pull(tag='<most recent>', force=False):
'''
Pull atlas from s3.
'''
import pomagma.io.s3
if not os.path.exists(pomagma.util.DATA):
os.makedirs(pomagma.util.DATA)
with pomagma.util.chdir(pomagma.util.DATA):
master = 'atlas'
if os.path.exists(master):
if force:
shutil.rmtree(master)
else:
raise IOError('atlas exists; first remove atlas')
if tag == '<most recent>':
snapshot = max(list_s3_atlases())
else:
snapshot = 'atlas.{}'.format(tag)
assert match_atlas(snapshot), 'invalid tag: {}'.format(tag)
print 'pulling {}'.format(snapshot)
pomagma.io.s3.pull('{}/'.format(snapshot))
blobs = [
os.path.join('blob', blob)
for blob in atlas.find_used_blobs(snapshot)
]
print 'pulling {} blobs'.format(len(blobs))
pomagma.io.s3.pull(*blobs)
pomagma.io.blobstore.validate_blobs()
pomagma.io.s3.snapshot(snapshot, master) # only after validation
def push(tag=default_tag, force=False):
'''
Push atlas to s3.
'''
import pomagma.io.s3
pomagma.io.blobstore.validate_blobs()
with pomagma.util.chdir(pomagma.util.DATA):
master = 'atlas'
assert os.path.exists(master), 'atlas does not exist'
snapshot = 'atlas.{}'.format(tag)
assert match_atlas(snapshot), 'invalid tag: {}'.format(tag)
if snapshot in list_s3_atlases() and not force:
raise IOError('snapshot already exists: {}'.format(snapshot))
print 'pushing {}'.format(snapshot)
pomagma.io.s3.snapshot(master, snapshot)
blobs = [
os.path.join('blob', blob)
for blob in atlas.find_used_blobs(snapshot)
]
pomagma.io.s3.push(snapshot, *blobs)