def pull(tag='<most recent>', force=False): ''' Pull atlas from s3. ''' import pomagma.io.s3 if not os.path.exists(pomagma.util.DATA): os.makedirs(pomagma.util.DATA) with pomagma.util.chdir(pomagma.util.DATA): master = 'atlas' if os.path.exists(master): if force: shutil.rmtree(master) else: raise IOError('atlas exists; first remove atlas') if tag == '<most recent>': snapshot = max(list_s3_atlases()) else: snapshot = 'atlas.{}'.format(tag) assert match_atlas(snapshot), 'invalid tag: {}'.format(tag) print 'pulling {}'.format(snapshot) pomagma.io.s3.pull('{}/'.format(snapshot)) blobs = [ os.path.join('blob', blob) for blob in atlas.find_used_blobs(snapshot) ] print 'pulling {} blobs'.format(len(blobs)) pomagma.io.s3.pull(*blobs) pomagma.io.blobstore.validate_blobs() pomagma.io.s3.snapshot(snapshot, master) # only after validation
def push(tag=default_tag, force=False): ''' Push atlas to s3. ''' import pomagma.io.s3 pomagma.io.blobstore.validate_blobs() with pomagma.util.chdir(pomagma.util.DATA): master = 'atlas' assert os.path.exists(master), 'atlas does not exist' snapshot = 'atlas.{}'.format(tag) assert match_atlas(snapshot), 'invalid tag: {}'.format(tag) if snapshot in list_s3_atlases() and not force: raise IOError('snapshot already exists: {}'.format(snapshot)) print 'pushing {}'.format(snapshot) pomagma.io.s3.snapshot(master, snapshot) blobs = [ os.path.join('blob', blob) for blob in atlas.find_used_blobs(snapshot) ] pomagma.io.s3.push(snapshot, *blobs)