def process(source, scene_root, verbose=False, clean=False, list_file=None,
overwrite=False):
if pusher.check_existance(scene_root):
print 'Scene %s already exists on destination bucket.' % scene_root
if not overwrite:
return collect_missing_entry(scene_root, verbose, clean, list_file)
if verbose:
print 'Processing scene: %s' % scene_root
scene_dict = {}
local_tarfile = puller.pull(source, scene_root, scene_dict,
verbose=verbose)
local_dir = splitter.split(scene_root, local_tarfile, verbose=verbose)
scene_info.add_mtl_info(scene_dict, scene_root, local_dir)
thumbnailer.thumbnail(scene_root, local_dir, verbose=verbose)
scene_index_maker.make_index(scene_root, local_dir, verbose=verbose)
pusher.push(scene_root, local_dir, scene_dict, verbose=verbose, overwrite=overwrite)
if clean:
os.unlink(local_tarfile)
shutil.rmtree(local_dir)
if list_file:
scene_info.append_scene_line(list_file, scene_dict)
return scene_dict
def process(source, scene_root, verbose=False):
s3_path = 'tarq/%s.tar.gz' % scene_root
if pusher.check_file_existance(s3_path):
raise Exception('%s already exists!' % s3_path)
scene_dict = {}
local_tarfile = puller.pull(source, scene_root, scene_dict,
verbose=verbose)
pusher.push_file(local_tarfile, s3_path, verbose=verbose)
os.unlink(local_tarfile)
def process(source, scene_root, verbose=False):
s3_path = 'tarq/%s.tar.gz' % scene_root
if pusher.check_file_existance(s3_path):
raise Exception('%s already exists!' % s3_path)
scene_dict = {}
local_tarfile = puller.pull(source,
scene_root,
scene_dict,
verbose=verbose)
pusher.push_file(local_tarfile, s3_path, verbose=verbose)
os.unlink(local_tarfile)
def process(source,
scene_root,
verbose=False,
clean=False,
list_file=None,
overwrite=False):
if pusher.check_existance(scene_root):
print 'Scene %s already exists on destination bucket.' % scene_root
if not overwrite:
return collect_missing_entry(scene_root, verbose, clean, list_file)
if verbose:
print 'Processing scene: %s' % scene_root
scene_dict = {}
local_tarfile = puller.pull(source,
scene_root,
scene_dict,
verbose=verbose)
try:
local_dir = splitter.split(scene_root, local_tarfile, verbose=verbose)
except:
if source == 's3queue':
# Remove problematic scenes from the queue directory
puller_s3queue.clean_queued_tarfile(scene_root)
return
scene_info.add_mtl_info(scene_dict, scene_root, local_dir)
thumbnailer.thumbnail(scene_root, local_dir, verbose=verbose)
scene_index_maker.make_index(scene_root, local_dir, verbose=verbose)
pusher.push(scene_root,
local_dir,
scene_dict,
verbose=verbose,
overwrite=overwrite)
if clean:
os.unlink(local_tarfile)
shutil.rmtree(local_dir)
if list_file:
scene_info.append_scene_line(list_file, scene_dict)
return scene_dict