def process(source, scene_root, verbose=False, clean=False, list_file=None,
overwrite=False):
if pusher.check_existance(scene_root):
print 'Scene %s already exists on destination bucket.' % scene_root
if not overwrite:
return collect_missing_entry(scene_root, verbose, clean, list_file)
if verbose:
print 'Processing scene: %s' % scene_root
scene_dict = {}
local_tarfile = puller.pull(source, scene_root, scene_dict,
verbose=verbose)
local_dir = splitter.split(scene_root, local_tarfile, verbose=verbose)
scene_info.add_mtl_info(scene_dict, scene_root, local_dir)
thumbnailer.thumbnail(scene_root, local_dir, verbose=verbose)
scene_index_maker.make_index(scene_root, local_dir, verbose=verbose)
pusher.push(scene_root, local_dir, scene_dict, verbose=verbose, overwrite=overwrite)
if clean:
os.unlink(local_tarfile)
shutil.rmtree(local_dir)
if list_file:
scene_info.append_scene_line(list_file, scene_dict)
return scene_dict
def process(source,
scene_root,
verbose=False,
clean=False,
list_file=None,
overwrite=False):
if pusher.check_existance(scene_root):
print 'Scene %s already exists on destination bucket.' % scene_root
if not overwrite:
return collect_missing_entry(scene_root, verbose, clean, list_file)
if verbose:
print 'Processing scene: %s' % scene_root
scene_dict = {}
local_tarfile = puller.pull(source,
scene_root,
scene_dict,
verbose=verbose)
try:
local_dir = splitter.split(scene_root, local_tarfile, verbose=verbose)
except:
if source == 's3queue':
# Remove problematic scenes from the queue directory
puller_s3queue.clean_queued_tarfile(scene_root)
return
scene_info.add_mtl_info(scene_dict, scene_root, local_dir)
thumbnailer.thumbnail(scene_root, local_dir, verbose=verbose)
scene_index_maker.make_index(scene_root, local_dir, verbose=verbose)
pusher.push(scene_root,
local_dir,
scene_dict,
verbose=verbose,
overwrite=overwrite)
if clean:
os.unlink(local_tarfile)
shutil.rmtree(local_dir)
if list_file:
scene_info.append_scene_line(list_file, scene_dict)
return scene_dict
def collect_missing_entry(scene_root, verbose, clean, list_file):
scene_dict = {}
mtl_file = scene_root + '_MTL.txt'
mtl_url = pusher.scene_url(scene_root) + '/' + mtl_file
rv = requests.get(mtl_url)
rv.raise_for_status()
with open(mtl_file, 'wb') as f:
for chunk in rv.iter_content(chunk_size=1000000):
f.write(chunk)
scene_info.add_mtl_info(scene_dict, scene_root, '.')
scene_dict['download_url'] = pusher.scene_url(scene_root) + '/index.html'
if list_file:
scene_info.append_scene_line(list_file, scene_dict)
if clean:
os.unlink(mtl_file)
return scene_dict
def collect_missing_entry(scene_root, verbose, clean, list_file):
scene_dict = {}
mtl_file = scene_root + '_MTL.txt'
mtl_url = pusher.scene_url(scene_root) + '/' + mtl_file
rv = requests.get(mtl_url)
rv.raise_for_status()
with open(mtl_file, 'wb') as f:
for chunk in rv.iter_content(chunk_size=1000000):
f.write(chunk)
scene_info.add_mtl_info(scene_dict, scene_root, '.')
scene_dict['download_url'] = pusher.scene_url(scene_root) + '/index.html'
if list_file:
scene_info.append_scene_line(list_file, scene_dict)
if clean:
os.unlink(mtl_file)
return scene_dict