def process(source, scene_root, verbose=False, clean=False, list_file=None,
            overwrite=False):

    if pusher.check_existance(scene_root):
        print 'Scene %s already exists on destination bucket.' % scene_root
        if not overwrite:
            return collect_missing_entry(scene_root, verbose, clean, list_file)

    if verbose:
        print 'Processing scene: %s' % scene_root
        
    scene_dict = {}
    
    local_tarfile = puller.pull(source, scene_root, scene_dict,
                                verbose=verbose)

    local_dir = splitter.split(scene_root, local_tarfile, verbose=verbose)

    scene_info.add_mtl_info(scene_dict, scene_root, local_dir)
    
    thumbnailer.thumbnail(scene_root, local_dir, verbose=verbose)
    scene_index_maker.make_index(scene_root, local_dir, verbose=verbose)
    pusher.push(scene_root, local_dir, scene_dict, verbose=verbose, overwrite=overwrite)

    if clean:
        os.unlink(local_tarfile)
        shutil.rmtree(local_dir)

    if list_file:
        scene_info.append_scene_line(list_file, scene_dict)

    return scene_dict
예제 #2
0
def process(source,
            scene_root,
            verbose=False,
            clean=False,
            list_file=None,
            overwrite=False):

    if pusher.check_existance(scene_root):
        print 'Scene %s already exists on destination bucket.' % scene_root
        if not overwrite:
            return collect_missing_entry(scene_root, verbose, clean, list_file)

    if verbose:
        print 'Processing scene: %s' % scene_root

    scene_dict = {}

    local_tarfile = puller.pull(source,
                                scene_root,
                                scene_dict,
                                verbose=verbose)

    try:
        local_dir = splitter.split(scene_root, local_tarfile, verbose=verbose)
    except:
        if source == 's3queue':
            # Remove problematic scenes from the queue directory
            puller_s3queue.clean_queued_tarfile(scene_root)
            return

    scene_info.add_mtl_info(scene_dict, scene_root, local_dir)

    thumbnailer.thumbnail(scene_root, local_dir, verbose=verbose)
    scene_index_maker.make_index(scene_root, local_dir, verbose=verbose)
    pusher.push(scene_root,
                local_dir,
                scene_dict,
                verbose=verbose,
                overwrite=overwrite)

    if clean:
        os.unlink(local_tarfile)
        shutil.rmtree(local_dir)

    if list_file:
        scene_info.append_scene_line(list_file, scene_dict)

    return scene_dict
예제 #3
0
def collect_missing_entry(scene_root, verbose, clean, list_file):
    scene_dict = {}

    mtl_file = scene_root + '_MTL.txt'
    mtl_url = pusher.scene_url(scene_root) + '/' + mtl_file

    rv = requests.get(mtl_url)
    rv.raise_for_status()

    with open(mtl_file, 'wb') as f:
        for chunk in rv.iter_content(chunk_size=1000000):
            f.write(chunk)

    scene_info.add_mtl_info(scene_dict, scene_root, '.')
    scene_dict['download_url'] = pusher.scene_url(scene_root) + '/index.html'

    if list_file:
        scene_info.append_scene_line(list_file, scene_dict)

    if clean:
        os.unlink(mtl_file)

    return scene_dict
def collect_missing_entry(scene_root, verbose, clean, list_file):
    scene_dict = {}

    mtl_file = scene_root + '_MTL.txt'
    mtl_url = pusher.scene_url(scene_root) + '/' + mtl_file

    rv = requests.get(mtl_url)
    rv.raise_for_status()

    with open(mtl_file, 'wb') as f:
        for chunk in rv.iter_content(chunk_size=1000000):
            f.write(chunk)
            
    scene_info.add_mtl_info(scene_dict, scene_root, '.')
    scene_dict['download_url'] = pusher.scene_url(scene_root) + '/index.html'
    
    if list_file:
        scene_info.append_scene_line(list_file, scene_dict)

    if clean:
        os.unlink(mtl_file)
        
    return scene_dict