def test_sync_tool_init(tmpdir): td = tmpdir.mkdir('sync-dest') aoi = read_fixture('aoi.geojson') st = _SyncTool(client, td.strpath, aoi, 'ortho', ('visual', 'analytic')) search_json = json.loads(read_fixture('search.geojson')) response = MagicMock(spec=Scenes) response.get.return_value = search_json client.get_scenes_list.return_value = response # init w/ no limit should return count from response count = st.init() # confusing but we'll download 2 products one for each scene assert search_json['count'] * 2 == count # expect limiting to 10 ids count = st.init(limit=10) # still replies with total jobs despite the limit assert search_json['count'] * 2 == count # this tracks the internal number of ids, still 10 assert 10 == st._scene_count # create a stored 'latest' date and ensure it's used latest = '2015-01-25T18:29:09.155671+00:00' td.join('sync.json').write(json.dumps({'latest': latest})) st.init() args = client.get_scenes_list.call_args # args are ((arguments),(kw)) assert args[1]['published.gt'] == latest td.remove(True)
def test_sync_tool_init(tmpdir): td = tmpdir.mkdir('sync-dest') aoi = read_fixture('aoi.geojson') st = _SyncTool(client, td.strpath, aoi, 'ortho', ('visual', 'analytic')) search_json = json.loads(read_fixture('search.geojson')) response = MagicMock(spec=Scenes) response.get.return_value = search_json client.get_scenes_list.return_value = response # init w/ no limit should return count from response count = st.init() # confusing but we'll download 2 products one for each scene assert search_json['count'] * 2 == count # expect limiting to 10 ids count = st.init(limit=10) # still replies with total jobs despite the limit assert search_json['count'] * 2 == count # this tracks the internal number of ids, still 10 assert 10 == st._scene_count # create a stored 'latest' date and ensure it's used latest = '2015-01-25T18:29:09.155671+00:00' td.join('sync.json').write(json.dumps({ 'latest': latest })) st.init() args = client.get_scenes_list.call_args # args are ((arguments),(kw)) assert args[1]['published.gt'] == latest td.remove(True)
def test_sync_tool_sync(tmpdir): td = tmpdir.mkdir('sync-dest') aoi = read_fixture('aoi.geojson') st = _SyncTool(client, td.strpath, aoi, 'ortho', ('visual',)) search_json = json.loads(read_fixture('search.geojson')) class Page: def get(self): return search_json class FakeScenes: def items_iter(self, limit): return (f for f in Page().get()['features'][:limit]) def iter(self): return iter([Page()]) class FakeBody: def __init__(self, val, name): self.val = val self.name = name def __len__(self): return len(self.val) class FakeGeoTiff: def __init__(self, val): self.body = FakeBody(val, val) self.name = val def await(self): pass def get_body(self): return self.body def write(self, f, cb): with open(f, 'w') as fp: fp.write(self.body.val) def __len__(self): return 371
def sync(destination, workspace, scene_type, limit, dryrun, products): '''Synchronize a directory to a specified AOI or workspace''' aoi = None filters = {'workspace': workspace} if 'all' in products: products = ORTHO_PRODUCTS else: products = products or ('visual',) sync_tool = _SyncTool(client(), destination, aoi, scene_type, products, **filters) try: to_fetch = sync_tool.init(limit) except ValueError as ve: raise click.ClickException(str(ve)) click.echo('total scene products to fetch: %s' % to_fetch) if limit > -1: click.echo('limiting to %s' % limit) if dryrun: click.echo('would download:') for scene in sync_tool.get_scenes_to_sync(): click.echo(scene['id']) return def progress_callback(name, remaining): click.echo('downloaded %s, remaining %s' % (name, remaining)) start_time = time.time() summary = sync_tool.sync(progress_callback) if summary.transferred: summarize_throughput(summary.transferred, start_time)
def sync(destination, workspace, scene_type, limit, dryrun, products): '''Synchronize a directory to a specified AOI or workspace''' aoi = None filters = {'workspace': workspace} if 'all' in products: products = ORTHO_PRODUCTS else: products = products or ('visual', ) sync_tool = _SyncTool(client(), destination, aoi, scene_type, products, **filters) try: to_fetch = sync_tool.init(limit) except ValueError as ve: raise click.ClickException(str(ve)) click.echo('total scene products to fetch: %s' % to_fetch) if limit > -1: click.echo('limiting to %s' % limit) if dryrun: click.echo('would download:') for scene in sync_tool.get_scenes_to_sync(): click.echo(scene['id']) return def progress_callback(name, remaining): click.echo('downloaded %s, remaining %s' % (name, remaining)) start_time = time.time() summary = sync_tool.sync(progress_callback) if summary.transferred: summarize_throughput(summary.transferred, start_time)
def test_sync_tool(tmpdir): # test non-existing destination try: _SyncTool(client, 'should-not-exist', None, None, None) except ValueError as ve: assert str(ve) == 'destination must exist and be a directory' # test existing destination, no aoi.geojson td = tmpdir.mkdir('sync-dest') try: _SyncTool(client, td.strpath, None, None, None) except ValueError as ve: assert str(ve) == 'no aoi provided and no aoi.geojson file' # test existing destination, invalid aoi.geojson aoi_file = td.join('aoi.geojson') aoi_file.write('not geojson') try: _SyncTool(client, td.strpath, None, None, None) except ValueError as ve: assert str(ve) == '%s does not contain valid JSON' % aoi_file td.remove(True)
def test_sync_tool_sync(tmpdir): td = tmpdir.mkdir('sync-dest') aoi = read_fixture('aoi.geojson') st = _SyncTool(client, td.strpath, aoi, 'ortho', ('visual', )) search_json = json.loads(read_fixture('search.geojson')) class Page: def get(self): return search_json class FakeScenes: def items_iter(self, limit): return (f for f in Page().get()['features'][:limit]) def iter(self): return iter([Page()]) class FakeBody: def __init__(self, val, name): self.val = val self.name = name def __len__(self): return len(self.val) class FakeGeoTiff: def __init__(self, val): self.body = FakeBody(val, val) self.name = val def await (self): pass def get_body(self): return self.body def write(self, f, cb): with open(f, 'w') as fp: fp.write(self.body.val) def __len__(self): return 371 st._scenes = FakeScenes() # record invocations to a callback (could be mock?) called_back = [] def callback(name, remaining): called_back.append((name, remaining)) # base - no items remaining, does nothing st._scene_count = 0 summary = st.sync(callback) assert summary.latest is None assert summary.transferred is 0 # 5 items to get items = 5 client.fetch_scene_geotiffs.reset_mock() st._scene_count = items responses = [FakeGeoTiff(str(i)) for i in range(st._scene_count)] # because process is normally async, as a sideeffect of this mock getting # called, we have to dispatch the callback def run_callbacks(ids, scene_type, product, callback): resp = responses.pop(0) callback(resp) return DEFAULT client.fetch_scene_geotiffs.side_effect = run_callbacks st.sync(callback) first_items = search_json['features'][:items] # should be 5 metadata files files = [td.join('%s_metadata.json' % f['id']) for f in first_items] assert len(files) == items assert all([f.exists() for f in files]) # and 5 'scenes' files = [td.join('%s' % f) for f in range(items)] assert len(files) == items assert all([f.exists() for f in files]) # and tiff requests were made args = client.fetch_scene_geotiffs.call_args_list # implementation detail - because we're making requests separately, # fetch_scene_geotiffs will be called once for each id (instead of in bulk) assert items == len(args) assert [a[0] for a in args] == \ [([f['id']], 'ortho', 'visual') for f in first_items] # callbacks should be made - arguments are 'tiff name', remaining assert called_back == [(str(i), 4 - i) for i in range(5)] td.remove(True)