def handle_root(dataset_prefix: str) -> None: """Handle writing a new dataset to the root catalog""" results = S3_CLIENT.list_objects( Bucket=ResourceName.STORAGE_BUCKET_NAME.value, Prefix=CATALOG_KEY) # create root catalog if it doesn't exist if CONTENTS_KEY in results: root_catalog = Catalog.from_file( f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}/{CATALOG_KEY}" ) else: root_catalog = Catalog( id=ROOT_CATALOG_ID, title=ROOT_CATALOG_TITLE, description=ROOT_CATALOG_DESCRIPTION, catalog_type=CatalogType.SELF_CONTAINED, ) root_catalog.set_self_href( f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}/{CATALOG_KEY}" ) dataset_path = f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}/{dataset_prefix}" dataset_catalog = Catalog.from_file(f"{dataset_path}/{CATALOG_KEY}") root_catalog.add_child(dataset_catalog, strategy=GeostoreSTACLayoutStrategy()) root_catalog.normalize_hrefs( f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}", strategy=GeostoreSTACLayoutStrategy(), ) root_catalog.save(catalog_type=CatalogType.SELF_CONTAINED)
def parse_stac(stac_uri: str) -> List[dict]: """Parse a STAC catalog JSON file to extract label URIs, images URIs, and AOIs. Note: This has been tested to be compatible with STAC version 1.0.0 but not any other versions. Args: stac_uri (str): Path to the STAC catalog JSON file. Returns: List[dict]: A lsit of dicts with keys: "label_uri", "image_uris", "label_bbox", "image_bbox", "bboxes_intersect", and "aoi_geometry". Each dict corresponds to one label item and its associated image assets in the STAC catalog. """ setup_stac_io() cat = Catalog.from_file(stac_uri) version: str = cat.to_dict()['stac_version'] if not version.startswith('1.0'): log.warning(f'Parsing is not guaranteed to work correctly for ' f'STAC version != 1.0.*. Found version: {version}.') cat.make_all_asset_hrefs_absolute() label_items = [item for item in cat.get_all_items() if is_label_item(item)] image_items = [get_linked_image_item(item) for item in label_items] if len(label_items) == 0: raise ValueError('Unable to find any label items in STAC catalog.') out = [] for label_item, image_item in zip(label_items, image_items): label_uri: str = list(label_item.assets.values())[0].href label_bbox = box(*label_item.bbox) aoi_geometry: Optional[dict] = label_item.geometry if image_item is not None: image_assets = [ asset for asset in image_item.get_assets().values() if 'image' in asset.media_type ] image_uris = [asset.href for asset in image_assets] image_bbox = box(*image_item.bbox) bboxes_intersect = label_bbox.intersects(image_bbox) else: image_uris = [] image_bbox = None bboxes_intersect = False out.append({ 'label_uri': label_uri, 'image_uris': image_uris, 'label_bbox': label_bbox, 'image_bbox': image_bbox, 'bboxes_intersect': bboxes_intersect, 'aoi_geometry': aoi_geometry }) return out
def test_map_assets_single(self): changed_asset = 'd43bead8-e3f8-4c51-95d6-e24e750a402b' def asset_mapper(key, asset): if key == changed_asset: asset.title = 'NEW TITLE' return asset with TemporaryDirectory() as tmp_dir: catalog = TestCases.test_case_2() new_cat = catalog.map_assets(asset_mapper) new_cat.normalize_hrefs(os.path.join(tmp_dir, 'cat')) new_cat.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED) result_cat = Catalog.from_file( os.path.join(tmp_dir, 'cat', 'catalog.json')) found = False for item in result_cat.get_all_items(): for key, asset in item.assets.items(): if key == changed_asset: found = True self.assertEqual(asset.title, 'NEW TITLE') else: self.assertNotEqual(asset.title, 'NEW TITLE') self.assertTrue(found)
def test_map_assets_tup(self): changed_assets = [] def asset_mapper(key, asset): if 'geotiff' in asset.media_type: asset.title = 'NEW TITLE' changed_assets.append(key) return ('{}-modified'.format(key), asset) else: return asset with TemporaryDirectory() as tmp_dir: catalog = TestCases.test_case_2() new_cat = catalog.map_assets(asset_mapper) new_cat.normalize_hrefs(os.path.join(tmp_dir, 'cat')) new_cat.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED) result_cat = Catalog.from_file( os.path.join(tmp_dir, 'cat', 'catalog.json')) found = False not_found = False for item in result_cat.get_all_items(): for key, asset in item.assets.items(): if key.replace('-modified', '') in changed_assets: found = True self.assertEqual(asset.title, 'NEW TITLE') else: not_found = True self.assertNotEqual(asset.title, 'NEW TITLE') self.assertTrue(found) self.assertTrue(not_found)
def mirror_collections(url, path='', **kwargs): API_RELS = ['search', 'collections', 'next'] cat = Catalog.from_file(url) empty_children = [] total_items = 0 for provider in cat.get_children(): links = provider.get_child_links() if len(links): # remove API specific links [provider.remove_links(rel) for rel in API_RELS] # remove links from the collections for collection in provider.get_children(): found = hits( collection.get_single_link('items').get_href(), {'limit': 0}) [ collection.remove_links(rel) for rel in ['child', 'next', 'items'] ] logger.info( f"{provider.id} - {collection.id}: {found} Items found") total_items += found else: empty_children.append(provider) [cat.remove_child(c.id) for c in empty_children] logger.info(f"{total_items} total Items found") cat.catalog_type = CatalogType.RELATIVE_PUBLISHED return cat
def lambda_handler(event, context): logger.debug('Event: %s' % json.dumps(event)) root_cat = get_root_catalog() # check if collection and if so, add to Cirrus if 'extent' in event: # add to static catalog root_cat.add_child(event) # send to Cirrus Publish SNS response = snsclient.publish(TopicArn=PUBLISH_TOPIC, Message=json.dumps(event)) logger.debug(f"SNS Publish response: {json.dumps(response)}") # check if URL to catalog if 'catalog_url' in event: cat = Catalog.from_file(event['catalog_url']) for child in cat.get_children(): if isinstance(child, Collection): child.remove_links('child') link = Link('copied_from', child) child.add_link(link, child.get_self_href()) root_cat.add_child(child) child_json = json.dumps(child.to_dict()) logger.debug(f"Publishing {child.id}: {child_json}") response = snsclient.publish(TopicArn=PUBLISH_TOPIC, Message=child_json) logger.debug(f"SNS Publish response: {json.dumps(response)}") root_cat.normalize_and_save(ROOT_URL, CatalogType.ABSOLUTE_PUBLISHED)
async def cli(): args = parse_args(sys.argv[1:]) logging.basicConfig( stream=sys.stdout, level=logging.DEBUG) #, format='%(asctime)-15s %(message)s') # quiet loggers for lg in [ 'httpx', 'urllib3', 'botocore', 'boto3', 'aioboto3', 'aiobotocore' ]: logging.getLogger(lg).propagate = False cmd = args.pop('command') if cmd == 'create': # create initial catalog through to collections cat = mirror_collections(args['url'], args['path']) cat.normalize_hrefs(args['path']) await cat.save() elif cmd == 'update': cat = Catalog.from_file(args['cat'], ) collection = cat.get_child(args['provider']).get_child( args['collection']) url = f"{args['url']}/{args['provider']}/collections/{args['collection']}/items" params = { 'limit': args['limit'], } if args['datetime'] is not None: params['datetime'] = args['datetime'] await mirror_items(collection, url, params, max_sync_queries=args['max_sync_queries'], item_template=args['item_template'])
def parse_stac(stac_uri): setup_stac_s3() cat = Catalog.from_file(stac_uri) cat.make_all_asset_hrefs_absolute() labels_uri = None geotiff_uris = [] for item in cat.get_all_items(): if isinstance(item, LabelItem): labels_uri = list(item.assets.values())[0].href labels_box = box(*item.bbox) # only use geotiffs that intersect with bbox of labels for item in cat.get_all_items(): if not isinstance(item, LabelItem): geotiff_uri = list(item.assets.values())[0].href geotiff_box = box(*item.bbox) if labels_box.intersects(geotiff_box): geotiff_uri = geotiff_uri.replace('%7C', '|') geotiff_uris.append(geotiff_uri) if not labels_uri: raise ValueError('Unable to read labels URI from STAC.') if not geotiff_uris: raise ValueError('Unable to read GeoTIFF URIs from STAC.') return labels_uri, labels_box, geotiff_uris
def test_case_4(): """Test case that is based on a local copy of the Tier 1 dataset from DrivenData's OpenCities AI Challenge. See: https://www.drivendata.org/competitions/60/building-segmentation-disaster-resilience """ return Catalog.from_file( TestCases.get_path('data-files/catalogs/test-case-4/catalog.json'))
def handler(event, context={}): # if this is batch, output to stdout if not hasattr(context, "invoked_function_arn"): logger.addHandler(logging.StreamHandler()) logger.debug('Event: %s' % json.dumps(event)) # parse input url = event.get('url') batch = event.get('batch', False) process = event['process'] if batch and hasattr(context, "invoked_function_arn"): submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch', name='feed-stac-crawl') return cat = Catalog.from_file(url) for item in cat.get_all_items(): payload = { 'type': 'FeatureCollection', 'features': [item.to_dict()], 'process': process } SNS_CLIENT.publish(TopicArn=SNS_TOPIC, Message=json.dumps(payload))
def test_read_remote(self): catalog_url = ('https://raw.githubusercontent.com/radiantearth/stac-spec/' 'v{}' '/extensions/label/examples/multidataset/catalog.json'.format(STAC_VERSION)) cat = Catalog.from_file(catalog_url) zanzibar = cat.get_child('zanzibar-collection') self.assertEqual(len(list(zanzibar.get_items())), 2)
def test_read_remote(self): # TODO: Move this URL to the main stac-spec repo once the example JSON is fixed. catalog_url = ( 'https://raw.githubusercontent.com/lossyrob/stac-spec/0.9.0/pystac-upgrade-fixes' '/extensions/label/examples/multidataset/catalog.json') cat = Catalog.from_file(catalog_url) zanzibar = cat.get_child('zanzibar-collection') self.assertEqual(len(list(zanzibar.get_items())), 2)
def get_scenes(json_file: str, class_config: ClassConfig, class_id_filter_dict: dict, catalog_dir: str, imagery_dir: str, train_crops: List[CropOffsets] = [], val_crops: List[CropOffsets] = [], N: int = None) -> Tuple[List[SceneConfig], List[SceneConfig]]: train_scenes = [] val_scenes = [] with open(json_file, 'r') as f: for catalog_imagery in json.load(f): catalog = catalog_imagery.get('catalog') catalog = catalog.strip() catalog = f'{catalog_dir}/{catalog}' catalog = catalog.replace('s3://', '/vsizip/vsis3/') (labelss, imagerys) = hrefs_from_catalog( Catalog.from_file(root_of_tarball(catalog)), N) imagery_name = imagery = catalog_imagery.get('imagery') if imagery_name is not None: imagery = imagery.strip() imagery = f'{imagery_dir}/{imagery}' if '.zip' in imagery: imagery = imagery.replace('s3://', '/vsizip/vsis3/') else: imagery = imagery.replace('s3://', '/vsis3/') imagerys = [imagery] * len(labelss) else: imagerys = map(lambda i: i.replace('_rgb', ''), imagerys) if not imagery_dir.endswith('/'): imagery_dir = imagery_dir + '/' imagerys = list( map(lambda i: i.replace('./', imagery_dir), imagerys)) h = hashlib.sha256(catalog.encode()).hexdigest() del imagery print('imagery', imagerys) print('labels', labelss) make_scene = partial(hrefs_to_sceneconfig, class_id_filter_dict=class_id_filter_dict) for j, (labels, imagery) in enumerate(zip(labelss, imagerys)): for i, crop in enumerate(train_crops): scene = make_scene(name=f'{h}-train-{i}-{j}', extent_crop=crop, imagery=imagery, labels=labels) train_scenes.append(scene) for i, crop in enumerate(val_crops): scene = make_scene(name=f'{h}-val-{i}-{j}', extent_crop=crop, imagery=imagery, labels=labels) val_scenes.append(scene) return train_scenes, val_scenes
def test_when_a_service_completes_it_writes_a_output_catalog_to_the_output_dir( self): with cli_parser('--harmony-action', 'invoke', '--harmony-input', '{"test": "input"}', '--harmony-sources', 'example/source/catalog.json', '--harmony-metadata-dir', self.workdir) as parser: args = parser.parse_args() cli.run_cli(parser, args, MockAdapter, cfg=self.config) output = Catalog.from_file( os.path.join(self.workdir, 'catalog.json')) self.assertTrue(output.validate)
def test_reading_iterating_and_writing_works_as_expected(self): """ Test case to cover issue #88 """ stac_uri = 'tests/data-files/catalogs/test-case-6/catalog.json' cat = Catalog.from_file(stac_uri) # Iterate over the items. This was causing failure in # in the later iterations as per issue #88 for item in cat.get_all_items(): pass with TemporaryDirectory() as tmp_dir: new_stac_uri = os.path.join(tmp_dir, 'test-case-6') cat.normalize_hrefs(new_stac_uri) cat.save(catalog_type=CatalogType.SELF_CONTAINED) # Open the local copy and iterate over it. cat2 = Catalog.from_file(os.path.join(new_stac_uri, 'catalog.json')) for item in cat2.get_all_items(): # Iterate again over the items. This would fail in #88 pass
def test_validate_label(self): with open(self.label_example_1_uri) as f: label_example_1_dict = json.load(f) pystac.validation.validate_dict(label_example_1_dict, "ITEM") with TemporaryDirectory() as tmp_dir: cat_dir = os.path.join(tmp_dir, 'catalog') catalog = TestCases.test_case_1() catalog.normalize_and_save(cat_dir, catalog_type=CatalogType.SELF_CONTAINED) cat_read = Catalog.from_file(os.path.join(cat_dir, 'catalog.json')) label_item_read = cat_read.get_item("area-2-2-labels", recursive=True) label_item_read.validate()
def get_root_catalog() -> Dict: """Get Cirrus root catalog from s3 Returns: Dict: STAC root catalog """ if s3().exists(ROOT_URL): cat = Catalog.from_file(ROOT_URL) else: catid = DATA_BUCKET.split('-data-')[0] cat = Catalog(id=catid, description=DESCRIPTION) logger.debug(f"Fetched {cat.describe()}") return cat
def test_set_hrefs_manually(self): catalog = TestCases.test_case_1() # Modify the datetimes year = 2004 month = 2 for item in catalog.get_all_items(): item.datetime = item.datetime.replace(year=year, month=month) year += 1 month += 1 with TemporaryDirectory() as tmp_dir: for root, _, items in catalog.walk(): # Set root's HREF based off the parent parent = root.get_parent() if parent is None: root_dir = tmp_dir else: d = os.path.dirname(parent.get_self_href()) root_dir = os.path.join(d, root.id) root_href = os.path.join(root_dir, root.DEFAULT_FILE_NAME) root.set_self_href(root_href) # Set each item's HREF based on it's datetime for item in items: item_href = '{}/{}-{}/{}.json'.format( root_dir, item.datetime.year, item.datetime.month, item.id) item.set_self_href(item_href) catalog.save(catalog_type=CatalogType.SELF_CONTAINED) read_catalog = Catalog.from_file( os.path.join(tmp_dir, 'catalog.json')) for root, _, items in read_catalog.walk(): parent = root.get_parent() if parent is None: self.assertEqual(root.get_self_href(), os.path.join(tmp_dir, 'catalog.json')) else: d = os.path.dirname(parent.get_self_href()) self.assertEqual( root.get_self_href(), os.path.join(d, root.id, root.DEFAULT_FILE_NAME)) for item in items: end = '{}-{}/{}.json'.format(item.datetime.year, item.datetime.month, item.id) self.assertTrue(item.get_self_href().endswith(end))
def get_item(catalog): cat = Catalog.from_file(catalog) try: collection = next(cat.get_children()) item = next(collection.get_items()) except StopIteration: item = next(cat.get_items()) return item
def test_create_and_read(self): with TemporaryDirectory() as tmp_dir: cat_dir = os.path.join(tmp_dir, 'catalog') catalog = TestCases.test_case_1() catalog.normalize_and_save(cat_dir, catalog_type=CatalogType.ABSOLUTE_PUBLISHED) read_catalog = Catalog.from_file('{}/catalog.json'.format(cat_dir)) collections = catalog.get_children() self.assertEqual(len(list(collections)), 2) items = read_catalog.get_all_items() self.assertEqual(len(list(items)), 8)
def lambda_handler(event, context={}): logger.debug('Event: %s' % json.dumps(event)) # check if collection and if so, add to Cirrus if 'extent' in event: stac.add_collections([Collection.from_dict(event)]) # check if URL to catalog - ingest all collections if 'catalog_url' in event: collections = [] cat = Catalog.from_file(event['catalog_url']) for child in cat.get_children(): if isinstance(child, Collection): collections.append(child) stac.add_collections(collections)
def get_root_catalog(): """Get Cirrus root catalog from s3 Returns: Dict: STAC root catalog """ caturl = f"{ROOT_URL}/catalog.json" if s3().exists(caturl): cat = Catalog.from_file(caturl) else: catid = DATA_BUCKET.split('-data-')[0] cat = Catalog(id=catid, description=DESCRIPTION) cat.normalize_and_save(ROOT_URL, CatalogType.ABSOLUTE_PUBLISHED) logger.debug(f"Fetched {cat.describe()}") return cat
def handle_dataset(version_metadata_key: str) -> None: """Handle writing a new dataset version to the dataset catalog""" storage_bucket_path = f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}" dataset_prefix = version_metadata_key.split("/", maxsplit=1)[0] dataset_catalog = Catalog.from_file( f"{storage_bucket_path}/{dataset_prefix}/{CATALOG_KEY}") dataset_version_metadata = STAC_IO.read_stac_object( f"{storage_bucket_path}/{version_metadata_key}") dataset_catalog.add_child(dataset_version_metadata, strategy=GeostoreSTACLayoutStrategy()) dataset_catalog.normalize_hrefs(f"{storage_bucket_path}/{dataset_prefix}", strategy=GeostoreSTACLayoutStrategy()) dataset_catalog.save(catalog_type=CatalogType.SELF_CONTAINED)
def test_validate_label(self) -> None: with open(self.label_example_1_uri, encoding="utf-8") as f: label_example_1_dict = json.load(f) pystac.validation.validate_dict(label_example_1_dict, pystac.STACObjectType.ITEM) with tempfile.TemporaryDirectory() as tmp_dir: cat_dir = os.path.join(tmp_dir, "catalog") catalog = TestCases.test_case_1() catalog.normalize_and_save(cat_dir, catalog_type=CatalogType.SELF_CONTAINED) cat_read = Catalog.from_file(os.path.join(cat_dir, "catalog.json")) label_item_read = cat_read.get_item("area-2-2-labels", recursive=True) assert label_item_read is not None label_item_read.validate()
def test_validate_label(self): sv = SchemaValidator() with open(self.label_example_1_uri) as f: label_example_1_dict = json.load(f) sv.validate_dict(label_example_1_dict, LabelItem) with TemporaryDirectory() as tmp_dir: cat_dir = os.path.join(tmp_dir, 'catalog') catalog = TestCases.test_case_1() label_item = LabelItem.from_dict(label_example_1_dict) catalog.add_item(label_item) catalog.normalize_and_save(cat_dir, catalog_type=CatalogType.SELF_CONTAINED) cat_read = Catalog.from_file(os.path.join(cat_dir, 'catalog.json')) label_item_read = cat_read.get_item("label-example-1-label-item") sv = SchemaValidator() sv.validate_object(label_item_read)
def __init__(self, csv_file, cat_path, root_dir, transform=None): """Initialisation of the dataset Create a dataset object with all required informations. Arguments: csv_file {str} -- [Path to the csv file with the dataset map] root_dir {str} -- [Root directorie of the images] cat_path {str} -- [Path of the catalog file] Keyword Arguments: transform {callable, optional} -- [Optional transform to be applied on a sample.] (default: {None}) """ self.houses_frame = pd.read_csv(csv_file) self.root_dir = root_dir self.transform = transform self.train_cat = Catalog.from_file(cat_path) self.cols = {cols.id: cols for cols in self.train_cat.get_children()} self.items_labels = {} items_size = 0 for col in list(self.cols.keys()): self.items_labels[str(col)] = {} for ii in self.cols[col].get_all_items(): print("Init dataset : ", col, ii.id) if "-labels" in str(ii.id): #rasterio.open(ii.make_asset_hrefs_absolute().assets['labels'].href).meta gpd.read_file( ii.make_asset_hrefs_absolute().assets['labels'].href) one_item_label = self.cols[col].get_item(id=ii.id) scene_labels_gdf = gpd.read_file( one_item_label.assets['labels'].href) self.items_labels[str(col)][str(ii.id)] = scene_labels_gdf items_size += sys.getsizeof(scene_labels_gdf) print("Size sum :", convert_bytes(items_size)) else: rasterio.open(ii.make_asset_hrefs_absolute(). assets['image'].href).meta
def main(): args = parse_args() data_path = Path(args.data_path) to_save = data_path / f'test_tiles_{args.win_sz}' if not to_save.exists(): to_save.mkdir(parents=True) test_cat = Catalog.from_file(str(data_path / 'test' / 'catalog.json')) with tqdm.tqdm(test_cat.get_items()) as pbar: for one_item in pbar: rst = rasterio.open(one_item.make_asset_hrefs_absolute().assets['image'].href) win_arr = rst.read() win_arr = np.transpose(win_arr, (1, 2, 0))[..., :3] win_arr = win_arr[..., ::-1] res = cv2.imwrite(str(to_save / f'{one_item.id}.jpg'), win_arr) assert res, f'Cannot write image {one_item.id}' rst.close()
def get_config(runner, root_uri, catalog_root, epochs='20', batch_size='8'): # Read STAC catalog catalog: Catalog = Catalog.from_file(pystac_workaround(catalog_root)) # TODO: pull desired channels from root collection properties channel_ordering: [int] = [0, 1, 2] # TODO: pull ClassConfig info from root collection properties class_config: ClassConfig = ClassConfig(names=["land", "water"], colors=["brown", "blue"]) dataset = build_dataset_from_catalog(catalog, channel_ordering, class_config) chip_sz = 512 backend = PyTorchSemanticSegmentationConfig( model=SemanticSegmentationModelConfig(backbone=Backbone.resnet50), solver=SolverConfig(lr=1e-4, num_epochs=int(epochs), batch_sz=int(batch_size), ignore_last_class=True), log_tensorboard=False, run_tensorboard=False, num_workers=0, ) chip_options = SemanticSegmentationChipOptions( window_method=SemanticSegmentationWindowMethod.sliding, target_class_ids=[1], negative_survival_prob=0.125, stride=chip_sz // 2) return SemanticSegmentationConfig( root_uri=root_uri, dataset=dataset, backend=backend, train_chip_sz=chip_sz, predict_chip_sz=chip_sz, chip_options=chip_options, img_format='npy', label_format='png', )
def test_map_items(self): def item_mapper(item): item.properties['ITEM_MAPPER'] = 'YEP' return item with TemporaryDirectory() as tmp_dir: catalog = TestCases.test_case_1() new_cat = catalog.map_items(item_mapper) new_cat.normalize_hrefs(os.path.join(tmp_dir, 'cat')) new_cat.save(catalog_type=CatalogType.ABSOLUTE_PUBLISHED) result_cat = Catalog.from_file(os.path.join(tmp_dir, 'cat', 'catalog.json')) for item in result_cat.get_all_items(): self.assertTrue('ITEM_MAPPER' in item.properties) for item in catalog.get_all_items(): self.assertFalse('ITEM_MAPPER' in item.properties)
def _build_adapter(AdapterClass, message_string, sources_path, data_location, config): """ Creates the adapter to be invoked for the given harmony input Parameters ---------- AdapterClass : class The BaseHarmonyAdapter subclass to use to handle service invocations message_string : string The Harmony input message sources_path : string A file location containing a STAC catalog corresponding to the input message sources data_location : string The name of the directory where output should be written config : harmony.util.Config A configuration instance for this service Returns ------- BaseHarmonyAdapter subclass instance The adapter to be invoked """ catalog = Catalog.from_file(sources_path) secret_key = config.shared_secret_key if bool(secret_key): decrypter = create_decrypter(bytes(secret_key, 'utf-8')) else: def identity(arg): return arg decrypter = identity message = Message(json.loads(message_string), decrypter) if data_location: message.stagingLocation = data_location adapter = AdapterClass(message, catalog) adapter.set_config(config) return adapter