async def transform_data(self, item: Item) -> None: cog_asset_list = [] for asset in item.assets: if not is_tiff(asset.source_path): return start_time = time_in_ms() output_path = os.path.join(item.collection.get_temp_dir(), f"{ulid.ULID()}.tiff") await create_cog(asset.source_path, output_path, compression_method="lzw").run() get_log().debug("Created COG", output_path=output_path, duration=time_in_ms() - start_time) asset.needs_upload = False cog_asset = Asset(output_path) cog_asset.content_type = pystac.MediaType.COG cog_asset.target = asset.target cog_asset_list.append(cog_asset) for asset in cog_asset_list: item.add_asset(asset)
def write_json(dictionary: str, target_json): start_time = time_in_ms() with get_fs(target_json).open(target_json, "w", ContentType=pystac.MediaType.JSON) as f1: f1.write(json.dumps(dictionary, indent=4)) get_log().debug("JSON Written", target_json=target_json, duration=time_in_ms() - start_time)
async def process_directory(source_dir: str) -> None: start_time = time_in_ms() await _create_assets(source_dir) get_log().debug("Assets Created", source_dir=source_dir, duration=time_in_ms() - start_time) await _create_items() get_log().debug("Items Created", source_dir=source_dir, duration=time_in_ms() - start_time)
def transfer_file(source_file: str, checksum: str, content_type, target_file: str): start_time = time_in_ms() with get_fs(source_file).open(source_file, "rb") as f1: data = f1.read() with get_fs(target_file).open(target_file, "wb", ContentType=content_type, Metadata={"hash": checksum}) as f2: f2.write(data) get_log().debug("File transferred", source_file=source_file, target_file=target_file, duration=time_in_ms() - start_time)
async def run(cmd: "Command"): start_time = time_in_ms() proc = await asyncio.create_subprocess_exec( *cmd.to_full_command(), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) stdout, stderr = await proc.communicate() if proc.returncode != 0: get_log().trace("Ran command failed", command=cmd.redacted_command(), duration=time_in_ms() - start_time) raise Exception(stderr.decode()) get_log().trace("Ran command succeeded", command=cmd.redacted_command(), duration=time_in_ms() - start_time) return proc.returncode, stdout.decode(), stderr.decode()
async def validate_metadata(self, item: Item) -> None: async with self.lock: for validator in self.validators: if validator.is_applicable(item): start_time = time_in_ms() try: await validator.validate_metadata(item) except Exception as e: item.add_error(str(e), validator.name, e) get_log().warning(f"Validation Failed: {e}", validator=validator.name) get_log().debug( "Validity Checked", validator=validator.name, duration=time_in_ms() - start_time, )
async def load_metadata(self, asset: Asset) -> None: async with self.lock: for loader in self.loaders: if loader.is_applicable(asset): start_time = time_in_ms() try: await loader.load_metadata(asset) if not asset.is_valid: break except Exception as e: asset.add_error(str(e), loader.name, e) get_log().warning(f"Metadata Load Failed: {e}", loader=loader.name) return get_log().debug( "Metadata Loaded", loader=loader.name, duration=time_in_ms() - start_time, )
async def main(source, datatype, target, verbose): if verbose: set_level(LogLevel.trace) start_time = time_in_ms() data_type = DataType(datatype) await process_directory(source) try: for collection in collection_store.values(): await transfer_collection(collection, target) finally: for collection in collection_store.values(): collection.delete_temp_dir() get_log().debug( "Job Completed", location=target, data_type=data_type.value, duration=time_in_ms() - start_time, )