async def transform_data(self, item: Item) -> None:
cog_asset_list = []
for asset in item.assets:
if not is_tiff(asset.source_path):
return
start_time = time_in_ms()
output_path = os.path.join(item.collection.get_temp_dir(),
f"{ulid.ULID()}.tiff")
await create_cog(asset.source_path,
output_path,
compression_method="lzw").run()
get_log().debug("Created COG",
output_path=output_path,
duration=time_in_ms() - start_time)
asset.needs_upload = False
cog_asset = Asset(output_path)
cog_asset.content_type = pystac.MediaType.COG
cog_asset.target = asset.target
cog_asset_list.append(cog_asset)
for asset in cog_asset_list:
item.add_asset(asset)
def write_json(dictionary: str, target_json):
start_time = time_in_ms()
with get_fs(target_json).open(target_json,
"w",
ContentType=pystac.MediaType.JSON) as f1:
f1.write(json.dumps(dictionary, indent=4))
get_log().debug("JSON Written",
target_json=target_json,
duration=time_in_ms() - start_time)
def get_temp_dir(self):
global TEMP_DIR
if not TEMP_DIR:
TEMP_DIR = mkdtemp()
get_log().debug("Temp directory created", path=TEMP_DIR)
temp_dir = os.path.join(TEMP_DIR, self.title)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
return temp_dir
async def process_directory(source_dir: str) -> None:
start_time = time_in_ms()
await _create_assets(source_dir)
get_log().debug("Assets Created",
source_dir=source_dir,
duration=time_in_ms() - start_time)
await _create_items()
get_log().debug("Items Created",
source_dir=source_dir,
duration=time_in_ms() - start_time)
def transfer_file(source_file: str, checksum: str, content_type,
target_file: str):
start_time = time_in_ms()
with get_fs(source_file).open(source_file, "rb") as f1:
data = f1.read()
with get_fs(target_file).open(target_file,
"wb",
ContentType=content_type,
Metadata={"hash": checksum}) as f2:
f2.write(data)
get_log().debug("File transferred",
source_file=source_file,
target_file=target_file,
duration=time_in_ms() - start_time)
async def run(cmd: "Command"):
start_time = time_in_ms()
proc = await asyncio.create_subprocess_exec(
*cmd.to_full_command(),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
get_log().trace("Ran command failed",
command=cmd.redacted_command(),
duration=time_in_ms() - start_time)
raise Exception(stderr.decode())
get_log().trace("Ran command succeeded",
command=cmd.redacted_command(),
duration=time_in_ms() - start_time)
return proc.returncode, stdout.decode(), stderr.decode()
async def validate_metadata(self, item: Item) -> None:
async with self.lock:
for validator in self.validators:
if validator.is_applicable(item):
start_time = time_in_ms()
try:
await validator.validate_metadata(item)
except Exception as e:
item.add_error(str(e), validator.name, e)
get_log().warning(f"Validation Failed: {e}",
validator=validator.name)
get_log().debug(
"Validity Checked",
validator=validator.name,
duration=time_in_ms() - start_time,
)
async def load_metadata(self, asset: Asset) -> None:
async with self.lock:
for loader in self.loaders:
if loader.is_applicable(asset):
start_time = time_in_ms()
try:
await loader.load_metadata(asset)
if not asset.is_valid:
break
except Exception as e:
asset.add_error(str(e), loader.name, e)
get_log().warning(f"Metadata Load Failed: {e}",
loader=loader.name)
return
get_log().debug(
"Metadata Loaded",
loader=loader.name,
duration=time_in_ms() - start_time,
)
async def main(source, datatype, target, verbose):
if verbose:
set_level(LogLevel.trace)
start_time = time_in_ms()
data_type = DataType(datatype)
await process_directory(source)
try:
for collection in collection_store.values():
await transfer_collection(collection, target)
finally:
for collection in collection_store.values():
collection.delete_temp_dir()
get_log().debug(
"Job Completed",
location=target,
data_type=data_type.value,
duration=time_in_ms() - start_time,
)
async def transfer_collection(collection: Collection, target: str):
stac_collection = collection.create_stac()
for item in collection.items.values():
if not item.is_valid():
get_log().warning("Invalid item was not uploaded:", error=item.log)
continue
stac_item = item.create_stac()
stac_collection.add_item(stac_item)
for asset in item.assets:
if asset.needs_upload:
checksum = await asset.get_checksum()
transfer_file(asset.source_path, checksum,
asset.get_content_type(),
os.path.join(target, asset.target))
write_json(
stac_item.to_dict(),
os.path.join(target, item.collection.title, f"{item.id}.json"))
write_json(stac_collection.to_dict(),
os.path.join(target, collection.title, "collection.json"))
