def _populate_products(changes_dict: ChangesDict, db: BundleDB, sv_deltas: COWFS) -> None: for lid, (vid, changed) in changes_dict.items(): if lid.is_product_lid(): lidvid = LIDVID.create_from_lid_and_vid(lid, vid) collection_lidvid = changes_dict.parent_lidvid(lidvid) if changed: product_path = lid_to_dirpath(lidvid.lid()) if collection_lidvid.lid().collection_id == "document": db.create_document_product(str(lidvid), str(collection_lidvid)) doc_files = [ doc_file for doc_file in sv_deltas.listdir(product_path) if (fs.path.splitext(doc_file)[1].lower() in DOCUMENT_SUFFIXES) ] for doc_file in doc_files: sys_filepath = sv_deltas.getsyspath( fs.path.join(product_path, doc_file)) db.create_document_file(sys_filepath, doc_file, str(lidvid)) else: db.create_fits_product(str(lidvid), str(collection_lidvid)) fits_files = [ fits_file for fits_file in sv_deltas.listdir(product_path) if fs.path.splitext(fits_file)[1].lower() == ".fits" ] for fits_file in fits_files: fits_file_path = fs.path.join(product_path, fits_file) fits_os_path = sv_deltas.getsyspath(fits_file_path) populate_database_from_fits_file( db, fits_os_path, str(lidvid)) else: if changes_dict.changed(collection_lidvid.lid()): db.create_collection_product_link(str(collection_lidvid), str(lidvid))
def _build_browse_collection( db: BundleDB, changes_dict: ChangesDict, browse_deltas: COWFS, bundle_lidvid: LIDVID, data_collection_lidvid: LIDVID, bundle_path: str, ) -> None: bundle_segment = bundle_lidvid.lid().parts()[0] collection_segment = data_collection_lidvid.lid().parts()[1] browse_collection_lid = data_collection_lidvid.lid().to_browse_lid() collection_path = f"{bundle_path}{collection_segment}$/" browse_collection_segment = browse_collection_lid.collection_id browse_collection_path = f"{bundle_path}{browse_collection_segment}$/" browse_collection_vid = data_collection_lidvid.vid() browse_collection_lidvid = LIDVID.create_from_lid_and_vid( browse_collection_lid, browse_collection_vid) changes_dict.set(browse_collection_lid, browse_collection_vid, True) browse_deltas.makedirs(browse_collection_path, recreate=True) db.create_other_collection(str(browse_collection_lidvid), str(bundle_lidvid)) db.create_bundle_collection_link(str(bundle_lidvid), str(browse_collection_lidvid)) product_segments = [ str(prod[:-1]) for prod in browse_deltas.listdir(collection_path) if "$" in prod ] for product_segment in product_segments: # These product_segments are from the data_collection product_lid = LID.create_from_parts( [bundle_segment, collection_segment, product_segment]) product_vid = changes_dict.vid(product_lid) product_path = f"{collection_path}{product_segment}$/" browse_product_path = f"{browse_collection_path}{product_segment}$/" browse_product_lidvid = _extend_lidvid(browse_collection_lid, product_vid, product_segment) if changes_dict.changed(product_lid): fits_product_lidvid = _extend_lidvid( data_collection_lidvid.lid(), data_collection_lidvid.vid(), product_segment, ) bpl = LIDVID(browse_product_lidvid) changes_dict.set(bpl.lid(), bpl.vid(), True) browse_deltas.makedirs(browse_product_path, recreate=True) db.create_browse_product( browse_product_lidvid, fits_product_lidvid, str(browse_collection_lidvid), ) db.create_collection_product_link(str(browse_collection_lidvid), browse_product_lidvid) for fits_file in browse_deltas.listdir(product_path): fits_filepath = fs.path.join(product_path, fits_file) fits_os_filepath = browse_deltas.getsyspath(fits_filepath) browse_file = fs.path.splitext(fits_file)[0] + ".jpg" browse_filepath = fs.path.join(browse_product_path, browse_file) # In a COWFS, a directory does not have a # syspath, only files. So we write a stub # file into the directory, find its syspath # and its directory's syspath. Then we remove # the stub file. browse_deltas.touch(browse_filepath) browse_product_os_filepath = browse_deltas.getsyspath( browse_filepath) browse_deltas.remove(browse_filepath) browse_product_os_dirpath = fs.path.dirname( browse_product_os_filepath) # Picmaker expects a list of strings. If you give it # str, it'll index into it and complain about '/' # not being a file. So don't do that! try: picmaker.ImagesToPics( [str(fits_os_filepath)], browse_product_os_dirpath, filter="None", percentiles=(1, 99), ) except IndexError as e: tb = traceback.format_exc() message = f"File {fits_file}: {e}\n{tb}" raise Exception(message) browse_os_filepath = fs.path.join(browse_product_os_dirpath, browse_file) size = os.stat(browse_os_filepath).st_size db.create_browse_file(browse_os_filepath, browse_file, browse_product_lidvid, size) else: bpl = LIDVID(browse_product_lidvid) changes_dict.set(bpl.lid(), bpl.vid(), False) db.create_collection_product_link(str(browse_collection_lidvid), browse_product_lidvid)