def activate_network(self, network=None, meta_path=None, temp_root=gettempdir()): """ Load files for specific networks Parameters ---------- network : list or str, optional (default: None) See __init__ description meta_path : str, optional (default: None) See __init__ description temp_root: str, optional (default: None) See __init__ description """ meta_csv_filename = f'{self.root.name}.csv' if meta_path is None: meta_path = Path(self.root.root_dir) / 'python_metadata' else: meta_path = Path(meta_path) meta_csv_file = meta_path / meta_csv_filename if os.path.isfile(meta_csv_file): self.__file_collection = IsmnFileCollection.from_metadata_csv( self.root, meta_csv_file, network=network) else: self.__file_collection = IsmnFileCollection.build_from_scratch( self.root, parallel=self.parallel, log_path=meta_path, temp_root=temp_root) self.__file_collection.to_metadata_csv(meta_csv_file) networks = self.__collect_networks(network) self.collection = NetworkCollection(networks)
def setUpClass(cls): testdata_path = os.path.join(testdata_root, 'zip_archives', 'header') testdata_zip_path = os.path.join(testdata_path, 'Data_seperate_files_header_20170810_20180809.zip') # clean existing metadata metadata_path = os.path.join(testdata_path, 'python_metadata') cleanup(metadata_path) cls.coll = IsmnFileCollection.build_from_scratch(testdata_zip_path)
def setUpClass(cls): """ get_some_resource() is slow, to avoid calling it for each test use setUpClass() and store the result as class variable """ super(Test_FileCollectionCeopSepUnzipped, cls).setUpClass() testdata_path_unzipped = os.path.join(testdata_root, 'Data_seperate_files_20170810_20180809') cleanup(os.path.join(testdata_path_unzipped, 'python_metadata')) print('Setup from scratch') cls.coll = IsmnFileCollection.build_from_scratch(Path(testdata_path_unzipped), parallel=True)
def setUpClass(cls): super(Test_FileCollectionCeopSepUnzipped, cls).setUpClass() testdata_path_unzipped = os.path.join(testdata_root, 'Data_seperate_files_header_20170810_20180809') # clean existing metadata metadata_path = os.path.join(testdata_path_unzipped, 'python_metadata') cleanup(metadata_path) cls.coll = IsmnFileCollection.build_from_scratch(testdata_path_unzipped)
def setUpClass(cls): testdata_path = os.path.join(testdata_root, "zip_archives", "ceop") testdata_zip_path = os.path.join( testdata_path, "Data_seperate_files_20170810_20180809.zip") # clean existing metadata metadata_path = os.path.join(testdata_path, "python_metadata") cleanup(metadata_path) cls.coll = IsmnFileCollection.build_from_scratch(testdata_zip_path)
def activate_network( self, network: Union[list, str] = None, meta_path: str = None, temp_root: str = gettempdir(), ): """ Load (file) collection for specific networks. """ meta_csv_filename = f"{self.root.name}.csv" if meta_path is None: meta_path = Path(self.root.root_dir) / "python_metadata" else: meta_path = Path(meta_path) meta_csv_file = meta_path / meta_csv_filename if network is not None: network = np.atleast_1d(network) if not os.path.isfile(meta_csv_file): self.__file_collection = IsmnFileCollection.build_from_scratch( self.root, parallel=self.parallel, log_path=meta_path, temp_root=temp_root, ) self.__file_collection.to_metadata_csv(meta_csv_file) self.__file_collection = IsmnFileCollection.from_metadata_csv( self.root, meta_csv_file, network=network ) metadata = self.__file_collection.metadata_df metadata.index = range(len(metadata.index)) self.metadata = metadata networks = self.__collect_networks(network) self.collection = NetworkCollection(networks)
def test_from_csv(self): # test alternative method to build collection, should get same result print('Setup from csv') with TemporaryDirectory() as temp: self.coll.to_metadata_csv(os.path.join(temp, 'meta.csv')) other = IsmnFileCollection.from_metadata_csv( self.coll.root.path, os.path.join(temp, 'meta.csv')) for thisfile, otherfile in zip(self.coll.iter_filehandlers(), other.iter_filehandlers()): assert thisfile.file_path == otherfile.file_path, "Paths dont match" assert thisfile.root.path == otherfile.root.path, "Paths dont match" assert thisfile.metadata == otherfile.metadata, "Meta dont match"