def test_load_materializer(self):
print("[Test]{}/test_load_materializer".format(
self.__class__.__name__))
materializer = Utils.load_materializer("noaa_materializer")
self.assertEqual(issubclass(type(materializer), MaterializerBase),
True)
self.assertIn(type(materializer).__name__, NoaaMaterializer.__name__)
print(colored('.Done', 'red'))
def updating(self,
description_path: str,
es_index: str,
document_id: int,
data_path: str = None,
query_data_for_updating: bool = False) -> dict:
"""Update document in elastic search.
By providing description file, index builder should be able to process it and create metadata json for the
dataset, update document in elastic search
Args:
description_path: Path to description json file.
es_index: str, es index for this dataset
document_id: int, document id of document which need to be updated
data_path: Path to data csv file.
query_data_for_updating: Bool. If no data is presented, and query_data_for_updating is False, will only
create metadata according to the description json. If query_data_for_updating is True and no data is
presented, will use Materialize to query data for profiling and indexing
Returns:
metadata dictionary
"""
self._check_es_index(es_index=es_index)
description, data = self._read_data(description_path, data_path)
if not data and query_data_for_updating:
try:
materializer_module = description["materialization"][
"python_path"]
materializer = Utils.load_materializer(materializer_module)
data = materializer.get(metadata=description)
except:
warnings.warn(
"Materialization Failed, index based on schema json only")
metadata = self.construct_global_metadata(
description=description,
data=data,
overwrite_datamart_id=document_id)
Utils.validate_schema(metadata.value)
self.im.update_doc(index=es_index,
doc_type='document',
body={"doc": metadata.value},
id=metadata.value['datamart_id'])
return metadata.value
def indexing(self,
description_path: str,
es_index: str,
data_path: str = None,
query_data_for_indexing: bool = False,
save_to_file: str = None,
save_to_file_mode: str = "a+",
delete_old_es_index: bool = False) -> dict:
"""API for the index builder.
By providing description file, index builder should be able to process it and create metadata json for the
dataset, create index in our index store
Args:
description_path: Path to description json file.
es_index: str, es index for this dataset
data_path: Path to data csv file.
query_data_for_indexing: Bool. If no data is presented, and query_data_for_indexing is False, will only
create metadata according to the description json. If query_data_for_indexing is True and no data is
presented, will use Materialize to query data for profiling and indexing
save_to_file: str, a path to the json line file
save_to_file_mode: str, mode for saving, default "a+"
delete_old_es_index: bool, boolean if delete original es index if it exist
Returns:
metadata dictionary
"""
self.check_es_index(es_index=es_index,
delete_old_es_index=delete_old_es_index)
if not self.current_global_index or delete_old_es_index:
self.current_global_index = self.im.current_global_datamart_id(
index=es_index)
description, data = self.read_data(description_path, data_path)
if not data and query_data_for_indexing:
try:
materializer_module = description["materialization"][
"python_path"]
materializer = Utils.load_materializer(materializer_module)
data = materializer.get(metadata=description)
except:
warnings.warn(
"Materialization Failed, index based on schema json only")
metadata = self.construct_global_metadata(description=description,
data=data)
Utils.validate_schema(metadata.value)
if save_to_file:
self.save_data(save_to_file=save_to_file,
save_mode=save_to_file_mode,
metadata=metadata)
self.im.create_doc(index=es_index,
doc_type='document',
body=metadata.value,
id=metadata.value['datamart_id'])
return metadata.value
def get_dataset(metadata, variables=None, constrains=None):
materializer = Utils.load_materializer(
metadata["materialization"]["python_path"])
return materializer.get(metadata=metadata,
variables=variables,
constrains=constrains)
def test_load_materializer(self):
materializer = Utils.load_materializer("noaa_materializer")
self.assertEqual(issubclass(type(materializer), MaterializerBase),
True)
self.assertIn(type(materializer).__name__, NoaaMaterializer.__name__)