def create_dashboard_neo4j_job(**kwargs):
tmp_folder = '/var/tmp/amundsen/table_metadata'
node_files_folder = '{tmp_folder}/nodes/'.format(tmp_folder=tmp_folder)
relationship_files_folder = '{tmp_folder}/relationships/'.format(
tmp_folder=tmp_folder)
job_config = ConfigFactory.from_dict({
'extractor.generic.{}'.format(GenericExtractor.EXTRACTION_ITEMS):
iter(input),
'extractor.generic.{}'.format('model_class'):
'databuilder.models.dashboard_metadata.DashboardMetadata',
'loader.filesystem_csv_neo4j.{}'.format(FsNeo4jCSVLoader.NODE_DIR_PATH):
node_files_folder,
'loader.filesystem_csv_neo4j.{}'.format(FsNeo4jCSVLoader.RELATION_DIR_PATH):
relationship_files_folder,
'publisher.neo4j.{}'.format(neo4j_csv_publisher.NODE_FILES_DIR):
node_files_folder,
'publisher.neo4j.{}'.format(neo4j_csv_publisher.RELATION_FILES_DIR):
relationship_files_folder,
'publisher.neo4j.{}'.format(neo4j_csv_publisher.NEO4J_END_POINT_KEY):
neo4j_endpoint,
'publisher.neo4j.{}'.format(neo4j_csv_publisher.NEO4J_USER):
neo4j_user,
'publisher.neo4j.{}'.format(neo4j_csv_publisher.NEO4J_PASSWORD):
neo4j_password,
'publisher.neo4j.{}'.format(neo4j_csv_publisher.JOB_PUBLISH_TAG):
'unique_tag', # should use unique tag here like {ds}
})
job = DefaultJob(conf=job_config,
task=DefaultTask(extractor=GenericExtractor(),
loader=FsNeo4jCSVLoader()),
publisher=Neo4jCsvPublisher())
return job
def test_extraction_with_model_class(self):
# type: () -> None
"""
Test Extraction using model class
"""
extractor = GenericExtractor()
extractor.init(Scoped.get_scoped_conf(conf=self.conf,
scope=extractor.get_scope()))
result = extractor.extract()
self.assertEquals(result.timestamp, 10000000)
def test_extraction_without_model_class(self) -> None:
"""
Test Extraction using model class
"""
config_dict = {
'extractor.generic.extraction_items': [{'foo': 1}, {'bar': 2}],
}
conf = ConfigFactory.from_dict(config_dict)
extractor = GenericExtractor()
self.conf = ConfigFactory.from_dict(config_dict)
extractor.init(Scoped.get_scoped_conf(conf=conf,
scope=extractor.get_scope()))
self.assertEqual(extractor.extract(), {'foo': 1})
self.assertEqual(extractor.extract(), {'bar': 2})
def test_extraction_with_model_class(self) -> None:
"""
Test Extraction using model class
"""
config_dict = {
'extractor.generic.extraction_items': [{'timestamp': 10000000}],
'extractor.generic.model_class': 'databuilder.models.neo4j_es_last_updated.Neo4jESLastUpdated',
}
conf = ConfigFactory.from_dict(config_dict)
extractor = GenericExtractor()
self.conf = ConfigFactory.from_dict(config_dict)
extractor.init(Scoped.get_scoped_conf(conf=conf,
scope=extractor.get_scope()))
result = extractor.extract()
self.assertEqual(result.timestamp, 10000000)