def test_arc_set_data_loader_load_stats(arc_layer):
loader = ArcSetDataLoader(arc_layer)
loader.load_stats()
assert loader._ArcSetDataLoader__min_OID > -1
assert loader._ArcSetDataLoader__max_OID > 0
assert loader._ArcSetDataLoader__record_count > 0
assert loader._ArcSetDataLoader__record_count >= \
(loader._ArcSetDataLoader__max_OID - loader._ArcSetDataLoader__min_OID)
assert len(loader.errors) == 0
def load_data_for_arc_service(uri, db_client, schema, create_db_objects=False):
try:
arc_service = ArcService(
uri, schema, None, db_client
) # create the arc service and load all the map servers and layers
if create_db_objects:
p = {
'header_sql': arc_service.generate_sql_preamble(),
'body_sql': arc_service.generate_sql(),
'footer_sql': arc_service.generate_sql_extra(),
'statement_terminator':
arc_service.db_client.statement_terminator
}
sql = "{header_sql}{statement_terminator}{body_sql}{statement_terminator}{footer_sql}".format(
**p)
arc_service.db_client.exec_non_query(sql)
# look from the layers and load their data
for l in [l for ms in arc_service.arc_map_servers for l in ms.layers]:
loader = ArcSetDataLoader(l)
loader.load_data()
for l in [l for ms in arc_service.arc_map_servers for l in ms.tables]:
loader = ArcSetDataLoader(l)
loader.load_data()
except Exception as e:
exception_logging(e)
def load_data_for_arc_layer(params, db_client, schema, file_path):
try:
p = params.copy()
uri = p['uri']
arc_layer = ArcLayer(
uri, schema, db_client
) # create the arc service and load all the map servers and layers
del p['uri']
# look from the layers and load their data
loader = ArcSetDataLoader(arc_layer, **p)
with open(file_path, 'r') as f:
# content = f.read()
json = load(f)
loader.parse_and_save_json_data(json)
except Exception as e:
exception_logging(e)
def load_data_for_arc_layer(params, db_client, schema, create_db_objects=False, load_data=True):
try:
p = params.copy()
uri = p['uri']
arc_layer = ArcLayer(uri, schema, db_client) # create the arc service and load all the map servers and layers
del p['uri']
if create_db_objects:
header_sql = arc_layer.generate_sql_preamble()
body_sql = arc_layer.generate_sql()
footer_sql = arc_layer.generate_sql_extra()
statement_terminator = arc_layer.db_client.statement_terminator
sql = f"{header_sql}{statement_terminator}{body_sql}{statement_terminator}{footer_sql}"
arc_layer.db_client.exec_non_query(sql, autocommit=True) # set autocommit to True to run DDL
# look from the layers and load their data
if load_data:
loader = ArcSetDataLoader(arc_layer, **p)
loader.load_data()
except Exception as e:
exception_logging(e)
def load_data(
self, **kargs
): # int concurrency=5, int resultRecordCount=10, int maxResultRecordCount=100)
try:
layers = [
l for s in self._arc_services for m in s.arc_map_servers
for l in m.layers if l.loaded
]
for l in layers:
loader = ArcSetDataLoader(l, self.db_client, **kargs)
loader.load_data()
layers = [
l for s in self._arc_services for m in s.arc_map_servers
for l in m.tables
]
for l in layers:
loader = ArcSetDataLoader(l, self.db_client, **kargs)
loader.load_data()
except Exception as e:
self.add_error(e)
def test_arc_set_data_loader_load_data(arc_layer):
loader = ArcSetDataLoader(arc_layer)
truncate_table = arc_layer.db_client.sql_generator_templates['truncate_table']\
.format(table_name=arc_layer.sql_full_table_name)
arc_layer.db_client.exec_non_query(truncate_table)
loader.load_data()
# assert len(loader.errors) == 0
assert loader.record_count == loader.loaded_record_count
loader._ArcSetDataLoader__get_object_ids_from_db()
assert len(
loader._ArcSetDataLoader__loaded_OIDs) == loader.loaded_record_count
select = arc_layer.db_client.sql_generator_templates['select_object_ids']\
.format(table_name=arc_layer.sql_full_table_name, object_id_field_name=loader.object_id_field_name)
results = arc_layer.db_client.exec_result_set_query(select) or ()
assert len(results) == loader.loaded_record_count
sleep(1)
loader.load_data()
# assert len(loader.errors) == 0
assert loader.record_count == loader.loaded_record_count
assert len(
loader._ArcSetDataLoader__loaded_OIDs) == loader.loaded_record_count
def test_arc_set_data_loader_init(arc_layer):
loader = ArcSetDataLoader(arc_layer)
assert loader.db_client is not None
assert loader.arc_set == arc_layer