def test_set_source_name_updates_source_if_not_none(self):
crate = Crate('foo', 'bar', 'baz', 'goo')
crate.set_source_name(None)
self.assertEqual(crate.source_name, 'foo')
self.assertEqual(crate.source, path.join('bar', 'foo'))
def test_try_to_find_data_source_by_name_returns_False_if_duplicate(self, walk):
walk.return_value = [(None, None, ['db.owner.Counties', 'db.owner2.Counties'])]
crate = Crate(
source_name='duplicate',
source_workspace='Database Connections\\something.sde',
destination_workspace='c:\\something.gdb',
destination_name='Counties')
self.assertFalse(crate._try_to_find_data_source_by_name()[0])
def test_init_with_coordinate_system_does_not_change(self):
crate = Crate('foo', 'bar', 'baz', 'qux', SpatialReference(26921))
self.assertEqual(crate.source_name, 'foo')
self.assertEqual(crate.source_workspace, 'bar')
self.assertEqual(crate.destination_workspace, 'baz')
self.assertEqual(crate.destination_name, 'qux')
self.assertIsInstance(crate.destination_coordinate_system, SpatialReference)
def test_init_with_coordinate_system_as_number_becomes_spatial_reference(self):
crate = Crate('foo', 'bar', 'baz', 'qux', 26912)
self.assertEqual(crate.source_name, 'foo')
self.assertEqual(crate.source_workspace, 'bar')
self.assertEqual(crate.destination_workspace, 'baz')
self.assertEqual(crate.destination_name, 'qux')
self.assertIsInstance(crate.destination_coordinate_system, SpatialReference)
def test_update_custom_validation_that_fails(self, arcpy_exists):
arcpy_exists.return_value = True
crate = Crate('', '', '', describer=arcpy_mocks.Describe)
self.assertEqual(
core.update(crate, raise_validation_exception)[0],
Crate.INVALID_DATA)
def test_is_ready_to_ship_crates_with_any_exception_returns_false(self):
updated = Crate('', '', '', '')
updated.result = (Crate.UPDATED, None)
no_changes = Crate('', '', '', '')
no_changes.result = (Crate.NO_CHANGES, None)
unhandled_exception = Crate('', '', '', '')
unhandled_exception.result = (Crate.UNHANDLED_EXCEPTION, None)
self.patient._crates = [updated, no_changes, unhandled_exception]
self.assertFalse(self.patient.is_ready_to_ship())
def test_deleted_destination_between_updates(self):
crate = Crate('ZipCodes', check_for_changes_gdb, test_gdb, 'ImNotHere')
core.update(crate, lambda x: True)
delete_if_arcpy_exists(crate.destination)
self.assertEqual(core.update(crate, lambda x: True)[0], Crate.CREATED)
self.assertEqual(arcpy.Exists(crate.destination), True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 299)
def run(name):
return core._hash(
Crate(name,
UPDATE_TESTS_SDE,
test_gdb,
name,
destination_coordinate_system=arcpy.SpatialReference(3857),
geographic_transformation='NAD_1983_To_WGS_1984_5'))
def test_move_data_feature_class(self):
skip_if_no_local_sde()
crate = Crate('DNROilGasWells', update_tests_sde, test_gdb) #: feature class
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 5)
def test_move_data_table():
skip_if_no_local_sde()
crate = Crate('Providers', UPDATE_TESTS_SDE, TEMP_GDB) #: table
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 57
def test_create_name_is_combined_hash_and_table_three_values(self):
destination_workspace = 'dw'
source_name = 'sn'
crate = Crate(source_name, 'source', destination_workspace)
hash = source_name + '_' + xxh64(path.join(destination_workspace, source_name)).hexdigest()
self.assertEqual(crate.name, hash)
def test_create_destination_data_table(test_gdb):
arcpy.CreateFileGDB_management(path.join(CURRENT_FOLDER, 'data'),
'test.gdb')
tbl_crate = Crate('Providers', test_gdb, TEMP_GDB)
core._create_destination_data(tbl_crate)
assert arcpy.Exists(tbl_crate.destination) == True
def test_move_data_table(self):
skip_if_no_local_sde()
crate = Crate('Providers', update_tests_sde, test_gdb) #: table
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 57)
def test_check_schema_ignore_length_for_all_except_text(self):
skip_if_no_local_sde()
# only worry about length on text fields
result = core.check_schema(
Crate(r'UPDATE_TESTS.DBO.Hello\UPDATE_TESTS.DBO.DNROilGasWells',
update_tests_sde, check_for_changes_gdb, 'DNROilGasWells'))
self.assertEqual(result, True)
def test_schema_changes_in_sde(self):
skip_if_no_local_sde()
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
result = core.check_schema(
Crate('FieldTypeFloat', test_gdb, update_tests_sde,
'FieldTypeFloat'))
self.assertEqual(result, True)
def test_check_schema_ignore_length_for_all_except_text(test_gdb):
skip_if_no_local_sde()
# only worry about length on text fields
result = core.check_schema(
Crate(r'UPDATE_TESTS.DBO.Hello\UPDATE_TESTS.DBO.DNROilGasWells',
UPDATE_TESTS_SDE, test_gdb, 'DNROilGasWells'))
assert result == True
def test_create_destination_data_feature_class(test_gdb):
arcpy.CreateFileGDB_management(path.join(CURRENT_FOLDER, 'data'),
'test.gdb')
fc_crate = Crate('DNROilGasWells', test_gdb, TEMP_GDB)
core._create_destination_data(fc_crate)
assert arcpy.Exists(fc_crate.destination) == True
def test_is_ready_to_ship_crates_with_any_schema_changed_returns_false(
self):
updated = Crate('', '', '', '')
updated.result = (Crate.UPDATED, None)
no_changes = Crate('', '', '', '')
no_changes.result = (Crate.NO_CHANGES, None)
schema_change = Crate('', '', '', '')
schema_change.result = (Crate.INVALID_DATA, None)
self.patient._crates = [updated, no_changes, schema_change]
self.assertFalse(self.patient.is_ready_to_ship())
def test_deleted_destination_between_updates(test_gdb):
crate = Crate('ZipCodes', test_gdb, TEMP_GDB, 'ImNotHere')
core.update(crate, lambda x: True, CHANGE_DETECTION)
delete_if_arcpy_exists(crate.destination)
assert core.update(crate, lambda x: True,
CHANGE_DETECTION)[0] == Crate.CREATED
assert arcpy.Exists(crate.destination) == True
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 14
def test_move_data_feature_class():
skip_if_no_local_sde()
crate = Crate('DNROilGasWells', UPDATE_TESTS_SDE,
TEMP_GDB) #: feature class
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 5
def test_invalid_source_primary_key_name(self):
skip_if_no_local_sde()
crate = Crate('NO_OBJECTID_TEST',
update_tests_sde,
'',
'',
source_primary_key='NOTAFIELD')
self.assertEqual(crate.result[0], Crate.INVALID_DATA)
def test_hash_custom_source_key_float(self):
skip_if_no_local_sde()
tbl = 'FLOAT_ID'
#: has changes
crate = Crate('UPDATE_TESTS.dbo.{}'.format(tbl), update_tests_sde, check_for_changes_gdb, tbl, source_primary_key='TEST')
changes = core._hash(crate)
self.assertEqual(len(changes.adds), 1)
def test_invalid_data(test_gdb):
hash_table = str(Path(test_gdb) / 'TableHashes')
change_detection = ChangeDetection(['ChangeDetection'], test_gdb, hash_table=hash_table)
table = 'update_tests.dbo.providers'
crate = Crate(table, 'someWorkspace', arcpy.env.scratchGDB, Path(test_gdb).name)
result = change_detection.update(crate)
assert result[0] == Crate.INVALID_DATA
def run(name):
return core._hash(
Crate(
name,
update_tests_sde,
check_for_changes_gdb,
name + '_NEW',
destination_coordinate_system=arcpy.SpatialReference(3857),
geographic_transformation='NAD_1983_To_WGS_1984_5'))
def test_crate_ctor_doesnt_alter_destination_name(self):
source_name = 'name'
source_workspace = 'does not matter'
destination_workspace = env.scratchGDB
destination_name = 'db.owner.name'
x = Crate(source_name, source_workspace, destination_workspace, destination_name)
self.assertEqual(x.destination_name, destination_name)
def test_move_data_skip_empty_geometry(test_gdb):
empty_points = 'EmptyPointTest'
crate = Crate(empty_points, test_gdb, TEMP_GDB)
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 4
def test_move_data_skip_empty_geometry(self):
empty_geometry_gdb = path.join(current_folder, 'data', 'EmptyGeometry.gdb')
empty_points = 'EmptyPointTest'
crate = Crate(empty_points, empty_geometry_gdb, test_gdb)
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 4)
def test_update_default_validation_that_fails(self, arcpy_exists):
arcpy_exists.return_value = True
core.check_schema = Mock(side_effect=ValidationException())
def custom(crate):
return NotImplemented
crate = Crate('', '', '', describer=mocks.Describe)
self.assertEqual(core.update(crate, custom)[0], Crate.INVALID_DATA)
def test_try_to_find_data_source_by_name_filters_common_duplicates(self, list_feature_classes):
list_feature_classes.return_value = ['db.owner.Counties', 'db.owner.duplicateCounties']
crate = Crate(
source_name='Counties',
source_workspace='Database Connections\\something.sde',
destination_workspace='c:\\something.gdb',
destination_name='Counties')
#: reset values because _try_to_find_data_source_by_name is called in the init
crate.set_source_name('Counties')
ok, name = crate._try_to_find_data_source_by_name()
self.assertTrue(ok)
self.assertEqual(name, 'db.owner.Counties')
self.assertEqual(crate.source_name, name)
self.assertEqual(crate.destination_name, 'Counties')
self.assertEqual(crate.source, path.join(crate.source_workspace, crate.source_name))
def test_update_default_validation_that_fails(arcpy_exists):
arcpy_exists.return_value = True
def custom(crate):
return NotImplemented
crate = Crate('', '', '', describer=mocks.Describe)
assert core.update(crate, custom,
CHANGE_DETECTION)[0] == Crate.INVALID_DATA
def test_preserve_metadata(test_gdb):
crate = Crate('PreserveMetadata', test_gdb, test_gdb,
'PreserveMetadata_Dest')
source_metadata = arcpy.metadata.Metadata(crate.source)
core._create_destination_data(crate)
destination_metadata = arcpy.metadata.Metadata(crate.destination)
assert source_metadata.description == destination_metadata.description
assert source_metadata.title == destination_metadata.title
assert source_metadata.summary == destination_metadata.summary
def test_hash_custom_source_key_float(test_gdb):
skip_if_no_local_sde()
tbl = 'FLOAT_ID'
#: has changes
crate = Crate('UPDATE_TESTS.dbo.{}'.format(tbl), UPDATE_TESTS_SDE,
test_gdb, tbl)
changes = core._hash(crate)
assert len(changes.adds) == 1
def test_try_to_find_data_source_by_name_filters_common_duplicates(self, walk):
walk.return_value = [(None, None, ['db.owner.Counties', 'db.owner.duplicateCounties'])]
crate = Crate(
source_name='Counties',
source_workspace='Database Connections\\something.sde',
destination_workspace='c:\\something.gdb',
destination_name='Counties')
#: reset values because _try_to_find_data_source_by_name is called in the init
crate.set_source_name('Counties')
ok, name = crate._try_to_find_data_source_by_name()
self.assertTrue(ok)
self.assertEqual(name, 'db.owner.Counties')
self.assertEqual(crate.source_name, name)
self.assertEqual(crate.destination_name, 'Counties')
self.assertEqual(crate.source, path.join(crate.source_workspace, crate.source_name))
def update_problem_layers(self):
for source_name, source_workspace, destination_workspace, destination_name in self.problem_layer_infos:
if self.test_layer and self.test_layer.split('.')[-1] != destination_name:
continue
try:
crate = Crate(source_name, source_workspace, destination_workspace, destination_name)
source = path.join(source_workspace, source_name)
destination = path.join(destination_workspace, destination_name)
if not arcpy.Exists(destination):
self.log.info('creating %s', destination)
arcpy.Copy_management(source, destination)
crate.result = (Crate.CREATED, None)
else:
self.log.info('manually updating %s', destination)
arcpy.TruncateTable_management(destination)
arcpy.Append_management(source, destination, 'TEST')
crate.result = (Crate.UPDATED, None)
except Exception as ex:
self.log.error('error manually updating %s!', destination)
crate.result = (Crate.UNHANDLED_EXCEPTION, ex)
self._crates.append(crate)
def test_try_to_find_data_source_by_name_returns_None_if_not_sde(self):
crate = Crate(source_name='something.shp', source_workspace='c:\\temp', destination_workspace='c:\\something.gdb', destination_name='Counties')
self.assertIsNone(crate._try_to_find_data_source_by_name()[0])
def test_set_result_with_invalid_result_returns_result(self):
crate = Crate('foo', 'bar', 'baz', 'goo')
self.assertEqual(crate.set_result(('wat?', 'some crazy message'))[0], 'unknown result')
self.assertEqual(crate.result[0], 'unknown result')
def test_set_result_with_valid_result_returns_result(self):
crate = Crate('foo', 'bar', 'baz', 'goo')
self.assertEqual(crate.set_result((Crate.UPDATED, 'Yay!'))[0], Crate.UPDATED)
self.assertEqual(crate.result[0], Crate.UPDATED)
return log
try:
sgid_name = 'DAQAirMonitorData'
sgid_db = settings.sgid['ENVIRONMENT']
stage_db = r'C:\forklift\data\hashed\deqquerylayers.gdb'
source_db = path.join(settings.dbConnects, r'AVData.sde')
source_name = 'AVData.dbo.interactive_map_monitoring_data'
bad_results = [Crate.INVALID_DATA, Crate.UNHANDLED_EXCEPTION, Crate.UNINITIALIZED, Crate.ERROR]
log = _setup_logging()
log.info('creating crate')
crate = Crate(sgid_name, sgid_db, stage_db, sgid_name)
log.info('processing crate')
core.init(log)
crate.set_result(core.update(crate, validate_crate))
if crate.was_updated():
log.info('updating data in SDE')
sgid_destination = path.join(sgid_db, 'SGID10.ENVIRONMENT.{}'.format(sgid_name))
arcpy.management.TruncateTable(sgid_destination)
arcpy.management.Append(crate.destination, sgid_destination, 'NO_TEST')
log.info('updating prod fgdbs')
for dest_fgdb in [settings.mapData1, settings.mapData2]:
dest = path.join(dest_fgdb, 'deqquerylayers.gdb', sgid_name)
arcpy.management.TruncateTable(dest)
arcpy.management.Append(crate.destination, dest, 'NO_TEST')
