def test_update_duplicate_features(self):
arcpy.Copy_management(duplicates_gdb, duplicates_gdb_copy)
crate = Crate('Duplicates', duplicates_gdb_copy, test_gdb, 'DuplicatesDest')
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(crate.destination).getOutput(0), '4')
#: remove feature
with arcpy.da.UpdateCursor(crate.source, '*') as delete_cursor:
for row in delete_cursor:
delete_cursor.deleteRow()
break
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(crate.destination).getOutput(0), '3')
#: change feature
with arcpy.da.UpdateCursor(crate.source, ['TEST']) as update_cursor:
for row in update_cursor:
row[0] = 'change'
update_cursor.updateRow(row)
break
self.assertEqual(core.update(crate, lambda x: True)[0], Crate.WARNING)
self.assertEqual(arcpy.GetCount_management(crate.destination).getOutput(0), '3')
def test_update_duplicate_features(test_gdb):
arcpy.management.Copy(test_gdb, TEMP_GDB)
crate = Crate('Duplicates', TEMP_GDB, TEMP_GDB, 'DuplicatesDest')
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert arcpy.GetCount_management(crate.destination).getOutput(0) == '4'
#: remove feature
with arcpy.da.UpdateCursor(crate.source, '*') as delete_cursor:
for row in delete_cursor:
delete_cursor.deleteRow()
break
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert arcpy.GetCount_management(crate.destination).getOutput(0) == '3'
#: change feature
with arcpy.da.UpdateCursor(crate.source, ['TEST']) as update_cursor:
for row in update_cursor:
row[0] = 'change'
update_cursor.updateRow(row)
break
assert core.update(
crate, lambda x: True,
CHANGE_DETECTION)[0] == Crate.UPDATED_OR_CREATED_WITH_WARNINGS
assert arcpy.GetCount_management(crate.destination).getOutput(0) == '3'
def test_move_data_table():
skip_if_no_local_sde()
crate = Crate('Providers', UPDATE_TESTS_SDE, TEMP_GDB) #: table
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 57
def test_deleted_destination_between_updates(self):
crate = Crate('ZipCodes', check_for_changes_gdb, test_gdb, 'ImNotHere')
core.update(crate, lambda x: True)
delete_if_arcpy_exists(crate.destination)
self.assertEqual(core.update(crate, lambda x: True)[0], Crate.CREATED)
self.assertEqual(arcpy.Exists(crate.destination), True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 299)
def test_move_data_table(self):
skip_if_no_local_sde()
crate = Crate('Providers', update_tests_sde, test_gdb) #: table
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 57)
def test_move_data_feature_class(self):
skip_if_no_local_sde()
crate = Crate('DNROilGasWells', update_tests_sde, test_gdb) #: feature class
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 5)
def test_deleted_destination_between_updates(self):
crate = Crate('ZipCodes', check_for_changes_gdb, test_gdb, 'ImNotHere')
core.update(crate, lambda x: True)
delete_if_arcpy_exists(crate.destination)
self.assertEqual(core.update(crate, lambda x: True)[0], Crate.CREATED)
self.assertEqual(arcpy.Exists(crate.destination), True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 299)
def test_move_data_feature_class():
skip_if_no_local_sde()
crate = Crate('DNROilGasWells', UPDATE_TESTS_SDE,
TEMP_GDB) #: feature class
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 5
def test_deleted_destination_between_updates(test_gdb):
crate = Crate('ZipCodes', test_gdb, TEMP_GDB, 'ImNotHere')
core.update(crate, lambda x: True, CHANGE_DETECTION)
delete_if_arcpy_exists(crate.destination)
assert core.update(crate, lambda x: True,
CHANGE_DETECTION)[0] == Crate.CREATED
assert arcpy.Exists(crate.destination) == True
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 14
def test_move_data_skip_empty_geometry(test_gdb):
empty_points = 'EmptyPointTest'
crate = Crate(empty_points, test_gdb, TEMP_GDB)
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 4
def test_move_data_feature_class(self):
skip_if_no_local_sde()
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('DNROilGasWells', update_tests_sde, test_gdb) #: feature class
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 5)
def test_move_data_skip_empty_geometry(self):
empty_geometry_gdb = path.join(current_folder, 'data', 'EmptyGeometry.gdb')
empty_points = 'EmptyPointTest'
crate = Crate(empty_points, empty_geometry_gdb, test_gdb)
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 4)
def test_move_data_table(self):
skip_if_no_local_sde()
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('Providers', update_tests_sde, test_gdb) #: table
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 57)
def test_move_data_skip_empty_geometry(self):
empty_geometry_gdb = path.join(current_folder, 'data', 'EmptyGeometry.gdb')
empty_points = 'EmptyPointTest'
crate = Crate(empty_points, empty_geometry_gdb, test_gdb)
core.update(crate, lambda x: True)
self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 4)
def test_move_data_no_objectid(self):
skip_if_no_local_sde()
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('NO_OBJECTID_TEST', update_tests_sde, test_gdb, source_primary_key='TEST')
core.update(crate, lambda x: True)
with arcpy.da.SearchCursor(crate.destination, '*') as cur:
row = cur.next()
self.assertEqual('this is ', row[1])
def test_move_data_no_objectid():
skip_if_no_local_sde()
crate = Crate('NO_OBJECTID_TEST', UPDATE_TESTS_SDE, TEMP_GDB)
core.update(crate, lambda x: True, CHANGE_DETECTION)
with arcpy.da.SearchCursor(crate.destination, '*') as cur:
for row in cur:
assert 'this is ' == row[1]
break
def test_move_data_no_objectid(self):
skip_if_no_local_sde()
crate = Crate('NO_OBJECTID_TEST', update_tests_sde, test_gdb, source_primary_key='TEST')
core.update(crate, lambda x: True)
with arcpy.da.SearchCursor(crate.destination, '*') as cur:
for row in cur:
self.assertEqual('this is ', row[1])
break
def test_destination_exists_hash_not_exist(self):
#: If there is no existing hash then the dest table should be truncated
#: and all feature should be added as new.
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('ExistingDest', test_gdb, test_gdb, 'ExistingDest_Dest')
changes = core._hash(crate, core.hash_gdb_path, False)
self.assertTrue(arcpy.Exists(path.join(core.hash_gdb_path, crate.name)))
self.assertEqual(len(changes.adds), 4)
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(crate.destination)[0], '4')
def test_source_row_added(test_gdb):
arcpy.Copy_management(test_gdb, TEMP_GDB)
crate = Crate('RowAdd', TEMP_GDB, TEMP_GDB, 'RowAdd_Dest')
core.update(crate, lambda x: True, CHANGE_DETECTION)
with arcpy.da.InsertCursor(crate.source, 'URL') as cur:
cur.insertRow(('newrow', ))
changes = core._hash(crate)
assert len(changes.adds) == 1
assert len(changes._deletes) == 0
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert arcpy.GetCount_management(crate.destination)[0] == '6'
def test_update_custom_validation_that_fails(self, arcpy_exists):
arcpy_exists.return_value = True
crate = Crate('', '', '', describer=arcpy_mocks.Describe)
self.assertEqual(
core.update(crate, raise_validation_exception)[0],
Crate.INVALID_DATA)
def test_source_row_added(self):
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('RowAdd', test_gdb, test_gdb, 'RowAdd_Dest')
core.update(crate, lambda x: True)
with arcpy.da.InsertCursor(crate.source, 'URL') as cur:
cur.insertRow(('newrow',))
changes = core._hash(crate)
self.assertEqual(len(changes.adds), 1)
self.assertEqual(len(changes._deletes), 0)
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(crate.destination)[0], '6')
def test_update_new_dataset_with_change_detection(test_gdb):
change_detection = ChangeDetection([], 'blah')
change_detection.has_table = MagicMock(name='has_table', return_value=True)
change_detection.has_changed = MagicMock(name='has_changed',
return_value=False)
crate = Crate('Counties', test_gdb, test_gdb, 'Counties_Destination')
core.update(crate, lambda c: True, change_detection)
source_count = arcpy.management.GetCount(str(Path(test_gdb) /
'Counties'))[0]
destination_count = arcpy.management.GetCount(
str(Path(test_gdb) / 'Counties_Destination'))[0]
assert source_count == destination_count
def test_source_row_geometry_changed(self):
row_api = '4300311427'
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('GeometryChange', test_gdb, test_gdb, 'GeometryChange_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, 'Shape@XY', 'API = \'{}\''.format(row_api)) as cur:
for row in cur:
row[0] = (row[0][0] + 10, row[0][1] + 10)
cur.updateRow(row)
break
changes = core._hash(crate)
self.assertEqual(len(changes.adds), 1)
self.assertEqual(len(changes._deletes), 1)
def test_update_error(self, arcpy_exists):
arcpy_exists.return_value = True
crate = Crate('', '', '', describer=arcpy_mocks.Describe)
self.assertEqual(
core.update(crate, lambda c: True)[0], Crate.UNHANDLED_EXCEPTION)
def test_update_error(arcpy_exists):
arcpy_exists.return_value = True
crate = Crate('', '', '', describer=mocks.Describe)
assert core.update(crate, lambda c: True,
CHANGE_DETECTION)[0] == Crate.UNHANDLED_EXCEPTION
def test_source_row_geometry_changed(test_gdb):
row_api = '4300311427'
arcpy.Copy_management(test_gdb, TEMP_GDB)
crate = Crate('GeometryChange', TEMP_GDB, TEMP_GDB, 'GeometryChange_Dest')
core.update(crate, lambda x: True, CHANGE_DETECTION)
with arcpy.da.UpdateCursor(crate.source, 'Shape@XY',
'API = \'{}\''.format(row_api)) as cur:
for row in cur:
row[0] = (row[0][0] + 10, row[0][1] + 10)
cur.updateRow(row)
break
changes = core._hash(crate)
assert len(changes.adds) == 1
assert len(changes._deletes) == 1
def test_source_row_added(self):
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('RowAdd', test_gdb, test_gdb, 'RowAdd_Dest')
core.update(crate, lambda x: True)
with arcpy.da.InsertCursor(crate.source, 'URL') as cur:
cur.insertRow(('newrow',))
changes = core._hash(crate, core.hash_gdb_path, False)
self.assertEqual(len(changes.adds), 1)
self.assertEqual(len(changes._deletes), 0)
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(path.join(core.hash_gdb_path, crate.name))[0], '6')
self.assertEqual(arcpy.GetCount_management(crate.destination)[0], '6')
def test_source_row_attribute_changed(self):
row_name = 'MALTA'
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('AttributeChange', test_gdb, test_gdb, 'AttributeChange_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, 'SYMBOL', 'NAME = \'{}\''.format(row_name)) as cur:
for row in cur:
row[0] = 99
cur.updateRow(row)
break
changes = core._hash(crate)
self.assertEqual(len(changes.adds), 1)
self.assertEqual(len(changes._deletes), 1)
def test_source_row_geometry_changed_to_none(test_gdb):
arcpy.Copy_management(test_gdb, TEMP_GDB)
crate = Crate('GeometryToNull', TEMP_GDB, TEMP_GDB, 'GeometryToNull_Dest')
core.update(crate, lambda x: True, CHANGE_DETECTION)
with arcpy.da.UpdateCursor(crate.source, 'Shape@XY') as cur:
for row in cur:
row[0] = None
cur.updateRow(row)
break
changes = core._hash(crate)
assert len(changes._deletes) == 1
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert arcpy.GetCount_management(crate.destination)[0] == '3'
def test_source_row_geometry_changed(self):
row_api = '4300311427'
row_id = '4164826'
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('GeometryChange', test_gdb, test_gdb, 'GeometryChange_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, 'Shape@XY', 'API = \'{}\''.format(row_api)) as cur:
row = cur.next()
row[0] = (row[0][0] + 10, row[0][1] + 10)
cur.updateRow(row)
changes = core._hash(crate, core.hash_gdb_path, False)
self.assertEqual(len(changes.adds), 1)
self.assertEqual(changes.adds.keys()[0], row_id)
self.assertEqual(len(changes._deletes), 1)
self.assertEqual(list(changes._deletes)[0], '3')
def test_source_row_geometry_changed_to_none(self):
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('GeometryToNull', test_gdb, test_gdb, 'GeometryToNull_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, 'Shape@XY') as cur:
for row in cur:
row[0] = None
cur.updateRow(row)
break
changes = core._hash(crate)
self.assertEqual(len(changes._deletes), 1)
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(crate.destination)[0], '3')
def test_source_row_geometry_changed_to_none(self):
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('GeometryToNull', test_gdb, test_gdb, 'GeometryToNull_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, 'Shape@XY') as cur:
row = cur.next()
row[0] = None
cur.updateRow(row)
changes = core._hash(crate, core.hash_gdb_path, False)
self.assertEqual(len(changes._deletes), 1)
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(path.join(core.hash_gdb_path, crate.name))[0], '3')
self.assertEqual(arcpy.GetCount_management(crate.destination)[0], '3')
def test_source_row_attribute_changed(self):
row_name = 'MALTA'
row_id = '588'
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('AttributeChange', test_gdb, test_gdb, 'AttributeChange_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, 'SYMBOL', 'NAME = \'{}\''.format(row_name)) as cur:
row = cur.next()
row[0] = 99
cur.updateRow(row)
changes = core._hash(crate, core.hash_gdb_path, False)
self.assertEqual(len(changes.adds), 1)
self.assertEqual(changes.adds.keys()[0], row_id)
self.assertEqual(len(changes._deletes), 1)
self.assertEqual(list(changes._deletes)[0], '4')
def test_source_row_attribute_changed(test_gdb):
row_name = 'MALTA'
arcpy.Copy_management(test_gdb, TEMP_GDB)
crate = Crate('AttributeChange', TEMP_GDB, TEMP_GDB,
'AttributeChange_Dest')
core.update(crate, lambda x: True, CHANGE_DETECTION)
with arcpy.da.UpdateCursor(crate.source, 'SYMBOL',
'NAME = \'{}\''.format(row_name)) as cur:
for row in cur:
row[0] = 99
cur.updateRow(row)
break
changes = core._hash(crate)
assert len(changes.adds) == 1
assert len(changes._deletes) == 1
def test_update_default_validation_that_fails(arcpy_exists):
arcpy_exists.return_value = True
def custom(crate):
return NotImplemented
crate = Crate('', '', '', describer=mocks.Describe)
assert core.update(crate, custom,
CHANGE_DETECTION)[0] == Crate.INVALID_DATA
def test_update_default_validation_that_fails(self, arcpy_exists):
arcpy_exists.return_value = True
core.check_schema = Mock(side_effect=ValidationException())
def custom(crate):
return NotImplemented
crate = Crate('', '', '', describer=mocks.Describe)
self.assertEqual(core.update(crate, custom)[0], Crate.INVALID_DATA)
def test_update_default_validation_that_fails(self, arcpy_exists):
arcpy_exists.return_value = True
core.check_schema = Mock(side_effect=ValidationException())
def custom(crate):
return NotImplemented
crate = Crate('', '', '', describer=arcpy_mocks.Describe)
self.assertEqual(core.update(crate, custom)[0], Crate.INVALID_DATA)
def test_source_row_deleted(self):
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('RowDelete', test_gdb, test_gdb, 'RowDelete_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, '*') as cur:
for row in cur:
cur.deleteRow()
break
changes = core._hash(crate)
#: all features hashes are invalid since we deleted the first row
#: which changes the salt for all following rows
self.assertEqual(len(changes.adds), 0)
self.assertEqual(len(changes._deletes), 1)
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(crate.destination)[0], '4')
def test_source_row_deleted(test_gdb):
arcpy.Copy_management(test_gdb, TEMP_GDB)
crate = Crate('RowDelete', TEMP_GDB, TEMP_GDB, 'RowDelete_Dest')
core.update(crate, lambda x: True, CHANGE_DETECTION)
with arcpy.da.UpdateCursor(crate.source, '*') as cur:
for _ in cur:
cur.deleteRow()
break
changes = core._hash(crate)
#: all features hashes are invalid since we deleted the first row
#: which changes the salt for all following rows
assert len(changes.adds) == 0
assert len(changes._deletes) == 1
core.update(crate, lambda x: True, CHANGE_DETECTION)
assert arcpy.GetCount_management(crate.destination)[0] == '4'
def test_source_row_deleted(self):
arcpy.Copy_management(check_for_changes_gdb, test_gdb)
crate = Crate('RowDelete', test_gdb, test_gdb, 'RowDelete_Dest')
core.update(crate, lambda x: True)
with arcpy.da.UpdateCursor(crate.source, '*') as cur:
cur.next()
cur.deleteRow()
changes = core._hash(crate, core.hash_gdb_path, False)
#: all features hashes are invalid since we deleted the first row
#: which changes the salt for all following rows
self.assertEqual(len(changes.adds), 4)
self.assertEqual(len(changes._deletes), 5)
core.update(crate, lambda x: True)
self.assertEqual(arcpy.GetCount_management(path.join(core.hash_gdb_path, crate.name))[0], '4')
self.assertEqual(arcpy.GetCount_management(crate.destination)[0], '4')
try:
sgid_name = 'DAQAirMonitorData'
sgid_db = settings.sgid['ENVIRONMENT']
stage_db = r'C:\forklift\data\hashed\deqquerylayers.gdb'
source_db = path.join(settings.dbConnects, r'AVData.sde')
source_name = 'AVData.dbo.interactive_map_monitoring_data'
bad_results = [Crate.INVALID_DATA, Crate.UNHANDLED_EXCEPTION, Crate.UNINITIALIZED, Crate.ERROR]
log = _setup_logging()
log.info('creating crate')
crate = Crate(sgid_name, sgid_db, stage_db, sgid_name)
log.info('processing crate')
core.init(log)
crate.set_result(core.update(crate, validate_crate))
if crate.was_updated():
log.info('updating data in SDE')
sgid_destination = path.join(sgid_db, 'SGID10.ENVIRONMENT.{}'.format(sgid_name))
arcpy.management.TruncateTable(sgid_destination)
arcpy.management.Append(crate.destination, sgid_destination, 'NO_TEST')
log.info('updating prod fgdbs')
for dest_fgdb in [settings.mapData1, settings.mapData2]:
dest = path.join(dest_fgdb, 'deqquerylayers.gdb', sgid_name)
arcpy.management.TruncateTable(dest)
arcpy.management.Append(crate.destination, dest, 'NO_TEST')
if crate.result[0] in bad_results:
send_email(reportEmail,
'DEQ Hourly Crate Error',
def test_update_no_existing_destination(test_gdb):
crate = Crate('ZipCodes', test_gdb, TEMP_GDB, 'ImNotHere')
assert core.update(crate, lambda x: True,
CHANGE_DETECTION)[0] == Crate.CREATED
assert arcpy.Exists(crate.destination) == True
def test_update_no_existing_destination(self):
crate = Crate('ZipCodes', check_for_changes_gdb, test_gdb, 'ImNotHere')
self.assertEqual(core.update(crate, lambda x: True)[0], Crate.CREATED)
self.assertEqual(arcpy.Exists(crate.destination), True)
def test_update_no_existing_destination(self):
crate = Crate('ZipCodes', check_for_changes_gdb, test_gdb, 'ImNotHere')
self.assertEqual(core.update(crate, lambda x: True)[0], Crate.CREATED)
self.assertEqual(arcpy.Exists(crate.destination), True)
def test_update_custom_validation_that_fails(self, arcpy_exists):
arcpy_exists.return_value = True
crate = Crate('', '', '', describer=arcpy_mocks.Describe)
self.assertEqual(core.update(crate, raise_validation_exception)[0], Crate.INVALID_DATA)
def test_update_error(self, arcpy_exists):
arcpy_exists.return_value = True
crate = Crate('', '', '', describer=arcpy_mocks.Describe)
self.assertEqual(core.update(crate, lambda c: True)[0], Crate.UNHANDLED_EXCEPTION)