def test_set_source_name_updates_source_if_not_none(self): crate = Crate('foo', 'bar', 'baz', 'goo') crate.set_source_name(None) self.assertEqual(crate.source_name, 'foo') self.assertEqual(crate.source, path.join('bar', 'foo'))
def test_try_to_find_data_source_by_name_returns_False_if_duplicate(self, walk): walk.return_value = [(None, None, ['db.owner.Counties', 'db.owner2.Counties'])] crate = Crate( source_name='duplicate', source_workspace='Database Connections\\something.sde', destination_workspace='c:\\something.gdb', destination_name='Counties') self.assertFalse(crate._try_to_find_data_source_by_name()[0])
def test_init_with_coordinate_system_does_not_change(self): crate = Crate('foo', 'bar', 'baz', 'qux', SpatialReference(26921)) self.assertEqual(crate.source_name, 'foo') self.assertEqual(crate.source_workspace, 'bar') self.assertEqual(crate.destination_workspace, 'baz') self.assertEqual(crate.destination_name, 'qux') self.assertIsInstance(crate.destination_coordinate_system, SpatialReference)
def test_init_with_coordinate_system_as_number_becomes_spatial_reference(self): crate = Crate('foo', 'bar', 'baz', 'qux', 26912) self.assertEqual(crate.source_name, 'foo') self.assertEqual(crate.source_workspace, 'bar') self.assertEqual(crate.destination_workspace, 'baz') self.assertEqual(crate.destination_name, 'qux') self.assertIsInstance(crate.destination_coordinate_system, SpatialReference)
def test_update_custom_validation_that_fails(self, arcpy_exists): arcpy_exists.return_value = True crate = Crate('', '', '', describer=arcpy_mocks.Describe) self.assertEqual( core.update(crate, raise_validation_exception)[0], Crate.INVALID_DATA)
def test_is_ready_to_ship_crates_with_any_exception_returns_false(self): updated = Crate('', '', '', '') updated.result = (Crate.UPDATED, None) no_changes = Crate('', '', '', '') no_changes.result = (Crate.NO_CHANGES, None) unhandled_exception = Crate('', '', '', '') unhandled_exception.result = (Crate.UNHANDLED_EXCEPTION, None) self.patient._crates = [updated, no_changes, unhandled_exception] self.assertFalse(self.patient.is_ready_to_ship())
def test_deleted_destination_between_updates(self): crate = Crate('ZipCodes', check_for_changes_gdb, test_gdb, 'ImNotHere') core.update(crate, lambda x: True) delete_if_arcpy_exists(crate.destination) self.assertEqual(core.update(crate, lambda x: True)[0], Crate.CREATED) self.assertEqual(arcpy.Exists(crate.destination), True) self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 299)
def run(name): return core._hash( Crate(name, UPDATE_TESTS_SDE, test_gdb, name, destination_coordinate_system=arcpy.SpatialReference(3857), geographic_transformation='NAD_1983_To_WGS_1984_5'))
def test_move_data_feature_class(self): skip_if_no_local_sde() crate = Crate('DNROilGasWells', update_tests_sde, test_gdb) #: feature class core.update(crate, lambda x: True) self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 5)
def test_move_data_table(): skip_if_no_local_sde() crate = Crate('Providers', UPDATE_TESTS_SDE, TEMP_GDB) #: table core.update(crate, lambda x: True, CHANGE_DETECTION) assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 57
def test_create_name_is_combined_hash_and_table_three_values(self): destination_workspace = 'dw' source_name = 'sn' crate = Crate(source_name, 'source', destination_workspace) hash = source_name + '_' + xxh64(path.join(destination_workspace, source_name)).hexdigest() self.assertEqual(crate.name, hash)
def test_create_destination_data_table(test_gdb): arcpy.CreateFileGDB_management(path.join(CURRENT_FOLDER, 'data'), 'test.gdb') tbl_crate = Crate('Providers', test_gdb, TEMP_GDB) core._create_destination_data(tbl_crate) assert arcpy.Exists(tbl_crate.destination) == True
def test_move_data_table(self): skip_if_no_local_sde() crate = Crate('Providers', update_tests_sde, test_gdb) #: table core.update(crate, lambda x: True) self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 57)
def test_check_schema_ignore_length_for_all_except_text(self): skip_if_no_local_sde() # only worry about length on text fields result = core.check_schema( Crate(r'UPDATE_TESTS.DBO.Hello\UPDATE_TESTS.DBO.DNROilGasWells', update_tests_sde, check_for_changes_gdb, 'DNROilGasWells')) self.assertEqual(result, True)
def test_schema_changes_in_sde(self): skip_if_no_local_sde() arcpy.Copy_management(check_for_changes_gdb, test_gdb) result = core.check_schema( Crate('FieldTypeFloat', test_gdb, update_tests_sde, 'FieldTypeFloat')) self.assertEqual(result, True)
def test_check_schema_ignore_length_for_all_except_text(test_gdb): skip_if_no_local_sde() # only worry about length on text fields result = core.check_schema( Crate(r'UPDATE_TESTS.DBO.Hello\UPDATE_TESTS.DBO.DNROilGasWells', UPDATE_TESTS_SDE, test_gdb, 'DNROilGasWells')) assert result == True
def test_create_destination_data_feature_class(test_gdb): arcpy.CreateFileGDB_management(path.join(CURRENT_FOLDER, 'data'), 'test.gdb') fc_crate = Crate('DNROilGasWells', test_gdb, TEMP_GDB) core._create_destination_data(fc_crate) assert arcpy.Exists(fc_crate.destination) == True
def test_is_ready_to_ship_crates_with_any_schema_changed_returns_false( self): updated = Crate('', '', '', '') updated.result = (Crate.UPDATED, None) no_changes = Crate('', '', '', '') no_changes.result = (Crate.NO_CHANGES, None) schema_change = Crate('', '', '', '') schema_change.result = (Crate.INVALID_DATA, None) self.patient._crates = [updated, no_changes, schema_change] self.assertFalse(self.patient.is_ready_to_ship())
def test_deleted_destination_between_updates(test_gdb): crate = Crate('ZipCodes', test_gdb, TEMP_GDB, 'ImNotHere') core.update(crate, lambda x: True, CHANGE_DETECTION) delete_if_arcpy_exists(crate.destination) assert core.update(crate, lambda x: True, CHANGE_DETECTION)[0] == Crate.CREATED assert arcpy.Exists(crate.destination) == True assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 14
def test_move_data_feature_class(): skip_if_no_local_sde() crate = Crate('DNROilGasWells', UPDATE_TESTS_SDE, TEMP_GDB) #: feature class core.update(crate, lambda x: True, CHANGE_DETECTION) assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 5
def test_invalid_source_primary_key_name(self): skip_if_no_local_sde() crate = Crate('NO_OBJECTID_TEST', update_tests_sde, '', '', source_primary_key='NOTAFIELD') self.assertEqual(crate.result[0], Crate.INVALID_DATA)
def test_hash_custom_source_key_float(self): skip_if_no_local_sde() tbl = 'FLOAT_ID' #: has changes crate = Crate('UPDATE_TESTS.dbo.{}'.format(tbl), update_tests_sde, check_for_changes_gdb, tbl, source_primary_key='TEST') changes = core._hash(crate) self.assertEqual(len(changes.adds), 1)
def test_invalid_data(test_gdb): hash_table = str(Path(test_gdb) / 'TableHashes') change_detection = ChangeDetection(['ChangeDetection'], test_gdb, hash_table=hash_table) table = 'update_tests.dbo.providers' crate = Crate(table, 'someWorkspace', arcpy.env.scratchGDB, Path(test_gdb).name) result = change_detection.update(crate) assert result[0] == Crate.INVALID_DATA
def run(name): return core._hash( Crate( name, update_tests_sde, check_for_changes_gdb, name + '_NEW', destination_coordinate_system=arcpy.SpatialReference(3857), geographic_transformation='NAD_1983_To_WGS_1984_5'))
def test_crate_ctor_doesnt_alter_destination_name(self): source_name = 'name' source_workspace = 'does not matter' destination_workspace = env.scratchGDB destination_name = 'db.owner.name' x = Crate(source_name, source_workspace, destination_workspace, destination_name) self.assertEqual(x.destination_name, destination_name)
def test_move_data_skip_empty_geometry(test_gdb): empty_points = 'EmptyPointTest' crate = Crate(empty_points, test_gdb, TEMP_GDB) core.update(crate, lambda x: True, CHANGE_DETECTION) assert int(arcpy.GetCount_management(crate.destination).getOutput(0)) == 4
def test_move_data_skip_empty_geometry(self): empty_geometry_gdb = path.join(current_folder, 'data', 'EmptyGeometry.gdb') empty_points = 'EmptyPointTest' crate = Crate(empty_points, empty_geometry_gdb, test_gdb) core.update(crate, lambda x: True) self.assertEqual(int(arcpy.GetCount_management(crate.destination).getOutput(0)), 4)
def test_update_default_validation_that_fails(self, arcpy_exists): arcpy_exists.return_value = True core.check_schema = Mock(side_effect=ValidationException()) def custom(crate): return NotImplemented crate = Crate('', '', '', describer=mocks.Describe) self.assertEqual(core.update(crate, custom)[0], Crate.INVALID_DATA)
def test_try_to_find_data_source_by_name_filters_common_duplicates(self, list_feature_classes): list_feature_classes.return_value = ['db.owner.Counties', 'db.owner.duplicateCounties'] crate = Crate( source_name='Counties', source_workspace='Database Connections\\something.sde', destination_workspace='c:\\something.gdb', destination_name='Counties') #: reset values because _try_to_find_data_source_by_name is called in the init crate.set_source_name('Counties') ok, name = crate._try_to_find_data_source_by_name() self.assertTrue(ok) self.assertEqual(name, 'db.owner.Counties') self.assertEqual(crate.source_name, name) self.assertEqual(crate.destination_name, 'Counties') self.assertEqual(crate.source, path.join(crate.source_workspace, crate.source_name))
def test_update_default_validation_that_fails(arcpy_exists): arcpy_exists.return_value = True def custom(crate): return NotImplemented crate = Crate('', '', '', describer=mocks.Describe) assert core.update(crate, custom, CHANGE_DETECTION)[0] == Crate.INVALID_DATA
def test_preserve_metadata(test_gdb): crate = Crate('PreserveMetadata', test_gdb, test_gdb, 'PreserveMetadata_Dest') source_metadata = arcpy.metadata.Metadata(crate.source) core._create_destination_data(crate) destination_metadata = arcpy.metadata.Metadata(crate.destination) assert source_metadata.description == destination_metadata.description assert source_metadata.title == destination_metadata.title assert source_metadata.summary == destination_metadata.summary
def test_hash_custom_source_key_float(test_gdb): skip_if_no_local_sde() tbl = 'FLOAT_ID' #: has changes crate = Crate('UPDATE_TESTS.dbo.{}'.format(tbl), UPDATE_TESTS_SDE, test_gdb, tbl) changes = core._hash(crate) assert len(changes.adds) == 1
def test_try_to_find_data_source_by_name_filters_common_duplicates(self, walk): walk.return_value = [(None, None, ['db.owner.Counties', 'db.owner.duplicateCounties'])] crate = Crate( source_name='Counties', source_workspace='Database Connections\\something.sde', destination_workspace='c:\\something.gdb', destination_name='Counties') #: reset values because _try_to_find_data_source_by_name is called in the init crate.set_source_name('Counties') ok, name = crate._try_to_find_data_source_by_name() self.assertTrue(ok) self.assertEqual(name, 'db.owner.Counties') self.assertEqual(crate.source_name, name) self.assertEqual(crate.destination_name, 'Counties') self.assertEqual(crate.source, path.join(crate.source_workspace, crate.source_name))
def update_problem_layers(self): for source_name, source_workspace, destination_workspace, destination_name in self.problem_layer_infos: if self.test_layer and self.test_layer.split('.')[-1] != destination_name: continue try: crate = Crate(source_name, source_workspace, destination_workspace, destination_name) source = path.join(source_workspace, source_name) destination = path.join(destination_workspace, destination_name) if not arcpy.Exists(destination): self.log.info('creating %s', destination) arcpy.Copy_management(source, destination) crate.result = (Crate.CREATED, None) else: self.log.info('manually updating %s', destination) arcpy.TruncateTable_management(destination) arcpy.Append_management(source, destination, 'TEST') crate.result = (Crate.UPDATED, None) except Exception as ex: self.log.error('error manually updating %s!', destination) crate.result = (Crate.UNHANDLED_EXCEPTION, ex) self._crates.append(crate)
def test_try_to_find_data_source_by_name_returns_None_if_not_sde(self): crate = Crate(source_name='something.shp', source_workspace='c:\\temp', destination_workspace='c:\\something.gdb', destination_name='Counties') self.assertIsNone(crate._try_to_find_data_source_by_name()[0])
def test_set_result_with_invalid_result_returns_result(self): crate = Crate('foo', 'bar', 'baz', 'goo') self.assertEqual(crate.set_result(('wat?', 'some crazy message'))[0], 'unknown result') self.assertEqual(crate.result[0], 'unknown result')
def test_set_result_with_valid_result_returns_result(self): crate = Crate('foo', 'bar', 'baz', 'goo') self.assertEqual(crate.set_result((Crate.UPDATED, 'Yay!'))[0], Crate.UPDATED) self.assertEqual(crate.result[0], Crate.UPDATED)
return log try: sgid_name = 'DAQAirMonitorData' sgid_db = settings.sgid['ENVIRONMENT'] stage_db = r'C:\forklift\data\hashed\deqquerylayers.gdb' source_db = path.join(settings.dbConnects, r'AVData.sde') source_name = 'AVData.dbo.interactive_map_monitoring_data' bad_results = [Crate.INVALID_DATA, Crate.UNHANDLED_EXCEPTION, Crate.UNINITIALIZED, Crate.ERROR] log = _setup_logging() log.info('creating crate') crate = Crate(sgid_name, sgid_db, stage_db, sgid_name) log.info('processing crate') core.init(log) crate.set_result(core.update(crate, validate_crate)) if crate.was_updated(): log.info('updating data in SDE') sgid_destination = path.join(sgid_db, 'SGID10.ENVIRONMENT.{}'.format(sgid_name)) arcpy.management.TruncateTable(sgid_destination) arcpy.management.Append(crate.destination, sgid_destination, 'NO_TEST') log.info('updating prod fgdbs') for dest_fgdb in [settings.mapData1, settings.mapData2]: dest = path.join(dest_fgdb, 'deqquerylayers.gdb', sgid_name) arcpy.management.TruncateTable(dest) arcpy.management.Append(crate.destination, dest, 'NO_TEST')