def test_copy_data(self, copytree_mock, rmtree_mock, move, exists_mock,
compact_mock, describe_mock):
describe_mock.side_effect = describe_side_effect
exists_mock.return_value = True
three = 'C:\\MapData\\three.gdb'
two = 'C:\\MapData\\two.gdb'
pallet_one = Pallet()
pallet_one.copy_data = ['C:\\MapData\\one.gdb', two]
pallet_one.requires_processing = Mock(return_value=True)
pallet_two = Pallet()
pallet_two.copy_data = ['C:\\MapData\\one.gdb', three]
pallet_two.requires_processing = Mock(return_value=True)
pallet_three = Pallet()
pallet_three.copy_data = ['C:\\MapData\\four', three]
lift.copy_data([pallet_one, pallet_two, pallet_three],
[pallet_one, pallet_two, pallet_three],
['dest1', 'dest2'])
self.assertEqual(copytree_mock.call_count, 6)
self.assertEqual(rmtree_mock.call_count, 6)
self.assertEqual(compact_mock.call_count, 3)
def test_copy_data_turns_off_and_on_services(self, copytree_mock,
rmtree_mock, move,
exists_mock, compact_mock,
describe_mock,
lightswitch_mock):
describe_mock.side_effect = describe_side_effect
exists_mock.return_value = True
lightswitch_mock().ensure.return_value = (True, [])
three = 'C:\\MapData\\three.gdb'
two = 'C:\\MapData\\two.gdb'
pallet_one = Pallet()
pallet_one.copy_data = ['C:\\MapData\\one.gdb', two]
pallet_one.arcgis_services = [('Pallet', 'MapServer')]
pallet_one.requires_processing = Mock(return_value=True)
pallet_two = Pallet()
pallet_two.copy_data = ['C:\\MapData\\one.gdb', three]
pallet_two.arcgis_services = [('Pallet', 'MapServer')]
pallet_two.requires_processing = Mock(return_value=True)
pallet_three = Pallet()
pallet_three.copy_data = ['C:\\MapData\\four', three]
lift.copy_data([pallet_one, pallet_two, pallet_three],
[pallet_one, pallet_two, pallet_three],
['dest1', 'dest2'])
self.assertEqual(copytree_mock.call_count, 6)
self.assertEqual(rmtree_mock.call_count, 6)
self.assertEqual(compact_mock.call_count, 3)
lightswitch_mock().ensure.assert_has_calls([
call('off', set([('Pallet', 'MapServer')])),
call('on', set([('Pallet', 'MapServer')]))
])
def test_prepare_packaging_for_pallets(self):
pallet_good = Pallet()
pallet_bad = Pallet()
pallet_bad.prepare_packaging = lambda: 1 + 't'
lift.prepare_packaging_for_pallets([pallet_good, pallet_bad])
self.assertTrue(pallet_good.success[0])
self.assertFalse(pallet_bad.success[0])
def test_get_lift_status(self):
git_errors = ['a', 'b']
p1 = Pallet()
p1.success = (False, '')
p2 = Pallet()
p3 = Pallet()
report = lift.get_lift_status([p1, p2, p3], 10, git_errors, [])
self.assertEqual(report['total_pallets'], 3)
self.assertEqual(report['num_success_pallets'], 2)
self.assertEqual(report['git_errors'], git_errors)
def test_hydrate_data_structures_unions_all_copy_data_locations(self):
pallets = [Pallet(), Pallet(), Pallet()]
pallets[0].requires_processing = Mock(return_value=True)
pallets[1].requires_processing = Mock(return_value=True)
pallets[2].requires_processing = Mock(return_value=True)
pallets[0].copy_data = ['location']
pallets[1].copy_data = ['location']
pallets[2].copy_data = ['location2']
affected_services, data_being_moved, destination_to_pallet = lift._hydrate_data_structures(
pallets, pallets)
self.assertEqual(data_being_moved, set(['location', 'location2']))
def test_hydrate_data_structures_normalizes_paths(self):
pallets = [Pallet(), Pallet()]
pallets[0].requires_processing = Mock(return_value=True)
pallets[0].copy_data = ['C:\\\\Scheduled\\staging\\political_utm.gdb']
pallets[0].arcgis_services = ['service1']
pallets[1].copy_data = ['C:\\Scheduled\\staging\\political_utm.gdb']
pallets[1].arcgis_services = ['service2']
affected_services, data_being_moved, destination_to_pallet = lift._hydrate_data_structures(
pallets[:1], pallets)
self.assertEqual(affected_services, set(['service2', 'service1']))
def test_hydrate_data_structures_unions_arcgis_services(self):
pallets = [Pallet(), Pallet()]
pallets[0].requires_processing = Mock(return_value=True)
pallets[1].requires_processing = Mock(return_value=True)
pallets[0].copy_data = ['location']
pallets[1].copy_data = ['location']
pallets[0].arcgis_services = [('a', 'a'), ('b', 'b')]
pallets[1].arcgis_services = [('a', 'a'), ('c', 'c')]
affected_services, data_being_moved, destination_to_pallet = lift._hydrate_data_structures(
pallets, pallets)
self.assertEqual(affected_services,
set([('a', 'a'), ('b', 'b'), ('c', 'c')]))
def test_create_report_object(self):
git_errors = ['a', 'b']
copy_results = ['c', 'd']
p1 = Pallet()
p1.success = (False, '')
p2 = Pallet()
p3 = Pallet()
report = lift.create_report_object([p1, p2, p3], 10, copy_results,
git_errors, '')
self.assertEqual(report['total_pallets'], 3)
self.assertEqual(report['num_success_pallets'], 2)
self.assertEqual(report['git_errors'], git_errors)
self.assertEqual(report['copy_results'], copy_results)
def test_copy_data_scrub_hash_field(self):
copy_data_fgdb_name = 'CopyData.gdb'
copied_data = path.join(current_folder, copy_data_fgdb_name)
def cleanup():
print('cleaning up')
if arcpy.Exists(copied_data):
arcpy.Delete_management(copied_data)
cleanup()
pallet = Pallet()
pallet.copy_data = [
path.join(current_folder, 'data', copy_data_fgdb_name)
]
pallet.requires_processing = Mock(return_value=True)
lift.copy_data([pallet], [pallet], [current_folder])
feature_class_fields = [
field.name for field in arcpy.Describe(
path.join(copied_data, 'DNROilGasWells_adds')).fields
]
table_fields = [
field.name for field in arcpy.Describe(
path.join(copied_data, 'Providers_adds')).fields
]
self.assertNotIn(core.hash_field, feature_class_fields)
self.assertNotIn(core.hash_field, table_fields)
cleanup()
def test_hydrate_data_structures_creates_destination_dictionary(self):
pallets = [Pallet(), Pallet(), Pallet()]
pallets[0].requires_processing = Mock(return_value=True)
pallets[1].requires_processing = Mock(return_value=True)
pallets[2].requires_processing = Mock(return_value=True)
pallets[0].copy_data = ['location']
pallets[1].copy_data = ['location']
pallets[2].copy_data = ['location2']
affected_services, data_being_moved, destination_to_pallet = lift._hydrate_data_structures(
pallets, pallets)
self.assertEqual(destination_to_pallet['location'],
[pallets[0], pallets[1]])
self.assertEqual(destination_to_pallet['location2'], [pallets[2]])
def test_hydrate_data_structures_prevents_duplicate_copy_datas(self):
pallets = [Pallet(), Pallet(), Pallet()]
pallets[0].requires_processing = Mock(return_value=True)
pallets[1].requires_processing = Mock(return_value=True)
pallets[2].requires_processing = Mock(return_value=True)
pallets[0].copy_data = ['location']
pallets[1].copy_data = ['Location']
pallets[2].copy_data = ['location2']
affected_services, data_being_moved, destination_to_pallet = lift._hydrate_data_structures(
pallets, pallets)
data_being_moved = list(data_being_moved)
self.assertEqual(len(data_being_moved), 2)
self.assertIn('location', data_being_moved)
self.assertIn('location2', data_being_moved)
def test_no_existing_data(self, _stop_services_mock, _start_services_mock,
_copy_with_overwrite_mock):
pallet = Pallet()
pallet.static_data = [check_for_changes_gdb]
lift.update_static_for([pallet], ['blah'], True)
_stop_services_mock.assert_not_called()
_start_services_mock.assert_not_called()
_copy_with_overwrite_mock.assert_called_once_with(
check_for_changes_gdb, path.join('blah', 'checkForChanges.gdb'))
def test_existing_data_should_stop_services(self, _stop_services_mock,
_start_services_mock,
_copy_with_overwrite_mock):
pallet = Pallet()
pallet.static_data = [check_for_changes_gdb]
lift.update_static_for([pallet], [path.join(current_folder, 'data')],
True)
_stop_services_mock.assert_called_once()
_start_services_mock.assert_called_once()
def test_process_crate_doesnt_call_update_def_on_duplicate_crates(self):
crate1 = Crate('DNROilGasWells', test_gdb, test_gdb, 'a')
crate2 = Crate('DNROilGasWells', test_gdb, test_gdb, 'a')
pallet = Pallet()
pallet._crates = [crate1, crate2]
update_def = Mock(return_value=(Crate.UPDATED, 'message'))
lift.process_crates_for([pallet], update_def)
self.assertEqual(update_def.call_count, 1)
self.assertEqual(crate1.result[0], Crate.UPDATED)
self.assertEqual(crate2.result[0], Crate.UPDATED)
def test_copy_data_error(self, copytree_mock, rmtree_mock, move, compact_mock, describe_mock, listdir_mock):
describe_mock.side_effect = describe_side_effect
error_message = 'there was an error'
copytree_mock.side_effect = Exception(error_message)
pallet = Pallet()
pallet.copy_data = ['C:\\MapData\\one.gdb']
pallet.requires_processing = Mock(return_value=True)
success, failed = lift.copy_data('from_location', 'to_location', engine.packing_slip_file)
self.assertTrue(failed['testfile'].startswith(error_message))
def test_no_force_existing_data_does_not_copy(self, _stop_services_mock,
_start_services_mock,
_copy_with_overwrite_mock):
pallet = Pallet()
pallet.static_data = [check_for_changes_gdb]
lift.update_static_for([pallet], [path.join(current_folder, 'data')],
False)
_stop_services_mock.assert_not_called()
_start_services_mock.assert_not_called()
_copy_with_overwrite_mock.assert_not_called()
def test_process_crate_for_set_results(self):
crate1 = Crate('DNROilGasWells', check_for_changes_gdb,
check_for_changes_gdb, 'a')
crate2 = Crate('DNROilGasWells', check_for_changes_gdb,
check_for_changes_gdb, 'b')
pallet = Pallet()
pallet._crates = [crate1, crate2]
update_def = Mock(return_value=(Crate.UPDATED, 'message'))
lift.process_crates_for([pallet], update_def)
self.assertEqual(update_def.call_count, 2)
self.assertEqual(crate1.result[0], Crate.UPDATED)
self.assertEqual(crate2.result[0], Crate.UPDATED)
def test_copy_data_error(self, copytree_mock, rmtree_mock, move,
compact_mock, describe_mock):
describe_mock.side_effect = describe_side_effect
error_message = 'there was an error'
copytree_mock.side_effect = Exception(error_message)
pallet = Pallet()
pallet.copy_data = ['C:\\MapData\\one.gdb']
pallet.requires_processing = Mock(return_value=True)
lift.copy_data([pallet], [pallet], ['hello'])
self.assertEqual(pallet.success, (False, error_message))
def test_hydrate_data_structures_only_includes_copy_data_in_specific_pallets(
self):
pallets = [Pallet(), Pallet(), Pallet()]
pallets[0].requires_processing = Mock(return_value=True)
pallets[0].copy_data = ['C:\location']
pallets[0].arcgis_services = ['service1']
pallets[1].copy_data = ['C:\Location']
pallets[1].arcgis_services = ['service2']
pallets[2].copy_data = ['C:\location2']
affected_services, data_being_moved, destination_to_pallet = lift._hydrate_data_structures(
pallets[:1], pallets)
data_being_moved = list(data_being_moved)
self.assertEqual(data_being_moved[0], 'c:\location')
self.assertEqual(len(data_being_moved), 1)
self.assertEqual(destination_to_pallet['c:\\location'],
[pallets[0], pallets[1]])
self.assertEqual(affected_services, set(['service2', 'service1']))
def test_processing_time(self):
pallet = Pallet()
first = 'first'
second = 'second'
pallet.start_timer(first)
sleep(2)
pallet.stop_timer(first)
pallet.start_timer(second)
sleep(3)
pallet.stop_timer(second)
self.assertEqual(round(pallet.total_processing_time), 5)
self.assertEqual(round(pallet.processing_times[first]), 2)
self.assertEqual(round(pallet.processing_times[second]), 3)
def test_successful_pallet(self):
pallet = Pallet()
pallet.add_crates(
['fc1', 'fc2', 'fc3'], {
'source_workspace': 'Z:\\a\\path\\to\\database.sde',
'destination_workspace': 'Z:\\a\\path\\to\\database.gdb'
})
pallet.success = (True, None)
pallet.name = 'name'
pallet._crates[0].result = (Crate.CREATED, None)
pallet._crates[1].result = (Crate.UPDATED, None)
pallet._crates[2].result = (Crate.NO_CHANGES, None)
report = pallet.get_report()
self.assertEqual(report['name'], 'name')
self.assertEqual(report['success'], True)
self.assertEqual(len(report['crates']), 2)
self.assertEqual(report['crates'][0]['result'], Crate.CREATED, None)
self.assertEqual(report['crates'][0]['name'], 'fc1')
def test_failed_pallet(self):
pallet = Pallet()
pallet.add_crates(
['fc4', 'fc5', 'fc6'], {
'source_workspace': 'Z:\\a\\path\\to\\database.sde',
'destination_workspace': 'Z:\\a\\path\\to\\database.gdb'
})
pallet.success = (False, 'Failed message')
pallet._crates[0].result = (Crate.UPDATED, None)
pallet._crates[1].result = (Crate.INVALID_DATA, 'Invalid data message')
pallet._crates[2].result = (Crate.UNHANDLED_EXCEPTION, None)
report = pallet.get_report()
self.assertEqual(report['success'], False)
self.assertEqual(report['message'], 'Failed message')
self.assertEqual(len(report['crates']), 3)
self.assertEqual(report['crates'][1]['result'], Crate.INVALID_DATA)
self.assertEqual(report['crates'][1]['crate_message'],
'Invalid data message')
def test_successful_crate(self):
slip = {
"name":
"c:\\forklift\\warehouse\\warehouse\\sgid\\AGOLPallet.py:AGOLPallet",
"success":
True,
"is_ready_to_ship":
True,
"requires_processing":
True,
"message":
"",
"crates": [{
"name": "fc5",
"result": "Data updated successfully.",
"crate_message": "",
"message_level": "",
"source":
"C:\\\\forklift-garage\\sgid10.sde\\SGID10.BOUNDARIES.ZipCodes",
"destination":
"\\\\123.456.789.123\\agrc\\sgid_to_agol\\sgid10mercator.gdb\\ZipCodes",
"was_updated": True
}],
"total_processing_time":
"55.06 seconds"
}
pallet = Pallet()
pallet.add_crates(
['fc4', 'fc5', 'fc6'], {
'source_workspace': 'Z:\\a\\path\\to\\database.sde',
'destination_workspace': 'Z:\\a\\path\\to\\database.gdb'
})
self.assertFalse(pallet.get_crates()[1].was_updated())
pallet.add_packing_slip(slip)
self.assertTrue(pallet.get_crates()[1].was_updated())
self.assertEqual(pallet.get_crates()[1].result[0], Crate.UPDATED)
def setUp(self):
self.patient = Pallet()
def test_hydrate_data_structures_with_empty_is_ok(self):
pallets = [Pallet(), Pallet()]
pallets[0].requires_processing = Mock(return_value=True)
pallets[1].requires_processing = Mock(return_value=True)
lift._hydrate_data_structures([], pallets)