def process_pallets(pallets, is_post_copy=False):
'''
pallets: Pallet[]
is_post_copy: Boolean
Loop over all pallets, check if data has changed and determine whether to process.
Call `process` if this is not the post copy. Otherwise call `post_copy_process`.
Finally, call ship.
'''
if not is_post_copy:
verb = 'processing'
else:
verb = 'post copy processing'
log.info('%s pallets...', verb)
for pallet in pallets:
if pallet.is_ready_to_ship(): #: checks for schema changes or errors
if pallet.requires_processing() and pallet.success[0]: #: checks for data that was updated
log.info('%s pallet: %r', verb, pallet)
start_seconds = clock()
try:
ResetEnvironments()
if not is_post_copy:
pallet.process()
else:
pallet.post_copy_process()
except Exception as e:
pallet.success = (False, e.message)
log.error('error %s pallet: %s for pallet: %r', verb, e.message, pallet, exc_info=True)
log.debug('%s pallet %s', verb.replace('ing', 'ed'), seat.format_time(clock() - start_seconds))
if not is_post_copy:
log.debug('shipping pallet...')
start_seconds = clock()
try:
log.info('shipping pallet: %r', pallet)
ResetEnvironments()
pallet.ship()
log.debug('shipped pallet %s', seat.format_time(clock() - start_seconds))
except Exception as e:
pallet.success = (False, e.message)
log.error('error shipping pallet: %s for pallet: %r', e.message, pallet, exc_info=True)
def speedtest(pallet_location):
print('{0}{1}Setting up speed test...{0}'.format(Fore.RESET, Fore.MAGENTA))
#: remove logging
log.handlers = [logging.NullHandler()]
#: spoof hashes location so there is no caching
core.garage = speedtest_destination
core.hash_gdb_path = join(core.garage, core._hash_gdb)
core.scratch_gdb_path = join(core.garage, core._scratch_gdb)
#: delete destination and other artifacts form prior runs
import arcpy
if arcpy.Exists(join(speedtest_destination, 'DestinationData.gdb')):
arcpy.Delete_management(join(speedtest_destination, 'DestinationData.gdb'))
arcpy.CreateFileGDB_management(speedtest_destination, 'DestinationData.gdb')
else:
arcpy.CreateFileGDB_management(speedtest_destination, 'DestinationData.gdb')
if arcpy.Exists(core.hash_gdb_path):
arcpy.Delete_management(core.hash_gdb_path)
if arcpy.Exists(core.scratch_gdb_path):
arcpy.Delete_management(core.scratch_gdb_path)
print('{0}{1}Tests ready starting dry run...{0}'.format(Fore.RESET, Fore.MAGENTA))
start_seconds = clock()
dry_report = start_lift(pallet_location)
dry_run = seat.format_time(clock() - start_seconds)
print('{0}{1}Repeating test...{0}'.format(Fore.RESET, Fore.MAGENTA))
start_seconds = clock()
repeat_report = start_lift(pallet_location)
repeat = seat.format_time(clock() - start_seconds)
#: clean up so git state is unchanged
if arcpy.Exists(join(speedtest_destination, 'DestinationData.gdb')):
arcpy.Delete_management(join(speedtest_destination, 'DestinationData.gdb'))
if arcpy.Exists(core.hash_gdb_path):
arcpy.Delete_management(core.hash_gdb_path)
if arcpy.Exists(core.scratch_gdb_path):
arcpy.Delete_management(core.scratch_gdb_path)
print('{1}Dry Run Output{0}{2}{3}'.format(Fore.RESET, Fore.CYAN, linesep, dry_report))
print('{1}Repeat Run Output{0}{2}{3}'.format(Fore.RESET, Fore.CYAN, linesep, repeat_report))
print('{3}{0}{1}Speed Test Results{3}{0}{2}Dry Run:{0} {4}{3}{2}Repeat:{0} {5}'.format(Fore.RESET, Fore.GREEN, Fore.CYAN, linesep, dry_run, repeat))
def start_lift(file_path=None, pallet_arg=None):
log.info('starting forklift')
git_errors = git_update()
start_seconds = clock()
pallets_to_lift, all_pallets = _sort_pallets(file_path, pallet_arg)
start_process = clock()
core.init(log)
lift.process_crates_for(pallets_to_lift, core.update, config.get_config_prop('configuration'))
log.info('process_crates time: %s', seat.format_time(clock() - start_process))
start_process = clock()
lift.process_pallets(pallets_to_lift)
log.info('process_pallets time: %s', seat.format_time(clock() - start_process))
start_copy = clock()
copy_results = lift.copy_data(pallets_to_lift, all_pallets, config.get_config_prop('copyDestinations'))
log.info('copy_data time: %s', seat.format_time(clock() - start_copy))
start_post_copy_process = clock()
lift.process_pallets(pallets_to_lift, is_post_copy=True)
log.info('post_copy_process time: %s', seat.format_time(clock() - start_post_copy_process))
elapsed_time = seat.format_time(clock() - start_seconds)
report_object = lift.create_report_object(pallets_to_lift, elapsed_time, copy_results, git_errors)
_send_report_email(report_object)
log.info('Finished in {}.'.format(elapsed_time))
report = _format_dictionary(report_object)
log.info('%s', report)
return report
def process_crates_for(pallets, update_def, configuration='Production'):
'''
pallets: Pallet[]
update_def: Function. core.update
configuration: string. Production, Staging, Dev
Calls update_def on all crates (excluding duplicates) in pallets
'''
processed_crates = {}
log.info('processing crates for %d pallets.', len(pallets))
for pallet in pallets:
log.info('processing crates for pallet: %r', pallet)
try:
log.debug('building pallet: %r', pallet)
pallet.build(configuration)
except Exception as e:
pallet.success = (False, e.message)
log.error('error building pallet: %s for pallet: %r', e.message, pallet, exc_info=True)
continue
for crate in pallet.get_crates():
log.info('crate: %s', crate.destination_name)
if crate.result[0] == Crate.INVALID_DATA:
log.warn('result: %s', crate.result)
continue
if crate.destination not in processed_crates:
log.debug('%r', crate)
start_seconds = clock()
processed_crates[crate.destination] = crate.set_result(update_def(crate, pallet.validate_crate))
log.debug('finished crate %s', seat.format_time(clock() - start_seconds))
log.info('result: %s', crate.result)
else:
log.info('skipping crate')
crate.set_result(processed_crates[crate.destination])
def copy_data(specific_pallets, all_pallets, config_copy_destinations):
#: we're lifting everything
if len(specific_pallets) == 0:
specific_pallets = all_pallets
#: filter out pallets whose data did not change
specific_pallets = [pallet for pallet in specific_pallets if pallet.requires_processing() is True]
#: no pallets to process. we are done here
if len(specific_pallets) == 0:
return
lightswitch = LightSwitch()
services_affected, data_being_moved, destination_to_pallet = _hydrate_data_structures(specific_pallets, all_pallets)
results = ''
log.info('stopping %s dependent services.', len(services_affected))
ok, problem_children = lightswitch.ensure('off', services_affected)
service_msg = 'Service(s) will not {}: {}. '
if not ok:
stop_msg = service_msg.format('stop', problem_children) + 'This will affect data copy.'
results = stop_msg
log.error(stop_msg)
for source in data_being_moved:
if Describe(source).workspaceFactoryProgID.startswith('esriDataSourcesGDB.FileGDBWorkspaceFactory'):
log.info('compacting %s', source)
Compact_management(source)
for destination in config_copy_destinations:
destination_workspace = path.join(destination, path.basename(source))
log.info('copying {} to {}...'.format(source, destination_workspace))
start_seconds = clock()
try:
if path.exists(destination_workspace):
log.debug('%s exists moving', destination_workspace)
shutil.move(destination_workspace, destination_workspace + 'x')
log.debug('copying source to destination')
shutil.copytree(source, destination_workspace)
if path.exists(destination_workspace + 'x'):
log.debug('removing temporary gdb: %s', destination_workspace + 'x')
shutil.rmtree(destination_workspace + 'x')
log.info('copy successful in %s', seat.format_time(clock() - start_seconds))
except Exception as e:
try:
#: There is still a lock?
#: The service probably wasn't shut down
#: if there was a problem and the temp gdb exists
#: since we couln't delete it before we probably can't delete it now
#: so take what is in x and copy it over what it can in the original
#: that _should_ leave the gdb in a functioning state
if path.exists(destination_workspace) and path.exists(destination_workspace + 'x'):
log.debug('cleaning up %s', destination_workspace)
_copy_with_overwrite(destination_workspace + 'x', destination_workspace)
shutil.rmtree(destination_workspace + 'x')
except Exception:
log.error('%s might be in a corrupted state', destination_workspace, exc_info=True)
if source.lower() in destination_to_pallet:
for pallet in destination_to_pallet[source.lower()]:
pallet.success = (False, str(e))
log.error('there was an error copying %s to %s', source, destination_workspace, exc_info=True)
log.info('starting %s dependent services.', len(services_affected))
ok, problem_children = lightswitch.ensure('on', services_affected)
if not ok:
start_msg = service_msg.format('start', problem_children)
results += ' ' + start_msg
log.error(start_msg)
return results
