def bootstrap():
"""Bootstrap the TsuDAT run into existence.
The following globals are used (all are strings):
User user name
Project the TsuDAT project
Scenario the scenario
Setup the run setup ('trial', etc)
BaseDir base of the tsudat working directory
"""
log.info('bootstrap start, user_data globals:')
log.info(' User=%s' % User)
log.info(' UserDir=%s' % UserDir)
log.info(' Project=%s' % Project)
log.info(' Scenario=%s' % Scenario)
log.info(' Setup=%s' % Setup)
log.info(' BaseDir=%s' % BaseDir)
log.info(' Debug=%s' % Debug)
log.info(' Instance=%s' % Instance)
send_message(status=StatusStart)
# jigger the PYTHONPATH so we can import 'run_tsudat' from the common filesystem
new_pythonpath = os.path.join(UserDir, Project, Scenario, Setup,
ScriptsDirectory)
sys.path.append(new_pythonpath)
log.debug('Added additional import path=%s' % new_pythonpath)
# get the code for the rest of the simulation
import run_tsudat
# get path to the JSON file in scripts dir, pass to run_tsudat()
json_path = os.path.join(new_pythonpath, JSONFile)
log.info('Running run_tsudat.run_tsudat(%s)' % json_path)
gen_files = run_tsudat.run_tsudat(json_path)
# add local log files to the 'log' entry (*.log, *.out)
output_path = os.path.join(UserDir, Project, Scenario, Setup,
OutputsDirectory)
local_log_files = []
local_logs = glob.glob('*.log')
for l_l in local_logs:
dst = os.path.join(output_path, l_l)
shutil.copyfile(l_l, dst)
local_log_files.append(dst)
local_logs = glob.glob('*.out')
for l_l in local_logs:
dst = os.path.join(output_path, l_l)
shutil.copyfile(l_l, dst)
local_log_files.append(dst)
gen_files['log'] = local_log_files
# # before we possibly delete the gen_files['sww'], get output path
# save_zip_base = os.path.dirname(gen_files['sww'][0])[1:]
# log.debug('save_zip_base=%s' % save_zip_base)
# if user data shows 'getsww' as False, remove 'sww' key from dictionary
if not UserData.get('GETSWW', True):
msg = 'Userdata says not to save SWW files, deleting...'
log.info(msg)
del gen_files['sww']
# optionally dump returned file data
if Debug:
import pprint
pp = pprint.PrettyPrinter(indent=4)
gen_str = pprint.pformat(gen_files)
log.debug('Returned files:\n%s' % gen_str)
# convert gen_files to JSON and add to stopping message
send_message(status=StatusStop, payload=gen_files)
# stop this AMI
log.info('run_tsudat() finished, shutting down')
shutdown()
def main():
"""Behave like the UI and run a TsuDAT simulation locally."""
# build the appropriate json data file
(_, json_file) = tempfile.mkstemp(suffix='.json',
prefix='fake_ui_', text=True)
json_dict = {'user': User,
'project': Project,
'scenario': Scenario,
'setup': Setup,
'event_number': Event,
'working_directory': TsuDATBase,
'mux_directory': TsuDATMux,
'initial_tide': 0.0,
'start_time': 0,
'end_time': 27000,
'smoothing': 0.1,
'bounding_polygon_file': BoundingPolygon,
'elevation_data_list': RawElevationFiles,
'combined_elevation_filestem': CombinedElevationFileStem,
'mesh_friction': 0.01,
'raster_resolution': 250,
'layers': ['stage', 'depth'],
'area': ['All'],
'get_results_max': True,
'get_timeseries': True,
'gauge_file': GaugeFile,
'mesh_file': MeshFile,
'interior_regions_list': InteriorRegions,
'bounding_polygon_maxarea': 100000,
'urs_order_file': UrsOrder,
'landward_boundary_file': LandwardBoundary,
'ascii_grid_filenames': [],
'sts_filestem': STSFileStem,
'zone_number': 54,
'force_run': True, # if True, *forces* a simulation
'debug': True} # if True, forces DEBUG logging
with open(json_file, 'w') as fd:
json.dump(json_dict, fd, indent=2, separators=(',', ':'))
# create the user working directory
(run_dir, raw_elevations, boundaries, meshes,
polygons, gauges) = run_tsudat.make_tsudat_dir(TsuDATBase, User, Project,
Scenario, Setup, Event)
# copy data files to correct places in user directory
# maintain time/data stats
for f in RawElevationFiles:
shutil.copy2(os.path.join(DataFilesDir, 'raw_elevations', f),
raw_elevations)
shutil.copy2(os.path.join(DataFilesDir, 'polygons', BoundingPolygon), polygons)
for (f, _) in InteriorRegions:
shutil.copy2(os.path.join(DataFilesDir, 'polygons', f), polygons)
shutil.copy2(os.path.join(DataFilesDir, 'boundaries', LandwardBoundary), boundaries)
shutil.copy2(os.path.join(DataFilesDir, 'boundaries', UrsOrder), boundaries)
shutil.copy2(os.path.join(DataFilesDir, 'boundaries', STSFile), boundaries)
shutil.copy2(os.path.join(DataFilesDir, 'gauges', GaugeFile), gauges)
# now run the simulation
gen_files = run_tsudat.run_tsudat(json_file)
# remove temporary files
os.remove(json_file)
def bootstrap():
"""Bootstrap the TsuDAT run into existence.
The following globals are used (all are strings):
User user name
Project the TsuDAT project
Scenario the scenario
Setup the run setup ('trial', etc)
BaseDir base of the tsudat working directory
"""
log.info('bootstrap start, user_data globals:')
log.info(' User=%s' % User)
log.info(' Project=%s' % Project)
log.info(' Scenario=%s' % Scenario)
log.info(' Setup=%s' % Setup)
log.info(' BaseDir=%s' % BaseDir)
log.info(' Debug=%s' % Debug)
log.info(' Instance=%s' % Instance)
send_sqs_message(status=StatusStart)
# get name of ZIP working file
zip_name = DataFileFormat % (User, Project, Scenario, Setup)
# get an S3 connection
s3 = s3_connect()
# load the input data files from S3
key_str = ('%s/%s' % (InputS3DataDir, zip_name))
log.info('Loading %s from S3 ...' % key_str)
bucket = s3.get_bucket(S3Bucket)
if bucket is None:
abort("Can't find bucket '%s'" % S3Bucket)
try:
key = bucket.get_key(key_str)
except S3ResponseError:
abort("Can't find key '%s' in bucket '%s'" % (key_str, S3Bucket))
if key is None:
abort("Can't find key '%s' in bucket '%s'" % (key_str, S3Bucket))
key.get_contents_to_filename(InputZipFile)
log.info('Done')
# unzip the input data ZIP file into the local directory
log.debug('Unzipping %s ...' % InputZipFile)
z = zipfile.ZipFile(InputZipFile)
z.extractall(path='/')
if not Debug:
os.remove(InputZipFile)
log.debug('Done')
# now load any generated data from a previous run
key_str = ('%s/%s' % (OutputS3DataDir, zip_name))
log.info('Trying to load %s from S3 ...' % key_str)
try:
bucket = s3.get_bucket(S3Bucket)
key = bucket.get_key(key_str)
if key:
key.get_contents_to_filename(OutputZipFile)
log.info('Done')
# unpack generated data into working directory
log.debug('Unzipping %s ...' % OutputZipFile)
z = zipfile.ZipFile(OutputZipFile)
z.extractall(path='/')
if not Debug:
os.remove(OutputZipFile)
log.debug('Done')
else:
log.info('Previously generated data not found')
except S3ResponseError:
log.info('Previously generated data not found')
# jigger the PYTHONPATH so we can import 'run_tsudat' from the S3 data
new_pythonpath = os.path.join(BaseDir, User, Project, Scenario, Setup,
ScriptsDirectory)
sys.path.append(new_pythonpath)
log.debug('Added additional import path=%s' % new_pythonpath)
# get the code for the rest of the simulation
import run_tsudat
# get path to the JSON file in scripts dir, pass to run_tsudat()
json_path = os.path.join(new_pythonpath, JSONFile)
log.info('Running run_tsudat.run_tsudat()')
gen_files = run_tsudat.run_tsudat(json_path, logger=run_tsudat_log)
# add local log files to the 'log' entry
gen_files['log'] = glob.glob('*.log')
# before we possibly delete the gen_files['sww'], get output path
save_zip_base = os.path.dirname(gen_files['sww'][0])[1:]
log.debug('save_zip_base=%s' % save_zip_base)
# if user data shows 'getsww' as False, remove 'sww' key from dictionary
if not UserData.get('GETSWW', True):
msg = 'Userdata says not to save SWW files, deleting...'
log.info(msg)
send_sqs_message(status=StatusLog, msg=msg)
del gen_files['sww']
# optionally dump returned file data
if Debug:
import pprint
pp = pprint.PrettyPrinter(indent=4)
gen_str = pprint.pformat(gen_files)
log.debug('Returned files:\n%s' % gen_str)
# save generated data to a staging directory
# want same pathname for each file as in input ZIP archive
shutil.rmtree(save_zip_base, ignore_errors=True) # just in case
os.makedirs(save_zip_base)
for key in gen_files:
for f in gen_files[key]:
log.debug('Copying %s -> %s' % (f, save_zip_base))
shutil.copy2(f, save_zip_base)
# ZIP the generated directory
log.debug('zipping dir: %s' % save_zip_base)
make_dir_zip(save_zip_base, OutputZipFile)
# save generated directory back to S3
s3_name = '%s/%s' % (OutputS3DataDir, zip_name)
zip_size = os.path.getsize(OutputZipFile)
zip_size_mb = float(zip_size) / (1024*1024)
log.info('Saving %s (%.2fMB) to S3.' % (s3_name, zip_size_mb))
try:
bucket = s3.create_bucket(S3Bucket)
key = bucket.new_key(s3_name)
log.debug('Creating S3 file: %s/%s' % (S3Bucket, s3_name))
key.set_contents_from_filename(OutputZipFile)
log.debug('Done!')
key.set_acl('public-read')
except boto.exception.S3ResponseError, e:
log.critical('S3 error: %s' % str(e))
sys.exit(10)
