def bootstrap(): """Bootstrap the TsuDAT run into existence. The following globals are used (all are strings): User user name Project the TsuDAT project Scenario the scenario Setup the run setup ('trial', etc) BaseDir base of the tsudat working directory """ log.info('bootstrap start, user_data globals:') log.info(' User=%s' % User) log.info(' UserDir=%s' % UserDir) log.info(' Project=%s' % Project) log.info(' Scenario=%s' % Scenario) log.info(' Setup=%s' % Setup) log.info(' BaseDir=%s' % BaseDir) log.info(' Debug=%s' % Debug) log.info(' Instance=%s' % Instance) send_message(status=StatusStart) # jigger the PYTHONPATH so we can import 'run_tsudat' from the common filesystem new_pythonpath = os.path.join(UserDir, Project, Scenario, Setup, ScriptsDirectory) sys.path.append(new_pythonpath) log.debug('Added additional import path=%s' % new_pythonpath) # get the code for the rest of the simulation import run_tsudat # get path to the JSON file in scripts dir, pass to run_tsudat() json_path = os.path.join(new_pythonpath, JSONFile) log.info('Running run_tsudat.run_tsudat(%s)' % json_path) gen_files = run_tsudat.run_tsudat(json_path) # add local log files to the 'log' entry (*.log, *.out) output_path = os.path.join(UserDir, Project, Scenario, Setup, OutputsDirectory) local_log_files = [] local_logs = glob.glob('*.log') for l_l in local_logs: dst = os.path.join(output_path, l_l) shutil.copyfile(l_l, dst) local_log_files.append(dst) local_logs = glob.glob('*.out') for l_l in local_logs: dst = os.path.join(output_path, l_l) shutil.copyfile(l_l, dst) local_log_files.append(dst) gen_files['log'] = local_log_files # # before we possibly delete the gen_files['sww'], get output path # save_zip_base = os.path.dirname(gen_files['sww'][0])[1:] # log.debug('save_zip_base=%s' % save_zip_base) # if user data shows 'getsww' as False, remove 'sww' key from dictionary if not UserData.get('GETSWW', True): msg = 'Userdata says not to save SWW files, deleting...' log.info(msg) del gen_files['sww'] # optionally dump returned file data if Debug: import pprint pp = pprint.PrettyPrinter(indent=4) gen_str = pprint.pformat(gen_files) log.debug('Returned files:\n%s' % gen_str) # convert gen_files to JSON and add to stopping message send_message(status=StatusStop, payload=gen_files) # stop this AMI log.info('run_tsudat() finished, shutting down') shutdown()
def main(): """Behave like the UI and run a TsuDAT simulation locally.""" # build the appropriate json data file (_, json_file) = tempfile.mkstemp(suffix='.json', prefix='fake_ui_', text=True) json_dict = {'user': User, 'project': Project, 'scenario': Scenario, 'setup': Setup, 'event_number': Event, 'working_directory': TsuDATBase, 'mux_directory': TsuDATMux, 'initial_tide': 0.0, 'start_time': 0, 'end_time': 27000, 'smoothing': 0.1, 'bounding_polygon_file': BoundingPolygon, 'elevation_data_list': RawElevationFiles, 'combined_elevation_filestem': CombinedElevationFileStem, 'mesh_friction': 0.01, 'raster_resolution': 250, 'layers': ['stage', 'depth'], 'area': ['All'], 'get_results_max': True, 'get_timeseries': True, 'gauge_file': GaugeFile, 'mesh_file': MeshFile, 'interior_regions_list': InteriorRegions, 'bounding_polygon_maxarea': 100000, 'urs_order_file': UrsOrder, 'landward_boundary_file': LandwardBoundary, 'ascii_grid_filenames': [], 'sts_filestem': STSFileStem, 'zone_number': 54, 'force_run': True, # if True, *forces* a simulation 'debug': True} # if True, forces DEBUG logging with open(json_file, 'w') as fd: json.dump(json_dict, fd, indent=2, separators=(',', ':')) # create the user working directory (run_dir, raw_elevations, boundaries, meshes, polygons, gauges) = run_tsudat.make_tsudat_dir(TsuDATBase, User, Project, Scenario, Setup, Event) # copy data files to correct places in user directory # maintain time/data stats for f in RawElevationFiles: shutil.copy2(os.path.join(DataFilesDir, 'raw_elevations', f), raw_elevations) shutil.copy2(os.path.join(DataFilesDir, 'polygons', BoundingPolygon), polygons) for (f, _) in InteriorRegions: shutil.copy2(os.path.join(DataFilesDir, 'polygons', f), polygons) shutil.copy2(os.path.join(DataFilesDir, 'boundaries', LandwardBoundary), boundaries) shutil.copy2(os.path.join(DataFilesDir, 'boundaries', UrsOrder), boundaries) shutil.copy2(os.path.join(DataFilesDir, 'boundaries', STSFile), boundaries) shutil.copy2(os.path.join(DataFilesDir, 'gauges', GaugeFile), gauges) # now run the simulation gen_files = run_tsudat.run_tsudat(json_file) # remove temporary files os.remove(json_file)
def bootstrap(): """Bootstrap the TsuDAT run into existence. The following globals are used (all are strings): User user name Project the TsuDAT project Scenario the scenario Setup the run setup ('trial', etc) BaseDir base of the tsudat working directory """ log.info('bootstrap start, user_data globals:') log.info(' User=%s' % User) log.info(' Project=%s' % Project) log.info(' Scenario=%s' % Scenario) log.info(' Setup=%s' % Setup) log.info(' BaseDir=%s' % BaseDir) log.info(' Debug=%s' % Debug) log.info(' Instance=%s' % Instance) send_sqs_message(status=StatusStart) # get name of ZIP working file zip_name = DataFileFormat % (User, Project, Scenario, Setup) # get an S3 connection s3 = s3_connect() # load the input data files from S3 key_str = ('%s/%s' % (InputS3DataDir, zip_name)) log.info('Loading %s from S3 ...' % key_str) bucket = s3.get_bucket(S3Bucket) if bucket is None: abort("Can't find bucket '%s'" % S3Bucket) try: key = bucket.get_key(key_str) except S3ResponseError: abort("Can't find key '%s' in bucket '%s'" % (key_str, S3Bucket)) if key is None: abort("Can't find key '%s' in bucket '%s'" % (key_str, S3Bucket)) key.get_contents_to_filename(InputZipFile) log.info('Done') # unzip the input data ZIP file into the local directory log.debug('Unzipping %s ...' % InputZipFile) z = zipfile.ZipFile(InputZipFile) z.extractall(path='/') if not Debug: os.remove(InputZipFile) log.debug('Done') # now load any generated data from a previous run key_str = ('%s/%s' % (OutputS3DataDir, zip_name)) log.info('Trying to load %s from S3 ...' % key_str) try: bucket = s3.get_bucket(S3Bucket) key = bucket.get_key(key_str) if key: key.get_contents_to_filename(OutputZipFile) log.info('Done') # unpack generated data into working directory log.debug('Unzipping %s ...' % OutputZipFile) z = zipfile.ZipFile(OutputZipFile) z.extractall(path='/') if not Debug: os.remove(OutputZipFile) log.debug('Done') else: log.info('Previously generated data not found') except S3ResponseError: log.info('Previously generated data not found') # jigger the PYTHONPATH so we can import 'run_tsudat' from the S3 data new_pythonpath = os.path.join(BaseDir, User, Project, Scenario, Setup, ScriptsDirectory) sys.path.append(new_pythonpath) log.debug('Added additional import path=%s' % new_pythonpath) # get the code for the rest of the simulation import run_tsudat # get path to the JSON file in scripts dir, pass to run_tsudat() json_path = os.path.join(new_pythonpath, JSONFile) log.info('Running run_tsudat.run_tsudat()') gen_files = run_tsudat.run_tsudat(json_path, logger=run_tsudat_log) # add local log files to the 'log' entry gen_files['log'] = glob.glob('*.log') # before we possibly delete the gen_files['sww'], get output path save_zip_base = os.path.dirname(gen_files['sww'][0])[1:] log.debug('save_zip_base=%s' % save_zip_base) # if user data shows 'getsww' as False, remove 'sww' key from dictionary if not UserData.get('GETSWW', True): msg = 'Userdata says not to save SWW files, deleting...' log.info(msg) send_sqs_message(status=StatusLog, msg=msg) del gen_files['sww'] # optionally dump returned file data if Debug: import pprint pp = pprint.PrettyPrinter(indent=4) gen_str = pprint.pformat(gen_files) log.debug('Returned files:\n%s' % gen_str) # save generated data to a staging directory # want same pathname for each file as in input ZIP archive shutil.rmtree(save_zip_base, ignore_errors=True) # just in case os.makedirs(save_zip_base) for key in gen_files: for f in gen_files[key]: log.debug('Copying %s -> %s' % (f, save_zip_base)) shutil.copy2(f, save_zip_base) # ZIP the generated directory log.debug('zipping dir: %s' % save_zip_base) make_dir_zip(save_zip_base, OutputZipFile) # save generated directory back to S3 s3_name = '%s/%s' % (OutputS3DataDir, zip_name) zip_size = os.path.getsize(OutputZipFile) zip_size_mb = float(zip_size) / (1024*1024) log.info('Saving %s (%.2fMB) to S3.' % (s3_name, zip_size_mb)) try: bucket = s3.create_bucket(S3Bucket) key = bucket.new_key(s3_name) log.debug('Creating S3 file: %s/%s' % (S3Bucket, s3_name)) key.set_contents_from_filename(OutputZipFile) log.debug('Done!') key.set_acl('public-read') except boto.exception.S3ResponseError, e: log.critical('S3 error: %s' % str(e)) sys.exit(10)