def _load_spill_data(self, spill_data): """ load NetCDF file and add spill data back in - designed for savefiles """ if not os.path.exists(spill_data): return if self.uncertain: saveloc, spill_data_fname = os.path.split(spill_data) spill_data_fname, ext = os.path.splitext(spill_data_fname) u_spill_data = os.path.join(saveloc, '{0}_uncertain{1}'.format(spill_data_fname, ext)) array_types = {} for m in self.movers: array_types.update(m.array_types) for w in self.weatherers: array_types.update(w.array_types) for sc in self.spills.items(): if sc.uncertain: data = NetCDFOutput.read_data(u_spill_data, time=None, which_data='all') else: data = NetCDFOutput.read_data(spill_data, time=None, which_data='all') sc.current_time_stamp = data.pop('current_time_stamp').item() sc._data_arrays = data sc._array_types.update(array_types)
def _save_spill_data(self, datafile): """ save the data arrays for current timestep to NetCDF """ nc_out = NetCDFOutput(datafile, which_data='all', cache=self._cache) nc_out.prepare_for_model_run(model_start_time=self.start_time, uncertain=self.uncertain, spills=self.spills) nc_out.write_output(self.current_time_step)
def _load_spill_data(saveloc, model): """ load NetCDF file and add spill data back in """ spill_data = os.path.join(saveloc, "spills_data_arrays.nc") if not os.path.exists(spill_data): return if model.uncertain: u_spill_data = os.path.join(saveloc, "spills_data_arrays_uncertain.nc") array_types = {} for m in model.movers: array_types.update(m.array_types) for w in model.weatherers: array_types.update(w.array_types) for sc in model.spills.items(): if sc.uncertain: data = NetCDFOutput.read_data(u_spill_data, time=None, which_data="all") else: data = NetCDFOutput.read_data(spill_data, time=None, which_data="all") sc.current_time_stamp = data.pop("current_time_stamp").item() sc._data_arrays = data sc._array_types.update(array_types)
def test_write_output_post_run(model, output_ts_factor): """ Create netcdf file post run from the cache. Under the hood, it is simply calling write_output so no need to check the data is correctly written test_write_output_standard already checks data is correctly written. Instead, make sure if output_timestep is not same as model.time_step, then data is output at correct time stamps """ model.rewind() o_put = [model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput)][0] o_put.which_data = 'standard' o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor) del model.outputters[o_put.id] # remove from list of outputters _run_model(model) assert (not os.path.exists(o_put.netcdf_filename)) if o_put._u_netcdf_filename: assert (not os.path.exists(o_put._u_netcdf_filename)) # now write netcdf output o_put.write_output_post_run(model.start_time, model.num_time_steps, spills=model.spills, cache=model._cache, uncertain=model.uncertain) assert os.path.exists(o_put.netcdf_filename) if model.uncertain: assert os.path.exists(o_put._u_netcdf_filename) uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): for step in range(model.num_time_steps, int(output_ts_factor)): print "step: {0}".format(step) scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) nc_data = NetCDFOutput.read_data(file_, curr_time) assert curr_time == nc_data['current_time_stamp'].item() if o_put.output_last_step: scp = model._cache.load_timestep(model.num_time_steps - 1) curr_time = scp.LE('current_time_stamp', uncertain) nc_data = NetCDFOutput.read_data(file_, curr_time) assert curr_time == nc_data['current_time_stamp'].item() """ at least one matching time found """ print ('\nAll expected timestamps are written out for' ' output_ts_factor: {1}'.format(file_, output_ts_factor)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True # add this back in so cleanup script deletes the generated *.nc files model.outputters += o_put
def _read_data_file(self, filename, index, time): if time is not None: self._init_data = NetCDFOutput.read_data(filename, time, which_data='all')[0] elif index is not None: self._init_data = NetCDFOutput.read_data(filename, index=index, which_data='all')[0] else: self._init_data = NetCDFOutput.read_data(filename, index=-1, which_data='all')[0]
def test_init_exceptions(): ''' test exceptions raised during __init__ ''' with raises(ValueError): # must be filename, not dir name NetCDFOutput(os.path.abspath(os.path.dirname(__file__))) with raises(ValueError): NetCDFOutput('invalid_path_to_file/file.nc')
def _read_data_file(self, filename, index, time): if time is not None: self._init_data = NetCDFOutput.read_data(filename, time, which_data='all')[0] elif index is not None: self._init_data = NetCDFOutput.read_data(filename, index=index, which_data='all')[0] else: self._init_data = NetCDFOutput.read_data(filename, index=-1, which_data='all')[0] # if init_mass is not there, set it to mass # fixme: should this be a required data array? self._init_data.setdefault('init_mass', self._init_data['mass'].copy())
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'initializing the model' start_time = datetime(2012, 9, 15, 12, 0) mapfile = get_datafile(os.path.join(base_dir, './LongIslandSoundMap.BNA')) gnome_map = MapFromBNA(mapfile, refloat_halflife=6) # hours # # the image output renderer # global renderer # one hour timestep model = Model(start_time=start_time, duration=timedelta(hours=48), time_step=3600, map=gnome_map, uncertain=True, cache_enabled=True) netcdf_file = os.path.join(base_dir, 'script_long_island.nc') scripting.remove_netcdf(netcdf_file) print 'adding outputters' model.outputters += Renderer(mapfile, images_dir, size=(800, 600)) model.outputters += NetCDFOutput(netcdf_file, which_data='all') print 'adding a spill' spill = point_line_release_spill(num_elements=1000, start_position=(-72.419992, 41.202120, 0.0), release_time=start_time) model.spills += spill print 'adding a RandomMover:' model.movers += RandomMover(diffusion_coef=500000, uncertain_factor=2) print 'adding a wind mover:' series = np.zeros((5, ), dtype=datetime_value_2d) series[0] = (start_time, (10, 45)) series[1] = (start_time + timedelta(hours=18), (10, 90)) series[2] = (start_time + timedelta(hours=30), (10, 135)) series[3] = (start_time + timedelta(hours=42), (10, 180)) series[4] = (start_time + timedelta(hours=54), (10, 225)) wind = Wind(timeseries=series, units='m/s') model.movers += WindMover(wind) print 'adding a cats mover:' curr_file = get_datafile(os.path.join(base_dir, r"./LI_tidesWAC.CUR")) tide_file = get_datafile(os.path.join(base_dir, r"./CLISShio.txt")) c_mover = CatsMover(curr_file, tide=Tide(tide_file)) model.movers += c_mover model.environment += c_mover.tide print 'viewport is:', [ o.viewport for o in model.outputters if isinstance(o, Renderer) ] return model
def test_read_standard_arrays(model, output_ts_factor): """ tests the data returned by read_data is correct when `which_data` flag is 'standard'. It is only reading the standard_arrays Test will only verify the data when time_stamp of model matches the time_stamp of data written out. output_ts_factor means not all data is written out. """ model.rewind() # check contents of netcdf File at multiple time steps (should only be 1!) o_put = [model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput)][0] o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor) _run_model(model) atol = 1e-5 rtol = 0 uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): _found_a_matching_time = False for step in range(0, model.num_time_steps, int(output_ts_factor)): scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) nc_data = NetCDFOutput.read_data(file_, curr_time) # check time if curr_time == nc_data['current_time_stamp'].item(): _found_a_matching_time = True # check standard variables assert np.allclose(scp.LE('positions', uncertain), nc_data['positions'], rtol, atol) assert np.all(scp.LE('spill_num', uncertain)[:] == nc_data['spill_num']) assert np.all(scp.LE('status_codes', uncertain)[:] == nc_data['status_codes']) # flag variable is not currently set or checked if 'mass' in scp.LE_data: assert np.all(scp.LE('mass', uncertain)[:] == nc_data['mass']) if 'age' in scp.LE_data: assert np.all(scp.LE('age', uncertain)[:] == nc_data['age']) if _found_a_matching_time: """ at least one matching time found """ print ('\ndata in model matches for output in \n{0} \nand' ' output_ts_factor: {1}'.format(file_, output_ts_factor)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True
def model(sample_model_fcn, output_filename): """ Use fixture model_surface_release_spill and add a few things to it for the test """ model = sample_model_fcn['model'] model.cache_enabled = True model.spills += \ point_line_release_spill(num_elements=5, start_position=sample_model_fcn['release_start_pos'], release_time=model.start_time, end_release_time=model.start_time + model.duration, substance=test_oil, amount=1000, units='kg') water = Water() model.movers += RandomMover(diffusion_coef=100000) model.movers += constant_wind_mover(1.0, 0.0) model.weatherers += Evaporation(water=water, wind=model.movers[-1].wind) model.outputters += NetCDFOutput(output_filename) model.rewind() return model
def allWeatherers(timeStep, start_time, duration, weatheringSteps, map, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, tidalFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td): print 'initializing the model:' model = Model(time_step=timeStep, start_time=start_time, duration=duration) print 'adding the map:' map_folder = os.path.join(data_path, map_path) if not(os.path.exists(map_folder)): print('The map folder is incorrectly set:', map_folder) mapfile = get_datafile( os.path.join(map_folder,map) ) model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife) print 'adding a renderer' model.outputters += Renderer(mapfile, output_path, size=(800, 600), output_timestep=timedelta(hours=1)) if save_nc: nc_outputter = NetCDFOutput(netcdf_file, which_data='most', output_timestep=timedelta(hours=1)) model.outputters += nc_outputter print 'adding a wind mover:' wind_file = get_datafile(os.path.join(data_path, wind_path, windFile)) wind = GridWindMover(wind_file) wind.wind_scale = wind_scale model.movers += wind print 'adding a current mover: ' curr_file = get_datafile(os.path.join(data_path, curr_path, currFile)) model.movers += GridCurrentMover(curr_file, num_method='RK4') if td: random_mover = RandomMover(diffusion_coef=10000) model.movers += random_mover print 'adding spill' model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration) print 'adding weatherers' water = Water(280.92) wind = constant_wind(20.0, 117, 'knots') waves = Waves(wind, water) model.weatherers += Evaporation(water, wind) model.weatherers += Emulsification(waves) model.weatherers += NaturalDispersion(waves, water) return model
def test_read_data_exception(model): """ tests the exception is raised by read_data when file contains more than one output time and read_data is not given the output time to read """ model.rewind() # check contents of netcdf File at multiple time steps (should only be 1!) o_put = [model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput)][0] _run_model(model) with raises(ValueError): NetCDFOutput.read_data(o_put.filename)
def CurrentsAndWinds(timeStep, start_time, duration, weatheringSteps, mapfile, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, tidalFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td): print 'initializing the model:' model = Model(time_step=timeStep, start_time=start_time, duration=duration) print 'adding the map:' print (data_path, map_path, mapfile) mapfile = get_datafile(os.path.join(data_path, map_path, mapfile)) model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife) print 'adding a renderer' model.outputters += Renderer(mapfile, output_path, size=(800, 600), output_timestep=timedelta(hours=timestep_outputs)) if save_nc: nc_outputter = NetCDFOutput('currentsAndWinds_example.nc', which_data='standard', output_timestep=timedelta(hours=timestep_outputs)) model.outputters += nc_outputter print 'adding a wind mover:' wind_file = get_datafile(os.path.join(data_path, wind_path, windFile)) wind = GridWindMover(wind_file) wind.wind_scale = wind_scale model.movers += wind print 'adding a current mover: ' curr_file = get_datafile(os.path.join(data_path, curr_path, currFile)) model.movers += GridCurrentMover(curr_file, num_method='RK4') if td: random_mover = RandomMover(diffusion_coef=10000) model.movers += random_mover print 'adding spill' model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration) return model
def test_read_data_exception(model): """ tests the exception is raised by read_data when file contains more than one output time and read_data is not given the output time to read """ model.rewind() # check contents of netcdf File at multiple time steps (should only be 1!) o_put = [model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput)][0] _run_model(model) with raises(ValueError): NetCDFOutput.read_data(o_put.netcdf_filename)
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'creating the maps' mapfile = get_datafile(os.path.join(base_dir, 'LowerMississippiMap.bna')) gnome_map = MapFromBNA(mapfile, refloat_halflife=6) # hours print 'initializing the model' start_time = datetime(2012, 9, 15, 12, 0) # default to now, rounded to the nearest hour model = Model(time_step=600, start_time=start_time, duration=timedelta(days=1), map=gnome_map, uncertain=True) print 'adding outputters' model.outputters += Renderer(mapfile, images_dir, image_size=(800, 600)) netcdf_file = os.path.join(base_dir, 'script_lower_mississippi.nc') scripting.remove_netcdf(netcdf_file) model.outputters += NetCDFOutput(netcdf_file, which_data='all') print 'adding a RandomMover:' model.movers += RandomMover(diffusion_coef=10000) print 'adding a wind mover:' series = np.zeros((5, ), dtype=datetime_value_2d) series[0] = (start_time, (2, 45)) series[1] = (start_time + timedelta(hours=18), (2, 90)) series[2] = (start_time + timedelta(hours=30), (2, 135)) series[3] = (start_time + timedelta(hours=42), (2, 180)) series[4] = (start_time + timedelta(hours=54), (2, 225)) w_mover = WindMover(Wind(timeseries=series, units='m/s')) model.movers += w_mover print 'adding a cats mover:' curr_file = get_datafile(os.path.join(base_dir, 'LMiss.CUR')) c_mover = CatsMover(curr_file) # but do need to scale (based on river stage) c_mover.scale = True c_mover.scale_refpoint = (-89.699944, 29.494558) # based on stage height 10ft (range is 0-18) c_mover.scale_value = 1.027154 model.movers += c_mover print 'adding a spill' spill = point_line_release_spill(num_elements=1000, start_position=(-89.699944, 29.494558, 0.0), release_time=start_time) model.spills += spill return model
def _make_run_model(spill, nc_name): 'internal funtion' m = Model() m.outputters += NetCDFOutput(nc_name) m.spills += spill _run_model(m) return m
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'initializing the model' start_time = datetime(2006, 3, 31, 20, 0) model = Model(start_time=start_time, duration=timedelta(days=3), time_step=30 * 60, uncertain=True) print 'adding the map' mapfile = get_datafile(os.path.join(base_dir, 'coastSF.bna')) model.map = MapFromBNA(mapfile, refloat_halflife=1) # seconds renderer = Renderer(mapfile, images_dir, image_size=(800, 600), draw_ontop='forecast') renderer.viewport = ((-124.5, 37.), (-120.5, 39)) print 'adding outputters' model.outputters += renderer netcdf_file = os.path.join(base_dir, 'script_sf_bay.nc') scripting.remove_netcdf(netcdf_file) model.outputters += NetCDFOutput(netcdf_file, which_data='all') print 'adding a spill' spill = point_line_release_spill( num_elements=1000, start_position=(-123.57152, 37.369436, 0.0), release_time=start_time, substance=NonWeatheringSubstance(windage_range=(0.01, .04)) #element_type=floating(windage_range=(0.01, # 0.04) # ) ) model.spills += spill # print 'adding a RandomMover:' # r_mover = gnome.movers.RandomMover(diffusion_coef=50000) # model.movers += r_mover print 'adding a grid wind mover:' wind_file = get_datafile(os.path.join(base_dir, 'WindSpeedDirSubset.nc')) topology_file = get_datafile( os.path.join(base_dir, 'WindSpeedDirSubsetTop.dat')) w_mover = GridWindMover(wind_file, topology_file) # w_mover.uncertain_time_delay = 6 # w_mover.uncertain_duration = 6 w_mover.uncertain_speed_scale = 1 w_mover.uncertain_angle_scale = 0.2 # default is .4 w_mover.wind_scale = 2 model.movers += w_mover return model
def _make_run_model(spill, nc_name): 'internal function' release_time = spill.release.release_time m = Model(start_time=release_time) m.outputters += NetCDFOutput(nc_name) m.spills += spill _run_model(m) return m
def test_serialize_deserialize(json_): ''' todo: this behaves in unexpected ways when using the 'model' testfixture. For now, define a model in here for the testing - not sure where the problem lies ''' s_time = datetime(2014, 1, 1, 1, 1, 1) model = Model(start_time=s_time) model.spills += point_line_release_spill(num_elements=5, start_position=(0, 0, 0), release_time=model.start_time) o_put = NetCDFOutput(os.path.join(base_dir, u'xtemp.nc')) model.outputters += o_put model.movers += RandomMover(diffusion_coef=100000) #========================================================================== # o_put = [model.outputters[outputter.id] # for outputter in model.outputters # if isinstance(outputter, NetCDFOutput)][0] #========================================================================== model.rewind() print "step: {0}, _start_idx: {1}".format(-1, o_put._start_idx) for ix in range(2): model.step() print "step: {0}, _start_idx: {1}".format(ix, o_put._start_idx) #for json_ in ('save', 'webapi'): dict_ = o_put.deserialize(o_put.serialize(json_)) o_put2 = NetCDFOutput.new_from_dict(dict_) if json_ == 'save': assert o_put == o_put2 else: # _start_idx and _middle_of_run should not match assert o_put._start_idx != o_put2._start_idx assert o_put._middle_of_run != o_put2._middle_of_run assert o_put != o_put2 if os.path.exists(o_put.netcdf_filename): print '\n{0} exists'.format(o_put.netcdf_filename)
def make_models(): print 'initializing the model' # start_time = datetime(2015, 12, 18, 06, 01) # 1 day of data in file # 1/2 hr in seconds models = [] start_time = datetime(2012, 10, 27, 0, 30) duration_hrs=23 time_step=450 num_steps = duration_hrs * 3600 / time_step names = [ 'Euler', 'Trapezoid', 'RK4', ] mapfile = get_datafile(os.path.join(base_dir, 'long_beach.bna')) print 'gen map' map = MapFromBNA(mapfile, refloat_halflife=0.0) # seconds fn = ('00_dir_roms_display.ncml.nc4') curr = GridCurrent.from_netCDF(filename=fn) models = [] for method in names: mod = Model(start_time=start_time, duration=timedelta(hours=duration_hrs), time_step=time_step) mod.map = map spill = point_line_release_spill(num_elements=1000, start_position=(-74.1, 39.7525, 0.0), release_time=start_time) mod.spills += spill mod.movers += RandomMover(diffusion_coef=100) mod.movers += PyGridCurrentMover(current=curr, default_num_method=method) images_dir = method + '-' + str(time_step / 60) + 'min-' + str(num_steps) + 'steps' renderer = Renderer(mapfile, images_dir, image_size=(1024, 768)) renderer.delay = 25 # renderer.add_grid(curr.grid) mod.outputters += renderer netCDF_fn = os.path.join(base_dir, images_dir + '.nc') mod.outputters += NetCDFOutput(netCDF_fn, which_data='all') models.append(mod) print 'returning models' return models
def test_serialize_deserialize(json_, output_filename): ''' todo: this behaves in unexpected ways when using the 'model' testfixture. For now, define a model in here for the testing - not sure where the problem lies ''' s_time = datetime(2014, 1, 1, 1, 1, 1) model = Model(start_time=s_time) model.spills += point_line_release_spill(num_elements=5, start_position=(0, 0, 0), release_time=model.start_time) o_put = NetCDFOutput(output_filename) model.outputters += o_put model.movers += RandomMover(diffusion_coef=100000) # ========================================================================== # o_put = [model.outputters[outputter.id] # for outputter in model.outputters # if isinstance(outputter, NetCDFOutput)][0] # ========================================================================== model.rewind() print "step: {0}, _start_idx: {1}".format(-1, o_put._start_idx) for ix in range(2): model.step() print "step: {0}, _start_idx: {1}".format(ix, o_put._start_idx) dict_ = o_put.deserialize(o_put.serialize(json_)) o_put2 = NetCDFOutput.new_from_dict(dict_) if json_ == 'save': assert o_put == o_put2 else: # _start_idx and _middle_of_run should not match assert o_put._start_idx != o_put2._start_idx assert o_put._middle_of_run != o_put2._middle_of_run assert o_put != o_put2 if os.path.exists(o_put.netcdf_filename): print '\n{0} exists'.format(o_put.netcdf_filename)
def test_read_all_arrays(model): """ tests the data returned by read_data is correct when `which_data` flag is 'all'. """ model.rewind() o_put = [ model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput) ][0] o_put.which_data = 'all' _run_model(model) atol = 1e-5 rtol = 0 uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): _found_a_matching_time = False for step in range(model.num_time_steps): scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) (nc_data, mb) = NetCDFOutput.read_data(file_, curr_time, which_data='all') if curr_time == nc_data['current_time_stamp'].item(): _found_a_matching_time = True for key in scp.LE_data: if key == 'current_time_stamp': """ already matched """ continue elif key == 'positions': assert np.allclose(scp.LE('positions', uncertain), nc_data['positions'], rtol, atol) elif key == 'mass_balance': assert scp.LE(key, uncertain) == mb else: if key not in ['surface_concentration' ]: # not always there assert np.all( scp.LE(key, uncertain)[:] == nc_data[key]) if _found_a_matching_time: print('\ndata in model matches for output in \n{0}'.format(file_)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True
def test_read_all_arrays(model): """ tests the data returned by read_data is correct when `which_data` flag is 'all'. """ model.rewind() o_put = [model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput)][0] o_put.which_data = 'all' _run_model(model) atol = 1e-5 rtol = 0 uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): _found_a_matching_time = False for step in range(model.num_time_steps): scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) (nc_data, mb) = NetCDFOutput.read_data(file_, curr_time, which_data='all') if curr_time == nc_data['current_time_stamp'].item(): _found_a_matching_time = True for key in scp.LE_data: if key == 'current_time_stamp': """ already matched """ continue elif key == 'positions': assert np.allclose(scp.LE('positions', uncertain), nc_data['positions'], rtol, atol) elif key == 'mass_balance': assert scp.LE(key, uncertain) == mb else: # if key not in ['last_water_positions', # 'next_positions']: assert np.all(scp.LE(key, uncertain)[:] == nc_data[key]) if _found_a_matching_time: print ('\ndata in model matches for output in \n{0}'.format(file_)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True
def make_model(images_dir=os.path.join(base_dir,"images")): print "initializing the model" start_time = datetime(2013, 7, 23, 0) model = Model(start_time = start_time, duration = timedelta(hours=47), # n+1 of data in file time_step = 900, # 4 hr in seconds uncertain = False, ) mapfile = os.path.join(base_dir, './coast.bna') print "adding the map" gnome_map = MapFromBNA(mapfile, refloat_halflife=6) # hours print "adding renderer" model.outputters += Renderer(mapfile, images_dir, size=(1800, 1600)) print "adding a wind mover from a time-series" ## this is wind wind_file=get_datafile(os.path.join(base_dir, 'wind.WND')) wind = Wind(filename=wind_file) w_mover = WindMover(wind) model.movers += w_mover print "adding a current mover:" ## this is currents curr_file = get_datafile(os.path.join(base_dir, 'current.txt')) model.movers += GridCurrentMover(curr_file) ## ## Add some spills (sources of elements) ## print "adding 13 points in a cluster that has some small initial separation as the source of spill" for i in range(len(coor)): aaa=utmToLatLng(14,coor[i][0],coor[i][1],northernHemisphere=True) model.spills += point_line_release_spill(num_elements=1, start_position = (aaa[1],aaa[0], 0.0), release_time = start_time, ) print "adding netcdf output" netcdf_output_file = os.path.join(base_dir,'GNOME_output.nc') scripting.remove_netcdf(netcdf_output_file) model.outputters += NetCDFOutput(netcdf_output_file, which_data='all') return model
def only_Winds(timeStep, start_time, duration, weatheringSteps, map, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, tidalFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td): print 'initializing the model:' model = Model(time_step=timeStep, start_time=start_time, duration=duration) print 'adding the map:' mapfile = get_datafile(os.path.join(data_path, map_path, map)) model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife) print 'adding a renderer' model.outputters += Renderer(mapfile, output_path, size=(800, 600), output_timestep=timedelta(hours=timestep_outputs)) if save_nc: nc_outputter = NetCDFOutput(netcdf_file, which_data='most', output_timestep=timedelta(hours=timestep_outputs)) model.outputters += nc_outputter print 'adding a wind mover:' wind_file = get_datafile(os.path.join(data_path, wind_path, windFile)) wind = GridWindMover(wind_file) wind.wind_scale = wind_scale model.movers += wind print 'adding a spill' model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration) return model
def make_model(images_dir): print 'initializing the model' timestep = timedelta(minutes=15) # this is already default start_time = datetime(2012, 9, 15, 12, 0) model = Model(timestep, start_time) # timeseries for wind data. The value is interpolated if time is between # the given datapoints series = np.zeros((4, ), dtype=datetime_value_2d) series[:] = [(start_time, (5, 180)), (start_time + timedelta(hours=6), (10, 180)), (start_time + timedelta(hours=12), (12, 180)), (start_time + timedelta(hours=18), (8, 180))] wind = Wind(timeseries=series, units='m/s') model.environment += wind # include a wind mover and random diffusion print 'adding movers' model.movers += [WindMover(wind), RandomMover()] # add particles print 'adding particles' release = release_from_splot_data(start_time, 'GL.2013267._LE_WHOLELAKE.txt') model.spills += Spill(release) # output data as png images and in netcdf format print 'adding outputters' netcdf_file = os.path.join(base_dir, 'script_example.nc') # ignore renderer for now model.outputters += [ Renderer(images_dir=images_dir, size=(800, 800), projection_class=GeoProjection), NetCDFOutput(netcdf_file) ] print 'model complete' return model
def make_modelF(timeStep, start_time, duration, weatheringSteps, map, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td, dif_coef,temp_water): print 'initializing the model:' model = Model(time_step=timeStep, start_time=start_time, duration=duration, uncertain=uncertain) print 'adding the map:' mapfile = get_datafile(os.path.join(data_path, map_path, map)) model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife) print 'adding a renderer' if save_nc: scripting.remove_netcdf(output_path+'/'+'output.nc') nc_outputter = NetCDFOutput(output_path+'/'+'output.nc', which_data='standard', output_timestep=timedelta(hours=timestep_outputs)) model.outputters += nc_outputter print 'adding a wind mover:' wind_file = get_datafile(os.path.join(data_path, wind_path, windFile)) wind = GridWindMover(wind_file) # wind.wind_scale = wind_scale model.movers += wind print 'adding a current mover:' curr_file = get_datafile(os.path.join(data_path, curr_path, currFile)) model.movers += GridCurrentMover(curr_file, num_method='RK4') if td: random_mover = RandomMover(diffusion_coef=dif_coef) model.movers += random_mover print 'adding spill' model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration)#, substance='AD04001', amount=9600000, units='kg') if weatherers: print 'adding weatherers' water = Water(temp_water) wind = constant_wind(0.0001, 0, 'knots') waves = Waves(wind, water) model.weatherers += Evaporation(water, wind) # model.weatherers += Emulsification(waves) model.weatherers += NaturalDispersion(waves, water) return model
def make_model(): duration_hrs = 48 time_step = 900 num_steps = duration_hrs * 3600 / time_step mod = Model(start_time=t, duration=timedelta(hours=duration_hrs), time_step=time_step) spill = point_line_release_spill(num_elements=1000, amount=1600, units='kg', start_position=(0.5, 0.5, 0.0), release_time=t, end_release_time=t + timedelta(hours=4)) mod.spills += spill method = 'Trapezoid' images_dir = method + '-' + str( time_step / 60) + 'min-' + str(num_steps) + 'steps' renderer = Renderer(output_dir=images_dir, image_size=(800, 800)) renderer.delay = 5 renderer.add_grid(g) renderer.add_vec_prop(vg) renderer.graticule.set_max_lines(max_lines=0) mod.outputters += renderer mod.movers += PyCurrentMover(current=vg, default_num_method=method, extrapolate=True) mod.movers += RandomMover(diffusion_coef=10) netCDF_fn = os.path.join(base_dir, images_dir + '.nc') mod.outputters += NetCDFOutput(netCDF_fn, which_data='all') return mod
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'initializing the model' start_time = datetime(2004, 12, 31, 13, 0) model = Model(start_time=start_time, duration=timedelta(days=3), time_step=30 * 60, uncertain=False) print 'adding the map' model.map = GnomeMap() # draw_ontop can be 'uncertain' or 'forecast' # 'forecast' LEs are in black, and 'uncertain' are in red # default is 'forecast' LEs draw on top renderer = Renderer( output_dir=images_dir, # size=(800, 600), output_timestep=timedelta(hours=1), draw_ontop='uncertain') renderer.viewport = ((-76.5, 37.), (-75.8, 38.)) print 'adding outputters' model.outputters += renderer netcdf_file = os.path.join(base_dir, 'script_plume.nc') scripting.remove_netcdf(netcdf_file) model.outputters += NetCDFOutput(netcdf_file, which_data='most', output_timestep=timedelta(hours=2)) print 'adding two spills' # Break the spill into two spills, first with the larger droplets # and second with the smaller droplets. # Split the total spill volume (100 m^3) to have most # in the larger droplet spill. # Smaller droplets start at a lower depth than larger wd = WeibullDistribution(alpha=1.8, lambda_=.00456, min_=.0002) # 200 micron min end_time = start_time + timedelta(hours=24) # spill = point_line_release_spill(num_elements=10, # amount=90, # default volume_units=m^3 # units='m^3', # start_position=(-76.126872, 37.680952, # 1700), # release_time=start_time, # end_release_time=end_time, # element_type=plume(distribution=wd, # density=600) # ) spill = subsurface_plume_spill( num_elements=10, start_position=(-76.126872, 37.680952, 1700), release_time=start_time, distribution=wd, amount=90, # default volume_units=m^3 units='m^3', end_release_time=end_time, density=600) model.spills += spill wd = WeibullDistribution(alpha=1.8, lambda_=.00456, max_=.0002) # 200 micron max spill = point_line_release_spill( num_elements=10, amount=90, units='m^3', start_position=(-76.126872, 37.680952, 1800), release_time=start_time, element_type=plume(distribution=wd, substance_name='oil_crude')) model.spills += spill print 'adding a RandomMover:' model.movers += RandomMover(diffusion_coef=50000) print 'adding a RiseVelocityMover:' model.movers += RiseVelocityMover() print 'adding a RandomVerticalMover:' model.movers += RandomVerticalMover(vertical_diffusion_coef_above_ml=5, vertical_diffusion_coef_below_ml=.11, mixed_layer_depth=10) # print 'adding a wind mover:' # series = np.zeros((2, ), dtype=gnome.basic_types.datetime_value_2d) # series[0] = (start_time, (30, 90)) # series[1] = (start_time + timedelta(hours=23), (30, 90)) # wind = Wind(timeseries=series, units='knot') # # default is .4 radians # w_mover = gnome.movers.WindMover(wind, uncertain_angle_scale=0) # # model.movers += w_mover print 'adding a simple mover:' s_mover = SimpleMover(velocity=(0.0, -.3, 0.0)) model.movers += s_mover return model
def test_exceptions(model): t_file = os.path.join(base_dir, 'temp.nc') spill_pair = model.spills def _del_temp_file(): # clean up temporary file from previous run try: print 'remove temporary file {0}'.format(t_file) os.remove(t_file) except: pass # begin tests _del_temp_file() netcdf = NetCDFOutput(t_file, which_data='all') with raises(TypeError): # need to pass in model start time netcdf.prepare_for_model_run(num_time_steps=4) with raises(TypeError): # need to pass in model start time and spills netcdf.prepare_for_model_run() with raises(ValueError): # need a cache object netcdf.write_output(0) with raises(ValueError): netcdf.which_data = 'some random string' with raises(ValueError): # raise error because file 'temp.nc' should exist after 1st call netcdf.prepare_for_model_run(model_start_time=datetime.now(), spills=spill_pair, num_time_steps=4) netcdf.prepare_for_model_run(model_start_time=datetime.now(), spills=spill_pair, num_time_steps=4) with raises(AttributeError): 'cannot change after prepare_for_model_run has been called' netcdf.which_data = 'all' _del_temp_file()
def test_exceptions(output_filename): spill_pair = SpillContainerPair() print "output_filename:", output_filename # begin tests netcdf = NetCDFOutput(output_filename, which_data='all') netcdf.rewind() # delete temporary files with raises(TypeError): # need to pass in model start time netcdf.prepare_for_model_run(num_time_steps=4) with raises(TypeError): # need to pass in model start time and spills netcdf.prepare_for_model_run() with raises(ValueError): # need a cache object netcdf.write_output(0) with raises(ValueError): netcdf.which_data = 'some random string' # changed renderer and netcdf ouputter to delete old files in # prepare_for_model_run() rather than rewind() # -- rewind() was getting called a lot # -- before there was time to change the ouput file names, etc. # So for this unit test, there should be no exception if we do it twice. netcdf.prepare_for_model_run(model_start_time=datetime.now(), spills=spill_pair, num_time_steps=4) netcdf.prepare_for_model_run(model_start_time=datetime.now(), spills=spill_pair, num_time_steps=4) with raises(AttributeError): 'cannot change after prepare_for_model_run has been called' netcdf.prepare_for_model_run(model_start_time=datetime.now(), spills=spill_pair, num_time_steps=4) netcdf.which_data = 'most'
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'initializing the model' start_time = datetime(1985, 1, 1, 13, 31) # 1 day of data in file # 1/2 hr in seconds model = Model(start_time=start_time, duration=timedelta(days=4), time_step=7200) # mapfile = get_datafile(os.path.join(base_dir, 'ak_arctic.bna')) mapfile = get_datafile('arctic_coast3.bna') print 'adding the map' model.map = MapFromBNA(mapfile, refloat_halflife=0.0) # seconds print 'adding a spill' # for now subsurface spill stays on initial layer # - will need diffusion and rise velocity # - wind doesn't act # - start_position = (-76.126872, 37.680952, 5.0), # spill1 = point_line_release_spill(num_elements=10000, # start_position=(-163.75, # 69.75, # 0.0), # release_time=start_time) # spill1 = point_line_release_spill(num_elements=50000, start_position=(196.25, 69.75, 0.0), release_time=start_time) model.spills += spill1 # model.spills += spill2 print 'adding a wind mover:' # model.movers += constant_wind_mover(0.5, 0, units='m/s') print 'adding a current mover:' fn = ['arctic_avg2_0001_gnome.nc', 'arctic_avg2_0002_gnome.nc'] # fn = ['C:\\Users\\jay.hennen\\Documents\\Code\\pygnome\\py_gnome\\scripts\\script_TAP\\arctic_avg2_0001_gnome.nc', # 'C:\\Users\\jay.hennen\\Documents\\Code\\pygnome\\py_gnome\\scripts\\script_TAP\\arctic_avg2_0002_gnome.nc'] gt = {'node_lon': 'lon', 'node_lat': 'lat'} # fn='arctic_avg2_0001_gnome.nc' wind_method = 'Euler' method = 'RK2' print 'adding outputters' # draw_ontop can be 'uncertain' or 'forecast' # 'forecast' LEs are in black, and 'uncertain' are in red # default is 'forecast' LEs draw on top renderer = Renderer(mapfile, images_dir, image_size=(1024, 768)) model.outputters += renderer netcdf_file = os.path.join(base_dir, str(model.time_step / 60) + method + '.nc') scripting.remove_netcdf(netcdf_file) print 'adding movers' model.outputters += NetCDFOutput(netcdf_file, which_data='all') print 'loading entire current data' ice_aware_curr = IceAwareCurrent.from_netCDF(filename=fn, grid_topology=gt) # env1 = get_env_from_netCDF(filename) # mov = PyCurrentMover.from_netCDF(filename) ice_aware_curr.ice_velocity.variables[0].dimension_ordering = [ 'time', 'x', 'y' ] ice_aware_wind = IceAwareWind.from_netCDF( filename=fn, ice_velocity=ice_aware_curr.ice_velocity, ice_concentration=ice_aware_curr.ice_concentration, grid=ice_aware_curr.grid) curr = GridCurrent.from_netCDF(filename=fn) # GridCurrent.is_gridded() # import pprint as pp # from gnome.utilities.orderedcollection import OrderedCollection # model.environment = OrderedCollection(dtype=Environment) # model.environment.add(ice_aware_curr) # from gnome.environment import WindTS print 'loading entire wind data' # i_c_mover = PyCurrentMover(current=ice_aware_curr) # i_c_mover = PyCurrentMover(current=ice_aware_curr, default_num_method='Euler') i_c_mover = PyCurrentMover(current=ice_aware_curr, default_num_method=method, extrapolate=True) i_w_mover = PyWindMover(wind=ice_aware_wind, default_num_method=wind_method) # ice_aware_curr.grid.node_lon = ice_aware_curr.grid.node_lon[:]-360 # ice_aware_curr.grid.build_celltree() model.movers += i_c_mover model.movers += i_w_mover print 'adding an IceAwareRandomMover:' model.movers += IceAwareRandomMover( ice_concentration=ice_aware_curr.ice_concentration, diffusion_coef=1000) # renderer.add_grid(ice_aware_curr.grid) # renderer.add_vec_prop(ice_aware_curr) # curr_file = get_datafile(os.path.join(base_dir, 'COOPSu_CREOFS24.nc')) # c_mover = GridCurrentMover(curr_file) # model.movers += c_mover # model.environment.add(WindTS.constant(10, 300)) # print('Saving') # model.environment[0].ice_velocity.variables[0].serialize() # IceVelocity.deserialize(model.environment[0].ice_velocity.serialize()) # model.save('.') # from gnome.persist.save_load import load # print('Loading') # model2 = load('./Model.zip') return model
def test_read_standard_arrays(model, output_ts_factor, use_time): """ tests the data returned by read_data is correct when `which_data` flag is 'standard'. It is only reading the standard_arrays Test will only verify the data when time_stamp of model matches the time_stamp of data written out. output_ts_factor means not all data is written out. The use_time flag says data is read by timestamp. If false, then it is read by step number - either way, the result should be the same """ model.rewind() # check contents of netcdf File at multiple time steps (should only be 1!) o_put = [ model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput) ][0] o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor) _run_model(model) atol = 1e-5 rtol = 0 uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): _found_a_matching_time = False for idx, step in enumerate( range(0, model.num_time_steps, int(ceil(output_ts_factor)))): scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) if use_time: (nc_data, weathering_data) = NetCDFOutput.read_data(file_, curr_time) else: (nc_data, weathering_data) = NetCDFOutput.read_data(file_, index=idx) # check time if curr_time == nc_data['current_time_stamp'].item(): _found_a_matching_time = True # check standard variables assert np.allclose(scp.LE('positions', uncertain), nc_data['positions'], rtol, atol) assert np.all( scp.LE('spill_num', uncertain)[:] == nc_data['spill_num']) assert np.all( scp.LE('status_codes', uncertain)[:] == nc_data['status_codes']) # flag variable is not currently set or checked if 'mass' in scp.LE_data: assert np.all( scp.LE('mass', uncertain)[:] == nc_data['mass']) if 'age' in scp.LE_data: assert np.all( scp.LE('age', uncertain)[:] == nc_data['age']) if uncertain: sc = scp.items()[1] else: sc = scp.items()[0] assert sc.mass_balance == weathering_data else: raise Exception( 'Assertions not tested since no data found ' 'in NetCDF file for timestamp: {0}'.format(curr_time)) if _found_a_matching_time: print( '\n' 'data in model matches for output in\n' '{0}\n' 'and output_ts_factor: {1}'.format(file_, output_ts_factor)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True
def test_write_output_post_run(model, output_ts_factor): """ Create netcdf file post run from the cache. Under the hood, it is simply calling write_output so no need to check the data is correctly written test_write_output_standard already checks data is correctly written. Instead, make sure if output_timestep is not same as model.time_step, then data is output at correct time stamps """ model.rewind() o_put = [model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput)][0] o_put.which_data = 'standard' o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor) del model.outputters[o_put.id] # remove from list of outputters _run_model(model) # clear out old files... o_put.clean_output_files() assert not os.path.exists(o_put.netcdf_filename) if o_put._u_netcdf_filename: assert (not os.path.exists(o_put._u_netcdf_filename)) # now write netcdf output o_put.write_output_post_run(model.start_time, model.num_time_steps, spills=model.spills, cache=model._cache, uncertain=model.uncertain) assert os.path.exists(o_put.netcdf_filename) if model.uncertain: assert os.path.exists(o_put._u_netcdf_filename) uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): ix = 0 # index for grabbing record from NetCDF file for step in range(0, model.num_time_steps, int(ceil(output_ts_factor))): print "step: {0}".format(step) scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) (nc_data, mb) = NetCDFOutput.read_data(file_, curr_time) assert curr_time == nc_data['current_time_stamp'].item() # test to make sure data_by_index is consistent with _cached data # This is just to double check that getting the data by curr_time # does infact give the next consecutive index (data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix) assert curr_time == data_by_index['current_time_stamp'].item() assert scp.LE('mass_balance', uncertain) == mb ix += 1 if o_put.output_last_step and step < model.num_time_steps - 1: ''' Last timestep written to NetCDF wasn't tested - do that here ''' scp = model._cache.load_timestep(model.num_time_steps - 1) curr_time = scp.LE('current_time_stamp', uncertain) (nc_data, mb) = NetCDFOutput.read_data(file_, curr_time) assert curr_time == nc_data['current_time_stamp'].item() assert scp.LE('mass_balance', uncertain) == mb # again, check that last time step (data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix) assert curr_time == data_by_index['current_time_stamp'].item() assert scp.LE('mass_balance', uncertain) == mb with pytest.raises(IndexError): # check that no more data exists in NetCDF NetCDFOutput.read_data(file_, index=ix + 1) """ at least one matching time found """ print ('All expected timestamps in {0} for output_ts_factor: {1}' .format(os.path.split(file_)[1], output_ts_factor)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True # add this back in so cleanup script deletes the generated *.nc files model.outputters += o_put
def test_read_standard_arrays(model, output_ts_factor, use_time): """ tests the data returned by read_data is correct when `which_data` flag is 'standard'. It is only reading the standard_arrays Test will only verify the data when time_stamp of model matches the time_stamp of data written out. output_ts_factor means not all data is written out. The use_time flag says data is read by timestamp. If false, then it is read by step number - either way, the result should be the same """ model.rewind() # check contents of netcdf File at multiple time steps (should only be 1!) o_put = [model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput)][0] o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor) _run_model(model) atol = 1e-5 rtol = 0 uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): _found_a_matching_time = False for idx, step in enumerate(range(0, model.num_time_steps, int(ceil(output_ts_factor)))): scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) if use_time: (nc_data, weathering_data) = NetCDFOutput.read_data(file_, curr_time) else: (nc_data, weathering_data) = NetCDFOutput.read_data(file_, index=idx) # check time if curr_time == nc_data['current_time_stamp'].item(): _found_a_matching_time = True # check standard variables assert np.allclose(scp.LE('positions', uncertain), nc_data['positions'], rtol, atol) assert np.all(scp.LE('spill_num', uncertain)[:] == nc_data['spill_num']) assert np.all(scp.LE('status_codes', uncertain)[:] == nc_data['status_codes']) # flag variable is not currently set or checked if 'mass' in scp.LE_data: assert np.all(scp.LE('mass', uncertain)[:] == nc_data['mass']) if 'age' in scp.LE_data: assert np.all(scp.LE('age', uncertain)[:] == nc_data['age']) if uncertain: sc = scp.items()[1] else: sc = scp.items()[0] assert sc.mass_balance == weathering_data else: raise Exception('Assertions not tested since no data found ' 'in NetCDF file for timestamp: {0}' .format(curr_time)) if _found_a_matching_time: print ('\n' 'data in model matches for output in\n' '{0}\n' 'and output_ts_factor: {1}' .format(file_, output_ts_factor)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True
def test_write_output_post_run(model, output_ts_factor): """ Create netcdf file post run from the cache. Under the hood, it is simply calling write_output so no need to check the data is correctly written test_write_output_standard already checks data is correctly written. Instead, make sure if output_timestep is not same as model.time_step, then data is output at correct time stamps """ model.rewind() o_put = [ model.outputters[outputter.id] for outputter in model.outputters if isinstance(outputter, NetCDFOutput) ][0] o_put.which_data = 'standard' o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor) del model.outputters[o_put.id] # remove from list of outputters _run_model(model) # clear out old files... o_put.clean_output_files() assert not os.path.exists(o_put.netcdf_filename) if o_put._u_netcdf_filename: assert (not os.path.exists(o_put._u_netcdf_filename)) # now write netcdf output o_put.write_output_post_run(model.start_time, model.num_time_steps, spills=model.spills, cache=model._cache, uncertain=model.uncertain) assert os.path.exists(o_put.netcdf_filename) if model.uncertain: assert os.path.exists(o_put._u_netcdf_filename) uncertain = False for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename): ix = 0 # index for grabbing record from NetCDF file for step in range(0, model.num_time_steps, int(ceil(output_ts_factor))): print "step: {0}".format(step) scp = model._cache.load_timestep(step) curr_time = scp.LE('current_time_stamp', uncertain) (nc_data, mb) = NetCDFOutput.read_data(file_, curr_time) assert curr_time == nc_data['current_time_stamp'].item() # test to make sure data_by_index is consistent with _cached data # This is just to double check that getting the data by curr_time # does infact give the next consecutive index (data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix) assert curr_time == data_by_index['current_time_stamp'].item() assert scp.LE('mass_balance', uncertain) == mb ix += 1 if o_put.output_last_step and step < model.num_time_steps - 1: ''' Last timestep written to NetCDF wasn't tested - do that here ''' scp = model._cache.load_timestep(model.num_time_steps - 1) curr_time = scp.LE('current_time_stamp', uncertain) (nc_data, mb) = NetCDFOutput.read_data(file_, curr_time) assert curr_time == nc_data['current_time_stamp'].item() assert scp.LE('mass_balance', uncertain) == mb # again, check that last time step (data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix) assert curr_time == data_by_index['current_time_stamp'].item() assert scp.LE('mass_balance', uncertain) == mb with pytest.raises(IndexError): # check that no more data exists in NetCDF NetCDFOutput.read_data(file_, index=ix + 1) """ at least one matching time found """ print( 'All expected timestamps in {0} for output_ts_factor: {1}'.format( os.path.split(file_)[1], output_ts_factor)) # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True # add this back in so cleanup script deletes the generated *.nc files model.outputters += o_put
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'initializing the model' # set up the modeling environment start_time = datetime(2016, 9, 23, 0, 0) model = Model(start_time=start_time, duration=timedelta(days=2), time_step=30 * 60, uncertain=False) print 'adding the map' model.map = GnomeMap() # this is a "water world -- no land anywhere" # renderere is only top-down view on 2d -- but it's something renderer = Renderer(output_dir=images_dir, image_size=(1024, 768), output_timestep=timedelta(hours=1), ) renderer.viewport = ((196.14, 71.89), (196.18, 71.93)) print 'adding outputters' model.outputters += renderer # Also going to write the results out to a netcdf file netcdf_file = os.path.join(base_dir, 'script_arctic_plume.nc') scripting.remove_netcdf(netcdf_file) model.outputters += NetCDFOutput(netcdf_file, which_data='most', # output most of the data associated with the elements output_timestep=timedelta(hours=2)) print "adding Horizontal and Vertical diffusion" # Horizontal Diffusion model.movers += RandomMover(diffusion_coef=500) # vertical diffusion (different above and below the mixed layer) model.movers += RandomMover3D(vertical_diffusion_coef_above_ml=5, vertical_diffusion_coef_below_ml=.11, mixed_layer_depth=10) print 'adding Rise Velocity' # droplets rise as a function of their density and radius model.movers += TamocRiseVelocityMover() print 'adding a circular current and eastward current' fn = 'hycom_glb_regp17_2016092300_subset.nc' fn_ice = 'hycom-cice_ARCu0.08_046_2016092300_subset.nc' iconc = IceConcentration.from_netCDF(filename=fn_ice) ivel = IceVelocity.from_netCDF(filename=fn_ice, grid = iconc.grid) ic = IceAwareCurrent.from_netCDF(ice_concentration = iconc, ice_velocity= ivel, filename=fn) model.movers += PyCurrentMover(current = ic) model.movers += SimpleMover(velocity=(0., 0., 0.)) model.movers += constant_wind_mover(20, 315, units='knots') # Now to add in the TAMOC "spill" print "Adding TAMOC spill" model.spills += tamoc_spill.TamocSpill(release_time=start_time, start_position=(196.16, 71.91, 40.0), num_elements=1000, end_release_time=start_time + timedelta(days=1), name='TAMOC plume', TAMOC_interval=None, # how often to re-run TAMOC ) model.spills[0].data_sources['currents'] = ic return model
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'initializing the model' start_time = datetime(2014, 6, 9, 0, 0) mapfile = get_datafile(os.path.join(base_dir, 'PassamaquoddyMap.bna')) gnome_map = MapFromBNA(mapfile, refloat_halflife=1) # hours # # the image output renderer # global renderer # one hour timestep model = Model(start_time=start_time, duration=timedelta(hours=24), time_step=360, map=gnome_map, uncertain=False, cache_enabled=True) print 'adding outputters' renderer = Renderer(mapfile, images_dir, size=(800, 600), # output_timestep=timedelta(hours=1), draw_ontop='uncertain') renderer.viewport = ((-67.15, 45.), (-66.9, 45.2)) model.outputters += renderer netcdf_file = os.path.join(base_dir, 'script_passamaquoddy.nc') scripting.remove_netcdf(netcdf_file) model.outputters += NetCDFOutput(netcdf_file, which_data='all') print 'adding a spill' spill = point_line_release_spill(num_elements=1000, start_position=(-66.991344, 45.059316, 0.0), release_time=start_time) model.spills += spill print 'adding a RandomMover:' model.movers += RandomMover(diffusion_coef=30000, uncertain_factor=2) print 'adding a wind mover:' series = np.zeros((5, ), dtype=datetime_value_2d) series[0] = (start_time, (5, 90)) series[1] = (start_time + timedelta(hours=18), (5, 180)) series[2] = (start_time + timedelta(hours=30), (5, 135)) series[3] = (start_time + timedelta(hours=42), (5, 180)) series[4] = (start_time + timedelta(hours=54), (5, 225)) wind = Wind(timeseries=series, units='m/s') model.movers += WindMover(wind) print 'adding a current mover:' curr_file = get_datafile(os.path.join(base_dir, 'PQBayCur.nc4')) topology_file = get_datafile(os.path.join(base_dir, 'PassamaquoddyTOP.dat') ) tide_file = get_datafile(os.path.join(base_dir, 'EstesHead.txt')) cc_mover = CurrentCycleMover(curr_file, topology_file, tide=Tide(tide_file)) model.movers += cc_mover model.environment += cc_mover.tide print 'viewport is:', [o.viewport for o in model.outputters if isinstance(o, Renderer)] return model
def main(RootDir, Data_Dir, StartSite, RunSite, NumStarts, RunStarts, ReleaseLength, TrajectoryRunLength, StartTimeFiles, TrajectoriesPath, NumLEs, MapFileName, refloat, current_files, wind_files, diffusion_coef, model_timestep, windage_range, windage_persist, OutputTimestep): timingRecord = open(os.path.join(RootDir, "timing.txt"), "w") count = len(StartTimeFiles) * len(RunStarts) timingRecord.write("This file tracks the time to process " + str(count) + " gnome runs") # model timing release_duration = timedelta(hours=ReleaseLength) run_time = timedelta(hours=TrajectoryRunLength) # initiate model model = Model(duration=run_time, time_step=model_timestep, uncertain=False) # determine boundary for model print "Adding the map:", MapFileName mapfile = get_datafile(os.path.join(Data_Dir, MapFileName)) # model.map = MapFromBNA(mapfile, refloat_halflife=refloat) no, model map needs to inclde mudflats. later # loop through seasons for Season in StartTimeFiles: timer1 = datetime.now() SeasonName = Season[1] start_times = open(Season[0], 'r').readlines()[:NumStarts] SeasonTrajDir = os.path.join(RootDir, TrajectoriesPath, SeasonName) if not os.path.isdir(SeasonTrajDir): print "Creating directory: ", SeasonTrajDir make_dir(SeasonTrajDir) print " Season:", SeasonName # get and parse start times in this season start_dt = [] for start_time in start_times: start_time = [int(i) for i in start_time.split(',')] start_time = datetime(start_time[0], start_time[1], start_time[2], start_time[3], start_time[4]) start_dt.append(start_time) ## loop through start times for time_idx in RunStarts: timer2 = datetime.now() gc.collect() model.movers.clear() ## set the start location start_time = start_dt[time_idx] end_time = start_time + run_time model.start_time = start_time print " ", start_time, "to", end_time ## get a list of the only data files needed for the start time (less data used) ## note: requires data files in year increments #Todo: needs fixing before real run years = range(start_time.year, end_time.year + 1) years = [str(i) for i in years] wind = [s for s in wind_files if any(xs in s for xs in years)] current = [ s for s in current_files if any(xs in s for xs in years) ] #Todo: add mudflats. Does it work like this? topology = {'node_lon': 'x', 'node_lat': 'y'} ## add wind movers w_mover = PyWindMover(filename=wind) model.movers += w_mover ## add current movers current_mover = gs.GridCurrent.from_netCDF(current, grid_topology=topology) c_mover = PyCurrentMover(current=current_mover) model.movers += c_mover tideflat = Matroos_Mudflats(current, grid_topology=topology) land_map = gs.MapFromBNA(mapfile) model.map = TideflatMap(land_map, tideflat) ## add diffusion model.movers += RandomMover(diffusion_coef=diffusion_coef) ## loop through start locations timer3 = datetime.now() #Todo: can it deal with the test.location.txt file?? start_position = [float(i) for i in StartSite.split(',')] OutDir = os.path.join(RootDir, TrajectoriesPath, SeasonName, 'pos_%03i' % (RunSite + 1)) make_dir(OutDir) print " ", RunSite, time_idx print " Running: start time:", start_time, print "at start location:", start_position ## set the spill to the location spill = surface_point_line_spill( num_elements=NumLEs, start_position=(start_position[0], start_position[1], 0.0), release_time=start_time, end_release_time=start_time + release_duration, windage_range=windage_range, windage_persist=windage_persist) # print "adding netcdf output" netcdf_output_file = os.path.join( OutDir, 'pos_%03i-t%03i_%08i.nc' % (RunSite + 1, time_idx, int(start_time.strftime('%y%m%d%H'))), ) model.outputters.clear() model.outputters += NetCDFOutput( netcdf_output_file, output_timestep=timedelta(hours=OutputTimestep)) model.spills.clear() model.spills += spill model.full_run(rewind=True) timer4 = datetime.now() diff = round((timer4 - timer3).total_seconds() / 60, 2) timingRecord.write("\t\t" + str(RunSite) + " took " + str(diff) + " minutes to complete") diff = round((timer4 - timer1).total_seconds() / 3600, 2) count = len(RunStarts) timingRecord.write("\t" + str(SeasonName) + " took " + str(diff) + " hours to finish " + str(count) + " Gnome runs") #OutDir.close timingRecord.close
def test_exceptions(): t_file = os.path.join(base_dir, 'temp.nc') # clean up temporary file from previos run if os.path.exists(t_file): print 'remove temporary file {0}'.format(t_file) os.remove(t_file) netcdf = NetCDFOutput(t_file, which_data='all') with raises(TypeError): # need to pass in model start time netcdf.prepare_for_model_run(num_time_steps=4) with raises(TypeError): # need to pass in model start time netcdf.prepare_for_model_run() with raises(ValueError): # needs a spills object for "all" netcdf.prepare_for_model_run(model_start_time=datetime.now(), num_time_steps=4,) with raises(ValueError): # need a cache object netcdf.write_output(0) with raises(ValueError): # raise error because file 'temp.nc' should already exist netcdf.prepare_for_model_run(model_start_time=datetime.now(), num_time_steps=4) with raises(ValueError): # which_data is 'all' but spills are not provided so raise an error netcdf.rewind() netcdf.which_data = 'all' netcdf.prepare_for_model_run(model_start_time=datetime.now(), num_time_steps=4) with raises(ValueError): netcdf.which_data = 'some random string'
def make_model(images_dir=os.path.join(base_dir, 'images')): # create the maps: print 'creating the maps' mapfile = get_datafile(os.path.join(base_dir, 'DelawareRiverMap.bna')) gnome_map = MapFromBNA(mapfile, refloat_halflife=1) # hours renderer = Renderer(mapfile, images_dir, image_size=(800, 800), projection_class=GeoProjection) print 'initializing the model' start_time = datetime(2012, 8, 20, 13, 0) # 15 minutes in seconds # Default to now, rounded to the nearest hour model = Model(time_step=900, start_time=start_time, duration=timedelta(days=1), map=gnome_map, uncertain=False) print 'adding outputters' model.outputters += renderer netcdf_file = os.path.join(base_dir, 'script_delaware_bay.nc') scripting.remove_netcdf(netcdf_file) model.outputters += NetCDFOutput(netcdf_file, which_data='all') print 'adding a RandomMover:' model.movers += RandomMover(diffusion_coef=100000) print 'adding a wind mover:' # wind_file = get_datafile(os.path.join(base_dir, 'ConstantWind.WND')) # wind = Wind(filename=wind_file) series = np.zeros((2, ), dtype=datetime_value_2d) series[0] = (start_time, (5, 270)) series[1] = (start_time + timedelta(hours=25), (5, 270)) wind = Wind(timeseries=series, units='m/s') # w_mover = WindMover(Wind(timeseries=series, units='knots')) w_mover = WindMover(wind) model.movers += w_mover print 'adding a cats shio mover:' curr_file = get_datafile(os.path.join(base_dir, 'FloodTides.cur')) tide_file = get_datafile(os.path.join(base_dir, 'FloodTidesShio.txt')) c_mover = CatsMover(curr_file, tide=Tide(tide_file)) # this is the value in the file (default) c_mover.scale_refpoint = (-75.081667, 38.7995) c_mover.scale = True c_mover.scale_value = 1 model.movers += c_mover # TODO: cannot add this till environment base class is created model.environment += c_mover.tide print 'adding a cats mover:' curr_file = get_datafile(os.path.join(base_dir, 'Offshore.cur')) c_mover = CatsMover(curr_file) # but do need to scale (based on river stage) c_mover.scale = True c_mover.scale_refpoint = (-74.7483333, 38.898333) c_mover.scale_value = .03 model.movers += c_mover # # these are from windows they don't match Mac values... # pat1Angle 315; # pat1Speed 30; pat1SpeedUnits knots; # pat1ScaleToValue 0.314426 # # pat2Angle 225; # pat2Speed 30; pat2SpeedUnits knots; # pat2ScaleToValue 0.032882 # scaleBy WindStress print 'adding a component mover:' # if only using one current pattern # comp_mover = ComponentMover(curr_file1, None, wind) # # todo: following is not working when model is saved out - fix # comp_mover = ComponentMover(curr_file1, curr_file2, # Wind(timeseries=series, units='m/s')) # comp_mover = ComponentMover(curr_file1, curr_file2, # wind=Wind(filename=wind_file)) curr_file1 = get_datafile(os.path.join(base_dir, 'NW30ktwinds.cur')) curr_file2 = get_datafile(os.path.join(base_dir, 'SW30ktwinds.cur')) comp_mover = ComponentMover(curr_file1, curr_file2, wind) comp_mover.scale_refpoint = (-75.263166, 39.1428333) comp_mover.pat1_angle = 315 comp_mover.pat1_speed = 30 comp_mover.pat1_speed_units = 1 # comp_mover.pat1ScaleToValue = .314426 comp_mover.pat1_scale_to_value = .502035 comp_mover.pat2_angle = 225 comp_mover.pat2_speed = 30 comp_mover.pat2_speed_units = 1 # comp_mover.pat2ScaleToValue = .032882 comp_mover.pat2_scale_to_value = .021869 model.movers += comp_mover print 'adding a spill' end_time = start_time + timedelta(hours=12) spill = point_line_release_spill(num_elements=1000, release_time=start_time, # end_release_time=end_time, start_position=(-75.262319, 39.142987, 0.0), ) model.spills += spill return model
def make_model(images_dir=os.path.join(base_dir, 'images')): print 'initializing the model' start_time = datetime(1985, 1, 1, 13, 31) # 1 day of data in file # 1/2 hr in seconds model = Model(start_time=start_time, duration=timedelta(days=4), time_step=3600 * 2) # mapfile = get_datafile(os.path.join(base_dir, 'ak_arctic.bna')) mapfile = get_datafile('arctic_coast3.bna') print 'adding the map' model.map = MapFromBNA(mapfile, refloat_halflife=0.0) # seconds print 'adding a spill' # for now subsurface spill stays on initial layer # - will need diffusion and rise velocity # - wind doesn't act # - start_position = (-76.126872, 37.680952, 5.0), # spill1 = point_line_release_spill(num_elements=10000, # start_position=(-163.75, # 69.75, # 0.0), # release_time=start_time) # spill1 = point_line_release_spill(num_elements=100, start_position=(196.25, 69.75, 0.0), release_time=start_time) model.spills += spill1 # model.spills += spill2 print 'adding a wind mover:' # model.movers += constant_wind_mover(0.5, 0, units='m/s') print 'adding a current mover:' fn = ['arctic_avg2_0001_gnome.nc', 'arctic_avg2_0002_gnome.nc'] gt = {'node_lon': 'lon', 'node_lat': 'lat'} # fn='arctic_avg2_0001_gnome.nc' wind_method = 'Euler' method = 'Trapezoid' print 'adding outputters' # draw_ontop can be 'uncertain' or 'forecast' # 'forecast' LEs are in black, and 'uncertain' are in red # default is 'forecast' LEs draw on top # renderer = Renderer(mapfile, images_dir, image_size=(1024, 768)) # model.outputters += renderer netcdf_file = os.path.join(base_dir, str(model.time_step / 60) + method + '.nc') scripting.remove_netcdf(netcdf_file) print 'adding movers' model.outputters += NetCDFOutput(netcdf_file, which_data='all') ice_aware_curr = IceAwareCurrent.from_netCDF(filename=fn, grid_topology=gt) ice_aware_wind = IceAwareWind.from_netCDF( filename=fn, ice_var=ice_aware_curr.ice_var, ice_conc_var=ice_aware_curr.ice_conc_var, grid=ice_aware_curr.grid, ) # i_c_mover = PyGridCurrentMover(current=ice_aware_curr) # i_c_mover = PyGridCurrentMover(current=ice_aware_curr, default_num_method='Euler') i_c_mover = PyGridCurrentMover(current=ice_aware_curr, default_num_method=method) i_w_mover = PyWindMover(wind=ice_aware_wind, default_num_method=wind_method) # ice_aware_curr.grid.node_lon = ice_aware_curr.grid.node_lon[:]-360 # ice_aware_curr.grid.build_celltree() model.movers += i_c_mover model.movers += i_w_mover print 'adding an IceAwareRandomMover:' model.movers += IceAwareRandomMover( ice_conc_var=ice_aware_curr.ice_conc_var, diffusion_coef=1000) # renderer.add_grid(ice_aware_curr.grid) # renderer.add_vec_prop(ice_aware_curr) # renderer.set_viewport(((-190.9, 60), (-72, 89))) # curr_file = get_datafile(os.path.join(base_dir, 'COOPSu_CREOFS24.nc')) # c_mover = GridCurrentMover(curr_file) # model.movers += c_mover return model