def parse_time(self): """Parse the time configuration """ self.start_date = pd.to_datetime(self.config['time']['start_date']) self.end_date = pd.to_datetime(self.config['time']['end_date']) self.time_step = self.config['time']['time_step'] self.tzinfo = pytz.timezone(self.config['time']['time_zone']) # date to use for finding wy self.start_date = self.start_date.replace(tzinfo=self.tzinfo) self.end_date = self.end_date.replace(tzinfo=self.tzinfo) # find water year hour of start and end date self.start_wyhr = int(utils.water_day(self.start_date)[0]*24) self.end_wyhr = int(utils.water_day(self.end_date)[0]*24)
def initialize_aso_updates(self, myawsm, update_fp): """ Read in the ASO update file and parse images by date Argument: myawsm: instantiated awsm class update_fp: file pointer to netCDF with all flights in it Return: update_info: dictionary of updates """ # Update the snow depths in the initialization file using ASO lidar: ds = Dataset(update_fp, 'r') ds.set_always_mask(False) # get all depths, x, y, time D_all = ds.variables['depth'][:] D_all = D_all.filled(fill_value=np.nan) D_all[np.isinf(D_all)] = np.nan D_all[D_all > 200.0] = np.nan if np.any(D_all) > 100: print('Check D_all') D_all[D_all > 100.0] = np.nan x = ds.variables['x'][:] y = ds.variables['y'][:] times = ds.variables['time'] ts = times[:] # convert time index to dates t = nc.num2date(ts, times.units, times.calendar, only_use_cftime_datetimes=False) # find wyhr of dates t_wyhr = [] for t1 in t: tmp_date = t1.replace(tzinfo=myawsm.tzinfo) # get wyhr tmpwyhr = int(utils.water_day(tmp_date)[0] * 24) t_wyhr.append(tmpwyhr) t_wyhr = np.array(t_wyhr) # make dictionary of updates update_info = OrderedDict() keys = range(1, len(t_wyhr) + 1) for idk, k in enumerate(keys): # make dictionary for each update update_info[k] = {} # set update number update_info[k]['number'] = k update_info[k]['date_time'] = t[idk].replace(tzinfo=myawsm.tzinfo) update_info[k]['wyhr'] = t_wyhr[idk] # set depth update_info[k]['depth'] = D_all[idk, :] return update_info, x, y
def get_timestep_ipw(tstep, input_list, ppt_list, myawsm): """ Pull out a time step from the forcing files (IPW) and place that time step into a dict Args: tstep: datetime of time step input_list: numpy array (1D) of integer time steps given ppt_list: numpy array(1D) of integer time steps for ppt_list myawsm: AWSM instance for current run Returns: inpt: dictionary of forcing variable images """ inpt = {} # map function from these values to the ones required by snobal map_val = {1: 'T_a', 5: 'S_n', 0: 'I_lw', 2: 'e_a', 3: 'u'} map_val_prec = {0: 'm_pp', 1: 'percent_snow', 2: 'rho_snow', 3: 'T_pp'} # get wy hour wyhr = int(utils.water_day(tstep)[0] * 24) # if we have inputs matching this water year hour if np.any(input_list == wyhr): i_in = ipw.IPW(os.path.join(myawsm.pathi, 'in.%04i' % (wyhr))) # assign soil temp inpt['T_g'] = myawsm.soil_temp * np.ones( (myawsm.topo.ny, myawsm.topo.nx)) # myawsm._logger.info('T_g: {}'.format(myawsm.soil_temp)) # inpt['T_g'] = -2.5*np.ones((myawsm.topo.ny, myawsm.topo.nx)) for f, v in map_val.items(): # if no solar data, give it zero if f == 5 and len(i_in.bands) < 6: # myawsm._logger.info('No solar data for {}'.format(tstep)) inpt[v] = np.zeros((myawsm.topo.ny, myawsm.topo.nx)) else: inpt[v] = i_in.bands[f].data # assign ppt data if there else: raise ValueError('No input time steps for {}'.format(tstep)) if np.any(ppt_list == wyhr): i_ppt = ipw.IPW(os.path.join(myawsm.path_ppt, 'ppt.4b_%04i' % (wyhr))) for f, v in map_val_prec.items(): inpt[v] = i_ppt.bands[f].data else: for f, v in map_val_prec.items(): inpt[v] = np.zeros((myawsm.topo.ny, myawsm.topo.nx)) # convert from C to K inpt['T_a'] += FREEZE inpt['T_pp'] += FREEZE inpt['T_g'] += FREEZE return inpt
def get_netcdf_out(self): """ Get init fields from output netcdf at correct time index """ i = nc.Dataset(self.init_file) # find time step indices to grab time = i.variables['time'][:] t_units = i.variables['time'].units nc_calendar = i.variables['time'].calendar nc_dates = nc.num2date( time, t_units, nc_calendar, only_use_cftime_datetimes=False, only_use_python_datetimes=True, ) if self.restart_crash: tmpwyhr = self.restart_hr else: # start date water year hour tmpwyhr = self.start_wyhr # make sure we account for time zones if hasattr(i.variables['time'], 'time_zone'): tzn = pytz.timezone(i.variables['time'].time_zone) nc_dates = [tzn.localize(ndt) for ndt in nc_dates] if self.tzinfo != tzn: nc_dates = [self.tzinfo.localize(ndt) for ndt in nc_dates] else: nc_dates = [ndt.replace(tzinfo=self.tzinfo) for ndt in nc_dates] # find water year hours nc_wyhr = np.array( [utils.water_day(ndt)[0] * 24.0 for ndt in nc_dates]) # find closest location that the water year hours equal the restart hr idt = np.argmin(np.absolute(nc_wyhr - tmpwyhr)) # returns index if np.min(np.absolute(nc_wyhr - tmpwyhr)) > 24.0: self.logger.error( 'No time in restart file that is within a day of restart time') self.logger.warning( 'Initializing PySnobal with state from water year hour {}'.format( nc_wyhr[idt])) self.init['z_s'] = i.variables['thickness'][idt, :] self.init['rho'] = i.variables['snow_density'][idt, :] self.init['T_s_0'] = i.variables['temp_surf'][idt, :] self.init['T_s'] = i.variables['temp_snowcover'][idt, :] self.init['T_s_l'] = i.variables['temp_lower'][idt, :] self.init['h2o_sat'] = i.variables['water_saturation'][idt, :] i.close()
def run_single_fist_step(self, s): """ mimic the main.c from the Snobal model. Recieves forcing data from SMRF in non-threaded application and initializes very first step. Args: s: smrf class instance """ # loop through the input # do_data_tstep needs two input records so only go # to the last record-1 self.data_tstep = self.tstep_info[0]['time_step'] self.timeSinceOut = 0.0 tmp_date = self.date_time[0].replace(tzinfo=self.tzinfo) wyhr = utils.water_day(tmp_date)[0] * 24.0 start_step = wyhr # if restart then it would be higher if this were iSnobal # start_step = 0 # if restart then it would be higher if this were iSnobal step_time = start_step * self.data_tstep # step_time = start_step * 60.0 self.output_rec['current_time'] = step_time * \ np.ones(self.output_rec['elevation'].shape) self.output_rec['time_since_out'] = self.timeSinceOut * \ np.ones(self.output_rec['elevation'].shape) # get first time step self.input1 = {} for var, v in self.variable_list.items(): # get the data desired data = getattr(s.distribute[v['module']], v['variable']) if data is None: data = np.zeros((self.ny, self.nx)) self._logger.info( 'No data from smrf to iSnobal for {} in {}'.format( v, self.date_time[0])) self.input1[self.map_val[var]] = data else: self.input1[self.map_val[var]] = data # set ground temp self.input1['T_g'] = self.soil_temp * np.ones((self.ny, self.nx)) self.input1['T_a'] += FREEZE self.input1['T_pp'] += FREEZE self.input1['T_g'] += FREEZE # for counting how many steps since the start of the run self.j = 1 self._logger.info( 'Finished initializing first time step for iPySnobal')
def initialize_aso_updates(self, myawsm, update_fp): """ Read in the ASO update file and parse images by date Argument: myawsm: instantiated awsm class update_fp: file pointer to netCDF with all flights in it Return: update_info: dictionary of updates """ # last_snow_image = ipw.IPW('/home/micahsandusky/Code/awsfTesting/newupdatetest/snow.2879') ## Update the snow depths in the initialization file using ASO lidar: fp = update_fp # read in update files ds = Dataset(fp, 'r') # get all depths, x, y, time D_all = ds.variables['depth'][:] if np.any(D_all) > 100: print('Check D_all') x = ds.variables['x'][:] y = ds.variables['y'][:] times = ds.variables['time'] ts = times[:] # convert time index to dates t = nc.num2date(ts, times.units, times.calendar) # find wyhr of dates t_wyhr = [] for t1 in t: tmp_date = t1.replace(tzinfo=myawsm.tzinfo) # get wyhr tmpwyhr = int(utils.water_day(tmp_date)[0] * 24) t_wyhr.append(tmpwyhr) t_wyhr = np.array(t_wyhr) # make dictionary of updates update_info = OrderedDict() keys = range(1, len(t_wyhr) + 1) for idk, k in enumerate(keys): # make dictionary for each update update_info[k] = {} # set update number update_info[k]['number'] = k update_info[k]['date_time'] = t[idk].replace(tzinfo=myawsm.tzinfo) update_info[k]['wyhr'] = t_wyhr[idk] # set depth update_info[k]['depth'] = D_all[idk, :] return update_info, x, y
def distribute_for_susong1999(self, data, ppt_temp, time, mask=None): """ Docs for susong1999 """ if data.sum() > 0: # distribute data and set the min/max self._distribute(data) # see if the mass threshold has been passed self.precip = utils.set_min_max(self.precip, self.min, self.max) # determine the precip phase and den snow_den, perc_snow = snow.calc_phase_and_density( ppt_temp, self.precip, nasde_model=self.nasde_model) # determine the time since last storm stormDays, stormPrecip = storms.time_since_storm( self.precip, perc_snow, time_step=self.time_step / 60 / 24, mass=self.ppt_threshold, time=self.time_to_end_storm, stormDays=self.storm_days, stormPrecip=self.storm_total) # save the model state self.percent_snow = perc_snow self.snow_density = snow_den self.storm_days = stormDays self.storm_total = stormPrecip else: self.storm_days += self.time_step / 60 / 24 # make everything else zeros self.precip = np.zeros(self.storm_days.shape) self.percent_snow = np.zeros(self.storm_days.shape) self.snow_density = np.zeros(self.storm_days.shape) # day of last storm, this will be used in albedo self.last_storm_day = utils.water_day(data.name)[0] - \ self.storm_days - 0.001 # get the time since most recent storm if mask is not None: self.last_storm_day_basin = np.max(mask * self.last_storm_day) else: self.last_storm_day_basin = np.max(self.last_storm_day)
def radiation_dates(self, date_time): """ Calculate some times based on the date for ``stoporad`` Args: date_time: date time object Returns: (tuple): tuple containing: * **wy_day** - day of water year from October 1 * **wyear** - water year * **tz_min_west** - minutes west of UTC for timezone """ # get the current day of water year wy_day, wyear = utils.water_day(date_time) # determine the minutes west of timezone tz_min_west = np.abs(date_time.utcoffset().total_seconds() / 60) return wy_day, wyear, tz_min_west
def parse_folder_structure(self): """Parse the config to get the folder structure Raises: ValueError: daily_folders can only be ran with smrf_ipysnobal """ if self.config['paths']['path_dr'] is not None: self.path_dr = os.path.abspath(self.config['paths']['path_dr']) else: print('No base path to drive given. Exiting now!') sys.exit() self.basin = self.config['paths']['basin'] self.water_year = utils.water_day(self.start_date)[1] self.project_name = self.config['paths']['project_name'] self.project_description = self.config['paths']['project_description'] self.folder_date_style = self.config['paths']['folder_date_style'] # setting to output in seperate daily folders self.daily_folders = self.config['awsm system']['daily_folders'] if self.daily_folders and not self.run_smrf_ipysnobal: raise ValueError('Cannot run daily_folders with anything other' ' than run_smrf_ipysnobal')
def run_awsm_daily(myawsm): """ This function is used to run smrf and pysnobal on an hourly scale with outputs seperated into daily folders. This will run hourly and allow for forecasts like the 18 hour HRRR forecast. """ # get the array of time steps over which to simulate d = utils.date_range(myawsm.start_date, myawsm.end_date, pd.to_timedelta(myawsm.time_step, unit='m'), myawsm.tzinfo) if myawsm.do_forecast: myawsm._logger.warning('Changing PySnobal output to hourly to allow' ' for forecast on each hour') myawsm.output_freq = 1 # set variables for adding a day or hour add_day = pd.to_timedelta(23, unit='h') add_hour = pd.to_timedelta(1, unit='h') start_day = pd.to_datetime(d[0].strftime("%Y%m%d")) end_day = pd.to_datetime(d[-1].strftime("%Y%m%d")) # if we're starting on an intermediate hour, find timesteps # up to first full day if d[0] != start_day: start_diff = start_day + add_day - d[0] else: start_diff = add_day # find timesteps to end run on last, incomplete day if d[-1] != end_day: end_diff = d[-1] - end_day else: end_diff = add_day # find total days to run model total_days = int(len(d) * myawsm.time_step / (60 * 24)) # loop through timesteps and initialize runs for each day for day in range(total_days): # set variable output names myawsm.snow_name = 'snow_00' myawsm.em_name = 'em_00' # set start and end appropriately if day == 0: myawsm.start_date = d[0] myawsm.end_date = d[0] + start_diff elif day == total_days - 1: myawsm.start_date = start_day + pd.to_timedelta(24 * day, unit='h') myawsm.end_date = myawsm.start_date + end_diff else: myawsm.start_date = start_day + pd.to_timedelta(24 * day, unit='h') myawsm.end_date = myawsm.start_date + pd.to_timedelta(23, unit='h') # recalculate start and end water year hour tmp_date = myawsm.start_date.replace(tzinfo=myawsm.tzinfo) tmp_end_date = myawsm.end_date.replace(tzinfo=myawsm.tzinfo) myawsm.start_wyhr = int(utils.water_day(tmp_date)[0] * 24) myawsm.end_wyhr = int(utils.water_day(tmp_end_date)[0] * 24) # find day for labelling the output folder nested one more level in daily_append = '{}'.format(myawsm.start_date.strftime("%Y%m%d")) myawsm.pathro = os.path.join(myawsm.pathrr, 'output' + daily_append) if not os.path.exists(myawsm.pathro): os.makedirs(myawsm.pathro) # turn off forecast for daily run (will be turned on later if it was true) myawsm.config['gridded']['hrrr_forecast_flag'] = False # ################# run_model for day ############################### myawsm.run_smrf_ipysnobal() # reset restart to be last output for next time step myawsm.ipy_init_type = 'netcdf_out' myawsm.config['ipysnobal initial conditions']['init_file'] = \ os.path.join(myawsm.pathro, myawsm.snow_name + '.nc') # do the 18hr forecast on each hour if forecast is true if myawsm.do_forecast: # turn forecast back on in smrf config myawsm.config['gridded']['hrrr_forecast_flag'] = True # now loop through the forecast hours for 18hr forecasts d_inner = utils.date_range( myawsm.start_date, myawsm.end_date, pd.to_timedelta(myawsm.time_step, unit='m'), myawsm.tzinfo) for t in d_inner: # find hour from start of day day_hour = t - pd.to_datetime(d_inner[0].strftime("%Y%m%d")) day_hour = int(day_hour / np.timedelta64(1, 'h')) # reset output names myawsm.snow_name = 'snow_{:02d}'.format(day_hour) myawsm.em_name = 'em_{:02d}'.format(day_hour) # reset start and end days myawsm.start_date = t myawsm.end_date = t + pd.to_timedelta(myawsm.n_forecast_hours, unit='h') # recalculate start and end water year hour tmp_date = myawsm.start_date.replace(tzinfo=myawsm.tzinfo) tmp_end_date = myawsm.end_date.replace(tzinfo=myawsm.tzinfo) myawsm.start_wyhr = int(utils.water_day(tmp_date)[0] * 24) myawsm.end_wyhr = int(utils.water_day(tmp_end_date)[0] * 24) # run the model for the forecast times myawsm.run_smrf_ipysnobal()
def run(self): """ mimic the main.c from the Snobal model. Runs Pysnobal while recieving forcing data from SMRF queue. """ force_variables = [ 'thermal', 'air_temp', 'vapor_pressure', 'wind_speed', 'net_solar', 'soil_temp', 'precip', 'percent_snow', 'snow_density', 'precip_temp' ] # loop through the input # do_data_tstep needs two input records so only go # to the last record-1 data_tstep = self.tstep_info[0]['time_step'] timeSinceOut = 0.0 tmp_date = self.date_time[0].replace(tzinfo=self.tzinfo) wyhr = utils.water_day(tmp_date)[0] * 24.0 start_step = wyhr # if restart then it would be higher if this were iSnobal # start_step = 0 # if restart then it would be higher if this were iSnobal step_time = start_step * data_tstep # step_time = start_step * 60.0 self.output_rec['current_time'] = step_time * np.ones( self.output_rec['elevation'].shape) self.output_rec['time_since_out'] = timeSinceOut * np.ones( self.output_rec['elevation'].shape) # map function from these values to the ones requried by snobal map_val = { 'air_temp': 'T_a', 'net_solar': 'S_n', 'thermal': 'I_lw', 'vapor_pressure': 'e_a', 'wind_speed': 'u', 'soil_temp': 'T_g', 'precip': 'm_pp', 'percent_snow': 'percent_snow', 'snow_density': 'rho_snow', 'precip_temp': 'T_pp' } # get first timestep input1 = {} for v in force_variables: if v in self.queue.keys(): data = self.queue[v].get(self.date_time[0], block=True, timeout=None) if data is None: data = np.zeros((self.ny, self.nx)) self._logger.info( 'No data from smrf to iSnobal for {} in {}'.format( v, self.date_time[0])) input1[map_val[v]] = data else: input1[map_val[v]] = data elif v != 'soil_temp': self._logger.error('Value not in keys: {}'.format(v)) # set ground temp input1['T_g'] = self.soil_temp * np.ones((self.ny, self.nx)) input1['T_a'] += FREEZE input1['T_pp'] += FREEZE input1['T_g'] += FREEZE # tell queue we assigned all the variables self.queue['isnobal'].put([self.date_time[0], True]) self._logger.info('Finished initializing first timestep for iPySnobal') j = 1 # for tstep in options['time']['date_time'][953:958]: for tstep in self.date_time[1:]: # get the output variables then pass to the function # this avoids zeroing of the energetics every timestep first_step = j input2 = {} for v in force_variables: if v in self.queue.keys(): # get variable from smrf queue data = self.queue[v].get(tstep, block=True, timeout=None) if data is None: data = np.zeros((self.ny, self.nx)) self._logger.info( 'No data from smrf to iSnobal for {} in {}'.format( v, tstep)) input2[map_val[v]] = data else: input2[map_val[v]] = data # set ground temp input2['T_g'] = self.soil_temp * np.ones((self.ny, self.nx)) # convert variables to Kelvin input2['T_a'] += FREEZE input2['T_pp'] += FREEZE input2['T_g'] += FREEZE first_step = j if self.updater is not None: #if tstep.tz_localize(None) in self.updater.update_dates: if tstep in self.updater.update_dates: # self.output_rec = \ # self.updater.do_update_pysnobal(self.output_rec, # tstep.tz_localize(None)) self.output_rec = \ self.updater.do_update_pysnobal(self.output_rec, tstep) first_step = 1 self._logger.info( 'running PySnobal for timestep: {}'.format(tstep)) rt = snobal.do_tstep_grid(input1, input2, self.output_rec, self.tstep_info, self.options['constants'], self.params, first_step=first_step, nthreads=self.nthreads) if rt != -1: self.logger.error( 'ipysnobal error on time step {}, pixel {}'.format( tstep, rt)) break self._logger.info('Finished timestep: {}'.format(tstep)) input1 = input2.copy() # output at the frequency and the last time step if ((j)*(data_tstep/3600.0) % self.options['output']['frequency'] == 0)\ or (j == len(self.options['time']['date_time']) - 1): io_mod.output_timestep(self.output_rec, tstep, self.options, self.awsm_output_vars) self.output_rec['time_since_out'] = \ np.zeros(self.output_rec['elevation'].shape) j += 1 # put the value into the output queue so clean knows it's done self.queue['isnobal'].put([tstep, True])
def run_awsm_daily_ops(config_file): """ Run each day seperately. Calls run_awsm """ # define some formats fmt_day = '%Y%m%d' fmt_cfg = '%Y-%m-%d %H:%M' add_day = pd.to_timedelta(24, unit='h') # get config instance config = get_user_config(config_file, modules=['smrf', 'awsm']) # copy the config and get total start and end # config = deepcopy(base_config) # set naming style config.raw_cfg['paths']['folder_date_style'] = 'day' config.apply_recipes() config = cast_all_variables(config, config.mcfg) # get the water year cfg_start_date = pd.to_datetime(config.cfg['time']['start_date']) tzinfo = pytz.timezone(config.cfg['time']['time_zone']) wy = utils.water_day(cfg_start_date.replace(tzinfo=tzinfo))[1] # find the model start depending on restart if config.cfg['isnobal restart']['restart_crash']: offset_wyhr = int(config.cfg['isnobal restart']['wyh_restart_output']) wy_start = pd.to_datetime('{:d}-10-01'.format(wy - 1)) model_start = wy_start + pd.to_timedelta(offset_wyhr, unit='h') else: model_start = config.cfg['time']['start_date'] model_end = config.cfg['time']['end_date'] isops = config.cfg['paths']['isops'] if isops: devops = 'ops' else: devops = 'devel' # find output location for previous output paths = config.cfg['paths'] prev_out_base = os.path.join(paths['path_dr'], paths['basin'], devops, 'wy{}'.format(wy), paths['proj'], 'runs') prev_data_base = os.path.join(paths['path_dr'], paths['basin'], devops, 'wy{}'.format(wy), paths['proj'], 'data') # find day of start and end start_day = pd.to_datetime(model_start.strftime(fmt_day)) end_day = pd.to_datetime(model_end.strftime(fmt_day)) # find total range of run ndays = int((end_day-start_day).days) + 1 date_list = [start_day + pd.to_timedelta(x, unit='D') for x in range(0, ndays)] # loop through daily runs and run awsm for idd, sd in enumerate(date_list): new_config = copy.deepcopy(config) if idd > 0: new_config.raw_cfg['isnobal restart']['restart_crash'] = False new_config.raw_cfg['grid']['thresh_normal'] = 60 new_config.raw_cfg['grid']['thresh_medium'] = 10 new_config.raw_cfg['grid']['thresh_small'] = 1 # get the end of the day ed = sd + add_day # make sure we're in the model date range if sd < model_start: sd = model_start if ed > model_end: ed = model_end # set the start and end dates new_config.raw_cfg['time']['start_date'] = sd.strftime(fmt_cfg) new_config.raw_cfg['time']['end_date'] = ed.strftime(fmt_cfg) # reset the initialization if idd > 0: # find previous output file prev_day = sd - pd.to_timedelta(1, unit='D') prev_out = os.path.join(prev_out_base, 'run{}'.format(prev_day.strftime(fmt_day)), 'snow.nc') # reset if running the model if new_config.cfg['awsm master']['model_type'] is not None: new_config.raw_cfg['files']['init_type'] = 'netcdf_out' new_config.raw_cfg['files']['init_file'] = prev_out # if we have a previous storm day file, use it prev_storm = os.path.join(prev_data_base, 'data{}'.format( prev_day.strftime(fmt_day)), 'smrfOutputs', 'storm_days.nc') if os.path.isfile(prev_storm): new_config.raw_cfg['precip']['storm_days_restart'] = prev_storm # apply recipes with new settings new_config.apply_recipes() new_config = cast_all_variables(new_config, new_config.mcfg) # run awsm for the day run_awsm(new_config)
def __init__(self, config): """ Initialize the model, read config file, start and end date, and logging Args: config: string path to the config file or inicheck UserConfig instance """ # read the config file and store awsm_mcfg = MasterConfig(modules='awsm') smrf_mcfg = MasterConfig(modules='smrf') if isinstance(config, str): if not os.path.isfile(config): raise Exception('Configuration file does not exist --> {}' .format(config)) configFile = config try: combined_mcfg = MasterConfig(modules=['smrf', 'awsm']) # Read in the original users config self.ucfg = get_user_config(configFile, mcfg=combined_mcfg) self.configFile = configFile except UnicodeDecodeError as e: print(e) raise Exception(('The configuration file is not encoded in ' 'UTF-8, please change and retry')) elif isinstance(config, UserConfig): self.ucfg = config configFile = '' else: raise Exception( 'Config passed to AWSM is neither file name nor UserConfig instance') # get the git version self.gitVersion = awsm_utils.getgitinfo() # create blank log and error log because logger is not initialized yet self.tmp_log = [] self.tmp_err = [] self.tmp_warn = [] # Check the user config file for errors and report issues if any self.tmp_log.append("Checking config file for issues...") warnings, errors = check_config(self.ucfg) print_config_report(warnings, errors) self.config = self.ucfg.cfg # Exit AWSM if config file has errors if len(errors) > 0: print("Errors in the config file. " "See configuration status report above.") # sys.exit() # ################## Decide which modules to run ##################### self.do_smrf = self.config['awsm master']['run_smrf'] #self.do_isnobal = self.config['awsm master']['run_isnobal'] self.model_type = self.config['awsm master']['model_type'] # self.do_smrf_ipysnobal = \ # self.config['awsm master']['run_smrf_ipysnobal'] # self.do_ipysnobal = self.config['awsm master']['run_ipysnobal'] self.do_forecast = False if 'gridded' in self.config and self.do_smrf: self.do_forecast = self.config['gridded']['hrrr_forecast_flag'] # WARNING: The value here is inferred in SMRF.data.loadGrid. A # change here requires a change there self.n_forecast_hours = 18 # Options for converting files self.do_make_in = self.config['awsm master']['make_in'] self.do_make_nc = self.config['awsm master']['make_nc'] # do report? # self.do_report = self.config['awsm master']['do_report'] self.snowav_config = self.config['awsm master']['snowav_config'] # options for masking isnobal self.mask_isnobal = self.config['awsm master']['mask_isnobal'] # prompt for making directories self.prompt_dirs = self.config['awsm master']['prompt_dirs'] # store smrf version if running smrf self.smrf_version = smrf.__version__ # ################ Time information ################## self.start_date = pd.to_datetime(self.config['time']['start_date']) self.end_date = pd.to_datetime(self.config['time']['end_date']) self.time_step = self.config['time']['time_step'] self.tmz = self.config['time']['time_zone'] self.tzinfo = pytz.timezone(self.config['time']['time_zone']) # date to use for finding wy tmp_date = self.start_date.replace(tzinfo=self.tzinfo) tmp_end_date = self.end_date.replace(tzinfo=self.tzinfo) # find water year hour of start and end date self.start_wyhr = int(utils.water_day(tmp_date)[0]*24) self.end_wyhr = int(utils.water_day(tmp_end_date)[0]*24) # find start of water year tmpwy = utils.water_day(tmp_date)[1] - 1 self.wy_start = pd.to_datetime('{:d}-10-01'.format(tmpwy)) # ################ Store some paths from config file ################## # path to the base drive (i.e. /data/blizzard) if self.config['paths']['path_dr'] is not None: self.path_dr = os.path.abspath(self.config['paths']['path_dr']) else: print('No base path to drive given. Exiting now!') sys.exit() # name of your basin (i.e. Tuolumne) self.basin = self.config['paths']['basin'] # water year of run self.wy = utils.water_day(tmp_date)[1] # if the run is operational or not self.isops = self.config['paths']['isops'] # name of project if not an operational run self.proj = self.config['paths']['proj'] # check for project description self.desc = self.config['paths']['desc'] # find style for folder date stamp self.folder_date_style = self.config['paths']['folder_date_style'] # setting to output in seperate daily folders self.daily_folders = self.config['awsm system']['daily_folders'] if self.daily_folders and not self.run_smrf_ipysnobal: raise ValueError('Cannot run daily_folders with anything other' ' than run_smrf_ipysnobal') if self.do_forecast: self.tmp_log.append('Forecasting set to True') # self.fp_forecastdata = self.config['gridded']['wrf_file'] # if self.fp_forecastdata is None: # self.tmp_err.append('Forecast set to true, ' # 'but no grid file given') # print("Errors in the config file. See configuration " # "status report above.") # print(self.tmp_err) # sys.exit() if self.config['system']['threading']: # Can't run threaded smrf if running forecast_data self.tmp_err.append('Cannot run SMRF threaded with' ' gridded input data') print(self.tmp_err) sys.exit() # Time step mass thresholds for iSnobal self.mass_thresh = [] self.mass_thresh.append(self.config['grid']['thresh_normal']) self.mass_thresh.append(self.config['grid']['thresh_medium']) self.mass_thresh.append(self.config['grid']['thresh_small']) # threads for running iSnobal self.ithreads = self.config['awsm system']['ithreads'] # how often to output form iSnobal self.output_freq = self.config['awsm system']['output_frequency'] # number of timesteps to run if ou don't want to run the whole thing self.run_for_nsteps = self.config['awsm system']['run_for_nsteps'] # pysnobal output variables self.pysnobal_output_vars = self.config['awsm system']['variables'] self.pysnobal_output_vars = [wrd.lower() for wrd in self.pysnobal_output_vars] # snow and emname self.snow_name = self.config['awsm system']['snow_name'] self.em_name = self.config['awsm system']['em_name'] # options for restarting iSnobal self.restart_crash = False if self.config['isnobal restart']['restart_crash']: self.restart_crash = True # self.new_init = self.config['isnobal restart']['new_init'] self.depth_thresh = self.config['isnobal restart']['depth_thresh'] self.restart_hr = \ int(self.config['isnobal restart']['wyh_restart_output']) self.restart_folder = self.config['isnobal restart']['output_folders'] # iSnobal active layer self.active_layer = self.config['grid']['active_layer'] # if we are going to run ipysnobal with smrf if self.model_type in ['ipysnobal', 'smrf_ipysnobal']: self.ipy_threads = self.ithreads self.ipy_init_type = \ self.config['files']['init_type'] self.forcing_data_type = \ self.config['ipysnobal']['forcing_data_type'] # parameters needed for restart procedure self.restart_run = False if self.config['isnobal restart']['restart_crash']: self.restart_run = True # find restart hour datetime reset_offset = pd.to_timedelta(self.restart_hr, unit='h') # set a new start date for this run self.restart_date = self.wy_start + reset_offset self.tmp_log.append('Restart date is {}'.format(self.start_date)) # read in update depth parameters self.update_depth = False if 'update depth' in self.config: self.update_depth = self.config['update depth']['update'] if self.update_depth: self.update_file = self.config['update depth']['update_file'] self.update_buffer = self.config['update depth']['buffer'] self.flight_numbers = self.config['update depth']['flight_numbers'] # if flights to use is not list, make it a list if self.flight_numbers is not None: if not isinstance(self.flight_numbers, list): self.flight_numbers = [self.flight_numbers] # list of sections releated to AWSM # These will be removed for smrf config self.sec_awsm = awsm_mcfg.cfg.keys() self.sec_smrf = smrf_mcfg.cfg.keys() # Make rigid directory structure self.mk_directories() # ################ Topo data for iSnobal ################## # get topo stats self.csys = self.config['grid']['csys'].upper() self.nbits = int(self.config['grid']['nbits']) self.soil_temp = self.config['soil_temp']['temp'] # get topo class self.topo = mytopo(self.config['topo'], self.mask_isnobal, self.model_type, self.csys, self.pathdd) # ################ Generate config backup ################## # if self.config['output']['input_backup']: # set location for backup and output backup of awsm sections config_backup_location = \ os.path.join(self.pathdd, 'awsm_config_backup.ini') generate_config(self.ucfg, config_backup_location) # create log now that directory structure is done self.createLog() # if we have a model, initialize it if self.model_type is not None: self.myinit = modelInit(self._logger, self.config, self.topo, self.start_wyhr, self.pathro, self.pathrr, self.pathinit, self.wy_start)
def distribute_for_marks2017(self, data, precip_temp, ta, time, mask=None): """ Specialized distribute function for working with the new accumulated snow density model Marks2017 requires storm total and a corrected precipitation as to avoid precip between storms. """ #self.corrected_precip # = data.mul(self.storm_correction) if data.sum() > 0.0: # Check for time in every storm for i, s in self.storms.iterrows(): storm_start = s['start'] storm_end = s['end'] if time >= storm_start and time <= storm_end: # establish storm info self.storm_id = i storm = self.storms.iloc[self.storm_id] self.storming = True break else: self.storming = False self._logger.debug("Storming? {0}".format(self.storming)) self._logger.debug("Current Storm ID = {0}".format(self.storm_id)) # distribute data and set the min/max self._distribute(data, zeros=None) self.precip = utils.set_min_max(self.precip, self.min, self.max) if time == storm_start: # Entered into a new storm period distribute the storm total self._logger.debug('{0} Entering storm #{1}'.format( data.name, self.storm_id + 1)) if precip_temp.min() < 2.0: self._logger.debug('''Distributing Total Precip for Storm #{0}'''.format( self.storm_id + 1)) self._distribute(storm[self.stations].astype(float), other_attribute='storm_total') self.storm_total = utils.set_min_max( self.storm_total, self.min, self.max) if self.storming and precip_temp.min() < 2.0: self._logger.debug('''Calculating new snow density for storm #{0}'''.format(self.storm_id + 1)) # determine the precip phase and den snow_den, perc_snow = snow.calc_phase_and_density( precip_temp, self.precip, nasde_model=self.nasde_model) else: snow_den = np.zeros(self.precip.shape) perc_snow = np.zeros(self.precip.shape) # calculate decimal days since last storm self.storm_days = storms.time_since_storm_pixel( self.precip, precip_temp, perc_snow, storming=self.storming, time_step=self.time_step / 60.0 / 24.0, stormDays=self.storm_days, mass=self.ppt_threshold) else: self.storm_days += self.time_step / 60.0 / 24.0 self.precip = np.zeros(self.storm_days.shape) perc_snow = np.zeros(self.storm_days.shape) snow_den = np.zeros(self.storm_days.shape) # save the model state self.percent_snow = perc_snow self.snow_density = snow_den # day of last storm, this will be used in albedo self.last_storm_day = utils.water_day(data.name)[0] - \ self.storm_days - 0.001 # get the time since most recent storm if mask is not None: self.last_storm_day_basin = np.max(mask * self.last_storm_day) else: self.last_storm_day_basin = np.max(self.last_storm_day)
def run(): fmt_file = fmt = '%Y%m%d' #basin = 'SJ' basin = 'TB' wy = 2017 fpdir = '/home/micahsandusky/Code/awsfTesting/newupdatetest' sj_updates = {} sj_updates[2018] = ['2018-04-23', '2018-05-28'] sj_updates[2017] = [] # date_lst = ['2018-04-23', '2018-05-28'] tuol_updates = {} tuol_updates[2013] = ['2013-04-03', '2013-04-29', '2013-05-03', '2013-05-25', '2013-06-01', '2013-06-08'] tuol_updates[2014] = ['2014-03-23', '2014-04-07', '2014-04-13', '2014-04-20', '2014-04-28', '2014-05-02', '2014-05-11', '2014-05-17', '2014-05-27', '2014-05-31', '2014-06-05'] tuol_updates[2015] = ['2015-02-18', '2015-03-06', '2015-03-25', '2015-04-03', '2015-04-09', '2015-04-15', '2015-04-27', '2015-05-01', '2015-05-28', '2015-06-08'] tuol_updates[2016] = ['2016-03-26', '2016-04-07', '2016-04-16', '2016-04-26', '2016-05-09', '2016-05-27', '2016-06-07', '2016-06-13', '2016-06-20', '2016-06-25', '2016-07-01', '2016-07-08'] tuol_updates[2017] = ['2017-01-29', '2017-03-03', '2017-04-01', '2017-05-02', '2017-06-04', '2017-07-09', '2017-07-17', '2017-08-16'] #tuol_updates[2016] = ['2016-04-16', '2016-04-26'] if basin == 'TB': date_lst = tuol_updates[wy] # put into datetime date_lst = [pd.to_datetime(dt+' 23:00') for dt in date_lst] tzinfo = pytz.timezone('UTC') tmp_date = date_lst[0] tmp_date = tmp_date.replace(tzinfo=tzinfo) # find start of water year tmpwy = utils.water_day(tmp_date)[1] wy = tmpwy start_date = pd.to_datetime('{:d}-10-01'.format(tmpwy-1)) # get the paths fp_lst = ['wy{}/{}{}_SUPERsnow_depth.asc'.format(wy, basin, dt.strftime(fmt_file)) for dt in date_lst] fp_lst = [os.path.join(fpdir,fpu) for fpu in fp_lst] if basin == 'TB': dem_fp = '/data/blizzard/tuolumne/common_data/topo/tuolx_dem_50m.ipw' gisPath = '/home/micahsandusky/Code/awsfTesting/initUpdate/' maskPath = os.path.join(gisPath, 'tuolx_mask_50m.ipw') if wy < 2017: maskPath = os.path.join(gisPath, 'tuolx_hetchy_mask_50m.ipw') elif basin == 'SJ': dem_fp = '/data/blizzard/sanjoaquin/common_data/topo/SJ_dem_50m.ipw' gisPath = '/data/blizzard/sanjoaquin/common_data/topo/' maskPath = os.path.join(gisPath, 'SJ_Millerton_mask_50m.ipw') else: raise ValueError('Wrong basin name') mask = ipw.IPW(maskPath).bands[0].data[:] output_path = os.path.join(fpdir, 'wy{}'.format(wy)) fname = 'flight_depths_{}'.format(basin) nanval = -9999.0 nanup = 1000.0 # #### Now actually do the stuff #### # date to use for finding wy fname = fname+'_{}'.format(tmpwy) # get topo stats from dem ts = get_topo_stats(dem_fp, filetype = 'ipw') x = ts['x'] # + ts['dv']*np.arange(ts['nx']) y = ts['y'] # + ts['du']*np.arange(ts['ny']) # get depth array depth_arr = read_flight(fp_lst, ts, nanval = nanval, nanup = nanup) print(depth_arr) # create netcdfs ds = output_files(output_path, fname, start_date, x, y) # write to file for idt, dt in enumerate(date_lst): data = depth_arr[idt,:]#*mask' data[mask == 0.0] = np.nan output_timestep(ds, data, dt, idt, start_date) # close file ds.close()