def create_outflow_old(dir_path, ts_start, ts_end): outflow_file_path = os.path.join(dir_path, 'OUTFLOW.DAT') init_tidal_path = os.path.join(os.getcwd(), 'outflow', 'INITTIDAL.CONF') config_path = os.path.join(os.getcwd(), 'outflow', 'config_old.json') print('create_outflow_old|outflow_file_path : ', outflow_file_path) print('create_outflow_old|init_tidal_path : ', init_tidal_path) print('create_outflow_old|config_path : ', config_path) with open(config_path) as json_file: config = json.load(json_file) adapter = MySQLAdapter(host=config['db_host'], user=config['db_user'], password=config['db_password'], db=config['db_name']) opts = { 'from': ts_start, 'to': ts_end, } tidal_timeseries = get_forecast_timeseries(adapter, TIDAL_FORECAST_ID, opts) if len(tidal_timeseries) > 0: print('tidal_timeseries::', len(tidal_timeseries), tidal_timeseries[0], tidal_timeseries[-1]) else: print('No data found for tidal timeseries: ', tidal_timeseries) sys.exit(1) adapter.close() print('Open FLO2D OUTFLOW ::', outflow_file_path) outflow_file = open(outflow_file_path, 'w') lines = [] print('Reading INIT TIDAL CONF...') with open(init_tidal_path) as initTidalConfFile: initTidalLevels = initTidalConfFile.readlines() for initTidalLevel in initTidalLevels: if len(initTidalLevel.split()): # Check if not empty line lines.append(initTidalLevel) if initTidalLevel[0] == 'N': lines.append('{0} {1:{w}} {2:{w}}\n'.format( 'S', 0, 0, w=DAT_WIDTH)) base_date_time = datetime.strptime( ts_start, '%Y-%m-%d %H:%M:%S') for step in tidal_timeseries: hours_so_far = (step[0] - base_date_time) hours_so_far = 24 * hours_so_far.days + hours_so_far.seconds / ( 60 * 60) lines.append('{0} {1:{w}} {2:{w}{b}}\n'.format( 'S', int(hours_so_far), float(step[1]), b='.2f', w=DAT_WIDTH)) outflow_file.writelines(lines) outflow_file.close() print('Finished writing OUTFLOW.DAT')
def setUpClass(cls): try: cls.root_dir = os.path.dirname(os.path.realpath(__file__)) config = json.loads(open(pjoin(cls.root_dir, '../../config/CONFIG.json')).read()) # Initialize Logger logging_config = json.loads(open(pjoin(cls.root_dir, '../../config/LOGGING_CONFIG.json')).read()) logging.config.dictConfig(logging_config) cls.logger = logging.getLogger('MySQLAdapterTest') cls.logger.addHandler(logging.StreamHandler()) cls.logger.info('setUpClass') MYSQL_HOST = "localhost" MYSQL_USER = "******" MYSQL_DB = "curw" MYSQL_PASSWORD = "" if 'MYSQL_HOST' in config: MYSQL_HOST = config['MYSQL_HOST'] if 'MYSQL_USER' in config: MYSQL_USER = config['MYSQL_USER'] if 'MYSQL_DB' in config: MYSQL_DB = config['MYSQL_DB'] if 'MYSQL_PASSWORD' in config: MYSQL_PASSWORD = config['MYSQL_PASSWORD'] cls.adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) cls.eventIds = [] cls.run_start_date = datetime.datetime(2018, 1, 2, 12, 0, 0) cls.run_end_date = datetime.datetime(2018, 1, 2, 16, 0, 0) except Exception as e: traceback.print_exc()
def get_curw_adapter(mysql_config=None, mysql_config_path=None): if mysql_config_path is None: mysql_config_path = res_mgr.get_resource_path( 'config/mysql_config.json') with open(mysql_config_path) as data_file: config = json.load(data_file) if mysql_config is not None and isinstance(mysql_config, dict): config.update(mysql_config) return MySQLAdapter(**config)
def upload_waterlevels_curw(dir_path, ts_start_date, ts_start_time, run_date, run_time): print( 'upload_waterlevels_curw|[ts_start_date, ts_start_time, run_date, run_time] : ', [ts_start_date, ts_start_time, run_date, run_time]) SERIES_LENGTH = 0 MISSING_VALUE = -999 try: config_path = os.path.join(os.getcwd(), 'extract', 'config.json') # print('config_path : ', config_path) utc_offset = '' with open(config_path) as json_file: config_data = json.load(json_file) output_dir = dir_path HYCHAN_OUT_FILE = config_data['hychan_out_file'] TIMDEP_FILE = config_data['timdep_out_file'] FLO2D_MODEL = config_data['flo2d_model'] RUN_NAME = config_data['run_name'] hychan_out_file_path = os.path.join(dir_path, HYCHAN_OUT_FILE) timdep_file_path = os.path.join(dir_path, TIMDEP_FILE) # print('hychan_out_file_path : ', hychan_out_file_path) # print('timdep_file_path : ', timdep_file_path) forceInsert = True MYSQL_HOST = config_data['db_host'] MYSQL_USER = config_data['db_user'] MYSQL_DB = config_data['db_name'] MYSQL_PASSWORD = config_data['db_password'] # if 'UTC_OFFSET' in config_data and len( # config_data['UTC_OFFSET']): # Use FLO2D Config file data, if available # UTC_OFFSET = config_data['UTC_OFFSET'] # if utc_offset: # UTC_OFFSET = config_data utcOffset = getUTCOffset('', default=True) adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) flo2d_source = adapter.get_source(name=FLO2D_MODEL) try: flo2d_source = json.loads(flo2d_source.get('parameters', "{}")) CHANNEL_CELL_MAP = {} if 'CHANNEL_CELL_MAP' in flo2d_source: CHANNEL_CELL_MAP = flo2d_source['CHANNEL_CELL_MAP'] FLOOD_PLAIN_CELL_MAP = {} if 'FLOOD_PLAIN_CELL_MAP' in flo2d_source: FLOOD_PLAIN_CELL_MAP = flo2d_source['FLOOD_PLAIN_CELL_MAP'] ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys() FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys() # Calculate the size of time series bufsize = 65536 with open(hychan_out_file_path) as infile: isWaterLevelLines = False isCounting = False countSeriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series while True: lines = infile.readlines(bufsize) if not lines or SERIES_LENGTH: break for line in lines: if line.startswith( 'CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5): isWaterLevelLines = True elif isWaterLevelLines: cols = line.split() if len(cols) > 0 and cols[0].replace( '.', '', 1).isdigit(): countSeriesSize += 1 isCounting = True elif isWaterLevelLines and isCounting: SERIES_LENGTH = countSeriesSize break # print('Series Length is :', SERIES_LENGTH) bufsize = 65536 ################################################################# # Extract Channel Water Level elevations from HYCHAN.OUT file # ################################################################# with open(hychan_out_file_path) as infile: isWaterLevelLines = False isSeriesComplete = False waterLevelLines = [] seriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series while True: lines = infile.readlines(bufsize) if not lines: break for line in lines: if line.startswith( 'CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5): seriesSize = 0 elementNo = line.split()[5] if elementNo in ELEMENT_NUMBERS: isWaterLevelLines = True waterLevelLines.append(line) else: isWaterLevelLines = False elif isWaterLevelLines: cols = line.split() if len(cols) > 0 and isfloat(cols[0]): seriesSize += 1 waterLevelLines.append(line) if seriesSize == SERIES_LENGTH: isSeriesComplete = True if isSeriesComplete: baseTime = datetime.strptime( '%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S') timeseries = [] elementNo = waterLevelLines[0].split()[5] # print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo]) for ts in waterLevelLines[1:]: v = ts.split() if len(v) < 1: continue # Get flood level (Elevation) value = v[1] # Get flood depth (Depth) # value = v[2] if not isfloat(value): value = MISSING_VALUE continue # If value is not present, skip if value == 'NaN': continue # If value is NaN, skip timeStep = float(v[0]) currentStepTime = baseTime + timedelta( hours=timeStep) dateAndTime = currentStepTime.strftime( "%Y-%m-%d %H:%M:%S") timeseries.append([dateAndTime, value]) # Save Forecast values into Database opts = { 'forceInsert': forceInsert, 'station': CHANNEL_CELL_MAP[elementNo], 'run_name': RUN_NAME } # print('>>>>>', opts) if utcOffset != timedelta(): opts['utcOffset'] = utcOffset # folder_path_ts = os.path.join(dir_path, 'time_series') # if not os.path.exists(folder_path_ts): # try: # os.makedirs(folder_path_ts) # except OSError as e: # print(str(e)) # file_path = os.path.join(folder_path_ts, 'time_series_' + elementNo + '.txt') # with open(file_path, 'w') as f: # for item in timeseries: # f.write("%s\n" % item) save_forecast_timeseries( adapter, timeseries, run_date, run_time, opts) isWaterLevelLines = False isSeriesComplete = False waterLevelLines = [] # -- END for loop # -- END while loop ################################################################# # Extract Flood Plain water elevations from BASE.OUT file # ################################################################# with open(timdep_file_path) as infile: waterLevelLines = [] waterLevelSeriesDict = dict.fromkeys( FLOOD_ELEMENT_NUMBERS, []) while True: lines = infile.readlines(bufsize) if not lines: break for line in lines: if len(line.split()) == 1: if len(waterLevelLines) > 0: waterLevels = get_water_level_of_channels( waterLevelLines, FLOOD_ELEMENT_NUMBERS) # Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060 # print(waterLevelLines[0].split()) ModelTime = float( waterLevelLines[0].split()[0]) baseTime = datetime.strptime( '%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S') currentStepTime = baseTime + timedelta( hours=ModelTime) dateAndTime = currentStepTime.strftime( "%Y-%m-%d %H:%M:%S") for elementNo in FLOOD_ELEMENT_NUMBERS: tmpTS = waterLevelSeriesDict[ elementNo][:] if elementNo in waterLevels: tmpTS.append([ dateAndTime, waterLevels[elementNo] ]) else: tmpTS.append( [dateAndTime, MISSING_VALUE]) waterLevelSeriesDict[elementNo] = tmpTS isWaterLevelLines = False # for l in waterLevelLines : # #print(l) waterLevelLines = [] waterLevelLines.append(line) for elementNo in FLOOD_ELEMENT_NUMBERS: opts = { 'forceInsert': forceInsert, 'station': FLOOD_PLAIN_CELL_MAP[elementNo], 'run_name': RUN_NAME, 'source': FLO2D_MODEL } if utcOffset != timedelta(): opts['utcOffset'] = utcOffset # folder_path_ts = os.path.join(dir_path, 'time_series') # if not os.path.exists(folder_path_ts): # try: # os.makedirs(folder_path_ts) # except OSError as e: # print(str(e)) # file_path = os.path.join(folder_path_ts, 'time_series_'+elementNo+'.txt') # with open(file_path, 'w') as f: # for item in waterLevelSeriesDict[elementNo]: # f.write("%s\n" % item) save_forecast_timeseries( adapter, waterLevelSeriesDict[elementNo], run_date, run_time, opts) # print('Extracted Cell No', elementNo, FLOOD_PLAIN_CELL_MAP[elementNo]) except Exception as ex: print('source data loading exception:', str(ex)) except Exception as e: print('config reading exception:', str(e))
fcst_db_configs = config['fcst_db_configs'] else: log.error('fcst_db_configs data not found') klb_observed_stations = config['klb_obs_stations'] kub_observed_stations = config['kub_obs_stations'] kub_basin_extent = config['KELANI_UPPER_BASIN_EXTENT'] klb_basin_extent = config['KELANI_LOWER_BASIN_EXTENT'] kelani_lower_basin_shp = get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') kelani_upper_basin_shp = get_resource_path('extraction/shp/kub-wgs84/kub-wgs84.shp') hourly_csv_file_dir = os.path.join(wrf_data_dir, run_date, run_time) create_dir_if_not_exists(hourly_csv_file_dir) raincsv_file_path = os.path.join(hourly_csv_file_dir, 'DailyRain.csv') if not os.path.isfile(raincsv_file_path): curw_adapter = MySQLAdapter(host=curw_db_config['host'], user=curw_db_config['user'], password=curw_db_config['password'], db=curw_db_config['db']) # fcst_adapter = MySQLAdapter(host=fcst_db_configs['host'], user=fcst_db_configs['user'], # password=fcst_db_configs['password'], db=fcst_db_configs['db']) obs_start = ts_start_datetime.strftime('%Y-%m-%d %H:%M:%S') obs_end = run_datetime.strftime('%Y-%m-%d %H:%M:%S') print('[obs_start, obs_end] : ', [obs_start, obs_end]) fcst_start = obs_end fcst_end = ts_end_datetime.strftime('%Y-%m-%d %H:%M:%S') print('[fcst_start, fcst_end] : ', [fcst_start, fcst_end]) forecast_duration = int((datetime.strptime(fcst_end, '%Y-%m-%d %H:%M:%S') - datetime.strptime( fcst_start, '%Y-%m-%d %H:%M:%S')).total_seconds() / (60 * 60)) obs_kub_mean_df = get_observed_kub_mean(curw_adapter, kub_observed_stations, obs_start, obs_end)
print('WARNING: Force Insert enabled') WU_DATA = json.loads(open(WU_CONFIG).read()) stations = WU_DATA['stations'] metaData = { 'station': 'Hanwella', 'variable': 'Precipitation', 'unit': 'mm', 'type': 'Observed', 'source': 'WeatherStation', 'name': 'WUnderground', } adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) for station in stations: print('station:', station) WUndergroundMeta, *timeseries = getTimeseries( BASE_URL, station['stationId'], now) # List Destructuring DateUTCIndex = WUndergroundMeta.index('DateUTC') if (len(timeseries) < 1): print('INFO: Timeseries doesn\'t have any data on :', now.strftime("%Y-%m-%d"), timeseries) continue print('Start Date :', timeseries[0][0]) print('End Date :', timeseries[-1][0]) startDateTime = datetime.datetime.strptime(timeseries[0][0],
month=data_array[1], day=data_array[2], hour=data_array[3]) return timestamp.strftime(Common_DateTime_Format) def prepare_timeseries(df): timeseries = [] for index, row in df.iterrows(): timeseries.append([row['timestamp'], row['value']]) return timeseries # Create database connections. mysql_adapter = MySQLAdapter(host=DB_CONFIG['MYSQL_HOST'], user=DB_CONFIG['MYSQL_USER'], password=DB_CONFIG['MYSQL_PASSWORD'], db=DB_CONFIG['MYSQL_DB']) print( '\n##########################################################################################' ) print("Extracting Observed data from Icharm on %s" % datetime.now().strftime(Common_DateTime_Format)) # Iterate over Icharm csv file download links and push date to DB for station_id, url in Links.items(): # Load the .csv to a Dataframe df = pd.read_csv(url, header=None, names=['year', 'month', 'day', 'hour', 'value']) df['timestamp'] = df[['year', 'month', 'day',
obs_end = '2019-03-20 00:00:00' forecast_end = '2019-03-22 23:00:00' points = np.genfromtxt(kelani_basin_points_file, delimiter=',') kel_lon_min = np.min(points, 0)[1] kel_lat_min = np.min(points, 0)[2] kel_lon_max = np.max(points, 0)[1] kel_lat_max = np.max(points, 0)[2] print('[kel_lon_min, kel_lat_min, kel_lon_max, kel_lat_max] : ', [kel_lon_min, kel_lat_min, kel_lon_max, kel_lat_max]) forecast_adapter = MySQLAdapter( host=forecast_db_config['host'], user=forecast_db_config['user'], password=forecast_db_config['password'], db=forecast_db_config['db']) # #min_lat, min_lon, max_lat, max_lon forecast_stations, station_points = get_forecast_stations_from_net_cdf( reference_net_cdf, kel_lat_min, kel_lon_min, kel_lat_max, kel_lon_max) print('forecast_stations length : ', len(forecast_stations)) forecast_precipitations = get_forecast_precipitation( forecast_stations, obs_end, forecast_end, forecast_adapter) # print('forecast_precipitations : ', forecast_precipitations) forecast_adapter.close() observed_adapter = MySQLAdapter( host=observed_db_config['host'],
CONFIG['WATER_DISCHARGE_DIR']) > 0: WATER_DISCHARGE_DIR = CONFIG['WATER_DISCHARGE_DIR'] if 'OUTPUT_DIR' in CONFIG: OUTPUT_DIR = CONFIG['OUTPUT_DIR'] if 'MYSQL_HOST' in CONFIG: MYSQL_HOST = CONFIG['MYSQL_HOST'] if 'MYSQL_USER' in CONFIG: MYSQL_USER = CONFIG['MYSQL_USER'] if 'MYSQL_DB' in CONFIG: MYSQL_DB = CONFIG['MYSQL_DB'] if 'MYSQL_PASSWORD' in CONFIG: MYSQL_PASSWORD = CONFIG['MYSQL_PASSWORD'] adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) # TODO: Pass source name as a paramter to script flo2d_source = adapter.get_source(name='FLO2D') try: flo2d_source = json.loads(flo2d_source.get('parameters', "{}")) except Exception as e: print(e) traceback.print_exc() CHANNEL_CELL_MAP = {} if 'CHANNEL_CELL_MAP' in flo2d_source: CHANNEL_CELL_MAP = flo2d_source['CHANNEL_CELL_MAP'] FLOOD_PLAIN_CELL_MAP = {} if 'FLOOD_PLAIN_CELL_MAP' in flo2d_source: FLOOD_PLAIN_CELL_MAP = flo2d_source['FLOOD_PLAIN_CELL_MAP'] """
def create_inflow(dir_path, run_date, run_time): try: # run_date = datetime.now().strftime("%Y-%m-%d") # run_time = datetime.now().strftime("%H:00:00") # FLO-2D parameters IHOURDAILY = 0 # 0-hourly interval, 1-daily interval IDEPLT = 0 # Set to 0 on running with Text mode. Otherwise cell number e.g. 8672 IFC = 'C' # foodplain 'F' or a channel 'C' INOUTFC = 0 # 0-inflow, 1-outflow KHIN = 8655 # inflow nodes HYDCHAR = 'H' # Denote line of inflow hydrograph time and discharge pairs # try: # opts, args = getopt.getopt(sys.argv[1:], "hd:t:T:f:b:", [ # "help", "date=", "time=", "forward=", "backward=", "wrf-rf=", "wrf-kub=", "tag=" # ]) # except getopt.GetoptError: # sys.exit(2) # for opt, arg in opts: # if opt in ("-h", "--help"): # sys.exit() # elif opt in ("-d", "--date"): # run_date = arg # 2018-05-24 # elif opt in ("-t", "--time"): # run_time = arg # 16:00:00 # elif opt in ("-f","--forward"): # forward = arg # elif opt in ("-b","--backward"): # backward = arg # elif opt in ("--wrf-rf"): # RF_DIR_PATH = arg # elif opt in ("--wrf-kub"): # KUB_DIR_PATH = arg # elif opt in ("-T", "--tag"): # tag = arg #run_date = '2019-04-29' print("WrfTrigger run_date : ", run_date) print("WrfTrigger run_time : ", run_time) # backward = 2 # forward = 3 startDateTime = datetime.strptime('%s %s' % (run_date, run_time), '%Y-%m-%d %H:%M:%S') print("startDateTime : ", startDateTime) config_path = os.path.join(os.getcwd(), 'inflowdat', 'config.json') print('config_path : ', config_path) with open(config_path) as json_file: config_data = json.load(json_file) output_dir = dir_path inflow_file = config_data["inflow_file"] # CONTROL_INTERVAL = config_data["CONTROL_INTERVAL"] # CSV_NUM_METADATA_LINES = config_data["CSV_NUM_METADATA_LINES"] DAT_WIDTH = config_data["DAT_WIDTH"] OBSERVED_WL_IDS = config_data["OBSERVED_WL_IDS"] MYSQL_HOST = config_data['db_host'] MYSQL_USER = config_data['db_user'] MYSQL_DB = config_data['db_name'] MYSQL_PASSWORD = config_data['db_password'] adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) try: #hourly_inflow_file_dir = os.path.join(output_dir, run_date, run_time) hourly_inflow_file = os.path.join(output_dir, inflow_file) #create_dir_if_not_exists(hourly_inflow_file_dir) print("hourly_outflow_file : ", hourly_inflow_file) if not os.path.isfile(hourly_inflow_file): discharge_df = get_discharge_data(adapter, startDateTime) print('discharge_df', discharge_df) initial_water_levels = update_initial_water_levels(OBSERVED_WL_IDS, adapter, startDateTime.strftime("%Y-%m-%d %H:%M:%S")) f = open(hourly_inflow_file, 'w') line1 = '{0} {1:{w}{b}}\n'.format(IHOURDAILY, IDEPLT, b='d', w=DAT_WIDTH) line2 = '{0} {1:{w}{b}} {2:{w}{b}}\n'.format(IFC, INOUTFC, KHIN, b='d', w=DAT_WIDTH) line3 = '{0} {1:{w}{b}} {2:{w}{b}}\n'.format(HYDCHAR, 0.0, 0.0, b='.1f', w=DAT_WIDTH) f.writelines([line1, line2, line3]) lines = []; i = 1.0 for time, row in discharge_df.iterrows(): lines.append( '{0} {1:{w}{b}} {2:{w}{b}}\n'.format(HYDCHAR, i, float(row["value"]), b='.1f', w=DAT_WIDTH)) i += 1.0 lines.extend(initial_water_levels) f.writelines(lines) f.close() adapter.close() except Exception as ex: adapter.close() print("Download required files|Exception: ", str(ex)) except Exception as e: print("Exception occurred: ", str(e))
def create_hybrid_raincell(dir_path, run_date, run_time, forward=3, backward=2): try: #60 120 2019-05-01 00:00:00 2019-05-05 23:00:00 #60 120 2019-05-01 23:00:00 2019-05-06 23:00:00 #60 120 2019-05-01 23:00:00 2019-05-06 23:00:00 #60 120 2019-04-30 23:00:00 2019-05-05 23:00:00 # run_date = datetime.now().strftime("%Y-%m-%d") # run_time = datetime.now().strftime("%H:00:00") # run_date = '2019-05-02' # run_time = '23:00:00' tag = '' # try: # opts, args = getopt.getopt(sys.argv[1:], "hd:t:T:f:b:", [ # "help", "date=", "time=", "forward=", "backward=", "wrf-rf=", "wrf-kub=", "tag=" # ]) # except getopt.GetoptError: # usage() # sys.exit(2) # for opt, arg in opts: # if opt in ("-h", "--help"): # usage() # sys.exit() # elif opt in ("-d", "--date"): # run_date = arg # 2018-05-24 # elif opt in ("-t", "--time"): # run_time = arg # 16:00:00 # elif opt in ("-f","--forward"): # forward = arg # elif opt in ("-b","--backward"): # backward = arg # elif opt in ("--wrf-rf"): # RF_DIR_PATH = arg # elif opt in ("--wrf-kub"): # KUB_DIR_PATH = arg # elif opt in ("-T", "--tag"): # tag = arg #run_date = '2019-04-29' print("WrfTrigger run_date : ", run_date) print("WrfTrigger run_time : ", run_time) # backward = 2 # forward = 3 start_ts_lk = datetime.strptime('%s %s' % (run_date, run_time), '%Y-%m-%d %H:%M:%S') start_ts_lk = start_ts_lk.strftime( '%Y-%m-%d_%H:00') # '2018-05-24_08:00' print("WrfTrigger start_ts_lk : ", start_ts_lk) duration_days = (int(backward), int(forward)) print("WrfTrigger duration_days : ", duration_days) config_path = os.path.join(os.getcwd(), 'raincelldat', 'config.json') print('config_path : ', config_path) with open(config_path) as json_file: config_data = json.load(json_file) key_file = os.path.join(os.getcwd(), 'raincelldat', 'uwcc-admin.json') bucket_name = config_data["BUCKET_NAME"] initial_path_prefix = config_data["INITIAL_PATH_PREFIX"] net_cdf_file_format = config_data["NET_CDF_FILE"] wrf_data_dir = dir_path print("wrf_data_dir : ", wrf_data_dir) net_cdf_date = datetime.strptime(run_date, '%Y-%m-%d') - timedelta(hours=24) net_cdf_date = net_cdf_date.strftime("%Y-%m-%d") download_location = os.path.join(os.getcwd(), run_date) print("download_location : ", download_location) print("net_cdf_date : ", net_cdf_date) MYSQL_HOST = config_data['db_host'] MYSQL_USER = config_data['db_user'] MYSQL_DB = config_data['db_name'] MYSQL_PASSWORD = config_data['db_password'] klb_observed_stations = config_data['klb_obs_stations'] klb_points = get_resource_path( 'extraction/local/kelani_basin_points_250m.txt') adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) name_list = net_cdf_file_format.split("-") net_cdf_file_name = name_list[ 0] + "_" + net_cdf_date + "_" + name_list[1] try: net_cdf_file_path = os.path.join(download_location, net_cdf_file_name) print("net_cdf_file_path : ", net_cdf_file_path) if not os.path.isfile(net_cdf_file_path): download_netcdf(initial_path_prefix, download_location, net_cdf_file_name, key_file, bucket_name) if os.path.isfile(net_cdf_file_path): raincell_file_path = os.path.join(wrf_data_dir, run_date, run_time, 'RAINCELL.DAT') if not os.path.isfile(raincell_file_path): create_raincell_file( run_time, adapter, net_cdf_file_path, start_ts_lk, os.path.join(wrf_data_dir, run_date), klb_observed_stations, klb_points, duration_days) adapter.close() except Exception as ex: adapter.close() print("Download required files|Exception: ", str(ex)) except Exception as e: print("Exception occurred: ", str(e))
def create_hybrid_mike_input(dir_path, run_date, run_time, forward, backward): try: res_mins = '60' model_prefix = 'wrf' forecast_source = 'wrf0' run_name = 'Cloud-1' forecast_adapter = None observed_adapter = None kelani_basin_mike_points_file = get_resource_path( 'extraction/local/metro_col_sub_catch_centroids.csv') kelani_basin_points_file = get_resource_path( 'extraction/local/kelani_basin_points_250m.txt') kelani_lower_basin_shp_file = get_resource_path( 'extraction/shp/klb-wgs84/klb-wgs84.shp') reference_net_cdf = get_resource_path( 'extraction/netcdf/wrf_wrfout_d03_2019-03-31_18_00_00_rf') #config_path = os.path.join(os.getcwd(), 'raincelldat', 'config.json') config_path = os.path.join(os.getcwd(), 'config.json') with open(config_path) as json_file: config = json.load(json_file) if 'forecast_db_config' in config: forecast_db_config = config['forecast_db_config'] if 'observed_db_config' in config: observed_db_config = config['observed_db_config'] if 'klb_obs_stations' in config: obs_stations = copy.deepcopy(config['klb_obs_stations']) res_mins = int(res_mins) print('[run_date, run_time] : ', [run_date, run_time]) start_ts_lk = datetime.strptime('%s %s' % (run_date, run_time), '%Y-%m-%d %H:%M:%S') start_ts_lk = start_ts_lk.strftime( '%Y-%m-%d_%H:00') # '2018-05-24_08:00' duration_days = (int(backward), int(forward)) obs_start = datetime.strptime( start_ts_lk, '%Y-%m-%d_%H:%M') - timedelta(days=duration_days[0]) obs_end = datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') forecast_end = datetime.strptime( start_ts_lk, '%Y-%m-%d_%H:%M') + timedelta(days=duration_days[1]) print([obs_start, obs_end, forecast_end]) fcst_duration_start = obs_end.strftime('%Y-%m-%d %H:%M:%S') fcst_duration_end = ( datetime.strptime(fcst_duration_start, '%Y-%m-%d %H:%M:%S') + timedelta(days=3)).strftime('%Y-%m-%d 00:00:00') obs_duration_start = ( datetime.strptime(fcst_duration_start, '%Y-%m-%d %H:%M:%S') - timedelta(days=2)).strftime('%Y-%m-%d 00:00:00') print('obs_duration_start : ', obs_duration_start) print('fcst_duration_start : ', fcst_duration_start) print('fcst_duration_end : ', fcst_duration_end) observed_duration = int( (datetime.strptime(fcst_duration_start, '%Y-%m-%d %H:%M:%S') - datetime.strptime(obs_duration_start, '%Y-%m-%d %H:%M:%S')).total_seconds() / (60 * res_mins)) forecast_duration = int( (datetime.strptime(fcst_duration_end, '%Y-%m-%d %H:%M:%S') - datetime.strptime(fcst_duration_start, '%Y-%m-%d %H:%M:%S')).total_seconds() / (60 * res_mins)) total_duration = int( (datetime.strptime(fcst_duration_end, '%Y-%m-%d %H:%M:%S') - datetime.strptime(obs_duration_start, '%Y-%m-%d %H:%M:%S')).total_seconds() / (60 * res_mins)) print('observed_duration : ', observed_duration) print('forecast_duration : ', forecast_duration) print('total_duration : ', total_duration) mike_input_file_path = os.path.join(dir_path, 'mike_input.txt') print('mike_input_file_path : ', mike_input_file_path) if not os.path.isfile(mike_input_file_path): points = np.genfromtxt(kelani_basin_points_file, delimiter=',') kel_lon_min = np.min(points, 0)[1] kel_lat_min = np.min(points, 0)[2] kel_lon_max = np.max(points, 0)[1] kel_lat_max = np.max(points, 0)[2] mike_points = np.genfromtxt(kelani_basin_mike_points_file, delimiter=',', names=True, dtype=None) print('mike_points : ', mike_points) print('mike_points : ', mike_points[0][0].decode()) print('mike_points : ', mike_points[1][0].decode()) print('mike_points : ', mike_points[2][0].decode()) def _get_points_names(mike_points): mike_point_names = [] for p in mike_points: mike_point_names.append(p[0].decode()) return mike_point_names #mike_point_names = get_centroid_names(kelani_basin_mike_points_file) mike_point_names = _get_points_names(mike_points) print('mike_point_names : ', mike_point_names) print('mike_point_names[0] : ', mike_point_names[0]) print('mike_point_names[1] : ', mike_point_names[1]) print('mike_point_names[2] : ', mike_point_names[2]) print('mike_point_names[-1] : ', mike_point_names[-1]) print( '[kel_lon_min, kel_lat_min, kel_lon_max, kel_lat_max] : ', [kel_lon_min, kel_lat_min, kel_lon_max, kel_lat_max]) #""" # #min_lat, min_lon, max_lat, max_lon forecast_stations, station_points = get_forecast_stations_from_net_cdf( model_prefix, reference_net_cdf, kel_lat_min, kel_lon_min, kel_lat_max, kel_lon_max) print('forecast_stations length : ', len(forecast_stations)) file_header = ','.join(mike_point_names) print('file_header : ', file_header) observed_adapter = MySQLAdapter( host=observed_db_config['host'], user=observed_db_config['user'], password=observed_db_config['password'], db=observed_db_config['db']) # print('obs_stations : ', obs_stations) observed_precipitations = get_observed_precip( obs_stations, obs_duration_start, fcst_duration_start, observed_duration, observed_adapter, forecast_source='wrf0') observed_adapter.close() observed_adapter = None validated_obs_station = {} # print('obs_stations.keys() : ', obs_stations.keys()) # print('observed_precipitations.keys() : ', observed_precipitations.keys()) for station_name in obs_stations.keys(): if station_name in observed_precipitations.keys(): validated_obs_station[station_name] = obs_stations[ station_name] else: print('invalid station_name : ', station_name) # if bool(observed_precipitations): if len(validated_obs_station) >= 1: thess_poly = get_voronoi_polygons( validated_obs_station, kelani_lower_basin_shp_file, add_total_area=False) forecast_adapter = MySQLAdapter( host=forecast_db_config['host'], user=forecast_db_config['user'], password=forecast_db_config['password'], db=forecast_db_config['db']) forecast_precipitations = get_forecast_precipitation( forecast_source, run_name, forecast_stations, forecast_adapter, obs_end.strftime('%Y-%m-%d %H:%M:%S'), forward_days=3) forecast_adapter.close() forecast_adapter = None if bool(forecast_precipitations): fcst_thess_poly = get_voronoi_polygons( station_points, kelani_lower_basin_shp_file, add_total_area=False) fcst_point_thess_idx = [] for point in mike_points: fcst_point_thess_idx.append( is_inside_geo_df(fcst_thess_poly, lon=point[1], lat=point[2])) pass # print('fcst_point_thess_idx : ', fcst_point_thess_idx) # create_dir_if_not_exists(dir_path) point_thess_idx = [] for point in mike_points: point_thess_idx.append( is_inside_geo_df(thess_poly, lon=point[1], lat=point[2])) pass print('len(mike_points)', len(mike_points)) print('len(point_thess_idx)', len(point_thess_idx)) print('len(fcst_point_thess_idx)', len(fcst_point_thess_idx)) print('point_thess_idx : ', point_thess_idx) print('fcst_point_thess_idx : ', fcst_point_thess_idx) print('mike_point_names : ', mike_point_names) with open(mike_input_file_path, mode='w') as output_file: output_writer = csv.writer(output_file, delimiter=',', dialect='excel') header = ['Times'] header.extend(mike_point_names) output_writer.writerow(header) print( 'range 1 : ', int(24 * 60 * duration_days[0] / res_mins) + 1) print( 'range 2 : ', int(24 * 60 * duration_days[1] / res_mins) - 1) obs_duration_end = None for t in range(observed_duration): date_time = datetime.strptime( obs_duration_start, '%Y-%m-%d %H:%M:%S') + timedelta(hours=t) obs_duration_end = date_time.strftime( '%Y-%m-%d %H:%M:%S') print(date_time.strftime('%Y-%m-%d %H:%M:%S')) obs_rf_list = [] for i, point in enumerate(mike_points): rf = float( observed_precipitations[ point_thess_idx[i]].values[t] ) if point_thess_idx[i] is not None else 0 obs_rf_list.append('%.6f' % rf) row = [date_time.strftime('%Y-%m-%d %H:%M:%S')] row.extend(obs_rf_list) output_writer.writerow(row) print( 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' ) next_time_step = datetime.strptime( obs_duration_end, '%Y-%m-%d %H:%M:%S') + timedelta(hours=1) for t in range(forecast_duration): date_time = next_time_step + timedelta(hours=t) print(date_time.strftime('%Y-%m-%d %H:%M:%S')) fcst_rf_list = [] for i, point in enumerate(mike_points): rf = float(forecast_precipitations[ fcst_point_thess_idx[i]].values[t] ) if fcst_point_thess_idx[ i] is not None else 0 fcst_rf_list.append('%.6f' % rf) row = [date_time.strftime('%Y-%m-%d %H:%M:%S')] row.extend(fcst_rf_list) output_writer.writerow(row) else: print('----------------------------------------------') print('No forecast data.') print('----------------------------------------------') else: print('----------------------------------------------') print('No observed data.') print('Available station count: ', len(validated_obs_station)) print('Proceed with forecast data.') print('----------------------------------------------') # """ except Exception as e: print('Raincell generation error|Exception:', str(e)) traceback.print_exc() try: if forecast_adapter is not None: forecast_adapter.close() if observed_adapter is not None: observed_adapter.close() except Exception as ex: print(str(ex))
def create_outflow(dir_path, run_date, run_time, forward = 3, backward = 2): try: # run_date = datetime.now().strftime("%Y-%m-%d") # run_time = datetime.now().strftime("%H:00:00") # tag = '' # try: # opts, args = getopt.getopt(sys.argv[1:], "hd:t:T:f:b:", [ # "help", "date=", "time=", "forward=", "backward=", "wrf-rf=", "wrf-kub=", "tag=" # ]) # except getopt.GetoptError: # sys.exit(2) # for opt, arg in opts: # if opt in ("-h", "--help"): # sys.exit() # elif opt in ("-d", "--date"): # run_date = arg # 2018-05-24 # elif opt in ("-t", "--time"): # run_time = arg # 16:00:00 # elif opt in ("-f","--forward"): # forward = arg # elif opt in ("-b","--backward"): # backward = arg # elif opt in ("--wrf-rf"): # RF_DIR_PATH = arg # elif opt in ("--wrf-kub"): # KUB_DIR_PATH = arg # elif opt in ("-T", "--tag"): # tag = arg #run_date = '2019-04-29' print("WrfTrigger run_date : ", run_date) print("WrfTrigger run_time : ", run_time) # backward = 2 # forward = 3 startDateTime = datetime.strptime('%s %s' % (run_date, run_time), '%Y-%m-%d %H:%M:%S') print("startDateTime : ", startDateTime) config_path = os.path.join(os.getcwd(), 'outflowdat', 'config.json') print('config_path : ', config_path) with open(config_path) as json_file: config_data = json.load(json_file) output_dir = dir_path inittidal_conf_path = os.path.join(os.getcwd(), 'outflowdat', 'INITTIDAL.CONF') CONTROL_INTERVAL = config_data["CONTROL_INTERVAL"] DAT_WIDTH = config_data["DAT_WIDTH"] TIDAL_FORECAST_ID = config_data["TIDAL_FORECAST_ID"] MYSQL_HOST = config_data['db_host'] MYSQL_USER = config_data['db_user'] MYSQL_DB = config_data['db_name'] MYSQL_PASSWORD = config_data['db_password'] adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) try: hourly_outflow_file = os.path.join(output_dir, 'OUTFLOW.DAT') print("hourly_outflow_file : ", hourly_outflow_file) if not os.path.isfile(hourly_outflow_file): opts = { 'from': (startDateTime - timedelta(minutes=0)).strftime("%Y-%m-%d %H:%M:%S"), 'to': (startDateTime + timedelta(minutes=CONTROL_INTERVAL)).strftime("%Y-%m-%d %H:%M:%S"), } tidal_timeseries = get_forecast_timeseries(adapter, TIDAL_FORECAST_ID, opts) if len(tidal_timeseries) > 0: print('tidal_timeseries::', len(tidal_timeseries), tidal_timeseries[0], tidal_timeseries[-1]) f = open(hourly_outflow_file, 'w') lines = [] print('Reading INIT TIDAL CONF...') with open(inittidal_conf_path) as initTidalConfFile: initTidalLevels = initTidalConfFile.readlines() for initTidalLevel in initTidalLevels: if len(initTidalLevel.split()): # Check if not empty line lines.append(initTidalLevel) if initTidalLevel[0] == 'N': lines.append('{0} {1:{w}} {2:{w}}\n'.format('S', 0, 0, w=DAT_WIDTH)) base_date_time = startDateTime.replace(minute=0, second=0, microsecond=0) for step in tidal_timeseries: hours_so_far = (step[0] - base_date_time) hours_so_far = 24 * hours_so_far.days + hours_so_far.seconds / (60 * 60) lines.append('{0} {1:{w}} {2:{w}{b}}\n' .format('S', int(hours_so_far), float(step[1]), b='.2f', w=DAT_WIDTH)) f.writelines(lines) f.close() print('Finished writing OUTFLOW.DAT') else: print('No data found for tidal timeseries: ', tidal_timeseries) sys.exit(1) adapter.close() except Exception as ex: adapter.close() print("Download required files|Exception: ", str(ex)) except Exception as e: print("Exception occurred: ", str(e))
def setUpClass(cls): try: root_dir = os.path.dirname(os.path.realpath(__file__)) config = json.loads(open(root_dir + '/CONFIG.json').read()) # Initialize Logger logging_config = json.loads( open(root_dir + '/LOGGING_CONFIG.json').read()) logging.config.dictConfig(logging_config) cls.logger = logging.getLogger('MySQLAdapterTest') cls.logger.addHandler(logging.StreamHandler()) cls.logger.info('setUpClass') MYSQL_HOST = "localhost" MYSQL_USER = "******" MYSQL_DB = "curw" MYSQL_PASSWORD = "" DAY_INTERVAL = 24 if 'MYSQL_HOST' in config: MYSQL_HOST = config['MYSQL_HOST'] if 'MYSQL_USER' in config: MYSQL_USER = config['MYSQL_USER'] if 'MYSQL_DB' in config: MYSQL_DB = config['MYSQL_DB'] if 'MYSQL_PASSWORD' in config: MYSQL_PASSWORD = config['MYSQL_PASSWORD'] cls.adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) cls.eventIds = [] # Store Rainfall Data stations = ['Colombo', 'Hanwella', 'Norwood'] types = [ 'Forecast-0-d', 'Forecast-1-d-after', 'Forecast-2-d-after' ] meta_data = { 'station': 'Hanwella', 'variable': 'Precipitation', 'unit': 'mm', 'type': 'Forecast-0-d', 'source': 'WRF', 'name': 'Forecast Test', 'start_date': '2017-05-01 00:00:00', 'end_date': '2017-05-03 23:00:00' } RAINFALL_DIR = os.path.join(root_dir, 'data', 'Rainfall') cls.logger.debug(RAINFALL_DIR) for station in stations: cls.logger.info('Inserting Rainfall at %s', station) for file in glob(os.path.join(RAINFALL_DIR, station + '*.csv')): timeseries = csv.reader(open(file, 'r'), delimiter=' ', skipinitialspace=True) timeseries = list(timeseries) filename = os.path.basename(file) date_arr = filename.split('.')[0].split('-') cls.logger.debug('-'.join(date_arr[1:])) start_date = datetime.datetime.strptime( '-'.join(date_arr[1:]), '%Y-%m-%d') end_date = start_date + datetime.timedelta( hours=len(timeseries)) cls.logger.debug('start_date: %s, end_date: %s', start_date, end_date) station_meta = dict(meta_data) station_meta['station'] = station station_meta['start_date'] = start_date.strftime( "%Y-%m-%d %H:%M:%S") station_meta['end_date'] = end_date.strftime( "%Y-%m-%d %H:%M:%S") for i in range(0, 3): station_meta['type'] = types[i] event_id = cls.adapter.get_event_id(station_meta) if event_id is None: event_id = cls.adapter.create_event_id( station_meta) cls.logger.debug('HASH SHA256 created: %s', event_id) else: cls.logger.debug('HASH SHA256 exists: %s', event_id) # for l in timeseries[:3] + timeseries[-2:] : # print(l) if event_id not in cls.eventIds: cls.eventIds.append(event_id) row_count = cls.adapter.insert_timeseries( event_id, timeseries[i * DAY_INTERVAL:(i + 1) * DAY_INTERVAL], True) cls.logger.debug('%s rows inserted.', row_count) cls.logger.info('Inserted Rainfall data.') # Store Discharge Data stations = ['Hanwella'] types = [ 'Forecast-0-d', 'Forecast-1-d-after', 'Forecast-2-d-after', 'Forecast-3-d-after', 'Forecast-4-d-after', 'Forecast-5-d-after' ] meta_data = { 'station': 'Hanwella', 'variable': 'Discharge', 'unit': 'm3/s', 'type': 'Forecast-0-d', 'source': 'HEC-HMS', 'name': 'Forecast Test', 'start_date': '2017-05-01 00:00:00', 'end_date': '2017-05-03 23:00:00' } DISCHARGE_DIR = os.path.join(root_dir, 'data', 'Discharge') for station in stations: cls.logger.info('Inserting Discharges at %s', station) for file in glob(os.path.join(DISCHARGE_DIR, station + '*.csv')): timeseries = csv.reader(open(file, 'r'), delimiter=',', skipinitialspace=True) timeseries = list(timeseries) filename = os.path.basename(file) date_arr = filename.split('.')[0].split('-') cls.logger.debug('-'.join(date_arr[1:])) start_date = datetime.datetime.strptime( '-'.join(date_arr[1:]), '%Y-%m-%d') end_date = start_date + datetime.timedelta( hours=len(timeseries)) cls.logger.debug('start_date: %s, end_date: %s', start_date, end_date) station_meta = dict(meta_data) station_meta['station'] = station station_meta['start_date'] = start_date.strftime( "%Y-%m-%d %H:%M:%S") station_meta['end_date'] = end_date.strftime( "%Y-%m-%d %H:%M:%S") for i in range(0, 6): station_meta['type'] = types[i] event_id = cls.adapter.get_event_id(station_meta) if event_id is None: event_id = cls.adapter.create_event_id( station_meta) cls.logger.debug('HASH SHA256 created: %s', event_id) else: cls.logger.debug('HASH SHA256 exists: %s', event_id) # for l in timeseries[:3] + timeseries[-2:] : # print(l) if event_id not in cls.eventIds: cls.eventIds.append(event_id) row_count = cls.adapter.insert_timeseries( event_id, timeseries[i * DAY_INTERVAL:(i + 1) * DAY_INTERVAL], True) cls.logger.debug('%s rows inserted.', row_count) cls.logger.info("Inserted Discharge data.") cls.contain_stations = [ 'Attanagalla', 'Daraniyagala', 'Glencourse', 'Holombuwa', 'Kitulgala' ] except Exception as e: traceback.print_exc()
def create_hybrid_raincell(dir_path, run_date, run_time, forward, backward, res_mins='60', model_prefix='wrf', forecast_source='wrf0', run_name='Cloud-1'): try: observed_adapter = None kelani_basin_points_file = get_resource_path( 'extraction/local/kelani_basin_points_250m.txt') kelani_lower_basin_shp_file = get_resource_path( 'extraction/shp/klb-wgs84/klb-wgs84.shp') reference_net_cdf = get_resource_path( 'extraction/netcdf/wrf_wrfout_d03_2019-03-31_18_00_00_rf') # config_path = os.path.join(os.getcwd(), 'raincelldat', 'config.json') config_path = os.path.join(os.getcwd(), 'config.json') with open(config_path) as json_file: config = json.load(json_file) if 'forecast_db_config' in config: forecast_db_config = config['forecast_db_config'] if 'observed_db_config' in config: observed_db_config = config['observed_db_config'] if 'klb_obs_stations' in config: obs_stations = copy.deepcopy(config['klb_obs_stations']) print('[run_date, run_time] : ', [run_date, run_time]) start_ts_lk = datetime.strptime('%s %s' % (run_date, run_time), '%Y-%m-%d %H:%M:%S') start_ts_lk = start_ts_lk.strftime( '%Y-%m-%d_%H:00') # '2018-05-24_08:00' duration_days = (int(backward), int(forward)) obs_start = datetime.strptime( start_ts_lk, '%Y-%m-%d_%H:%M') - timedelta(days=duration_days[0]) obs_end = datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') forecast_end = datetime.strptime( start_ts_lk, '%Y-%m-%d_%H:%M') + timedelta(days=duration_days[1]) print([obs_start, obs_end, forecast_end]) fcst_duration_start = obs_end.strftime('%Y-%m-%d %H:%M:%S') obs_duration_start = ( datetime.strptime(fcst_duration_start, '%Y-%m-%d %H:%M:%S') - timedelta(days=backward)).strftime('%Y-%m-%d 00:00:00') print('obs_duration_start : ', obs_duration_start) print('fcst_duration_start : ', fcst_duration_start) observed_duration = int( (datetime.strptime(fcst_duration_start, '%Y-%m-%d %H:%M:%S') - datetime.strptime(obs_duration_start, '%Y-%m-%d %H:%M:%S')).total_seconds() / (60 * res_mins)) print('observed_duration : ', observed_duration) raincell_file_path = os.path.join(dir_path, 'RAINCELL.DAT') if not os.path.isfile(raincell_file_path): points = np.genfromtxt(kelani_basin_points_file, delimiter=',') kel_lon_min = np.min(points, 0)[1] kel_lat_min = np.min(points, 0)[2] kel_lon_max = np.max(points, 0)[1] kel_lat_max = np.max(points, 0)[2] print( '[kel_lon_min, kel_lat_min, kel_lon_max, kel_lat_max] : ', [kel_lon_min, kel_lat_min, kel_lon_max, kel_lat_max]) observed_adapter = MySQLAdapter( host=observed_db_config['host'], user=observed_db_config['user'], password=observed_db_config['password'], db=observed_db_config['db']) # print('obs_stations : ', obs_stations) observed_precipitations = get_observed_precip( obs_stations, obs_duration_start, fcst_duration_start, observed_duration, observed_adapter, forecast_source='wrf0') #print('observed_precipitations : ', observed_precipitations) observed_adapter.close() observed_adapter = None validated_obs_station = {} print('observed_precipitations.keys() : ', observed_precipitations.keys()) #""" for station_name in obs_stations.keys(): if station_name in observed_precipitations.keys(): validated_obs_station[station_name] = obs_stations[ station_name] else: print('station_name : ', station_name) print('validated_obs_station : ', validated_obs_station) if len(validated_obs_station) >= 3 and bool( observed_precipitations): # if bool(observed_precipitations): thess_poly = get_voronoi_polygons( validated_obs_station, kelani_lower_basin_shp_file, add_total_area=False) print('thess_poly : ', thess_poly) point_thess_idx = [] for point in points: point_thess_idx.append( is_inside_geo_df(thess_poly, lon=point[1], lat=point[2])) pass print('len(points)', len(points)) print('len(point_thess_idx)', len(point_thess_idx)) print('point_thess_idx', point_thess_idx) with open(raincell_file_path, 'w') as output_file: output_file.write( "%d %d %s %s\n" % (res_mins, observed_duration, obs_duration_start, fcst_duration_start)) print('range 1 : ', int(24 * 60 * duration_days[0] / res_mins) + 1) print('range 2 : ', int(24 * 60 * duration_days[1] / res_mins) - 1) for t in range(observed_duration - 5): for i, point in enumerate(points): rf = float( observed_precipitations[ point_thess_idx[i]].values[t] ) if point_thess_idx[i] is not None else 0 output_file.write('%d %.1f\n' % (point[0], rf)) else: print('----------------------------------------------') print('No observed data.') print('----------------------------------------------') #""" except Exception as e: print('Raincell generation error.') traceback.print_exc() try: if observed_adapter is not None: observed_adapter.close() except Exception as ex: print(str(ex))
if args.config: CONFIG = json.loads(open(os.path.join(ROOT_DIR, args.config)).read()) else: CONFIG = json.loads( open(os.path.join(ROOT_DIR, './CONFIG.json')).read()) forceInsert = args.force weather_stations = CONFIG['weather_stations'] water_level_stations = CONFIG['water_level_stations'] stations = weather_stations + water_level_stations extract_from_db = CONFIG['extract_from'] push_to_db = CONFIG['push_to'] extract_adapter = MySQLAdapter(host=extract_from_db['MYSQL_HOST'], user=extract_from_db['MYSQL_USER'], password=extract_from_db['MYSQL_PASSWORD'], db=extract_from_db['MYSQL_DB']) push_adapter = MySQLAdapter(host=push_to_db['MYSQL_HOST'], user=push_to_db['MYSQL_USER'], password=push_to_db['MYSQL_PASSWORD'], db=push_to_db['MYSQL_DB']) # Prepare start and date times. now_date = utc_to_sl(datetime.now()) # now_date = datetime.now() start_datetime_obj = now_date - timedelta(hours=2) end_datetime_obj = now_date start_datetime = start_datetime_obj.strftime(COMMON_DATE_FORMAT) end_datetime = end_datetime_obj.strftime(COMMON_DATE_FORMAT) # start_datetime = '2018-07-04 00:00:00'
i += 1.0 print('Inserting Initial Water levels...') with open(INIT_WL_CONFIG) as initWLConfFile: initWaterLevels = initWLConfFile.readlines() for initWaterLevel in initWaterLevels: if len(initWaterLevel.split()): lines.append(initWaterLevel) f.writelines(lines) # Save Forecast values into Database opts = {'forceInsert': forceInsert, 'runName': runName} if storeData: adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) save_forecast_timeseries(adapter, csvList[CSV_NUM_METADATA_LINES:], date, time, opts) else: print( "---------------------------------------Mysql db not updated.-------------------------------------" ) except Exception as e: print(e) traceback.print_exc() finally: f.close() print('Completed ', DISCHARGE_CSV_FILE_PATH, ' to ', INFLOW_DAT_FILE_PATH)
print('WARNING: Force Insert enabled.') CON_DATA = json.loads(open(CONTINUOUS_CONFIG).read()) stations = CON_DATA['stations'] metaData = { 'station': '', 'variable': '', 'unit': '', 'type': '', 'source': '', 'name': '', } adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB) for station in stations: print('station:', station) # Check whether station exists is_station_exists = adapter.get_station({'name': station['name']}) if is_station_exists is None: print('Station %s does not exists.', station['name']) if 'station_meta' in station and 'station_type' in station: station_meta = station['station_meta'] station_meta.insert(0, Station.getType(station['station_type'])) row_count = adapter.create_station(station_meta) if row_count > 0: print('Created new station %s', station_meta)