예제 #1
0
        utc_offset = read_attribute_from_config_file('utc_offset', config,
                                                     False)
        if utc_offset is None:
            utc_offset = ''

        # sim tag
        if sim_tag is None:
            sim_tag = read_attribute_from_config_file('sim_tag', config, True)

        # source details
        model = read_attribute_from_config_file('model', config, True)
        version = "_".join(flo2d_model.split("_")[1:])

        # unit details
        unit = read_attribute_from_config_file('unit', config, True)
        unit_type = UnitType.getType(
            read_attribute_from_config_file('unit_type', config, True))

        # variable details
        variable = read_attribute_from_config_file('variable', config, True)

        hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE)
        timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)

        pool = get_Pool(host=con_params.CURW_FCST_HOST,
                        port=con_params.CURW_FCST_PORT,
                        db=con_params.CURW_FCST_DATABASE,
                        user=con_params.CURW_FCST_USERNAME,
                        password=con_params.CURW_FCST_PASSWORD)

        flo2d_model_name = '{}_{}'.format(model, version)
예제 #2
0
def extract_distrubuted_hechms_outputs(output_file_name, output_dir, run_date,
                                       run_time):
    """
    Config.json 
    {
      "output_file_name": "DailyDischarge.csv",
      "output_dir": "",

      "run_date": "2019-05-24",
      "run_time": "00:00:00",
      "utc_offset": "",

      "sim_tag": "hourly_run",

      "model": "HECHMS",
      "version": "single",

      "unit": "m3/s",
      "unit_type": "Instantaneous",

      "variable": "Discharge",

      "station_name": "Hanwella"
    }

    """
    try:

        config = json.loads(open('config.json').read())

        # output related details
        output_file_name = output_file_name
        output_dir = output_dir

        run_date = run_date
        run_time = run_time

        utc_offset = read_attribute_from_config_file('utc_offset', config,
                                                     False)
        if utc_offset is None:
            utc_offset = ''

        # sim tag
        sim_tag = read_attribute_from_config_file('sim_tag', config, True)

        # source details
        model = read_attribute_from_config_file('model', config, True)
        version = read_attribute_from_config_file('version', config, True)

        # unit details
        unit = read_attribute_from_config_file('unit', config, True)
        unit_type = UnitType.getType(
            read_attribute_from_config_file('unit_type', config, True))

        # variable details
        variable = read_attribute_from_config_file('variable', config, True)

        # station details
        station_name = read_attribute_from_config_file('station_name', config,
                                                       True)

        out_file_path = os.path.join(output_dir, output_file_name)

        if not os.path.exists(out_file_path):
            msg = 'no file :: {}'.format(out_file_path)
            logger.warning(msg)
            print(msg)
            exit(1)

        fgt = get_file_last_modified_time(out_file_path)
        print("fgt, ", fgt)

        timeseries = read_csv(out_file_path)

        pool = get_Pool(host=CURW_FCST_HOST,
                        port=CURW_FCST_PORT,
                        db=CURW_FCST_DATABASE,
                        user=CURW_FCST_USERNAME,
                        password=CURW_FCST_PASSWORD)

        hechms_stations = get_hechms_stations(pool=pool)

        station_id = hechms_stations.get(station_name)[0]
        lat = str(hechms_stations.get(station_name)[1])
        lon = str(hechms_stations.get(station_name)[2])

        source_id = get_source_id(pool=pool, model=model, version=version)

        variable_id = get_variable_id(pool=pool, variable=variable)

        unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)

        tms_meta = {
            'sim_tag': sim_tag,
            'model': model,
            'version': version,
            'variable': variable,
            'unit': unit,
            'unit_type': unit_type.value,
            'latitude': lat,
            'longitude': lon,
            'station_id': station_id,
            'source_id': source_id,
            'variable_id': variable_id,
            'unit_id': unit_id
        }

        utcOffset = getUTCOffset(utc_offset, default=True)

        if utcOffset != timedelta():
            tms_meta['utcOffset'] = utcOffset

        # Push timeseries to database
        save_forecast_timeseries_to_db(pool=pool,
                                       timeseries=timeseries,
                                       run_date=run_date,
                                       run_time=run_time,
                                       tms_meta=tms_meta,
                                       fgt=fgt)

    except Exception as e:
        logger.error('JSON config data loading error.')
        print('JSON config data loading error.')
        traceback.print_exc()
    finally:
        logger.info("Process finished.")
        print("Process finished.")
        if 'wrf_model_list' in config and (config['wrf_model_list'] != ""):
            wrf_model_list = config['wrf_model_list']
            wrf_model_list = wrf_model_list.split(',')
        else:
            logger.error("wrf_model_list not specified in config file.")
            exit(1)

        # unit details
        if 'unit' in config and (config['unit'] != ""):
            unit = config['unit']
        else:
            logger.error("unit not specified in config file.")
            exit(1)

        if 'unit_type' in config and (config['unit_type'] != ""):
            unit_type = UnitType.getType(config['unit_type'])
        else:
            logger.error("unit_type not specified in config file.")
            exit(1)

        # variable details
        if 'variable' in config and (config['variable'] != ""):
            variable = config['variable']
        else:
            logger.error("variable not specified in config file.")
            exit(1)

        # connection params
        if 'host' in config and (config['host'] != ""):
            host = config['host']
        else:
예제 #4
0
def extract_distrubuted_hechms_outputs(target_model, db_user, db_pwd, db_host,
                                       db_name, out_file_path, run_date,
                                       run_time):
    """
    Config.json
    {
      "output_file_name": "DailyDischarge.csv",
      "output_dir": "",
      "run_date": "2019-05-24",
      "run_time": "00:00:00",
      "utc_offset": "",
      "sim_tag": "hourly_run",
      "model": "HECHMS",
      "version": "single",
      "unit": "m3/s",
      "unit_type": "Instantaneous",
      "variable": "Discharge",
      "station_name": "Hanwella"
    }
    """
    try:

        config = json.loads(
            open('/home/curw/git/distributed_hechms/uploads/config.json').read(
            ))

        # output related details
        run_date = run_date
        run_time = run_time

        utc_offset = read_attribute_from_config_file('utc_offset', config,
                                                     False)
        if utc_offset is None:
            utc_offset = ''

        # sim tag
        sim_tag = read_attribute_from_config_file('sim_tag', config, True)
        print("extract_distrubuted_hechms_outputs|sim_tag : ", sim_tag)
        # source details
        model = read_attribute_from_config_file('model', config, True)
        print("extract_distrubuted_hechms_outputs|model : ", model)

        version_config = read_attribute_from_config_file(
            'version_config', config, True)
        print("extract_distrubuted_hechms_outputs|version_config : ",
              version_config)

        version = version_config[target_model]
        print("extract_distrubuted_hechms_outputs|version : ", version)

        # unit details
        unit = read_attribute_from_config_file('unit', config, True)
        print("extract_distrubuted_hechms_outputs|unit : ", unit)
        unit_type = UnitType.getType(
            read_attribute_from_config_file('unit_type', config, True))
        print("extract_distrubuted_hechms_outputs|unit_type : ", unit_type)

        # variable details
        variable = read_attribute_from_config_file('variable', config, True)
        print("extract_distrubuted_hechms_outputs|variable : ", variable)

        # station details
        station_name = read_attribute_from_config_file('station_name', config,
                                                       True)
        print("extract_distrubuted_hechms_outputs|station_name : ",
              station_name)

        if not os.path.exists(out_file_path):
            msg = 'no file :: {}'.format(out_file_path)
            logger.warning(msg)
            print(msg)
            exit(1)

        fgt = get_file_last_modified_time(out_file_path)
        print("extract_distrubuted_hechms_outputs|fgt : ", fgt)

        timeseries = read_csv(out_file_path)

        pool = get_Pool(host=db_host,
                        port=3306,
                        db=db_name,
                        user=db_user,
                        password=db_pwd)

        hechms_stations = get_hechms_stations(pool=pool)
        print("extract_distrubuted_hechms_outputs|hechms_stations : ",
              hechms_stations)

        station_id = hechms_stations.get(station_name)[0]
        lat = str(hechms_stations.get(station_name)[1])
        lon = str(hechms_stations.get(station_name)[2])
        print("extract_distrubuted_hechms_outputs|[station_id, lat, lon] : ",
              [station_id, lat, lon])

        source_id = get_source_id(pool=pool, model=model, version=version)
        print("extract_distrubuted_hechms_outputs|source_id : ", source_id)

        variable_id = get_variable_id(pool=pool, variable=variable)
        print("extract_distrubuted_hechms_outputs|variable_id : ", variable_id)

        unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
        print("extract_distrubuted_hechms_outputs|unit_id : ", unit_id)

        tms_meta = {
            'sim_tag': sim_tag,
            'model': model,
            'version': version,
            'variable': variable,
            'unit': unit,
            'unit_type': unit_type.value,
            'latitude': lat,
            'longitude': lon,
            'station_id': station_id,
            'source_id': source_id,
            'variable_id': variable_id,
            'unit_id': unit_id
        }

        print("extract_distrubuted_hechms_outputs|tms_meta : ", tms_meta)
        utcOffset = getUTCOffset(utc_offset, default=True)

        if utcOffset != timedelta():
            tms_meta['utcOffset'] = utcOffset

        # Push timeseries to database
        save_forecast_timeseries_to_db(pool=pool,
                                       timeseries=timeseries,
                                       run_date=run_date,
                                       run_time=run_time,
                                       tms_meta=tms_meta,
                                       fgt=fgt)
        return {'Result': 'Success'}
    except Exception as e:
        logger.error('JSON config data loading error.')
        print('JSON config data loading error.')
        traceback.print_exc()
        return {'Result': 'Fail'}
예제 #5
0
if __name__=="__main__":

    try:

        ################################################
        # Initialize parameters for DISTRIBUTED HECHMS #
        ################################################

        # source details
        model = 'HECHMS'
        version = 'distributed'

        # unit details
        unit = 'm3/s'
        unit_type = UnitType.getType('Instantaneous')

        # variable details
        variable = 'Discharge'

        # station details
        hechms_stations = read_csv('hechms_stations.csv')

        pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD,
                db=CURW_FCST_DATABASE)

        # ########
        # # test
        # ########

        # USERNAME = "******"
예제 #6
0
def upload_waterlevels(dir_path, ts_start_date, ts_start_time, run_date, run_time):
    """
    Config.json
    {
      "HYCHAN_OUT_FILE": "HYCHAN.OUT",
      "TIMDEP_FILE": "TIMDEP.OUT",
      "output_dir": "",

      "run_date": "2019-05-24",
      "run_time": "",
      "ts_start_date": "",
      "ts_start_time": "",
      "utc_offset": "",

      "sim_tag": "",

      "model": "WRF",
      "version": "v3",

      "unit": "mm",
      "unit_type": "Accumulative",

      "variable": "Precipitation"
    }

    """
    try:
        config_path = os.path.join(os.getcwd(), 'extract', 'config_curw_fcst.json')
        config = json.loads(open(config_path).read())

        # flo2D related details
        HYCHAN_OUT_FILE = read_attribute_from_config_file('HYCHAN_OUT_FILE', config, True)
        TIMDEP_FILE = read_attribute_from_config_file('TIMDEP_FILE', config, True)
        output_dir = dir_path

        run_date = run_date
        run_time = run_time
        ts_start_date = ts_start_date
        ts_start_time = ts_start_time
        utc_offset = read_attribute_from_config_file('utc_offset', config, False)
        if utc_offset is None:
            utc_offset = ''

        # sim tag
        sim_tag = read_attribute_from_config_file('sim_tag', config, True)

        # source details
        model = read_attribute_from_config_file('model', config, True)
        version = read_attribute_from_config_file('version', config, True)

        # unit details
        unit = read_attribute_from_config_file('unit', config, True)
        unit_type = UnitType.getType(read_attribute_from_config_file('unit_type', config, True))

        # variable details
        variable = read_attribute_from_config_file('variable', config, True)

        hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE)
        timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)

        pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME,
                        password=CURW_FCST_PASSWORD)

        # pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)

        flo2d_model_name = '{}_{}'.format(model, version)

        flo2d_source = json.loads(get_source_parameters(pool=pool, model=model, version=version))

        print("############### source ############", flo2d_source)

        flo2d_stations = get_flo2d_output_stations(pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name))

        print("############### 1st occurrence ############", flo2d_stations)

        source_id = get_source_id(pool=pool, model=model, version=version)

        variable_id = get_variable_id(pool=pool, variable=variable)

        unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)

        tms_meta = {
            'sim_tag': sim_tag,
            'model': model,
            'version': version,
            'variable': variable,
            'unit': unit,
            'unit_type': unit_type.value,
            'source_id': source_id,
            'variable_id': variable_id,
            'unit_id': unit_id
        }

        CHANNEL_CELL_MAP = flo2d_source["CHANNEL_CELL_MAP"]

        FLOOD_PLAIN_CELL_MAP = flo2d_source["FLOOD_PLAIN_CELL_MAP"]

        ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys()
        FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys()
        SERIES_LENGTH = 0
        MISSING_VALUE = -999

        utcOffset = getUTCOffset(utc_offset, default=True)

        print('Extract Water Level Result of FLO2D on', run_date, '@', run_time, 'with Base time of', ts_start_date,
              '@', ts_start_time)

        # Check HYCHAN.OUT file exists
        if not os.path.exists(hychan_out_file_path):
            print('Unable to find file : ', hychan_out_file_path)
            traceback.print_exc()

        #####################################
        # Calculate the size of time series #
        #####################################
        bufsize = 65536
        with open(hychan_out_file_path) as infile:
            isWaterLevelLines = False
            isCounting = False
            countSeriesSize = 0  # HACK: When it comes to the end of file, unable to detect end of time series
            while True:
                lines = infile.readlines(bufsize)
                if not lines or SERIES_LENGTH:
                    break
                for line in lines:
                    if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
                        isWaterLevelLines = True
                    elif isWaterLevelLines:
                        cols = line.split()
                        if len(cols) > 0 and cols[0].replace('.', '', 1).isdigit():
                            countSeriesSize += 1
                            isCounting = True
                        elif isWaterLevelLines and isCounting:
                            SERIES_LENGTH = countSeriesSize
                            break

        print('Series Length is :', SERIES_LENGTH)
        bufsize = 65536
        #################################################################
        # Extract Channel Water Level elevations from HYCHAN.OUT file   #
        #################################################################
        print('Extract Channel Water Level Result of FLO2D (HYCHAN.OUT) on', run_date, '@', run_time,
              'with Base time of',
              ts_start_date, '@', ts_start_time)
        with open(hychan_out_file_path) as infile:
            isWaterLevelLines = False
            isSeriesComplete = False
            waterLevelLines = []
            seriesSize = 0  # HACK: When it comes to the end of file, unable to detect end of time series
            while True:
                lines = infile.readlines(bufsize)
                if not lines:
                    break
                for line in lines:
                    if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
                        seriesSize = 0
                        elementNo = line.split()[5]

                        if elementNo in ELEMENT_NUMBERS:
                            isWaterLevelLines = True
                            waterLevelLines.append(line)
                        else:
                            isWaterLevelLines = False

                    elif isWaterLevelLines:
                        cols = line.split()
                        if len(cols) > 0 and isfloat(cols[0]):
                            seriesSize += 1
                            waterLevelLines.append(line)

                            if seriesSize == SERIES_LENGTH:
                                isSeriesComplete = True

                    if isSeriesComplete:
                        baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
                        timeseries = []
                        elementNo = waterLevelLines[0].split()[5]
                        # print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo])
                        for ts in waterLevelLines[1:]:
                            v = ts.split()
                            if len(v) < 1:
                                continue
                            # Get flood level (Elevation)
                            value = v[1]
                            # Get flood depth (Depth)
                            # value = v[2]
                            if not isfloat(value):
                                value = MISSING_VALUE
                                continue  # If value is not present, skip
                            if value == 'NaN':
                                continue  # If value is NaN, skip
                            timeStep = float(v[0])
                            currentStepTime = baseTime + timedelta(hours=timeStep)
                            dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")
                            timeseries.append([dateAndTime, value])

                        # Save Forecast values into Database
                        opts = {
                            'elementNo': elementNo,
                            'tms_meta': tms_meta
                        }
                        # print('>>>>>', opts)
                        if utcOffset != timedelta():
                            opts['utcOffset'] = utcOffset

                        # Push timeseries to database
                        save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries,
                                                       run_date=run_date, run_time=run_time, opts=opts,
                                                       flo2d_stations=flo2d_stations)

                        isWaterLevelLines = False
                        isSeriesComplete = False
                        waterLevelLines = []
                # -- END for loop
            # -- END while loop

        #################################################################
        # Extract Flood Plain water elevations from TIMEDEP.OUT file    #
        #################################################################

        if not os.path.exists(timdep_file_path):
            print('Unable to find file : ', timdep_file_path)
            traceback.print_exc()

        print('Extract Flood Plain Water Level Result of FLO2D (TIMEDEP.OUT) on', run_date, '@', run_time,
              'with Base time of', ts_start_date,
              '@', ts_start_time)

        with open(timdep_file_path) as infile:
            waterLevelLines = []
            waterLevelSeriesDict = dict.fromkeys(FLOOD_ELEMENT_NUMBERS, [])
            while True:
                lines = infile.readlines(bufsize)
                if not lines:
                    break
                for line in lines:
                    if len(line.split()) == 1:
                        # continue
                        if len(waterLevelLines) > 0:
                            waterLevels = get_water_level_of_channels(waterLevelLines, FLOOD_ELEMENT_NUMBERS)

                            # Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060
                            # print('waterLevelLines[0].split() : ', waterLevelLines[0].split())
                            ModelTime = float(waterLevelLines[0].split()[0])
                            baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
                            currentStepTime = baseTime + timedelta(hours=ModelTime)
                            dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")

                            for elementNo in FLOOD_ELEMENT_NUMBERS:
                                tmpTS = waterLevelSeriesDict[elementNo][:]
                                if elementNo in waterLevels:
                                    tmpTS.append([dateAndTime, waterLevels[elementNo]])
                                else:
                                    tmpTS.append([dateAndTime, MISSING_VALUE])
                                waterLevelSeriesDict[elementNo] = tmpTS

                            isWaterLevelLines = False
                            # for l in waterLevelLines :
                            # print(l)
                            waterLevelLines = []
                    waterLevelLines.append(line)

            # print('len(FLOOD_ELEMENT_NUMBERS) : ', len(FLOOD_ELEMENT_NUMBERS))
            for elementNo in FLOOD_ELEMENT_NUMBERS:

                # Save Forecast values into Database
                opts = {
                    'elementNo': elementNo,
                    'tms_meta': tms_meta
                }
                if utcOffset != timedelta():
                    opts['utcOffset'] = utcOffset

                # Push timeseries to database
                save_forecast_timeseries_to_db(pool=pool, timeseries=waterLevelSeriesDict[elementNo],
                                               run_date=run_date, run_time=run_time, opts=opts,
                                               flo2d_stations=flo2d_stations)

    except Exception as e:
        logger.error('JSON config data loading error.')
        print('JSON config data loading error.')
        traceback.print_exc()
    finally:
        logger.info("Process finished.")
        print("Process finished.")