Exemple #1
0
def load_era(var, year=None, STANDARDISE_COORDS=True, NORMALISE_TO_DATE_ONLY=True, ROLL_LON=False):

    """
    Get ERA-Interim data array (xr)
    Var list = ['T2', 't2max', 't2min']
    Name in array = ['T2', 'MX2T', 'MN2T'], just call by da[da.name]

    # 0: surface tempterature, T2_1979.nc, 1979-2016
    # 1: max temperature, t2max_1979.nc, 1979-2017
    # 2: min temperature, t2min_1979.nc, 1979-2017
    """

    tic()  # Measure time elapsed to load
    # path_parent = '/group_workspaces/jasmin4/bas_climate/data/ecmwf/era-interim/day/'
    path_parent = '/gws/nopw/j04/bas_climate/data/ecmwf1/era-interim/day/'
    # path_parent = '/group_workspaces/jasmin4/bas_climate/data/ecmwf/era5'

    if year is None:  # Get all years

        path = path_parent + var + '/*.nc'
        # Rounding coordinates for ERA-Interim as different years have slightly different coordinate values below 2 decimals
        # https://github.com/pydata/xarray/blob/aabda43c54c6336dc62cc9ec3c2050c9c656cff6/xarray/backends/api.py
        ds = xr.open_mfdataset(path, preprocess=round_coords)

    else:
        # Get specific year
        fname = path_parent + var + '/' + var + '_' + str(year) + '.nc'
        ds = xr.open_dataset(fname)

    da = ds[ds.name]   # Extract main data array from dataset
    da = standardise_da_coords(da, STANDARDISE_COORDS=STANDARDISE_COORDS, NORMALISE_TO_DATE_ONLY=NORMALISE_TO_DATE_ONLY, ROLL_LON=ROLL_LON)
    toc()

    return da.squeeze()
def convert_to_datetime64(da, verbose=True):
    """
    ! ONLY WORKS FOR 1D DATA (i.e. ONE GRID LOCATION)
    Convert datetime format to standard python datetime by simple interpolation
    """

    if type(da.time.values[0]) == np.datetime64:
        #print("Already in datetime64 format.")
        return da

    print("Converting to datetime64... (retrieving all values and interpolating if necessary, may take a few mins)")

    start_year = da.time.values[0].year
    end_year = da.time.values[-1].year

    # Create new data array
    times = np.arange(str(start_year), str(end_year + 1), dtype='datetime64[D]')
    # new_da = xr.DataArray(np.zeros(len(times)), coords=[('time', times)])

    new_da = xr.DataArray(np.zeros(len(times)), coords=[('time', times)], attrs=da.attrs)
    new_da['lon'] = da.coords['lon'].values
    new_da['lat'] = da.coords['lat'].values

    for yr in np.arange(start_year, end_year + 1):

        if len(da.sel(time=str(yr))) != len(new_da.sel(time=str(yr))):

            # Do interpolation of original array to fit standard datetime
            if verbose:
                print("    Processing year " + str(yr) + "/" + str(end_year))
                tic()
            leapyear = calendar.isleap(yr)
            size = 365 if not leapyear else 366
            arr = da.sel(time=str(yr)).values
            arr_new = helper.interp1d(arr, size)
            if verbose:
                toc()

        else:
            arr_new = da.sel(time=str(yr)).values

        # Replace array values with new data
        ind_start = (new_da.indexes['time'] == pd.Timestamp(str(yr) + '-01-01')).argmax()
        ind_end = (new_da.indexes['time'] == pd.Timestamp(str(yr) + '-12-31')).argmax()
        new_da[ind_start:ind_end + 1] = arr_new

    #print("Done")
    return new_da
Exemple #3
0
def tag2(tagid, begin_time, end_time, timestep, mode, utc, show, plot):
    """Parse user friendly tag query input and assemble wincc tag query"""
    if timestep and not end_time:
        end_time = datetime_to_str_without_ms(datetime.now())

    query = tag_query_builder(tagid, begin_time, end_time, timestep, mode, utc)
    if show:
        print(query)
        return

    toc = tic()
    try:
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)

        records = w.create_tag_records()
        print("Fetched data in {time}.".format(time=round(toc(), 3)))
        # print(tags)
        # tags.plot()
        for record in records:
            print(record)
        if plot:
            plot_tag_records(records)

    except Exception as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #4
0
def tag(tagid, begin_time, end_time, timestep, mode, utc, show):
    """Parse user friendly tag query and assemble userunfriendly wincc query"""
    query = tag_query_builder(tagid, begin_time, end_time, timestep, mode, utc)
    if show:
        print(query)
        return

    toc = tic()
    try:
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)

        if w.rowcount():
            print_tag_logging(w.fetchall())
            # for rec in w.fetchall():
            #    print rec

        print("Fetched data in {time}.".format(time=round(toc(), 3)))

    except Exception as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #5
0
def tag2(tagid, begin_time, end_time, timestep, mode, utc, show, plot):
    """Parse user friendly tag query input and assemble wincc tag query"""
    if timestep and not end_time:
        end_time = datetime_to_str_without_ms(datetime.now())

    query = tag_query_builder(tagid, begin_time, end_time, timestep, mode, utc)
    if show:
        print(query)
        return

    toc = tic()
    try:
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)

        records = w.create_tag_records()
        print("Fetched data in {time}.".format(time=round(toc(), 3)))
        # print(tags)
        # tags.plot()
        for record in records:
            print(record)
        if plot:
            plot_tag_records(records)

    except Exception as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #6
0
def tag(tagid, begin_time, end_time, timestep, mode, utc, show):
    """Parse user friendly tag query and assemble userunfriendly wincc query"""
    query = tag_query_builder(tagid, begin_time, end_time, timestep, mode, utc)
    if show:
        print(query)
        return

    toc = tic()
    try:
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)

        if w.rowcount():
            print_tag_logging(w.fetchall())
            # for rec in w.fetchall():
            #    print rec

        print("Fetched data in {time}.".format(time=round(toc(), 3)))

    except Exception as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #7
0
def tagid_by_name(tagname):
    """Search hosts db for tag entries matching the given name.
    Return tagid.
    """
    try:
        toc = tic()
        mssql_conn = mssql(host_info.address,
                           strip_R_from_db_name(host_info.database))
        mssql_conn.connect()
        mssql_conn.execute("SELECT TLGTAGID, VARNAME FROM PDE#TAGs WHERE "
                           "VARNAME LIKE '%{name}%'".format(name=tagname))
        if mssql_conn.rowcount():
            for rec in mssql_conn.fetchall():
                print rec
        print("Fetched data in {time}.".format(time=round(toc(), 3)))
    except Exception as e:
        print(e)
    finally:
        mssql_conn.close()
Exemple #8
0
def tagid_by_name(tagname):
    """Search hosts db for tag entries matching the given name.
    Return tagid.
    """
    try:
        toc = tic()
        mssql_conn = mssql(host_info.address,
                           strip_R_from_db_name(host_info.database))
        mssql_conn.connect()
        mssql_conn.execute("SELECT TLGTAGID, VARNAME FROM PDE#TAGs WHERE "
                           "VARNAME LIKE '%{name}%'".format(name=tagname))
        if mssql_conn.rowcount():
            for rec in mssql_conn.fetchall():
                print rec
        print("Fetched data in {time}.".format(time=round(toc(), 3)))
    except Exception as e:
        print(e)
    finally:
        mssql_conn.close()
Exemple #9
0
def operator_messages(begin_time, end_time, text, utc, show):
    """Query db for operator messages."""
    query = om_query_builder(eval_datetime(begin_time),
                             eval_datetime(end_time), text, utc)
    if show:
        print(query)
        return

    try:
        toc = tic()
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)
        w.print_operator_messages()
        print("Fetched data in {time}.".format(time=round(toc(), 3)))
    except WinCCException as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #10
0
def operator_messages(begin_time, end_time, text, utc, show):
    """Query db for operator messages."""
    query = om_query_builder(eval_datetime(begin_time),
                             eval_datetime(end_time), text, utc)
    if show:
        print(query)
        return

    try:
        toc = tic()
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)
        w.print_operator_messages()
        print("Fetched data in {time}.".format(time=round(toc(), 3)))
    except WinCCException as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #11
0
def tag2(tagid, begin_time, end_time, timestep, mode, utc, show, plot, outfile,
         outfile_col_name, outfile_time_zone):
    """Parse user friendly tag query input and assemble wincc tag query"""
    if timestep and not end_time:
        end_time = datetime_to_str_without_ms(datetime.now())

    query = tag_query_builder(tagid, begin_time, end_time, timestep, mode, utc)
    if show:
        print(query)
        return

    toc = tic()
    try:
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)

        records = w.create_tag_records(utc)
        print("Fetched data in {time}.".format(time=round(toc(), 3)))

        if records:
            if (outfile != ''):
                with open(outfile, "w") as f:
                    # print(records.to_csv().encode("UTF-8"))
                    for rec in records:
                        f.write(
                            rec.to_csv(name=outfile_col_name.encode("UTF-8"),
                                       tz=outfile_time_zone))
            else:
                for record in records:
                    print(record)
                if plot:
                    plot_tag_records(records)
        else:
            logging.warning("No data returned.")

    except Exception as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #12
0
def alarms(begin_time, end_time, text, utc, show, state, priority, priority2,
           report, report_hostname):
    """Read alarms from given host in given time."""
    query = alarm_query_builder(eval_datetime(begin_time),
                                eval_datetime(end_time),
                                text, utc, state, priority, priority2)

    if show:
        print(query)
        return

    try:
        toc = tic()
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)

        if report:
            alarms = w.create_alarm_record()
            if report_hostname:
                host_description = report_hostname
            else:
                host_description = host_info.description
            if not end_time:
                end_time = datetime_to_str_without_ms(datetime.now())
            generate_alarms_report(alarms, begin_time, end_time,
                                   host_description, text)
            print(unicode(alarms))
        else:
            w.print_alarms()

        print("Fetched data in {time}.".format(time=round(toc(), 3)))
    except WinCCException as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #13
0
def alarms(begin_time, end_time, text, utc, show, state, priority, priority2,
           report, report_hostname):
    """Read alarms from given host in given time."""
    query = alarm_query_builder(eval_datetime(begin_time),
                                eval_datetime(end_time), text, utc, state,
                                priority, priority2)

    if show:
        print(query)
        return

    try:
        toc = tic()
        w = wincc(host_info.address, host_info.database)
        w.connect()
        w.execute(query)

        if report:
            alarms = w.create_alarm_record()
            if report_hostname:
                host_description = report_hostname
            else:
                host_description = host_info.description
            if not end_time:
                end_time = datetime_to_str_without_ms(datetime.now())
            generate_alarms_report(alarms, begin_time, end_time,
                                   host_description, text)
            print(unicode(alarms))
        else:
            w.print_alarms()

        print("Fetched data in {time}.".format(time=round(toc(), 3)))
    except WinCCException as e:
        print(e)
        print(traceback.format_exc())
    finally:
        w.close()
Exemple #14
0
    ##########
    # 모델 저장
    ##########
    if (not epoch == 0):
        save_path = saver.save(sess, "./checkpoint/%s" % (TSET.name))
        print("Model saved in file: %s" % save_path)

    ##########
    # 레지듀얼 스케일 조절
    ##########
    rs_factor = min(1.0, epoch / 100)

    ##########
    # 트레이닝
    ##########
    train_time = tic()
    num_batch = TSET.num_batch_in_a_epoch
    batch_trn = []

    for i in range(num_batch):

        # 배치 가져오기
        batch_trn = dset_train.get_batch()

        # 학습 - generator
        [summary_train, _] = sess.run(
            [merged, train_sr],
            feed_dict={
                in_lowv: batch_trn.arr4d_LR_v,
                gt_highv: batch_trn.arr4d_HR_v,
                op_rs: rs_factor
Exemple #15
0
        save_path = saver.save(sess,
                               "./checkpoint/model-%s" % (exp.train_name),
                               global_step=epoch)
        print("Model saved in file: %s" % save_path)

    ##########
    # 레지듀얼 스케일 조절
    ##########
    rs_factor = 1.0
    if (epoch < 100):
        rs_factor = epoch / 100

    ##########
    # 트레이닝
    ##########
    train_time = tic()
    gpu_time = 0
    batch_time = 0
    loss = 0.0
    num_batch = exp.num_batch_in_a_epoch
    batch_trn = []

    for i in range(num_batch):

        batch_st = time.time()

        # 배치 가져오기
        batch_trn = dset.get_batch()

        st = tic()
        # 학습 - 오토엔코더