def plot_timeseries(cfg, metadata): """Plot timeseries data.""" short_name = 'pr' xaxis = 'time' datasets = read_input_data(metadata) ancestors = [info['filename'] for info in metadata] time_period = cfg['time_period'] var = datasets[short_name] time_unit = time_period[0].upper() start_date = np.datetime_as_string(datasets.time.min(), unit=time_unit) end_date = np.datetime_as_string(datasets.time.max(), unit=time_unit) name = f'{var.long_name}_{time_period}' caption = f"{var.long_name} per {time_period} for {start_date}:{end_date}" plot_data( cfg=cfg, datasets=datasets, xaxis=xaxis, yaxis=short_name, xlabel=f'{xaxis.capitalize()} / {time_period}', ylabel=f'{var.long_name} / {var.units}', caption=caption, name=name, ancestors=ancestors, ) filename_data = get_diagnostic_filename(name, cfg, extension='nc') datasets.to_netcdf(filename_data) log_provenance(caption, filename_data, cfg, ancestors)
def onselect(self, xmin, xmax): # print(xmin, xmax) indmin, indmax = 0, 0 if self.use_dates: xmin, xmax = xmin//1, xmax//1 indmin, indmax = self.get_index_from_datetime(xmin, xmax) else: indmin, indmax = np.searchsorted(self.x, (xmin, xmax)) indmax = min(len(self.x) - 1, indmax) thisx = self.x[indmin:indmax] thisy = self.y[indmin:indmax] poly_collection = ax1.fill_between(thisx, self.y_upper_lim, self.y_lower_lim, color="r", alpha=0.25) self.line2.set_data(thisx, thisy) ax2.set_xlim(thisx[0], thisx[-1]) ax2.set_ylim(thisy.min(), thisy.max()) begin_str, end_str = "", "" if self.use_dates: # Formatting Date String from ISO to German Notation begin_str = re.sub(':', ' ', np.datetime_as_string(thisx[0], unit='s')) end_str = re.sub(':', ' ', np.datetime_as_string(thisx[-1], unit='s')) begin_str = dt.strptime(begin_str, '%Y-%m-%dT%H %M %S').strftime('%d.%m.%Y, %H:%M:%S') end_str = dt.strptime(end_str, '%Y-%m-%dT%H %M %S').strftime('%d.%m.%Y, %H:%M:%S') else: begin_str = str(indmin) end_str = str(indmax) list_item = QtWidgets.QListWidgetItem(begin_str + " - " + end_str) list_item.setData(1, poly_collection) self.list_widget.addItem(list_item) self.canvas.draw()
def plot_E0_data(self, df): title = np.datetime_as_string(df.index.values[0], unit='D') + " to " title += np.datetime_as_string(df.index.values[-1], unit='D') file_name = os.path.join( "plots", "estimate_e0_" + title.replace(" ", "_") + ".png") current_backend = plt.get_backend() plt.switch_backend("agg") plt.ioff() fig, axs = plt.subplots() if "Sws" in list(self.df): ldf = self.df[self.df.index.isin(df.index)] sc = axs.scatter(df.TC.values, df.ER.values, c=ldf.Sws.values, s=10) else: sc = axs.scatter(df.TC.values, df.ER.values, s=10) axs.set_title(title) axs.set_xlabel("Temperature (degC)") axs.set_ylabel("ER (umol/m^2/s)") clb = plt.colorbar(sc) clb.ax.set_title("Sws") fig.savefig(file_name, format="png") plt.close(fig) plt.switch_backend(current_backend) plt.ion() return
def __init__(self, data_source, *args): """:parameter data_source: DataSource object""" self.data_source = data_source # call "read_xxx" functions of DataSource to read forcing,flow,attributes data if len(args) == 0: # read flow data_flow = data_source.read_streamflow() data_flow, usgs_id, t_range_list = data_source.screen_streamflow( data_flow) self.data_flow = data_flow # wrap gauges and time range to a dict. # To guarantee the time range is a left-closed and right-open interval, t_range_list[-1] + 1 day self.t_s_dict = OrderedDict( sites_id=usgs_id, t_final_range=[ np.datetime_as_string(t_range_list[0], unit='D'), np.datetime_as_string(t_range_list[-1] + np.timedelta64(1, 'D'), unit='D') ]) # statistics stat_dict = self.cal_stat_all() self.stat_dict = stat_dict else: self.data_flow = args[0] self.t_s_dict = args[1] self.stat_dict = args[2]
def test_get_time_sub_datacube(): start = '1984-03-01' end = '1984-03-03' sub = get_time_sub_datacube(ds, [start, end]) if start != np.datetime_as_string( sub["time"][0], unit='D') or end != np.datetime_as_string( sub["time"][-1], unit='D'): assert False
def getMarks(start, end, df, Nth=30): result = {} for i, date in enumerate(df['date'].unique()): if (i % Nth == 1): # Append value to dict result[np.datetime_as_string( date, unit='D')] = np.datetime_as_string(date, unit='D') return result
def example02(ws): # create TableWorkspace split_table_ws = CreateEmptyTableWorkspace() split_table_ws.addColumn('float', 'start') split_table_ws.addColumn('float', 'stop') split_table_ws.addColumn('str', 'target') #split_table_ws.addRow([0., 100., 'a']) #split_table_ws.addRow([200., 300., 'b']) #split_table_ws.addRow([400., 600., 'c']) #split_table_ws.addRow([600., 650., 'b']) split_table_ws.addRow([0., 33., 'a']) split_table_ws.addRow([34., 66., 'b']) split_table_ws.addRow([67., 99., 'c']) split_table_ws.addRow([100., 650., 'b']) # filter events time2 = time.time() FilterEvents(InputWorkspace=ws, SplitterWorkspace=split_table_ws, OutputWorkspaceBaseName='tempsplitws3', GroupWorkspaces=True, FilterByPulseTime=False, OutputWorkspaceIndexedFrom1=False, CorrectionToSample="None", SpectrumWithoutDetector="Skip", SplitSampleLogs=False, OutputTOFCorrectionWorkspace='mock', RelativeTime=True) time3 = time.time() # Print result wsgroup = mtd["tempsplitws3"] wsnames = wsgroup.getNames() for name in sorted(wsnames): tmpws = mtd[name] print("workspace %s has %d events" % (name, tmpws.getNumberEvents())) split_log = tmpws.run().getProperty('splitter') entry_0 = np.datetime_as_string(split_log.times[0].astype( np.dtype('M8[s]')), timezone='UTC') entry_1 = np.datetime_as_string(split_log.times[1].astype( np.dtype('M8[s]')), timezone='UTC') print( 'event splitter log: entry 0 and entry 1 are {0} and {1}.'.format( entry_0, entry_1)) DeleteWorkspace(tmpws) print("Example 2 Filter time: %.2f" % ((time3 - time2))) DeleteWorkspace(split_table_ws) print(divider) return
def divideByMonth(indices): tempCru = [] tempp = [] for i in range(len(indices) - 1): if int(np.datetime_as_string(indices[i], unit='D')[5:7]) != int( np.datetime_as_string(indices[i + 1], unit='D')[5:7]): tempCru += [indices[i].copy()] tempp += [i] return tempCru, tempp
def plot_skewt_icon(sounding, parcel=None, base=1000, top=100, skew=45): model_time = np.datetime_as_string(sounding.metadata.model_time, unit='m') valid_time = np.datetime_as_string(sounding.metadata.valid_time, unit='m') top_idx = find_closest_model_level(sounding.p * units.Pa, top * units("hPa")) fig = plt.figure(figsize=(11, 11), constrained_layout=True) skew = SkewT(fig, rotation=skew) skew.plot(sounding.p * units.Pa, sounding.T * units.K, 'r') skew.plot(sounding.p * units.Pa, sounding.Td, 'b') skew.plot_barbs(sounding.p[:top_idx] * units.Pa, sounding.U[:top_idx] * units.mps, sounding.V[:top_idx] * units.mps, plot_units=units.knot, alpha=0.6, xloc=1.13, x_clip_radius=0.3) if parcel == "surface-based": prof = mpcalc.parcel_profile(sounding.p * units.Pa, sounding.T[0] * units.K, sounding.Td[0]).to('degC') skew.plot(sounding.p * units.Pa, prof, 'y', linewidth=2) # Add the relevant special lines skew.plot_dry_adiabats() skew.plot_moist_adiabats() skew.plot_mixing_lines() skew.plot(sounding.p * units.Pa, np.zeros(len(sounding.p)) * units.degC, "#03d3fc", linewidth=1) skew.ax.set_ylim(base, top) plt.title(f"Model run: {model_time}Z", loc='left') plt.title(f"Valid time: {valid_time}Z", fontweight='bold', loc='right') plt.xlabel("Temperature [°C]") plt.ylabel("Pressure [hPa]") fig.suptitle(f"ICON-EU Model for {sounding.latitude_pretty}, {sounding.longitude_pretty}", fontsize=14) ax1 = plt.gca() ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis color = '#333333' ax2.set_ylabel('Geometric Altitude [kft]', color=color) # we already handled the x-label with ax1 ax2_data = (sounding.p * units.Pa).to('hPa') ax2.plot(np.zeros(len(ax2_data)), ax2_data, color=color, alpha=0.0) ax2.tick_params(axis='y', labelcolor=color) ax2.set_yscale('log') ax2.set_ylim((base, top)) ticks = np.linspace(base, top, num=10) ideal_ticks = np.geomspace(base, top, 20) real_tick_idxs = [find_closest_model_level(sounding.p * units.Pa, p_level * units("hPa")) for p_level in ideal_ticks] ticks = (sounding.p * units.Pa).to("hPa")[real_tick_idxs] full_levels = [full_level_height(sounding.HHL, idx) for idx in real_tick_idxs] tick_labels = np.around((full_levels * units.m).m_as("kft"), decimals=1) ax2.set_yticks(ticks) ax2.set_yticklabels(tick_labels) ax2.minorticks_off() return fig
def test_generate_sst_datacube(): filePath = fileDirectory + "sst1981-1982.nc" generate_sst_datacube(1981, 1982, fileDirectory, "sst1981-1982") if os.path.isfile(filePath): assert True start = "1981-09-01" end = "1982-12-31" x = xr.open_dataset(filePath) if np.datetime_as_string(x["time"][0], unit='D') == start and np.datetime_as_string( x["time"][-1], unit='D') == end: assert True
def time_since_start(date_time, games_schedule): date = np.datetime_as_string(date_time, unit='D') date = str(date) games_today = games_schedule.loc[games_schedule['date'] == date] d = games_today['date'].values[0] t = games_today['kickoff'].values[0] d_t = np.datetime_as_string(d, unit='D') + 'T' + t delta = date_time - np.datetime64(d_t) delta = delta.astype('timedelta64[m]') delta = delta / np.timedelta64(1, 'm') return delta
def build_discretization_vector(self): """ Set the discretization vector as date strings of 1 minute steps """ dt = numpy.datetime64('2020-01-01') self.discretization_vector = [] for x in self.rkutt_data_vector: numpy.datetime_as_string(dt, unit='h') self.discretization_vector.append(dt) dt += numpy.timedelta64(1, 'h')
def get_dates(data): if ('Date' in data.columns): data = data.set_index('Date') data = data.sort_index() dt1 = data.iloc[1:].index.values[0] dt1 = datetime.strptime(np.datetime_as_string(dt1, unit='s'), '%Y-%m-%dT%H:%M:%S') dt2 = data.iloc[-1:].index.values[0] dt2 = datetime.strptime(np.datetime_as_string(dt2, unit='s'), '%Y-%m-%dT%H:%M:%S') return dt1, dt2
def cargar_cotiz(self, fecha): self.logger.debug('Cargando cotizaciones ultimo cierre.') #lista_aux=[] start = fecha - timedelta(days=5) end = fecha for campo in self.lista: ## Tomo la ultima cotización local. local_ca = 0 try: pd_local_aux = yf.download(campo[1], start=start.strftime('%Y-%m-%d'), end=end.strftime('%Y-%m-%d'), interval="1d") pd_local = pd_local_aux.drop(fecha.date()).tail(1) except KeyError: self.logger.debug("No se pudo borrar el dia de hoy. " + campo[1]) pd_local = pd_local_aux.tail(1) if not pd_local.empty: local_ca = pd_local['Close'].values[0] np_date = pd_local.index.values[0] self.fechaUltimoCierreLocal = numpy.datetime_as_string( np_date, "D") self.logger.debug( self.fechaUltimoCierreLocal + " - " + campo[1] + " C. Loc ULT CIERRE {0:.2f}".format(local_ca)) ## Tomo la ultima cotizacion ADR. adr_ca = 0 try: pd_adr_aux = yf.download(campo[0], start=start.strftime('%Y-%m-%d'), end=end.strftime('%Y-%m-%d'), interval="1d") pd_adr = pd_adr_aux.drop(fecha.date()).tail(1) except KeyError: self.logger.debug("No se pudo borrar el dia de hoy. " + campo[0]) pd_adr = pd_adr_aux.tail(1) if not pd_adr.empty: adr_ca = pd_adr['Close'].values[0] np_date = pd_adr.index.values[0] self.fechaUltimoCierreAdr = numpy.datetime_as_string( np_date, "D") self.logger.debug(self.fechaUltimoCierreAdr + " - " + campo[0] + " C. ADR ULT CIERRE {0:.2f}".format(adr_ca)) ##Con esta linea me traigo el ADR del mismo dia del local #pd_adr = yf.download(campo[0], start=numpy.datetime_as_string(np_date, "D"), interval="1d").head(1) campo[3] = adr_ca campo[4] = local_ca return None
def get_main_data(data): """ """ lat = data['latitude'] lon = data['longitude'] # - 180 date_full = data['time'].values date = np.datetime_as_string(date_full, unit='s').partition('T')[0] time = np.datetime_as_string(date_full, unit='s').partition('T')[2] time = time.replace(":", "")[:-2] return lat, lon, date, time
def date(self, s, max=1000): a = np.datetime64(s) b = a + np.timedelta64(1, 'D') while True: f = self.ic.filterDate(np.datetime_as_string(a), np.datetime_as_string(b)) if f.size().getInfo() > 0: break a = a - np.timedelta64(1, 'D') b = b + np.timedelta64(1, 'D') c = iList(f.toList(max), self) print(c.times) return c
def __correct_starting_scan_index_time(self, runObj, abs_tolerance: float = 0.05 ) -> None: """Correct the DAS-issue for mis-record the first scan_index/sub run before the motor is in position This goes through a subset of logs and compares when they actually get to their specified setpoint, updating the start time for event filtering. When this is done ``self._starttime`` will have been updated. Parameters ---------- start_time: numpy.datetime64 The start time according to the scan_index log abs_tolerance: float When then log is within this absolute tolerance of the setpoint, it is correct Returns ------- float Corrected value or None """ start_time = self.times[0] # loop through the 'special' logs for log_name in ['sx', 'sy', 'sz', '2theta', 'omega', 'chi', 'phi']: if log_name not in runObj: continue # log doesn't exist - not a good one to look at if log_name + 'Setpoint' not in runObj: continue # log doesn't have a setpoint - not a good one to look at if runObj[log_name].size() == 1: continue # there is only one value # get the observed values of the log observed = runObj[log_name].value if observed.std() <= .5 * abs_tolerance: continue # don't bother if the log is constant within half of the tolerance # look for the setpoint and find when the log first got there # only look at first setpoint set_point = runObj[log_name + 'Setpoint'].value[0] for log_time, value in zip(runObj[log_name].times, observed): if abs(value - set_point) < abs_tolerance: # pick the larger of what was found and the previous largest value if log_time > start_time: start_time = log_time break self._log.debug('Shift from start_time {} to {}'.format( np.datetime_as_string(self.times[0]), np.datetime_as_string(start_time))) self.times[0] = start_time
def _compose_out_filename(self, str_beg, str_end): """Compose filename for output data file""" if not str_beg: # str_beg = self._datetime_beg.strftime('%Y-%m-%d_%H:%M') str_beg = np.datetime_as_string(self._datetime_beg, unit='s', timezone=pytz.timezone(self.TZ)) if not str_end: # str_end = self._datetime_end.strftime('%Y-%m-%d_%H:%M') str_end = np.datetime_as_string(self._datetime_end, unit='s', timezone=pytz.timezone(self.TZ)) return str_beg + '_' + str_end
def DBM(indices): tempCru = [] tempp = [] #print(indices) tempCru += [indices[0]] for k in range(len(indices) - 1): if int(np.datetime_as_string(indices[k], unit='D')[5:7]) != int( np.datetime_as_string(indices[k + 1], unit='D')[5:7]): tempCru += [indices[k + 1].copy()] tempp += [k] return tempp, tempCru
def chartSingleSeries(df, countryCode): ylabel = str(mt.inStateName(countryCode)) dates = df['Date'].values sd = np.datetime_as_string(dates[0], unit='D') ed = np.datetime_as_string(dates[dates.size - 1], unit='D') print("#:" + str(df[countryCode].size) + " From:" + sd + " to: " + ed) print("std dev:" + str(df[countryCode].std())) df.plot(x='Date', y=[countryCode], figsize=(16, 9), grid=True, xlabel="Date", ylabel=ylabel) plt.show()
def __repr__(self): start = np.datetime_as_string(self.coords['time'].values[0], unit='D') end = np.datetime_as_string(self.coords['time'].values[-1], unit='D') return ('<Cutout "{}">\n' ' x = {:.2f} ⟷ {:.2f}, dx = {:.2f}\n' ' y = {:.2f} ⟷ {:.2f}, dy = {:.2f}\n' ' time = {} ⟷ {}, dt = {}\n' ' module = {}\n' ' prepared_features = {}'.format( self.name, self.coords['x'].values[0], self.coords['x'].values[-1], self.dx, self.coords['y'].values[0], self.coords['y'].values[-1], self.dy, start, end, self.dt, self.module, list(self.prepared_features.index.unique('feature'))))
def test_get_dates_mutiple_tickers_multiple_fields(self, mock): mock.open.return_value = BytesIO( str.encode( dedent(""" START-OF-FILE ... START-OF-FIELDS FUT_NOTICE_FIRST LAST_TRADEABLE_DT END-OF-FIELDS ... START-OF-DATA SECURITIES|ERROR CODE|NUM FLDS|FUT_NOTICE_FIRST|LAST_TRADEABLE_DT| CTH4 Comdty|0|2|20240223|20240306| CTK4 Comdty|0|2|N.S.|N.S.| RTYU1 Index|0|2|20210917|20210917| RTYZ1 Index|0|2|20211217|20211217| RTYU2 Index|0|2|20220916|20220916| END-OF-DATA ... END-OF-FILE """))) parser = BloombergBeapHapiParser() df = parser.get_current_values_dates_fields_format(Mock()) self.assertEqual(type(df), QFDataFrame) self.assertEqual(df.shape, (5, 2)) self.assertTrue(len(df)) expected_tickers_str_list = [ 'CTH4 Comdty', 'CTK4 Comdty', 'RTYU1 Index', 'RTYZ1 Index', 'RTYU2 Index' ] self.assertCountEqual(df.index.tolist(), expected_tickers_str_list) expected_fields_list = ['FUT_NOTICE_FIRST', 'LAST_TRADEABLE_DT'] self.assertCountEqual(df.columns, expected_fields_list) # Compare string values of dates (to simplify the numpy.datetime64 comparison) expected_datetime_strings = [ ['2024-02-23', '2024-03-06'], ["NaT", "NaT"], ['2021-09-17', '2021-09-17'], ['2021-12-17', '2021-12-17'], ['2022-09-16', '2022-09-16'] ] # For datetime64[ns] types, NaT represents missing values. actual_datetime_strings = [[ datetime_as_string(datetime64(dt[0]), unit='D'), datetime_as_string(datetime64(dt[1]), unit='D') ] for dt in df.values] self.assertCountEqual(actual_datetime_strings, expected_datetime_strings)
def plot_title(init, valid, fhour, field_name, model_name="", field_units=""): try: init = np.datetime64(init.values) valid = np.datetime64(valid.values) except: init = np.datetime64(init) valid = np.datetime64(valid) init = np.datetime_as_string(init, unit='h', timezone='UTC') valid = np.datetime_as_string(valid, unit='h', timezone='UTC') fhour = str(fhour).zfill(2) return f"{model_name} Init: {init} Valid: {valid} {field_name} ({field_units}) Hour: {fhour}"
def default(self, obj): if isinstance(obj, np.integer): return int(obj) elif isinstance(obj, np.floating): return float(obj) elif isinstance(obj, np.ndarray): return obj.tolist() elif isinstance(obj, np.bool_): return bool(obj) elif isinstance(obj, np.datetime64): return np.datetime_as_string(obj, unit='s') elif isinstance(obj, pd.Timestamp): return np.datetime_as_string(obj.to_datetime64(), unit='s') else: return super(NpEncoder, self).default(obj)
def _hdf_array(channel, data): """ Convert data array into a format suitable for initialising HDF data """ if channel.data_type is types.TimeStamp: string_data = np.datetime_as_string(data, unit='us', timezone='UTC') return [s.encode('ascii') for s in string_data] return data
def summary_diff_dtype(x): if x.dtype.name in ['object', 'bool', 'category' ] and len(x.unique()) <= max_lev: vc = x.value_counts(dropna=False, normalize=True) s = '' for name, v in zip(vc.index, vc.values): s += f'{name} {v*100:>2.0f}%' s += '<br>' if in_cell_next_line else ', ' return s[:-2] elif x.dtype.name in ['float64', 'int64']: o = f'quantiles: {x.quantile(q=[0, 0.25, 0.5, 0.75, 1]).values.tolist()}{in_cell_next} \ mean: {x.mean():.2f}\ std: {x.std():.2f} \ cv: {x.std()/x.mean():.2f}{in_cell_next}\ skew: {skew(x[x.notnull()]):.2f}' if sum(x.notnull()) > 8: # requirement of skewtest p = skewtest(x[x.notnull()]).pvalue o += f'*' if p <= 0.05 else '' if min(x[x != 0]) > 0 and len(x[x != 0]) > 8: # take log o += f'{in_cell_next}log skew: {skew(np.log(x[x>0])):.2f}' p = skewtest(np.log(x[x != 0])).pvalue o += f'*' if p != p and p <= 0.05 else '' return o elif 'datetime' in x.dtype.name: # o = '' qs = x.quantile(q=[0, 0.25, 0.5, 0.75, 1]).values return print_list([np.datetime_as_string(q)[0:16] for q in qs], br=in_cell_next) else: return ''
def ThirtyOne(year): d1 = {} for i in s5: temp = SS.StockDataFrame.retype( yf.download(i, period='3mo', interval='1d', auto_adjust=False)) indices = list(temp.index.values) print(indices) temp2 = pd.DataFrame() crucialDate, ida = divideByMonth(indices) print(ida) for k in ida: print(temp.iloc[k]) #print(crucialDate) dataList = temp.loc[crucialDate] dataList.reset_index(drop=True, inplace=True) newDates = [] for j in crucialDate: newDates += [ np.datetime64(np.datetime_as_string(j, unit='D')[:-2] + '01') ] dataList['Date'] = newDates.copy() dataList.set_index('Date', inplace=True, drop=True) d1[i] = dataList return d1
def datetime64_str(time_str: Optional[str] = None) -> np.ndarray: """ Reformat time_str to ISO 8601 or to 'NaT'. Used here for input in funcs that converts str to numpy.datetime64 :param time_str: May be 'NaT' :return: ndarray of strings (tested for 1 element only) formatted by numpy. """ return np.datetime_as_string(np.datetime64(time_str, 's'))
def format_dt(dt): return str( np.datetime_as_string( arr=np.datetime64(dt.strftime("%Y-%m-%d %H:%M:%S.%f")), timezone="UTC", unit="us", ))
def dt64_to_str(val): try: return numpy.datetime_as_string(val, unit='D') except TypeError as err: print(type(val), val) print(err) return str(val)
def _get_metadata(self, quantity): date, time = np.datetime_as_string(self.start_datetime).split('T') return {'General': {'original_filename': os.path.split(self.filename)[1], 'title': '%s (%s)' % (quantity, self._parse_quantity_units(quantity)), 'date': date, 'time': time}, "Signal": {'signal_type': '', 'quantity': self._parse_quantity(quantity)}}
def _test_date_time(self, dt_str='now'): dt0 = np.datetime64(dt_str) data_str, time_str = np.datetime_as_string(dt0).split('T') self.tree.add_node("General") self.tree.General.date = data_str self.tree.General.time = time_str dt1 = np.datetime64('%sT%s' % (self.tree.General.date, self.tree.General.time)) np.testing.assert_equal(dt0, dt1) return dt1
def __init__(self, conf): ## TODO get the configuration values propogated to here GENERAL, SPECIALFILES = conf['GENERAL'], conf['SPECIALFILES'] flnm_tmplt = conf['FILETEMPLATES'] flnm_tags = conf['FILETAGS'] self.date_timestamp = np.datetime_as_string(np.datetime64('today', 'D')) self.numeric_timestamp = np.datetime64('now' ,'m').astype(int )-np.datetime64('2018-08-01T00:00' ,'m').astype(int) self.directory = DirectoryManager(conf) self.maskname = GENERAL['mask_name'] self.mtl_name = SPECIALFILES['mtl'] ## Setup Defaults self.current_read_template = None self.current_write_template = None self.calibration_template = flnm_tmplt['calibration'] self.default_calibration_template = flnm_tmplt['default_calibration'] self.pickled_datadump_name =flnm_tmplt['pickled_datadump'] self.lampline_template = flnm_tmplt['lampline'] self.redshift_fit_template = flnm_tmplt['redshift_fits'] self.save_plot_template = flnm_tmplt['save_plots'] self.tempname_dict = { 'bias': {'read': flnm_tmplt['raw'], 'write': flnm_tmplt['debiased']}, 'stitch': {'read': flnm_tmplt['debiased'], 'write': flnm_tmplt['stitched']},\ 'remove_crs': {'read': flnm_tmplt['stitched'], 'write': flnm_tmplt['stitched']}, \ 'apcut': {'read': flnm_tmplt['stitched'], 'write': flnm_tmplt['oneds']}, \ 'wavecalib': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['oneds']}, \ 'flatten': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['oneds']}, \ 'skysub': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['oneds']}, \ 'combine': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['combined']}, \ 'zfit': {'read': flnm_tmplt['combined'], 'write': flnm_tmplt['combined']} \ } self.tags_dict = { 'bias': {'read': '', 'write': flnm_tags['debiased']}, \ 'stitch': {'read':flnm_tags['debiased'], 'write': flnm_tags['debiased']},\ 'remove_crs': {'read':flnm_tags['debiased'], 'write': flnm_tags['crrmvd']},\ 'apcut': {'read':flnm_tags['crrmvd'], 'write': flnm_tags['crrmvd']}, \ 'wavecalib': {'read':flnm_tags['crrmvd'], 'write': flnm_tags['crrmvd']},\ 'flatten': {'read':flnm_tags['crrmvd'], 'write': flnm_tags['flatnd']}, \ 'skysub': {'read':flnm_tags['flatnd'], 'write': flnm_tags['skysubd']}, \ 'combine': {'read':flnm_tags['skysubd'], 'write': flnm_tags['skysubd']},\ 'zfit': {'read':flnm_tags['skysubd'], 'write': flnm_tags['skysubd']}\ } self.step = 'bias' self.update_templates_for()
def test_deltas_year(): start_list = pd.to_datetime(['2014-01-01', '2016-01-01']) start_array = np.array(start_list) new_time_1 = add_td(start_list, 2, 'Y') new_time_2 = add_td(start_array, 2, 'Y') values = [2016, 2018] for i, value in enumerate(values): assert new_time_1.dt.year.values[i] == value assert np.datetime_as_string(new_time_2[i])[:4] == str(value) error_text = 'Invalid Unit' with pytest.raises(ValueError, match=error_text) as excinfo: add_td(start_list, 2, 'M') assert 'Invalid Unit' in str(excinfo)
def _ar_prediction(self, data_frame, freq): """ Generate a prediction using an Autoregressive Model :return: A dataframe with predicted usage """ resampled_frame = data_frame.resample(freq, how='mean', closed='right') model = tsa.AR(resampled_frame, freq=freq) result = model.fit(maxlag=5, method='mle', transparams=True, disp=-1) # Currently the forecast requires 6 hours of readings with no NaN or inf values # So far it seems reasonably accurate to predict the usage for the next hour, but we can play with this end = to_datetime(datetime_as_string(resampled_frame.index.values[len(resampled_frame)-1])) # end of frame start_pred = str(end - relativedelta(hours=5)) # we can play around with this number a bit end_pred = str(end + relativedelta(hours=1)) # you can try increasing this if you want to return result.predict(start=start_pred, end=end_pred)
def update_date_time_in_metadata(dt, metadata): """ Update the date and time in a metadata tree. Parameters ---------- dt : date and time information: it can be a ISO 8601 string, a datetime.datetime or a numpy.datetime64 object metadata : metadata object to update Return ---------- metadata object Example ------- >>> s = hs.load("example1.msa") >>> dt = '2016-12-12T12:12:12-05:00' >>> s.metadata = update_date_time_in_metadata(dt, s.metadata) >>> s.metadata ├── General │ ├── date = 2016-12-12 │ ├── original_filename = example1.msa │ ├── time = 12:12:12 │ ├── time_zone = 'EST' │ └── title = NIO EELS OK SHELL """ time_zone = None if isinstance(dt, str): dt = parser.parse(dt) if isinstance(dt, np.datetime64): dt_split = np.datetime_as_string(dt).split('T') date = dt_split[0] time = dt_split[1] if isinstance(dt, datetime.datetime): date = dt.date().isoformat() time = dt.time().isoformat() if dt.tzname(): time_zone = dt.tzname() elif dt.tzinfo: time_zone = dt.isoformat()[-6:] metadata.set_item('General.date', date) metadata.set_item('General.time', time) if time_zone: metadata.set_item('General.time_zone', time_zone) elif metadata.has_item('General.time_zone'): del metadata.General.time_zone return metadata
def datetime_gas_cell(): dt_np = np.datetime64("2014-12-15T19:07:04.165000") dt_str = np.datetime_as_string(dt_np) date, time = dt_str.split("T") return date, time, dt_np
## CREATE DATETIME ARRAY ind = np.arange(0,ndays,dayint) #ind = np.arange(15,365,30) base = datetime(2010,1,1,0) arr = np.array([base + timedelta(days=i) for i in range(ndays)]) #arr = np.array([base + timedelta(days=i) for i in range(365)]) date_arr = [] for i in range(len(ind)): date_arr.append(pd.date_range(str(arr[ind][i]),freq='H',periods=24)) date_arr = np.datetime_as_string(date_arr,timezone='UTC') date_arr = date_arr.flatten().astype('datetime64') dates_pd = pd.DatetimeIndex(date_arr) ## WRITE TO NETCDF FILE dataset = netCDF4.Dataset('%spar%s_scope_out_diurnal.nc' % (path_to_output,param), 'w', format='NETCDF4_CLASSIC') # Create Dimensions dataset.createDimension('time',None) dataset.createDimension('vegp',nvp) # Create Variables nc_times = dataset.createVariable('time', np.float64, ('time',)) nc_vegp = dataset.createVariable('vegp', np.int32, ('vegp',)) nc_gridp = dataset.createVariable('gridp', np.int32, ('vegp',))
TsliceB=None data=sylv_flux_read(flptr) flptr.close() datax=record2xray(data) # get the last data point and figure out the time slice. # can add additional data variables to check if this does not work for i in range(len(datax)-1,1,-1): if not (np.isnan(datax[i].sel(varname='UST')) and np.isnan(datax[i].sel(varname='FC'))): TEnd=datax.time[i] TBegin= TEnd - np.timedelta64(10,'D') TsliceE=np.datetime_as_string(TEnd) TsliceB=np.datetime_as_string(TBegin) break # if you can't find a point just plot the whole year. if TsliceE==None and TsliceB==None: TsliceE=today.strftime('%Y-%m-%d') TsliceB=today.strftime('%Y-01-01') for var in plotvars: fig,ax=plt.subplots() datax.sel(varname=var, time=slice(TsliceB, TsliceE)).plot(marker='.') ax.xaxis.set_major_formatter(DateFormatter("%m/%d-%H")) plt.savefig(figdir + var + '.png', dpi=100) plt.close()
def get_datetime(dt): dt_np = np.datetime64(dt) dt_str = np.datetime_as_string(dt_np) date, time = dt_str.split('T') return date, time, dt_np