def plot_timeseries(cfg, metadata):
"""Plot timeseries data."""
short_name = 'pr'
xaxis = 'time'
datasets = read_input_data(metadata)
ancestors = [info['filename'] for info in metadata]
time_period = cfg['time_period']
var = datasets[short_name]
time_unit = time_period[0].upper()
start_date = np.datetime_as_string(datasets.time.min(), unit=time_unit)
end_date = np.datetime_as_string(datasets.time.max(), unit=time_unit)
name = f'{var.long_name}_{time_period}'
caption = f"{var.long_name} per {time_period} for {start_date}:{end_date}"
plot_data(
cfg=cfg,
datasets=datasets,
xaxis=xaxis,
yaxis=short_name,
xlabel=f'{xaxis.capitalize()} / {time_period}',
ylabel=f'{var.long_name} / {var.units}',
caption=caption,
name=name,
ancestors=ancestors,
)
filename_data = get_diagnostic_filename(name, cfg, extension='nc')
datasets.to_netcdf(filename_data)
log_provenance(caption, filename_data, cfg, ancestors)
def onselect(self, xmin, xmax):
# print(xmin, xmax)
indmin, indmax = 0, 0
if self.use_dates:
xmin, xmax = xmin//1, xmax//1
indmin, indmax = self.get_index_from_datetime(xmin, xmax)
else:
indmin, indmax = np.searchsorted(self.x, (xmin, xmax))
indmax = min(len(self.x) - 1, indmax)
thisx = self.x[indmin:indmax]
thisy = self.y[indmin:indmax]
poly_collection = ax1.fill_between(thisx, self.y_upper_lim, self.y_lower_lim, color="r", alpha=0.25)
self.line2.set_data(thisx, thisy)
ax2.set_xlim(thisx[0], thisx[-1])
ax2.set_ylim(thisy.min(), thisy.max())
begin_str, end_str = "", ""
if self.use_dates:
# Formatting Date String from ISO to German Notation
begin_str = re.sub(':', ' ', np.datetime_as_string(thisx[0], unit='s'))
end_str = re.sub(':', ' ', np.datetime_as_string(thisx[-1], unit='s'))
begin_str = dt.strptime(begin_str, '%Y-%m-%dT%H %M %S').strftime('%d.%m.%Y, %H:%M:%S')
end_str = dt.strptime(end_str, '%Y-%m-%dT%H %M %S').strftime('%d.%m.%Y, %H:%M:%S')
else:
begin_str = str(indmin)
end_str = str(indmax)
list_item = QtWidgets.QListWidgetItem(begin_str + " - " + end_str)
list_item.setData(1, poly_collection)
self.list_widget.addItem(list_item)
self.canvas.draw()
def plot_E0_data(self, df):
title = np.datetime_as_string(df.index.values[0], unit='D') + " to "
title += np.datetime_as_string(df.index.values[-1], unit='D')
file_name = os.path.join(
"plots", "estimate_e0_" + title.replace(" ", "_") + ".png")
current_backend = plt.get_backend()
plt.switch_backend("agg")
plt.ioff()
fig, axs = plt.subplots()
if "Sws" in list(self.df):
ldf = self.df[self.df.index.isin(df.index)]
sc = axs.scatter(df.TC.values,
df.ER.values,
c=ldf.Sws.values,
s=10)
else:
sc = axs.scatter(df.TC.values, df.ER.values, s=10)
axs.set_title(title)
axs.set_xlabel("Temperature (degC)")
axs.set_ylabel("ER (umol/m^2/s)")
clb = plt.colorbar(sc)
clb.ax.set_title("Sws")
fig.savefig(file_name, format="png")
plt.close(fig)
plt.switch_backend(current_backend)
plt.ion()
return
def __init__(self, data_source, *args):
""":parameter data_source: DataSource object"""
self.data_source = data_source
# call "read_xxx" functions of DataSource to read forcing，flow，attributes data
if len(args) == 0:
# read flow
data_flow = data_source.read_streamflow()
data_flow, usgs_id, t_range_list = data_source.screen_streamflow(
data_flow)
self.data_flow = data_flow
# wrap gauges and time range to a dict.
# To guarantee the time range is a left-closed and right-open interval, t_range_list[-1] + 1 day
self.t_s_dict = OrderedDict(
sites_id=usgs_id,
t_final_range=[
np.datetime_as_string(t_range_list[0], unit='D'),
np.datetime_as_string(t_range_list[-1] +
np.timedelta64(1, 'D'),
unit='D')
])
# statistics
stat_dict = self.cal_stat_all()
self.stat_dict = stat_dict
else:
self.data_flow = args[0]
self.t_s_dict = args[1]
self.stat_dict = args[2]
def test_get_time_sub_datacube():
start = '1984-03-01'
end = '1984-03-03'
sub = get_time_sub_datacube(ds, [start, end])
if start != np.datetime_as_string(
sub["time"][0], unit='D') or end != np.datetime_as_string(
sub["time"][-1], unit='D'):
assert False
def getMarks(start, end, df, Nth=30):
result = {}
for i, date in enumerate(df['date'].unique()):
if (i % Nth == 1):
# Append value to dict
result[np.datetime_as_string(
date, unit='D')] = np.datetime_as_string(date, unit='D')
return result
def example02(ws):
# create TableWorkspace
split_table_ws = CreateEmptyTableWorkspace()
split_table_ws.addColumn('float', 'start')
split_table_ws.addColumn('float', 'stop')
split_table_ws.addColumn('str', 'target')
#split_table_ws.addRow([0., 100., 'a'])
#split_table_ws.addRow([200., 300., 'b'])
#split_table_ws.addRow([400., 600., 'c'])
#split_table_ws.addRow([600., 650., 'b'])
split_table_ws.addRow([0., 33., 'a'])
split_table_ws.addRow([34., 66., 'b'])
split_table_ws.addRow([67., 99., 'c'])
split_table_ws.addRow([100., 650., 'b'])
# filter events
time2 = time.time()
FilterEvents(InputWorkspace=ws,
SplitterWorkspace=split_table_ws,
OutputWorkspaceBaseName='tempsplitws3',
GroupWorkspaces=True,
FilterByPulseTime=False,
OutputWorkspaceIndexedFrom1=False,
CorrectionToSample="None",
SpectrumWithoutDetector="Skip",
SplitSampleLogs=False,
OutputTOFCorrectionWorkspace='mock',
RelativeTime=True)
time3 = time.time()
# Print result
wsgroup = mtd["tempsplitws3"]
wsnames = wsgroup.getNames()
for name in sorted(wsnames):
tmpws = mtd[name]
print("workspace %s has %d events" % (name, tmpws.getNumberEvents()))
split_log = tmpws.run().getProperty('splitter')
entry_0 = np.datetime_as_string(split_log.times[0].astype(
np.dtype('M8[s]')),
timezone='UTC')
entry_1 = np.datetime_as_string(split_log.times[1].astype(
np.dtype('M8[s]')),
timezone='UTC')
print(
'event splitter log: entry 0 and entry 1 are {0} and {1}.'.format(
entry_0, entry_1))
DeleteWorkspace(tmpws)
print("Example 2 Filter time: %.2f" % ((time3 - time2)))
DeleteWorkspace(split_table_ws)
print(divider)
return
def divideByMonth(indices):
tempCru = []
tempp = []
for i in range(len(indices) - 1):
if int(np.datetime_as_string(indices[i], unit='D')[5:7]) != int(
np.datetime_as_string(indices[i + 1], unit='D')[5:7]):
tempCru += [indices[i].copy()]
tempp += [i]
return tempCru, tempp
def plot_skewt_icon(sounding, parcel=None, base=1000, top=100, skew=45):
model_time = np.datetime_as_string(sounding.metadata.model_time, unit='m')
valid_time = np.datetime_as_string(sounding.metadata.valid_time, unit='m')
top_idx = find_closest_model_level(sounding.p * units.Pa, top * units("hPa"))
fig = plt.figure(figsize=(11, 11), constrained_layout=True)
skew = SkewT(fig, rotation=skew)
skew.plot(sounding.p * units.Pa, sounding.T * units.K, 'r')
skew.plot(sounding.p * units.Pa, sounding.Td, 'b')
skew.plot_barbs(sounding.p[:top_idx] * units.Pa, sounding.U[:top_idx] * units.mps,
sounding.V[:top_idx] * units.mps, plot_units=units.knot, alpha=0.6, xloc=1.13, x_clip_radius=0.3)
if parcel == "surface-based":
prof = mpcalc.parcel_profile(sounding.p * units.Pa, sounding.T[0] * units.K, sounding.Td[0]).to('degC')
skew.plot(sounding.p * units.Pa, prof, 'y', linewidth=2)
# Add the relevant special lines
skew.plot_dry_adiabats()
skew.plot_moist_adiabats()
skew.plot_mixing_lines()
skew.plot(sounding.p * units.Pa, np.zeros(len(sounding.p)) * units.degC, "#03d3fc", linewidth=1)
skew.ax.set_ylim(base, top)
plt.title(f"Model run: {model_time}Z", loc='left')
plt.title(f"Valid time: {valid_time}Z", fontweight='bold', loc='right')
plt.xlabel("Temperature [°C]")
plt.ylabel("Pressure [hPa]")
fig.suptitle(f"ICON-EU Model for {sounding.latitude_pretty}, {sounding.longitude_pretty}", fontsize=14)
ax1 = plt.gca()
ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis
color = '#333333'
ax2.set_ylabel('Geometric Altitude [kft]', color=color) # we already handled the x-label with ax1
ax2_data = (sounding.p * units.Pa).to('hPa')
ax2.plot(np.zeros(len(ax2_data)), ax2_data, color=color, alpha=0.0)
ax2.tick_params(axis='y', labelcolor=color)
ax2.set_yscale('log')
ax2.set_ylim((base, top))
ticks = np.linspace(base, top, num=10)
ideal_ticks = np.geomspace(base, top, 20)
real_tick_idxs = [find_closest_model_level(sounding.p * units.Pa, p_level * units("hPa")) for p_level in
ideal_ticks]
ticks = (sounding.p * units.Pa).to("hPa")[real_tick_idxs]
full_levels = [full_level_height(sounding.HHL, idx) for idx in real_tick_idxs]
tick_labels = np.around((full_levels * units.m).m_as("kft"), decimals=1)
ax2.set_yticks(ticks)
ax2.set_yticklabels(tick_labels)
ax2.minorticks_off()
return fig
def test_generate_sst_datacube():
filePath = fileDirectory + "sst1981-1982.nc"
generate_sst_datacube(1981, 1982, fileDirectory, "sst1981-1982")
if os.path.isfile(filePath): assert True
start = "1981-09-01"
end = "1982-12-31"
x = xr.open_dataset(filePath)
if np.datetime_as_string(x["time"][0],
unit='D') == start and np.datetime_as_string(
x["time"][-1], unit='D') == end:
assert True
def time_since_start(date_time, games_schedule):
date = np.datetime_as_string(date_time, unit='D')
date = str(date)
games_today = games_schedule.loc[games_schedule['date'] == date]
d = games_today['date'].values[0]
t = games_today['kickoff'].values[0]
d_t = np.datetime_as_string(d, unit='D') + 'T' + t
delta = date_time - np.datetime64(d_t)
delta = delta.astype('timedelta64[m]')
delta = delta / np.timedelta64(1, 'm')
return delta
def build_discretization_vector(self):
""" Set the discretization vector as date strings of 1 minute steps
"""
dt = numpy.datetime64('2020-01-01')
self.discretization_vector = []
for x in self.rkutt_data_vector:
numpy.datetime_as_string(dt, unit='h')
self.discretization_vector.append(dt)
dt += numpy.timedelta64(1, 'h')
def get_dates(data):
if ('Date' in data.columns):
data = data.set_index('Date')
data = data.sort_index()
dt1 = data.iloc[1:].index.values[0]
dt1 = datetime.strptime(np.datetime_as_string(dt1, unit='s'),
'%Y-%m-%dT%H:%M:%S')
dt2 = data.iloc[-1:].index.values[0]
dt2 = datetime.strptime(np.datetime_as_string(dt2, unit='s'),
'%Y-%m-%dT%H:%M:%S')
return dt1, dt2
def cargar_cotiz(self, fecha):
self.logger.debug('Cargando cotizaciones ultimo cierre.')
#lista_aux=[]
start = fecha - timedelta(days=5)
end = fecha
for campo in self.lista:
## Tomo la ultima cotización local.
local_ca = 0
try:
pd_local_aux = yf.download(campo[1],
start=start.strftime('%Y-%m-%d'),
end=end.strftime('%Y-%m-%d'),
interval="1d")
pd_local = pd_local_aux.drop(fecha.date()).tail(1)
except KeyError:
self.logger.debug("No se pudo borrar el dia de hoy. " +
campo[1])
pd_local = pd_local_aux.tail(1)
if not pd_local.empty:
local_ca = pd_local['Close'].values[0]
np_date = pd_local.index.values[0]
self.fechaUltimoCierreLocal = numpy.datetime_as_string(
np_date, "D")
self.logger.debug(
self.fechaUltimoCierreLocal + " - " + campo[1] +
" C. Loc ULT CIERRE {0:.2f}".format(local_ca))
## Tomo la ultima cotizacion ADR.
adr_ca = 0
try:
pd_adr_aux = yf.download(campo[0],
start=start.strftime('%Y-%m-%d'),
end=end.strftime('%Y-%m-%d'),
interval="1d")
pd_adr = pd_adr_aux.drop(fecha.date()).tail(1)
except KeyError:
self.logger.debug("No se pudo borrar el dia de hoy. " +
campo[0])
pd_adr = pd_adr_aux.tail(1)
if not pd_adr.empty:
adr_ca = pd_adr['Close'].values[0]
np_date = pd_adr.index.values[0]
self.fechaUltimoCierreAdr = numpy.datetime_as_string(
np_date, "D")
self.logger.debug(self.fechaUltimoCierreAdr + " - " +
campo[0] +
" C. ADR ULT CIERRE {0:.2f}".format(adr_ca))
##Con esta linea me traigo el ADR del mismo dia del local
#pd_adr = yf.download(campo[0], start=numpy.datetime_as_string(np_date, "D"), interval="1d").head(1)
campo[3] = adr_ca
campo[4] = local_ca
return None
def get_main_data(data):
"""
"""
lat = data['latitude']
lon = data['longitude'] # - 180
date_full = data['time'].values
date = np.datetime_as_string(date_full, unit='s').partition('T')[0]
time = np.datetime_as_string(date_full, unit='s').partition('T')[2]
time = time.replace(":", "")[:-2]
return lat, lon, date, time
def date(self, s, max=1000):
a = np.datetime64(s)
b = a + np.timedelta64(1, 'D')
while True:
f = self.ic.filterDate(np.datetime_as_string(a), np.datetime_as_string(b))
if f.size().getInfo() > 0:
break
a = a - np.timedelta64(1, 'D')
b = b + np.timedelta64(1, 'D')
c = iList(f.toList(max), self)
print(c.times)
return c
def __correct_starting_scan_index_time(self,
runObj,
abs_tolerance: float = 0.05
) -> None:
"""Correct the DAS-issue for mis-record the first scan_index/sub run before the motor is in position
This goes through a subset of logs and compares when they actually
get to their specified setpoint, updating the start time for
event filtering. When this is done ``self._starttime`` will have been updated.
Parameters
----------
start_time: numpy.datetime64
The start time according to the scan_index log
abs_tolerance: float
When then log is within this absolute tolerance of the setpoint, it is correct
Returns
-------
float
Corrected value or None
"""
start_time = self.times[0]
# loop through the 'special' logs
for log_name in ['sx', 'sy', 'sz', '2theta', 'omega', 'chi', 'phi']:
if log_name not in runObj:
continue # log doesn't exist - not a good one to look at
if log_name + 'Setpoint' not in runObj:
continue # log doesn't have a setpoint - not a good one to look at
if runObj[log_name].size() == 1:
continue # there is only one value
# get the observed values of the log
observed = runObj[log_name].value
if observed.std() <= .5 * abs_tolerance:
continue # don't bother if the log is constant within half of the tolerance
# look for the setpoint and find when the log first got there
# only look at first setpoint
set_point = runObj[log_name + 'Setpoint'].value[0]
for log_time, value in zip(runObj[log_name].times, observed):
if abs(value - set_point) < abs_tolerance:
# pick the larger of what was found and the previous largest value
if log_time > start_time:
start_time = log_time
break
self._log.debug('Shift from start_time {} to {}'.format(
np.datetime_as_string(self.times[0]),
np.datetime_as_string(start_time)))
self.times[0] = start_time
def _compose_out_filename(self, str_beg, str_end):
"""Compose filename for output data file"""
if not str_beg:
# str_beg = self._datetime_beg.strftime('%Y-%m-%d_%H:%M')
str_beg = np.datetime_as_string(self._datetime_beg,
unit='s',
timezone=pytz.timezone(self.TZ))
if not str_end:
# str_end = self._datetime_end.strftime('%Y-%m-%d_%H:%M')
str_end = np.datetime_as_string(self._datetime_end,
unit='s',
timezone=pytz.timezone(self.TZ))
return str_beg + '_' + str_end
def DBM(indices):
tempCru = []
tempp = []
#print(indices)
tempCru += [indices[0]]
for k in range(len(indices) - 1):
if int(np.datetime_as_string(indices[k], unit='D')[5:7]) != int(
np.datetime_as_string(indices[k + 1], unit='D')[5:7]):
tempCru += [indices[k + 1].copy()]
tempp += [k]
return tempp, tempCru
def chartSingleSeries(df, countryCode):
ylabel = str(mt.inStateName(countryCode))
dates = df['Date'].values
sd = np.datetime_as_string(dates[0], unit='D')
ed = np.datetime_as_string(dates[dates.size - 1], unit='D')
print("#:" + str(df[countryCode].size) + " From:" + sd + " to: " + ed)
print("std dev:" + str(df[countryCode].std()))
df.plot(x='Date',
y=[countryCode],
figsize=(16, 9),
grid=True,
xlabel="Date",
ylabel=ylabel)
plt.show()
def __repr__(self):
start = np.datetime_as_string(self.coords['time'].values[0], unit='D')
end = np.datetime_as_string(self.coords['time'].values[-1], unit='D')
return ('<Cutout "{}">\n'
' x = {:.2f} ⟷ {:.2f}, dx = {:.2f}\n'
' y = {:.2f} ⟷ {:.2f}, dy = {:.2f}\n'
' time = {} ⟷ {}, dt = {}\n'
' module = {}\n'
' prepared_features = {}'.format(
self.name, self.coords['x'].values[0],
self.coords['x'].values[-1], self.dx,
self.coords['y'].values[0], self.coords['y'].values[-1],
self.dy, start, end, self.dt, self.module,
list(self.prepared_features.index.unique('feature'))))
def test_get_dates_mutiple_tickers_multiple_fields(self, mock):
mock.open.return_value = BytesIO(
str.encode(
dedent("""
START-OF-FILE
...
START-OF-FIELDS
FUT_NOTICE_FIRST
LAST_TRADEABLE_DT
END-OF-FIELDS
...
START-OF-DATA
SECURITIES|ERROR CODE|NUM FLDS|FUT_NOTICE_FIRST|LAST_TRADEABLE_DT|
CTH4 Comdty|0|2|20240223|20240306|
CTK4 Comdty|0|2|N.S.|N.S.|
RTYU1 Index|0|2|20210917|20210917|
RTYZ1 Index|0|2|20211217|20211217|
RTYU2 Index|0|2|20220916|20220916|
END-OF-DATA
...
END-OF-FILE
""")))
parser = BloombergBeapHapiParser()
df = parser.get_current_values_dates_fields_format(Mock())
self.assertEqual(type(df), QFDataFrame)
self.assertEqual(df.shape, (5, 2))
self.assertTrue(len(df))
expected_tickers_str_list = [
'CTH4 Comdty', 'CTK4 Comdty', 'RTYU1 Index', 'RTYZ1 Index',
'RTYU2 Index'
]
self.assertCountEqual(df.index.tolist(), expected_tickers_str_list)
expected_fields_list = ['FUT_NOTICE_FIRST', 'LAST_TRADEABLE_DT']
self.assertCountEqual(df.columns, expected_fields_list)
# Compare string values of dates (to simplify the numpy.datetime64 comparison)
expected_datetime_strings = [
['2024-02-23', '2024-03-06'], ["NaT", "NaT"],
['2021-09-17', '2021-09-17'], ['2021-12-17', '2021-12-17'],
['2022-09-16', '2022-09-16']
] # For datetime64[ns] types, NaT represents missing values.
actual_datetime_strings = [[
datetime_as_string(datetime64(dt[0]), unit='D'),
datetime_as_string(datetime64(dt[1]), unit='D')
] for dt in df.values]
self.assertCountEqual(actual_datetime_strings,
expected_datetime_strings)
def plot_title(init, valid, fhour, field_name, model_name="", field_units=""):
try:
init = np.datetime64(init.values)
valid = np.datetime64(valid.values)
except:
init = np.datetime64(init)
valid = np.datetime64(valid)
init = np.datetime_as_string(init, unit='h', timezone='UTC')
valid = np.datetime_as_string(valid, unit='h', timezone='UTC')
fhour = str(fhour).zfill(2)
return f"{model_name} Init: {init} Valid: {valid} {field_name} ({field_units}) Hour: {fhour}"
def default(self, obj):
if isinstance(obj, np.integer):
return int(obj)
elif isinstance(obj, np.floating):
return float(obj)
elif isinstance(obj, np.ndarray):
return obj.tolist()
elif isinstance(obj, np.bool_):
return bool(obj)
elif isinstance(obj, np.datetime64):
return np.datetime_as_string(obj, unit='s')
elif isinstance(obj, pd.Timestamp):
return np.datetime_as_string(obj.to_datetime64(), unit='s')
else:
return super(NpEncoder, self).default(obj)
def _hdf_array(channel, data):
""" Convert data array into a format suitable for initialising HDF data
"""
if channel.data_type is types.TimeStamp:
string_data = np.datetime_as_string(data, unit='us', timezone='UTC')
return [s.encode('ascii') for s in string_data]
return data
def summary_diff_dtype(x):
if x.dtype.name in ['object', 'bool', 'category'
] and len(x.unique()) <= max_lev:
vc = x.value_counts(dropna=False, normalize=True)
s = ''
for name, v in zip(vc.index, vc.values):
s += f'{name} {v*100:>2.0f}%'
s += '<br>' if in_cell_next_line else ', '
return s[:-2]
elif x.dtype.name in ['float64', 'int64']:
o = f'quantiles: {x.quantile(q=[0, 0.25, 0.5, 0.75, 1]).values.tolist()}{in_cell_next} \
mean: {x.mean():.2f}\
std: {x.std():.2f} \
cv: {x.std()/x.mean():.2f}{in_cell_next}\
skew: {skew(x[x.notnull()]):.2f}'
if sum(x.notnull()) > 8: # requirement of skewtest
p = skewtest(x[x.notnull()]).pvalue
o += f'*' if p <= 0.05 else ''
if min(x[x != 0]) > 0 and len(x[x != 0]) > 8: # take log
o += f'{in_cell_next}log skew: {skew(np.log(x[x>0])):.2f}'
p = skewtest(np.log(x[x != 0])).pvalue
o += f'*' if p != p and p <= 0.05 else ''
return o
elif 'datetime' in x.dtype.name:
# o = ''
qs = x.quantile(q=[0, 0.25, 0.5, 0.75, 1]).values
return print_list([np.datetime_as_string(q)[0:16] for q in qs],
br=in_cell_next)
else:
return ''
def ThirtyOne(year):
d1 = {}
for i in s5:
temp = SS.StockDataFrame.retype(
yf.download(i, period='3mo', interval='1d', auto_adjust=False))
indices = list(temp.index.values)
print(indices)
temp2 = pd.DataFrame()
crucialDate, ida = divideByMonth(indices)
print(ida)
for k in ida:
print(temp.iloc[k])
#print(crucialDate)
dataList = temp.loc[crucialDate]
dataList.reset_index(drop=True, inplace=True)
newDates = []
for j in crucialDate:
newDates += [
np.datetime64(np.datetime_as_string(j, unit='D')[:-2] + '01')
]
dataList['Date'] = newDates.copy()
dataList.set_index('Date', inplace=True, drop=True)
d1[i] = dataList
return d1
def datetime64_str(time_str: Optional[str] = None) -> np.ndarray:
"""
Reformat time_str to ISO 8601 or to 'NaT'. Used here for input in funcs that converts str to numpy.datetime64
:param time_str: May be 'NaT'
:return: ndarray of strings (tested for 1 element only) formatted by numpy.
"""
return np.datetime_as_string(np.datetime64(time_str, 's'))
def format_dt(dt):
return str(
np.datetime_as_string(
arr=np.datetime64(dt.strftime("%Y-%m-%d %H:%M:%S.%f")),
timezone="UTC",
unit="us",
))
def dt64_to_str(val):
try:
return numpy.datetime_as_string(val, unit='D')
except TypeError as err:
print(type(val), val)
print(err)
return str(val)
def _get_metadata(self, quantity):
date, time = np.datetime_as_string(self.start_datetime).split('T')
return {'General': {'original_filename': os.path.split(self.filename)[1],
'title': '%s (%s)' % (quantity,
self._parse_quantity_units(quantity)),
'date': date,
'time': time},
"Signal": {'signal_type': '',
'quantity': self._parse_quantity(quantity)}}
def _test_date_time(self, dt_str='now'):
dt0 = np.datetime64(dt_str)
data_str, time_str = np.datetime_as_string(dt0).split('T')
self.tree.add_node("General")
self.tree.General.date = data_str
self.tree.General.time = time_str
dt1 = np.datetime64('%sT%s' % (self.tree.General.date,
self.tree.General.time))
np.testing.assert_equal(dt0, dt1)
return dt1
def __init__(self, conf):
## TODO get the configuration values propogated to here
GENERAL, SPECIALFILES = conf['GENERAL'], conf['SPECIALFILES']
flnm_tmplt = conf['FILETEMPLATES']
flnm_tags = conf['FILETAGS']
self.date_timestamp = np.datetime_as_string(np.datetime64('today', 'D'))
self.numeric_timestamp = np.datetime64('now' ,'m').astype(int )-np.datetime64('2018-08-01T00:00' ,'m').astype(int)
self.directory = DirectoryManager(conf)
self.maskname = GENERAL['mask_name']
self.mtl_name = SPECIALFILES['mtl']
## Setup Defaults
self.current_read_template = None
self.current_write_template = None
self.calibration_template = flnm_tmplt['calibration']
self.default_calibration_template = flnm_tmplt['default_calibration']
self.pickled_datadump_name =flnm_tmplt['pickled_datadump']
self.lampline_template = flnm_tmplt['lampline']
self.redshift_fit_template = flnm_tmplt['redshift_fits']
self.save_plot_template = flnm_tmplt['save_plots']
self.tempname_dict = {
'bias': {'read': flnm_tmplt['raw'], 'write': flnm_tmplt['debiased']},
'stitch': {'read': flnm_tmplt['debiased'], 'write': flnm_tmplt['stitched']},\
'remove_crs': {'read': flnm_tmplt['stitched'], 'write': flnm_tmplt['stitched']}, \
'apcut': {'read': flnm_tmplt['stitched'], 'write': flnm_tmplt['oneds']}, \
'wavecalib': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['oneds']}, \
'flatten': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['oneds']}, \
'skysub': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['oneds']}, \
'combine': {'read': flnm_tmplt['oneds'], 'write': flnm_tmplt['combined']}, \
'zfit': {'read': flnm_tmplt['combined'], 'write': flnm_tmplt['combined']} \
}
self.tags_dict = {
'bias': {'read': '', 'write': flnm_tags['debiased']}, \
'stitch': {'read':flnm_tags['debiased'], 'write': flnm_tags['debiased']},\
'remove_crs': {'read':flnm_tags['debiased'], 'write': flnm_tags['crrmvd']},\
'apcut': {'read':flnm_tags['crrmvd'], 'write': flnm_tags['crrmvd']}, \
'wavecalib': {'read':flnm_tags['crrmvd'], 'write': flnm_tags['crrmvd']},\
'flatten': {'read':flnm_tags['crrmvd'], 'write': flnm_tags['flatnd']}, \
'skysub': {'read':flnm_tags['flatnd'], 'write': flnm_tags['skysubd']}, \
'combine': {'read':flnm_tags['skysubd'], 'write': flnm_tags['skysubd']},\
'zfit': {'read':flnm_tags['skysubd'], 'write': flnm_tags['skysubd']}\
}
self.step = 'bias'
self.update_templates_for()
def test_deltas_year():
start_list = pd.to_datetime(['2014-01-01', '2016-01-01'])
start_array = np.array(start_list)
new_time_1 = add_td(start_list, 2, 'Y')
new_time_2 = add_td(start_array, 2, 'Y')
values = [2016, 2018]
for i, value in enumerate(values):
assert new_time_1.dt.year.values[i] == value
assert np.datetime_as_string(new_time_2[i])[:4] == str(value)
error_text = 'Invalid Unit'
with pytest.raises(ValueError, match=error_text) as excinfo:
add_td(start_list, 2, 'M')
assert 'Invalid Unit' in str(excinfo)
def _ar_prediction(self, data_frame, freq):
"""
Generate a prediction using an Autoregressive Model
:return: A dataframe with predicted usage
"""
resampled_frame = data_frame.resample(freq, how='mean', closed='right')
model = tsa.AR(resampled_frame, freq=freq)
result = model.fit(maxlag=5, method='mle', transparams=True, disp=-1)
# Currently the forecast requires 6 hours of readings with no NaN or inf values
# So far it seems reasonably accurate to predict the usage for the next hour, but we can play with this
end = to_datetime(datetime_as_string(resampled_frame.index.values[len(resampled_frame)-1])) # end of frame
start_pred = str(end - relativedelta(hours=5)) # we can play around with this number a bit
end_pred = str(end + relativedelta(hours=1)) # you can try increasing this if you want to
return result.predict(start=start_pred, end=end_pred)
def update_date_time_in_metadata(dt, metadata):
""" Update the date and time in a metadata tree.
Parameters
----------
dt : date and time information: it can be a ISO 8601 string,
a datetime.datetime or a numpy.datetime64 object
metadata : metadata object to update
Return
----------
metadata object
Example
-------
>>> s = hs.load("example1.msa")
>>> dt = '2016-12-12T12:12:12-05:00'
>>> s.metadata = update_date_time_in_metadata(dt, s.metadata)
>>> s.metadata
├── General
│ ├── date = 2016-12-12
│ ├── original_filename = example1.msa
│ ├── time = 12:12:12
│ ├── time_zone = 'EST'
│ └── title = NIO EELS OK SHELL
"""
time_zone = None
if isinstance(dt, str):
dt = parser.parse(dt)
if isinstance(dt, np.datetime64):
dt_split = np.datetime_as_string(dt).split('T')
date = dt_split[0]
time = dt_split[1]
if isinstance(dt, datetime.datetime):
date = dt.date().isoformat()
time = dt.time().isoformat()
if dt.tzname():
time_zone = dt.tzname()
elif dt.tzinfo:
time_zone = dt.isoformat()[-6:]
metadata.set_item('General.date', date)
metadata.set_item('General.time', time)
if time_zone:
metadata.set_item('General.time_zone', time_zone)
elif metadata.has_item('General.time_zone'):
del metadata.General.time_zone
return metadata
def datetime_gas_cell():
dt_np = np.datetime64("2014-12-15T19:07:04.165000")
dt_str = np.datetime_as_string(dt_np)
date, time = dt_str.split("T")
return date, time, dt_np
## CREATE DATETIME ARRAY
ind = np.arange(0,ndays,dayint)
#ind = np.arange(15,365,30)
base = datetime(2010,1,1,0)
arr = np.array([base + timedelta(days=i) for i in range(ndays)])
#arr = np.array([base + timedelta(days=i) for i in range(365)])
date_arr = []
for i in range(len(ind)):
date_arr.append(pd.date_range(str(arr[ind][i]),freq='H',periods=24))
date_arr = np.datetime_as_string(date_arr,timezone='UTC')
date_arr = date_arr.flatten().astype('datetime64')
dates_pd = pd.DatetimeIndex(date_arr)
## WRITE TO NETCDF FILE
dataset = netCDF4.Dataset('%spar%s_scope_out_diurnal.nc' % (path_to_output,param), 'w', format='NETCDF4_CLASSIC')
# Create Dimensions
dataset.createDimension('time',None)
dataset.createDimension('vegp',nvp)
# Create Variables
nc_times = dataset.createVariable('time', np.float64, ('time',))
nc_vegp = dataset.createVariable('vegp', np.int32, ('vegp',))
nc_gridp = dataset.createVariable('gridp', np.int32, ('vegp',))
TsliceB=None
data=sylv_flux_read(flptr)
flptr.close()
datax=record2xray(data)
# get the last data point and figure out the time slice.
# can add additional data variables to check if this does not work
for i in range(len(datax)-1,1,-1):
if not (np.isnan(datax[i].sel(varname='UST')) and np.isnan(datax[i].sel(varname='FC'))):
TEnd=datax.time[i]
TBegin= TEnd - np.timedelta64(10,'D')
TsliceE=np.datetime_as_string(TEnd)
TsliceB=np.datetime_as_string(TBegin)
break
# if you can't find a point just plot the whole year.
if TsliceE==None and TsliceB==None:
TsliceE=today.strftime('%Y-%m-%d')
TsliceB=today.strftime('%Y-01-01')
for var in plotvars:
fig,ax=plt.subplots()
datax.sel(varname=var, time=slice(TsliceB, TsliceE)).plot(marker='.')
ax.xaxis.set_major_formatter(DateFormatter("%m/%d-%H"))
plt.savefig(figdir + var + '.png', dpi=100)
plt.close()
def get_datetime(dt):
dt_np = np.datetime64(dt)
dt_str = np.datetime_as_string(dt_np)
date, time = dt_str.split('T')
return date, time, dt_np
