def update_ts_temp_file(cache_dir, connection, id):
full_rewrite = False
afilename = os.path.join(cache_dir, '%d.hts'%(id,))
if os.path.exists(afilename):
if os.path.getsize(afilename)<3:
full_rewrite = True
#Update the file in the case of logged data, if this is possible
if os.path.exists(afilename) and not full_rewrite:
with open(afilename, 'r') as fileobject:
xr = xreverse(fileobject, 2048)
line = xr.next()
lastdate = datetime_from_iso(line.split(',')[0])
ts = Timeseries(id)
ts.read_from_db(connection, bottom_only=True)
if len(ts)>0:
db_start, db_end = ts.bounding_dates()
if db_start>lastdate:
full_rewrite = True
elif db_end>lastdate:
lastindex = ts.index(lastdate)
with open(afilename, 'a') as fileobject:
ts.write(fileobject, start=ts.keys()[lastindex+1])
#Check for tmmp file or else create it
if not os.path.exists(afilename) or full_rewrite:
ts = Timeseries(id)
ts.read_from_db(connection)
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
tempfile_handle, tempfile_name = tempfile.mkstemp(dir=cache_dir)
with os.fdopen(tempfile_handle, 'w') as afile:
ts.write(afile)
shutil.move(tempfile_name, afilename)
def update_ts_temp_file(cache_dir, connection, id):
full_rewrite = False
afilename = os.path.join(cache_dir, '%d.hts' % (id, ))
if os.path.exists(afilename):
if os.path.getsize(afilename) < 3:
full_rewrite = True
#Update the file in the case of logged data, if this is possible
if os.path.exists(afilename) and not full_rewrite:
with ropen(afilename) as fileobject:
line = fileobject.readline()
lastdate = datetime_from_iso(line.split(',')[0])
ts = Timeseries(id)
ts.read_from_db(connection, bottom_only=True)
if len(ts) > 0:
db_start, db_end = ts.bounding_dates()
if db_start > lastdate:
full_rewrite = True
elif db_end > lastdate:
lastindex = ts.index(lastdate)
with open(afilename, 'a') as fileobject:
ts.write(fileobject, start=ts.keys()[lastindex + 1])
#Check for tmmp file or else create it
if not os.path.exists(afilename) or full_rewrite:
ts = Timeseries(id)
ts.read_from_db(connection)
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
tempfile_handle, tempfile_name = tempfile.mkstemp(dir=cache_dir)
with os.fdopen(tempfile_handle, 'w') as afile:
ts.write(afile)
shutil.move(tempfile_name, afilename)
def MultiTimeseriesProcessDb(method, timeseries_arg, out_timeseries_id,
db, read_tstep_func, transaction=None,
commit=True, options={}):
out_timeseries = Timeseries(id = out_timeseries_id)
opts = copy.deepcopy(options)
if 'append_only' in opts and opts['append_only']:
bounds = timeseries_bounding_dates_from_db(db,
id = out_timeseries_id)
opts['start_date'] = bounds[1] if bounds else None;
opts['interval_exclusive'] = True
tseries_arg={}
for key in timeseries_arg:
ts = Timeseries(id=timeseries_arg[key])
if ('append_only' in opts and opts['append_only']) \
and opts['start_date'] is not None:
ts.read_from_db(db, bottom_only=True)
if ts.bounding_dates()[0]>opts['start_date']:
ts.read_from_db(db)
else:
ts.read_from_db(db)
ts.time_step = read_tstep_func(ts.id)
tseries_arg[key] = ts
MultiTimeseriesProcess(method, tseries_arg, out_timeseries, opts)
if 'append_only' in opts and opts['append_only']:
out_timeseries.append_to_db(db=db, transaction=transaction,
commit=commit)
else:
out_timeseries.write_to_db(db=db, transaction=transaction,
commit=commit)
def InterpolateDbTimeseries(source_id, dest_id, curve_type, curve_data,
db, data_columns=(0,1), logarithmic=False,
offset=0, append_only=False,
transaction=None, commit=True):
if append_only:
bounds = timeseries_bounding_dates_from_db(db, id = dest_id)
start_date = bounds[1] if bounds else None;
ts = Timeseries(id=source_id)
if append_only and start_date is not None:
ts.read_from_db(db, bottom_only=True)
if ts.bounding_dates()[0]>start_date:
ts.read_from_db(db)
while ts.bounding_dates()[0]<=start_date:
del(ts[ts.bounding_dates()[0]])
if len(ts)==0: return
else:
ts.read_from_db(db)
curve_list = TransientCurveList()
if curve_type=='SingleCurve':
curve_list.add(logarithmic=logarithmic,
offset=CurvePoint(offset, 0))
super(TransientCurve,
curve_list[0]).read_fp(StringIO(curve_data),
data_columns)
elif curve_type=='StageDischargeMulti':
curve_list.read_fp(StringIO(curve_data))
else:
assert(False)
out_timeseries = curve_list.interpolate_ts(ts)
out_timeseries.id = dest_id
if append_only:
out_timeseries.append_to_db(db=db, transaction=transaction,
commit=commit)
else:
out_timeseries.write_to_db(db=db, transaction=transaction,
commit=commit)
def update_one_timeseries(base_url, id, user=None, password=None):
if base_url[-1] != '/':
base_url += '/'
# Read timeseries from cache file
cache_filename = os.path.join(
settings.BITIA_TIMESERIES_CACHE_DIR, '{}.hts'.format(id))
ts1 = Timeseries()
if os.path.exists(cache_filename):
with open(cache_filename) as f:
try:
ts1.read_file(f)
except ValueError:
# File may be corrupted; continue with empty time series
ts1 = Timeseries()
# Get its end date
try:
end_date = ts1.bounding_dates()[1]
except TypeError:
# Timeseries is totally empty; no start and end date
end_date = datetime(1, 1, 1, 0, 0)
start_date = end_date + timedelta(minutes=1)
# Get newer timeseries and append it
session_cookies = enhydris_api.login(base_url, user, password)
url = base_url + 'timeseries/d/{}/download/{}/'.format(
id, start_date.isoformat())
r = requests.get(url, cookies=session_cookies)
r.raise_for_status()
ts2 = Timeseries()
ts2.read_file(StringIO(r.text))
ts1.append(ts2)
# Save it
with open(cache_filename, 'w') as f:
ts1.write(f)
def regularize_raw_series(raw_series_db, proc_series_db, rs, re, ps, pe ):
"""
This function regularize raw_series_db object from database and
writes a processed proc_series_db in database.
Raw series is a continuously increasing values time series,
aggregating the water consumption. Resulting processed timeseries
contains water consumption for each of its interval. I.e. if the
timeseries is of 15 minutes time step, then each record contains
the water consumption for each record period.
"""
raw_series = TSeries(id=raw_series_db.id)
raw_series.read_from_db(db.connection)
# We keep the last value for x-checking reasons, see last print
# command
test_value = raw_series[raw_series.bounding_dates()[1]]
time_step = ReadTimeStep(proc_series_db.id, proc_series_db)
proc_series = TSeries(id=proc_series_db.id, time_step = time_step)
# The following code can be used in real conditions to append only
# new records to db, in a next version
#if not pe:
# start = proc_series.time_step.down(rs)
#else:
# start = proc_series.time_step.up(pe)
# Instead of the above we use now:
start = proc_series.time_step.down(rs)
end = proc_series.time_step.up(re)
pointer = start
# Pass 1: Initialize proc_series
while pointer<=end:
proc_series[pointer] = float('nan')
pointer = proc_series.time_step.next(pointer)
# Pass 2: Transfer cummulative raw series to differences series:
prev_s = 0
for i in xrange(len(raw_series)):
dat, value = raw_series.items(pos=i)
if not math.isnan(value):
raw_series[dat] = value-prev_s
prev_s = value
# Pass 3: Regularize step: loop over raw series records and distribute
# floating point values to processed series
for i in xrange(len(raw_series)):
dat, value = raw_series.items(pos=i)
if not math.isnan(value):
# find previous, next timestamp of the proc time series
d1 = proc_series.time_step.down(dat)
d2 = proc_series.time_step.up(dat)
if math.isnan(proc_series[d1]): proc_series[d1] = 0
if math.isnan(proc_series[d2]): proc_series[d2] = 0
if d1==d2: # if dat on proc step then d1=d2
proc_series[d1] += value
continue
dif1 = _dif_in_secs(d1, dat)
dif2 = _dif_in_secs(dat, d2)
dif = dif1+dif2
# Distribute value to d1, d2
proc_series[d1] += (dif2/dif)*value
proc_series[d2] += (dif1/dif)*value
# Uncomment the following line in order to show debug information.
# Usually the three following sums are consistent by equality. If
# not equality is satisfied then there is a likelyhood of algorith
# error
print raw_series.sum(), proc_series.sum(), test_value
proc_series.write_to_db(db=db.connection, commit=True) #False)
#return the full timeseries
return proc_series
def regularize(raw_series_db, proc_series_db, rs, re):
"""
This function regularize raw_series_db object from database and
writes a processed proc_series_db in database.
Raw series is a continuously increasing values time series,
aggregating the water consumption. Resulting processed timeseries
contains water consumption for each of its interval. I.e. if the
timeseries is of 15 minutes time step, then each record contains
the water consumption for each record period.
"""
raw_series = TSeries(id=raw_series_db.id)
raw_series.read_from_db(db.connection)
# We keep the last value for x-checking reasons, see last print
# command
try:
test_value = raw_series[raw_series.bounding_dates()[1]]
except Exception as e:
#log.debug("Trying to get test value for raw series %s failed with %s. "
# "Skipping!" % (raw_series_db.id, repr(e)))
return None
time_step = ReadTimeStep(proc_series_db.id, proc_series_db)
proc_series = TSeries(id=proc_series_db.id, time_step=time_step)
# The following code can be used in real conditions to append only
# new records to db, in a next version
#if not pe:
# start = proc_series.time_step.down(rs)
#else:
# start = proc_series.time_step.up(pe)
# Instead of the above we use now:
start = proc_series.time_step.down(rs)
end = proc_series.time_step.up(re)
pointer = start
# Pass 1: Initialize proc_series
while pointer <= end:
proc_series[pointer] = float('nan')
pointer = proc_series.time_step.next(pointer)
# Pass 2: Transfer cummulative raw series to differences series:
prev_s = 0
for i in xrange(len(raw_series)):
dat, value = raw_series.items(pos=i)
d = datetime.today()
d = d.replace(month=11).replace(day=5)
if dat.date() == d.date():
pass
if not isnan(value):
# "if" Added by Chris Pantazis, because sometimes
# We get a negative small value by the meter
if prev_s > value:
prev_s = value
raw_series[dat] = value - prev_s
prev_s = value
# Pass 3: Regularize step: loop over raw series records and distribute
# floating point values to processed series
for i in xrange(len(raw_series)):
dat, value = raw_series.items(pos=i)
if not isnan(value):
# find previous, next timestamp of the proc time series
d1 = proc_series.time_step.down(dat)
d2 = proc_series.time_step.up(dat)
if isnan(proc_series[d1]):
proc_series[d1] = 0
if isnan(proc_series[d2]):
proc_series[d2] = 0
if d1 == d2: # if dat on proc step then d1=d2
proc_series[d1] += value
continue
dif1 = _dif_in_secs(d1, dat)
dif2 = _dif_in_secs(dat, d2)
dif = dif1 + dif2
# Distribute value to d1, d2
proc_series[d1] += (dif2 / dif) * value
proc_series[d2] += (dif1 / dif) * value
# Uncomment the following line in order to show debug information.
# Usually the three following sums are consistent by equality. If
# not equality is satisfied then there is a likelyhood of algorith
# error
# log.info("%s = %s = %s ?" % (raw_series.sum(),
# proc_series.sum(), test_value))
proc_series.write_to_db(db=db.connection, commit=True)
#return the full timeseries
return proc_series
def test_daily(self):
self.setup_input_file('temperature_max',
textwrap.dedent("""\
Title=Temperature Max
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,21.5,
"""))
self.setup_input_file('temperature_min',
textwrap.dedent("""\
Title=Temperature Min
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,12.3,
"""))
self.setup_input_file('humidity_max',
textwrap.dedent("""\
Title=Humidity Max
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,84.0,
"""))
self.setup_input_file('humidity_min',
textwrap.dedent("""\
Title=Humidity Min
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,63.0,
"""))
self.setup_input_file('wind_speed',
textwrap.dedent("""\
Title=Wind speed
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,2.078,
"""))
self.setup_input_file('sunshine_duration',
textwrap.dedent("""\
Title=Sunshine duration
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,9.25,
"""))
with open(self.config_file, 'w') as f:
f.write(textwrap.dedent('''\
base_dir = {self.tempdir}
albedo = 0.23
step_length = 1440
''').format(self=self))
application = VaporizeApp()
application.read_command_line()
application.read_configuration()
# Verify the output file doesn't exist yet
result_filename = os.path.join(self.tempdir, 'evaporation.hts')
self.assertFalse(os.path.exists(result_filename))
# Execute
application.run()
# Check that it has created a file and that the file is correct
t = Timeseries()
with open(result_filename) as f:
t.read_file(f)
self.assertEqual(len(t), 1)
adate = datetime(2014, 7, 6)
self.assertEqual(t.bounding_dates(), (adate, adate))
self.assertAlmostEqual(t[adate], 3.9)
def test_hourly(self):
self.setup_input_file('temperature',
textwrap.dedent("""\
Title=Temperature
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,38,
"""))
self.setup_input_file('humidity',
textwrap.dedent("""\
Title=Humidity
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,52,
"""))
self.setup_input_file('wind_speed',
textwrap.dedent("""\
Title=Wind speed
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,3.3,
"""))
self.setup_input_file('pressure',
textwrap.dedent("""\
Title=Pressure
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,1013.0,
"""))
self.setup_input_file('solar_radiation',
textwrap.dedent("""\
Title=Solar radiation
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,681.0,
"""))
with open(self.config_file, 'w') as f:
f.write(textwrap.dedent('''\
base_dir = {self.tempdir}
albedo = 0.23
nighttime_solar_radiation_ratio = 0.8
step_length = 60
unit_converter_pressure = x / 10.0
unit_converter_solar_radiation = x * 3600 / 1e6
''').format(self=self))
application = VaporizeApp()
application.read_command_line()
application.read_configuration()
# Verify the output file doesn't exist yet
result_filename = os.path.join(self.tempdir, 'evaporation.hts')
self.assertFalse(os.path.exists(result_filename))
# Execute
application.run()
# Check that it has created a file and that the file is correct
t = Timeseries()
with open(result_filename) as f:
t.read_file(f)
self.assertEqual(len(t), 1)
adate = datetime(2014, 10, 1, 15, 0, tzinfo=senegal_tzinfo)
self.assertEqual(t.bounding_dates(), (adate, adate))
self.assertAlmostEqual(t[adate], 0.63)
