def test_datasource_download_options(self):
(_, _, _, _, _, datasource) = self.setup_up_to_datasource()
datasource.config = '{"test1":0,"test2":"test3"}'
datasource.download()
file0 = datasource.sourcefiles.get(name='file.txt')
next_start_string = str({'start': aware(file0.stop)})[1:-1]
datasource.download()
file1 = datasource.sourcefiles.get(name='file.txt')
filecontent = file1.file.read()
self.assertTrue("u'test1': 0" in filecontent)
self.assertTrue("u'test2': u'test3'" in filecontent)
self.assertTrue(next_start_string in filecontent)
def download(self, **kwargs):
api = kwargs.get('url', settings.MUNISENSE_API)
endpoint = '/groundwaterwells/{id}/{property}/query/{start_timestamp}'
prop = kwargs.get('property', 'water_level_validated')
loc = kwargs.get('meetlocatie', None)
if not loc:
raise ValueError('Meetlocatie not defined')
if isinstance(loc, MeetLocatie):
loc = loc.name
object_id = kwargs.get('object_id', loc)
callback = kwargs.get('callback', None)
start = kwargs.get('start') or datetime(2010, 1, 1)
start = aware(start, pytz.utc)
username = kwargs.get('username', settings.MUNISENSE_USERNAME)
password = kwargs.get('password', settings.MUNISENSE_PASSWORD)
headers = {'Accept': 'application/json'}
url = '{api}{endpoint}'.format(api=api, endpoint=endpoint)
url = url.format(id=object_id,
property=prop,
start_timestamp=start.isoformat())
payload = {'rowcount': 1000}
result = {}
index = 0
while True:
response = requests.get(url,
headers=headers,
params=payload,
auth=HTTPBasicAuth(username, password))
response.raise_for_status()
index += 1
filename = kwargs.get(
'filename', '{id}_{timestamp}_{index}.json'.format(
id=slugify(loc),
timestamp=start.strftime('%y%m%d%H%M'),
index=index))
result[filename] = response.text
try:
data = response.json()
link_next = data['meta']['link_next']
if link_next:
url = '{api}{endpoint}'.format(api=api, endpoint=link_next)
else:
break
except:
break
if callback:
callback(result)
return result
def handle(self, *args, **options):
with SourceAdapter(logging.getLogger(__name__)) as logger:
logger.source = ''
logger.info('***UPDATE STARTED***')
thumb = options.get('thumb')
down = options.get('down')
force = options.get('force')
if down:
logger.info('Downloading data, updating parameters and related time series')
else:
logger.info('Updating parameters and related time series')
count = 0
pk = options.get('pk', None)
if pk is None:
datasources = Datasource.objects.all()
else:
datasources = Datasource.objects.filter(pk=pk)
replace = options.get('replace')
if replace:
logger.info('Recreating series')
# remember which series have changed during update
changed_series = []
for d in datasources:
if not d.autoupdate and pk is None:
continue
logger.source = d
logger.info('Updating datasource %s' % d.name)
try:
series = d.getseries()
if replace:
start = None
else:
# actualiseren (data toevoegen) vanaf laatste punt
# set starting date/time for update as last date/time in current sourcefiles
data_stop = d.stop()
if len(series) == 0:
last = {}
else:
# maak dict met een na laatste datapoint per tijdreeks
# (rekening houden met niet volledig gevulde laatste tijdsinterval bij accumulatie of sommatie)
last = {s: s.beforelast().date for s in series if s.aantal() > 0}
if down and d.autoupdate:
logger.info('Downloading datasource')
try:
# if start is in the future, use datetime.now as start to overwrite previous forecasts
start = min(data_stop, aware(datetime.now()))
newfiles = d.download(start)
except Exception as e:
logger.exception('ERROR downloading datasource: %s' % e)
newfiles = None
newfilecount = len(newfiles) if newfiles else 0
logger.info('Got %d new files' % newfilecount)
# for update use newfiles AND the existing sourcefiles that contain data for aggregation
if last:
after = min(last.values())
candidates = d.sourcefiles.filter(start__gte=after)
else:
after = None
candidates = d.sourcefiles.all()
if not candidates:
logger.warning('No new data for datasource {ds}'.format(ds=d))
if not force:
logger.debug('Update of timeseries skipped')
continue
else:
count += 1
logger.info('Reading datasource')
try:
data = d.get_data(files=candidates,start=after)
except Exception as e:
logger.exception('Error reading datasource: %s', e)
continue
if data is None:
logger.error('No new data for datasource {ds}'.format(ds=d))
# don't bother to continue: no data
continue
if replace:
logger.info('Updating parameters')
try:
d.update_parameters(data=data,files=candidates,limit=10)
if replace:
d.make_thumbnails(data=data)
except Exception as e:
logger.exception('ERROR updating parameters for datasource: %s' % e)
updated = 0
for s in series:
logger.info('Updating timeseries %s' % s.name)
try:
# replace timeseries or update after beforelast datapoint
start = last.get(s,None)
changes = s.replace() if replace else s.update(data,start=start,thumbnail=thumb)
if changes:
updated += 1
logger.debug('%d datapoints updated for %s' % (changes, s.name))
changed_series.append(s)
else:
logger.warning('No updates for %s' % s.name)
except Exception as e:
logger.exception('ERROR updating timeseries %s: %s' % (s.name, e))
if updated:
logger.info('{} time series updated for datasource {}'.format(updated,d.name))
except Exception as e:
logger.exception('ERROR updating datasource %s: %s' % (d.name, e))
logger.source = ''
logger.info('%d datasources were updated' % count)
calc = options.get('calc',True)
if calc:
def update_formula(f):
count = 0
# update dependent formulas first
for d in f.get_dependencies():
if d in formulas:
count += update_formula(d)
try:
logger.info('Updating calculated time series %s' % f.name)
if f.update(thumbnail=False) == 0:
logger.warning('No new data for %s' % f.name)
count += 1
except Exception as e:
logger.exception('ERROR updating calculated time series %s: %s' % (f.name, e))
formulas.remove(f)
return count
# get all unique formulas to update
formulas = set()
for f in Formula.objects.all():
for d in f.get_dependencies():
if d in changed_series:
formulas.add(f)
break
formulas = list(formulas)
count = 0
while formulas:
count += update_formula(formulas[0])
logger.info('%d calculated time series were updated' % count)
logger.info('***UPDATE COMPLETED***')
def handle(self, *args, **options):
with DatasourceAdapter(logging.getLogger('update')) as logger:
#logging.getLogger('acacia.data').addHandler(email_handler)
logger.datasource = ''
logger.info('***UPDATE STARTED***')
down = options.get('down')
if down:
logger.info('Downloading data, updating parameters and related time series')
else:
logger.info('Updating parameters and related time series')
count = 0
pk = options.get('pk', None)
if pk is None:
datasources = Datasource.objects.all()
else:
datasources = Datasource.objects.filter(pk=pk)
replace = options.get('replace')
if replace:
logger.info('Recreating series')
# remember which series have changed during update
changed_series = []
for d in datasources:
if not d.autoupdate and pk is None:
continue
logger.datasource = d
logger.info('Updating datasource %s' % d.name)
try:
series = d.getseries()
if replace:
start = None
else:
# actualiseren (data toevoegen) vanaf laatste punt
data_start = d.stop()
if len(series) == 0:
series_start = data_start
else:
# actialisatie vanaf een na laatste datapoint
# (rekening houden met niet volledig gevulde laatste tijdsinterval bij accumulatie of sommatie)
last = [p.date for p in [s.beforelast() for s in series if s.aantal()>0] if p is not None]
if len(last)>0:
series_start = min(last)
else:
series_start = data_start
if data_start is None:
start = series_start
else:
start = min(series_start,data_start)
# if start is in the future, use datetime.now as start to overwrite previous forecasts
if start:
start = min(start, aware(datetime.now()))
if down and d.autoupdate and d.url is not None:
logger.info('Downloading datasource')
try:
newfiles = d.download()
except Exception as e:
logger.exception('ERROR downloading datasource: %s' % e)
continue
if newfiles is None:
newfilecount = 0
else:
newfilecount = len(newfiles)
logger.info('Got %d new files' % newfilecount)
if newfilecount == 0:
newfiles = None
else:
newfilecount = 0
newfiles = None
if down and newfilecount == 0:
# we tried to download but there is no new data
logger.debug('Update of timeseries skipped')
continue
count = count + 1
# TODO: Avoid reading the data twice: d.download() has called sourcefile.save() which reads the data and updates dimensions
logger.info('Reading datasource')
try:
data = d.get_data(files=newfiles,start=start)
except Exception as e:
logger.exception('Error reading datasource: %s', e)
continue
if data is None:
logger.error('No data found for %s. Update of timeseries skipped' % d.name)
# don't bother to continue: no data
continue
if replace:
logger.info('Updating parameters')
try:
d.update_parameters(data=data,files=newfiles,limit=10)
if replace:
d.make_thumbnails(data=data)
except Exception as e:
logger.exception('ERROR updating parameters for datasource: %s' % e)
for s in series:
logger.info('Updating timeseries %s' % s.name)
try:
changes = s.replace() if replace else s.update(data,start=start)
if changes > 0:
changed_series.append(s)
# immediately trigger alarms (if any)
process_triggers(s)
else:
logger.warning('No new data for %s' % s.name)
except Exception as e:
logger.exception('ERROR updating timeseries %s: %s' % (s.name, e))
logger.info('Datasource %s updated' % d.name)
except Exception as e:
logger.exception('ERROR updating datasource %s: %s' % (d.name, e))
logger.datasource = ''
logger.info('%d datasources were updated' % count)
calc = options.get('calc',True)
if calc:
def update_formula(f):
count = 0
# update dependent formulas first
for d in f.get_dependencies():
if d in formulas:
count += update_formula(d)
try:
logger.info('Updating calculated time series %s' % f.name)
if f.update() == 0:
logger.warning('No new data for %s' % f.name)
else:
process_triggers(f)
count += 1
except Exception as e:
logger.exception('ERROR updating calculated time series %s: %s' % (f.name, e))
formulas.remove(f)
return count
# get all unique formulas to update
formulas = set()
for f in Formula.objects.all():
for d in f.get_dependencies():
if d in changed_series:
formulas.add(f)
break
formulas = list(formulas)
count = 0
while formulas:
count += update_formula(formulas[0])
logger.info('%d calculated time series were updated' % count)
logger.info('***UPDATE COMPLETED***')
#email_handler.flush()
#logging.getLogger('acacia.data.update').removeHandler(email_handler)
def handle(self, *args, **options):
baro = Series.objects.get(name='Luchtdruk Voorschoten')
# tolerantie voor aanpassingen
limit = 0.2
tolerance = timedelta(hours=6)
dry = options.get('dry')
well = options.get('well')
queryset = LoggerPos.objects.all()
if well:
queryset = LoggerPos.objects.filter(screen__well__name=well)
# nwe loggers: serienummer = 2005xxxx
queryset = queryset.filter(logger__serial__startswith='2005')
# Niet twee keer aanpassen...
queryset = queryset.exclude(remarks__contains='aangepast')
for pos in queryset.order_by('screen__well'):
if not uncompensated(pos):
continue
screen = pos.screen
start_date = aware(pos.start_date - timedelta(days=1),
'utc').replace(hour=0, minute=0, second=0)
hand = screen.get_manual_series(start=start_date)
if hand is None or hand.empty:
continue
hand_date = min(hand.index)
handpeiling = hand[hand_date]
levels = screen.get_compensated_series(start=hand_date).dropna()
if levels is None or levels.empty:
continue
logger_date = min(levels.index)
loggerwaarde = levels[logger_date]
hpa = baro.at(logger_date).value
if baro.unit.startswith('0.1'):
hpa *= 0.1
verschil = loggerwaarde - handpeiling
offset = 0.01 * (hpa -
1013) / 0.98123 # offset door luchtdruk verschil
if abs(verschil) < limit and (logger_date - hand_date < tolerance):
# pas bkb aan en rond af op mm
refpnt = round(pos.refpnt + offset, 3)
logger.debug('{},{},{},{:+.1f} cm'.format(
pos, pos.refpnt, refpnt, offset * 100))
if not dry:
pos.remarks = 'ophangpunt ({}) aangepast met {:+.2f} cm voor luchtdruk van {:.1f} hPa tijdens installatie.'.format(
pos.refpnt, offset * 100, hpa)
pos.refpnt = refpnt
pos.save()
series = screen.find_series()
if series:
logger.debug(
'Tijdreeks {} aanpassen...'.format(series))
success = recomp(screen,
series,
start=pos.start_date,
stop=pos.end_date)
def get_json(self, chart):
options = {
'chart': {'animation': False,
'zoomType': 'x',
'events': {'load': None},
},
'title': {'text': chart.title},
'xAxis': {'type': 'datetime','plotBands': []},
'yAxis': [],
'tooltip': {'valueDecimals': 2,
'shared': True,
},
'legend': {'enabled': chart.series.count() > 1},
'plotOptions': {'line': {'marker': {'enabled': False}}, 'series': {'connectNulls': True},
'column': {'allowpointSelect': True, 'pointPadding': 0.01, 'groupPadding': 0.01}},
'credits': {'enabled': True,
'text': 'acaciawater.com',
'href': 'http://www.acaciawater.com',
},
'exporting' :{
'sourceWidth': 1080,
'sourceHeight': 600,
# 'scale': 2,
# 'chartOptions' :{
# 'title': {'style': {'fontSize': 0 }}, # 0 gemaakt omdat titel niet wordt overgenomen
# 'xAxis': {'labels': {'style': {'fontSize': 15 }}},
# 'yAxis': {'labels': {'style': {'fontSize': 15 }}},
# 'legend': {'itemStyle': {'fontSize': 15 },'padding': 1,},
# 'credits': {'enabled': False}
# },
}
}
if chart.start:
options['xAxis']['min'] = chart.start
if chart.stop:
options['xAxis']['max'] = chart.stop
allseries = []
tmin = chart.start
tmax = chart.stop
ymin = None
ymax = None
num_series = chart.series.count()
for _,s in enumerate(chart.series.all()):
ser = s.series
if tmin:
tmin = min(tmin,s.t0 or ser.van() or chart.start or tmin)
else:
tmin = s.t0 or ser.van() or chart.start
if tmax:
tmax = max(tmax,s.t1 or ser.tot() or chart.stop or tmax)
else:
tmax = s.t1 or ser.tot() or chart.stop
if ymin:
ymin = min(ymin,s.y0 or ser.minimum())
else:
ymin = s.y0 or ser.minimum()
if ymax:
ymax = max(ymax,s.y1 or ser.maximum())
else:
ymax = s.y1 or ser.maximum()
try:
deltat = (ser.tot()-ser.van()).total_seconds() / ser.aantal() * 1000
except:
deltat = 24 * 3600000 # 1 day
options['yAxis'].append({
'title': {'text': s.label},
'opposite': 0 if s.axislr == 'l' else 1,
'min': s.y0,
'max': s.y1
})
pts = []
name = s.name
if name is None or name == '':
name = ser.name
if not ser.validated:
# append asterisk to name when series has not been validated
if isinstance(ser,Formula):
for dep in ser.get_dependencies():
if not dep.validated:
name += '*'
break
else:
name += '*'
sop = {'name': name,
'id': 'series_%d' % ser.id,
'type': s.type,
'yAxis': s.axis-1,
'data': pts}
if not s.color is None and len(s.color)>0:
sop['color'] = s.color
if s.type == 'scatter':
sop['tooltip'] = {'valueSuffix': ' '+ser.unit,
'headerFormat': '<small>{point.key}</small><br/><table>',
'pointFormat': '<tr><td style="color:{series.color}">{series.name}</td>\
<td style = "text-align: right">: <b>{point.y}</b></td></tr>'}
else:
sop['tooltip'] = {'valueSuffix': ' ' + ser.unit}
if s.type == 'column':
if s.stack is not None:
sop['stacking'] = s.stack
if num_series > 1:
sop['pointRange'] = deltat
if s.type == 'area' and s.series2:
sop['type'] = 'arearange'
sop['fillOpacity'] = 0.3
allseries.append(sop)
options['series'] = allseries
for band in chart.plotband_set.all():
if band.orientation == 'h':
ax = options['yAxis'][band.axis-1]
lo = float(band.low)
hi = float(band.high)
every = max(1,float(band.repetition))
b = []
for i in range(20):
if lo < ymax:
b.append(
{'color': band.style.fillcolor,
'borderWidth': band.style.borderwidth,
'borderColor': band.style.bordercolor,
'from': lo,
'to': hi,
'label': {'text':band.label},
'zIndex': band.style.zIndex
})
lo += every
hi += every
else:
ax = options['xAxis']
lo = parse(band.low)
hi = parse(band.high)
delta = parserep(band.repetition)
b = []
for i in range(20):
if aware(lo) < aware(tmax):
b.append({'color': band.style.fillcolor, 'borderWidth': band.style.borderwidth, 'borderColor': band.style.bordercolor, 'from': lo, 'to': hi, 'label': {'text':band.label}})
if delta:
lo += delta
hi += delta
if not 'plotBands' in ax:
ax['plotBands'] = []
ax['plotBands'].extend(b)
for line in chart.plotline_set.all():
if line.orientation == 'h':
ax = options['yAxis'][line.axis-1]
else:
ax = options['xAxis']
line_options = {
'color': line.style.color,
'dashStyle': line.style.dashstyle,
'label': {'text': line.label},
'value': line.value,
'width': line.style.width,
'zIndex': line.style.zIndex
}
if not 'plotLines' in ax:
ax['plotLines'] = []
ax['plotLines'].append(line_options)
return json.dumps(options,default=to_millis)
def get_json(self, chart):
options = {
# 'rangeSelector': { 'enabled': True,
# 'inputEnabled': True,
# 'selected': 5,
# },
# 'navigator': {'adaptToUpdatedData': False, 'enabled': False},
# 'loading': {'style': {'backgroundColor': 'white', 'fontFamily': 'Arial', 'fontSize': 'small'},
# 'labelStyle': {'fontWeight': 'normal'},
# 'hideDuration': 0,
# },
'chart': {'animation': False,
'zoomType': 'x',
'events': {'load': None},
},
'title': {'text': chart.title},
'xAxis': {'type': 'datetime','plotBands': []},
'yAxis': [],
'tooltip': {'valueDecimals': 2,
'shared': True,
},
'legend': {'enabled': chart.series.count() > 1},
'plotOptions': {'line': {'marker': {'enabled': False}},
'column': {'allowpointSelect': True, 'pointPadding': 0.01, 'groupPadding': 0.01}},
'credits': {'enabled': True,
'text': 'acaciawater.com',
'href': 'http://www.acaciawater.com',
}
}
if chart.start:
options['xAxis']['min'] = tojs(chart.start)
if chart.stop:
options['xAxis']['max'] = tojs(chart.stop)
allseries = []
tmin = chart.start
tmax = chart.stop
ymin = None
ymax = None
for _,s in enumerate(chart.series.all()):
ser = s.series
if tmin:
tmin = min(tmin,s.t0 or ser.van())
else:
tmin = s.t0 or ser.van()
if tmax:
tmax = max(tmax,s.t1 or ser.tot())
else:
tmax = s.t1 or ser.tot()
if ymin:
ymin = min(ymin,s.y0 or ser.minimum())
else:
ymin = s.y0 or ser.minimum()
if ymax:
ymax = max(ymax,s.y1 or ser.maximum())
else:
ymax = s.y1 or ser.maximum()
title = s.label #ser.name if len(ser.unit)==0 else '%s [%s]' % (ser.name, ser.unit) if chart.series.count()>1 else ser.unit
options['yAxis'].append({
'title': {'text': title},
'opposite': 0 if s.axislr == 'l' else 1,
'min': s.y0,
'max': s.y1
})
pts = [] #[[p.date,p.value] for p in ser.datapoints.filter(date__gte=start).order_by('date')]
name = s.name
if name is None or name == '':
name = ser.name
sop = {'name': name,
'id': 'series_%d' % ser.id,
'type': s.type,
'yAxis': s.axis-1,
'data': pts}
if not s.color is None and len(s.color)>0:
sop['color'] = s.color
if s.type == 'scatter':
sop['tooltip'] = {'valueSuffix': ' '+ser.unit,
'headerFormat': '<small>{point.key}</small><br/><table>',
'pointFormat': '<tr><td style="color:{series.color}">{series.name}</td>\
<td style = "text-align: right">: <b>{point.y}</b></td></tr>'}
else:
sop['tooltip'] = {'valueSuffix': ' ' + ser.unit}
if s.type == 'column' and s.stack is not None:
sop['stacking'] = s.stack
if s.type == 'area' and s.series2:
sop['type'] = 'arearange'
sop['fillOpacity'] = 0.3
allseries.append(sop)
options['series'] = allseries
for band in chart.plotband_set.all():
if band.orientation == 'h':
ax = options['yAxis'][band.axis-1]
lo = float(band.low)
hi = float(band.high)
every = max(1,float(band.repetition))
b = []
for i in range(20):
if lo < ymax:
b.append({'color': band.style.fillcolor, 'borderWidth': band.style.borderwidth, 'borderColor': band.style.bordercolor, 'from': lo, 'to': hi, 'label': {'text':band.label}})
lo += every
hi += every
else:
ax = options['xAxis']
lo = parse(band.low)
hi = parse(band.high)
def parserep(r):
from dateutil.relativedelta import relativedelta
pattern = r'(?P<rep>\d*)(?P<how>[hdwmy])'
match = re.match(pattern, r,re.IGNORECASE)
if match:
rep = match.group('rep') or 1
how = match.group('how')
if how == 'h':
delta = relativedelta(hours=int(rep))
elif how == 'd':
delta = relativedelta(days=int(rep))
elif how == 'w':
delta = relativedelta(weeks=int(rep))
elif how == 'm':
delta = relativedelta(months=int(rep))
elif how == 'y':
delta = relativedelta(years=int(rep))
return delta
return None
delta = parserep(band.repetition)
b = []
for i in range(20):
if aware(lo) < aware(tmax):
b.append({'color': band.style.fillcolor, 'borderWidth': band.style.borderwidth, 'borderColor': band.style.bordercolor, 'from': lo, 'to': hi, 'label': {'text':band.label}})
if delta:
lo += delta
hi += delta
if not 'plotBands' in ax:
ax['plotBands'] = []
ax['plotBands'].extend(b)
for line in chart.plotline_set.all():
pass
jop = json.dumps(options,default=date_handler)
# remove quotes around date stuff
jop = re.sub(r'\"(Date\.UTC\([\d,]+\))\"',r'\1', jop)
return jop
