def test_datasource_download_options(self): (_, _, _, _, _, datasource) = self.setup_up_to_datasource() datasource.config = '{"test1":0,"test2":"test3"}' datasource.download() file0 = datasource.sourcefiles.get(name='file.txt') next_start_string = str({'start': aware(file0.stop)})[1:-1] datasource.download() file1 = datasource.sourcefiles.get(name='file.txt') filecontent = file1.file.read() self.assertTrue("u'test1': 0" in filecontent) self.assertTrue("u'test2': u'test3'" in filecontent) self.assertTrue(next_start_string in filecontent)
def download(self, **kwargs): api = kwargs.get('url', settings.MUNISENSE_API) endpoint = '/groundwaterwells/{id}/{property}/query/{start_timestamp}' prop = kwargs.get('property', 'water_level_validated') loc = kwargs.get('meetlocatie', None) if not loc: raise ValueError('Meetlocatie not defined') if isinstance(loc, MeetLocatie): loc = loc.name object_id = kwargs.get('object_id', loc) callback = kwargs.get('callback', None) start = kwargs.get('start') or datetime(2010, 1, 1) start = aware(start, pytz.utc) username = kwargs.get('username', settings.MUNISENSE_USERNAME) password = kwargs.get('password', settings.MUNISENSE_PASSWORD) headers = {'Accept': 'application/json'} url = '{api}{endpoint}'.format(api=api, endpoint=endpoint) url = url.format(id=object_id, property=prop, start_timestamp=start.isoformat()) payload = {'rowcount': 1000} result = {} index = 0 while True: response = requests.get(url, headers=headers, params=payload, auth=HTTPBasicAuth(username, password)) response.raise_for_status() index += 1 filename = kwargs.get( 'filename', '{id}_{timestamp}_{index}.json'.format( id=slugify(loc), timestamp=start.strftime('%y%m%d%H%M'), index=index)) result[filename] = response.text try: data = response.json() link_next = data['meta']['link_next'] if link_next: url = '{api}{endpoint}'.format(api=api, endpoint=link_next) else: break except: break if callback: callback(result) return result
def handle(self, *args, **options): with SourceAdapter(logging.getLogger(__name__)) as logger: logger.source = '' logger.info('***UPDATE STARTED***') thumb = options.get('thumb') down = options.get('down') force = options.get('force') if down: logger.info('Downloading data, updating parameters and related time series') else: logger.info('Updating parameters and related time series') count = 0 pk = options.get('pk', None) if pk is None: datasources = Datasource.objects.all() else: datasources = Datasource.objects.filter(pk=pk) replace = options.get('replace') if replace: logger.info('Recreating series') # remember which series have changed during update changed_series = [] for d in datasources: if not d.autoupdate and pk is None: continue logger.source = d logger.info('Updating datasource %s' % d.name) try: series = d.getseries() if replace: start = None else: # actualiseren (data toevoegen) vanaf laatste punt # set starting date/time for update as last date/time in current sourcefiles data_stop = d.stop() if len(series) == 0: last = {} else: # maak dict met een na laatste datapoint per tijdreeks # (rekening houden met niet volledig gevulde laatste tijdsinterval bij accumulatie of sommatie) last = {s: s.beforelast().date for s in series if s.aantal() > 0} if down and d.autoupdate: logger.info('Downloading datasource') try: # if start is in the future, use datetime.now as start to overwrite previous forecasts start = min(data_stop, aware(datetime.now())) newfiles = d.download(start) except Exception as e: logger.exception('ERROR downloading datasource: %s' % e) newfiles = None newfilecount = len(newfiles) if newfiles else 0 logger.info('Got %d new files' % newfilecount) # for update use newfiles AND the existing sourcefiles that contain data for aggregation if last: after = min(last.values()) candidates = d.sourcefiles.filter(start__gte=after) else: after = None candidates = d.sourcefiles.all() if not candidates: logger.warning('No new data for datasource {ds}'.format(ds=d)) if not force: logger.debug('Update of timeseries skipped') continue else: count += 1 logger.info('Reading datasource') try: data = d.get_data(files=candidates,start=after) except Exception as e: logger.exception('Error reading datasource: %s', e) continue if data is None: logger.error('No new data for datasource {ds}'.format(ds=d)) # don't bother to continue: no data continue if replace: logger.info('Updating parameters') try: d.update_parameters(data=data,files=candidates,limit=10) if replace: d.make_thumbnails(data=data) except Exception as e: logger.exception('ERROR updating parameters for datasource: %s' % e) updated = 0 for s in series: logger.info('Updating timeseries %s' % s.name) try: # replace timeseries or update after beforelast datapoint start = last.get(s,None) changes = s.replace() if replace else s.update(data,start=start,thumbnail=thumb) if changes: updated += 1 logger.debug('%d datapoints updated for %s' % (changes, s.name)) changed_series.append(s) else: logger.warning('No updates for %s' % s.name) except Exception as e: logger.exception('ERROR updating timeseries %s: %s' % (s.name, e)) if updated: logger.info('{} time series updated for datasource {}'.format(updated,d.name)) except Exception as e: logger.exception('ERROR updating datasource %s: %s' % (d.name, e)) logger.source = '' logger.info('%d datasources were updated' % count) calc = options.get('calc',True) if calc: def update_formula(f): count = 0 # update dependent formulas first for d in f.get_dependencies(): if d in formulas: count += update_formula(d) try: logger.info('Updating calculated time series %s' % f.name) if f.update(thumbnail=False) == 0: logger.warning('No new data for %s' % f.name) count += 1 except Exception as e: logger.exception('ERROR updating calculated time series %s: %s' % (f.name, e)) formulas.remove(f) return count # get all unique formulas to update formulas = set() for f in Formula.objects.all(): for d in f.get_dependencies(): if d in changed_series: formulas.add(f) break formulas = list(formulas) count = 0 while formulas: count += update_formula(formulas[0]) logger.info('%d calculated time series were updated' % count) logger.info('***UPDATE COMPLETED***')
def handle(self, *args, **options): with DatasourceAdapter(logging.getLogger('update')) as logger: #logging.getLogger('acacia.data').addHandler(email_handler) logger.datasource = '' logger.info('***UPDATE STARTED***') down = options.get('down') if down: logger.info('Downloading data, updating parameters and related time series') else: logger.info('Updating parameters and related time series') count = 0 pk = options.get('pk', None) if pk is None: datasources = Datasource.objects.all() else: datasources = Datasource.objects.filter(pk=pk) replace = options.get('replace') if replace: logger.info('Recreating series') # remember which series have changed during update changed_series = [] for d in datasources: if not d.autoupdate and pk is None: continue logger.datasource = d logger.info('Updating datasource %s' % d.name) try: series = d.getseries() if replace: start = None else: # actualiseren (data toevoegen) vanaf laatste punt data_start = d.stop() if len(series) == 0: series_start = data_start else: # actialisatie vanaf een na laatste datapoint # (rekening houden met niet volledig gevulde laatste tijdsinterval bij accumulatie of sommatie) last = [p.date for p in [s.beforelast() for s in series if s.aantal()>0] if p is not None] if len(last)>0: series_start = min(last) else: series_start = data_start if data_start is None: start = series_start else: start = min(series_start,data_start) # if start is in the future, use datetime.now as start to overwrite previous forecasts if start: start = min(start, aware(datetime.now())) if down and d.autoupdate and d.url is not None: logger.info('Downloading datasource') try: newfiles = d.download() except Exception as e: logger.exception('ERROR downloading datasource: %s' % e) continue if newfiles is None: newfilecount = 0 else: newfilecount = len(newfiles) logger.info('Got %d new files' % newfilecount) if newfilecount == 0: newfiles = None else: newfilecount = 0 newfiles = None if down and newfilecount == 0: # we tried to download but there is no new data logger.debug('Update of timeseries skipped') continue count = count + 1 # TODO: Avoid reading the data twice: d.download() has called sourcefile.save() which reads the data and updates dimensions logger.info('Reading datasource') try: data = d.get_data(files=newfiles,start=start) except Exception as e: logger.exception('Error reading datasource: %s', e) continue if data is None: logger.error('No data found for %s. Update of timeseries skipped' % d.name) # don't bother to continue: no data continue if replace: logger.info('Updating parameters') try: d.update_parameters(data=data,files=newfiles,limit=10) if replace: d.make_thumbnails(data=data) except Exception as e: logger.exception('ERROR updating parameters for datasource: %s' % e) for s in series: logger.info('Updating timeseries %s' % s.name) try: changes = s.replace() if replace else s.update(data,start=start) if changes > 0: changed_series.append(s) # immediately trigger alarms (if any) process_triggers(s) else: logger.warning('No new data for %s' % s.name) except Exception as e: logger.exception('ERROR updating timeseries %s: %s' % (s.name, e)) logger.info('Datasource %s updated' % d.name) except Exception as e: logger.exception('ERROR updating datasource %s: %s' % (d.name, e)) logger.datasource = '' logger.info('%d datasources were updated' % count) calc = options.get('calc',True) if calc: def update_formula(f): count = 0 # update dependent formulas first for d in f.get_dependencies(): if d in formulas: count += update_formula(d) try: logger.info('Updating calculated time series %s' % f.name) if f.update() == 0: logger.warning('No new data for %s' % f.name) else: process_triggers(f) count += 1 except Exception as e: logger.exception('ERROR updating calculated time series %s: %s' % (f.name, e)) formulas.remove(f) return count # get all unique formulas to update formulas = set() for f in Formula.objects.all(): for d in f.get_dependencies(): if d in changed_series: formulas.add(f) break formulas = list(formulas) count = 0 while formulas: count += update_formula(formulas[0]) logger.info('%d calculated time series were updated' % count) logger.info('***UPDATE COMPLETED***') #email_handler.flush() #logging.getLogger('acacia.data.update').removeHandler(email_handler)
def handle(self, *args, **options): baro = Series.objects.get(name='Luchtdruk Voorschoten') # tolerantie voor aanpassingen limit = 0.2 tolerance = timedelta(hours=6) dry = options.get('dry') well = options.get('well') queryset = LoggerPos.objects.all() if well: queryset = LoggerPos.objects.filter(screen__well__name=well) # nwe loggers: serienummer = 2005xxxx queryset = queryset.filter(logger__serial__startswith='2005') # Niet twee keer aanpassen... queryset = queryset.exclude(remarks__contains='aangepast') for pos in queryset.order_by('screen__well'): if not uncompensated(pos): continue screen = pos.screen start_date = aware(pos.start_date - timedelta(days=1), 'utc').replace(hour=0, minute=0, second=0) hand = screen.get_manual_series(start=start_date) if hand is None or hand.empty: continue hand_date = min(hand.index) handpeiling = hand[hand_date] levels = screen.get_compensated_series(start=hand_date).dropna() if levels is None or levels.empty: continue logger_date = min(levels.index) loggerwaarde = levels[logger_date] hpa = baro.at(logger_date).value if baro.unit.startswith('0.1'): hpa *= 0.1 verschil = loggerwaarde - handpeiling offset = 0.01 * (hpa - 1013) / 0.98123 # offset door luchtdruk verschil if abs(verschil) < limit and (logger_date - hand_date < tolerance): # pas bkb aan en rond af op mm refpnt = round(pos.refpnt + offset, 3) logger.debug('{},{},{},{:+.1f} cm'.format( pos, pos.refpnt, refpnt, offset * 100)) if not dry: pos.remarks = 'ophangpunt ({}) aangepast met {:+.2f} cm voor luchtdruk van {:.1f} hPa tijdens installatie.'.format( pos.refpnt, offset * 100, hpa) pos.refpnt = refpnt pos.save() series = screen.find_series() if series: logger.debug( 'Tijdreeks {} aanpassen...'.format(series)) success = recomp(screen, series, start=pos.start_date, stop=pos.end_date)
def get_json(self, chart): options = { 'chart': {'animation': False, 'zoomType': 'x', 'events': {'load': None}, }, 'title': {'text': chart.title}, 'xAxis': {'type': 'datetime','plotBands': []}, 'yAxis': [], 'tooltip': {'valueDecimals': 2, 'shared': True, }, 'legend': {'enabled': chart.series.count() > 1}, 'plotOptions': {'line': {'marker': {'enabled': False}}, 'series': {'connectNulls': True}, 'column': {'allowpointSelect': True, 'pointPadding': 0.01, 'groupPadding': 0.01}}, 'credits': {'enabled': True, 'text': 'acaciawater.com', 'href': 'http://www.acaciawater.com', }, 'exporting' :{ 'sourceWidth': 1080, 'sourceHeight': 600, # 'scale': 2, # 'chartOptions' :{ # 'title': {'style': {'fontSize': 0 }}, # 0 gemaakt omdat titel niet wordt overgenomen # 'xAxis': {'labels': {'style': {'fontSize': 15 }}}, # 'yAxis': {'labels': {'style': {'fontSize': 15 }}}, # 'legend': {'itemStyle': {'fontSize': 15 },'padding': 1,}, # 'credits': {'enabled': False} # }, } } if chart.start: options['xAxis']['min'] = chart.start if chart.stop: options['xAxis']['max'] = chart.stop allseries = [] tmin = chart.start tmax = chart.stop ymin = None ymax = None num_series = chart.series.count() for _,s in enumerate(chart.series.all()): ser = s.series if tmin: tmin = min(tmin,s.t0 or ser.van() or chart.start or tmin) else: tmin = s.t0 or ser.van() or chart.start if tmax: tmax = max(tmax,s.t1 or ser.tot() or chart.stop or tmax) else: tmax = s.t1 or ser.tot() or chart.stop if ymin: ymin = min(ymin,s.y0 or ser.minimum()) else: ymin = s.y0 or ser.minimum() if ymax: ymax = max(ymax,s.y1 or ser.maximum()) else: ymax = s.y1 or ser.maximum() try: deltat = (ser.tot()-ser.van()).total_seconds() / ser.aantal() * 1000 except: deltat = 24 * 3600000 # 1 day options['yAxis'].append({ 'title': {'text': s.label}, 'opposite': 0 if s.axislr == 'l' else 1, 'min': s.y0, 'max': s.y1 }) pts = [] name = s.name if name is None or name == '': name = ser.name if not ser.validated: # append asterisk to name when series has not been validated if isinstance(ser,Formula): for dep in ser.get_dependencies(): if not dep.validated: name += '*' break else: name += '*' sop = {'name': name, 'id': 'series_%d' % ser.id, 'type': s.type, 'yAxis': s.axis-1, 'data': pts} if not s.color is None and len(s.color)>0: sop['color'] = s.color if s.type == 'scatter': sop['tooltip'] = {'valueSuffix': ' '+ser.unit, 'headerFormat': '<small>{point.key}</small><br/><table>', 'pointFormat': '<tr><td style="color:{series.color}">{series.name}</td>\ <td style = "text-align: right">: <b>{point.y}</b></td></tr>'} else: sop['tooltip'] = {'valueSuffix': ' ' + ser.unit} if s.type == 'column': if s.stack is not None: sop['stacking'] = s.stack if num_series > 1: sop['pointRange'] = deltat if s.type == 'area' and s.series2: sop['type'] = 'arearange' sop['fillOpacity'] = 0.3 allseries.append(sop) options['series'] = allseries for band in chart.plotband_set.all(): if band.orientation == 'h': ax = options['yAxis'][band.axis-1] lo = float(band.low) hi = float(band.high) every = max(1,float(band.repetition)) b = [] for i in range(20): if lo < ymax: b.append( {'color': band.style.fillcolor, 'borderWidth': band.style.borderwidth, 'borderColor': band.style.bordercolor, 'from': lo, 'to': hi, 'label': {'text':band.label}, 'zIndex': band.style.zIndex }) lo += every hi += every else: ax = options['xAxis'] lo = parse(band.low) hi = parse(band.high) delta = parserep(band.repetition) b = [] for i in range(20): if aware(lo) < aware(tmax): b.append({'color': band.style.fillcolor, 'borderWidth': band.style.borderwidth, 'borderColor': band.style.bordercolor, 'from': lo, 'to': hi, 'label': {'text':band.label}}) if delta: lo += delta hi += delta if not 'plotBands' in ax: ax['plotBands'] = [] ax['plotBands'].extend(b) for line in chart.plotline_set.all(): if line.orientation == 'h': ax = options['yAxis'][line.axis-1] else: ax = options['xAxis'] line_options = { 'color': line.style.color, 'dashStyle': line.style.dashstyle, 'label': {'text': line.label}, 'value': line.value, 'width': line.style.width, 'zIndex': line.style.zIndex } if not 'plotLines' in ax: ax['plotLines'] = [] ax['plotLines'].append(line_options) return json.dumps(options,default=to_millis)
def get_json(self, chart): options = { # 'rangeSelector': { 'enabled': True, # 'inputEnabled': True, # 'selected': 5, # }, # 'navigator': {'adaptToUpdatedData': False, 'enabled': False}, # 'loading': {'style': {'backgroundColor': 'white', 'fontFamily': 'Arial', 'fontSize': 'small'}, # 'labelStyle': {'fontWeight': 'normal'}, # 'hideDuration': 0, # }, 'chart': {'animation': False, 'zoomType': 'x', 'events': {'load': None}, }, 'title': {'text': chart.title}, 'xAxis': {'type': 'datetime','plotBands': []}, 'yAxis': [], 'tooltip': {'valueDecimals': 2, 'shared': True, }, 'legend': {'enabled': chart.series.count() > 1}, 'plotOptions': {'line': {'marker': {'enabled': False}}, 'column': {'allowpointSelect': True, 'pointPadding': 0.01, 'groupPadding': 0.01}}, 'credits': {'enabled': True, 'text': 'acaciawater.com', 'href': 'http://www.acaciawater.com', } } if chart.start: options['xAxis']['min'] = tojs(chart.start) if chart.stop: options['xAxis']['max'] = tojs(chart.stop) allseries = [] tmin = chart.start tmax = chart.stop ymin = None ymax = None for _,s in enumerate(chart.series.all()): ser = s.series if tmin: tmin = min(tmin,s.t0 or ser.van()) else: tmin = s.t0 or ser.van() if tmax: tmax = max(tmax,s.t1 or ser.tot()) else: tmax = s.t1 or ser.tot() if ymin: ymin = min(ymin,s.y0 or ser.minimum()) else: ymin = s.y0 or ser.minimum() if ymax: ymax = max(ymax,s.y1 or ser.maximum()) else: ymax = s.y1 or ser.maximum() title = s.label #ser.name if len(ser.unit)==0 else '%s [%s]' % (ser.name, ser.unit) if chart.series.count()>1 else ser.unit options['yAxis'].append({ 'title': {'text': title}, 'opposite': 0 if s.axislr == 'l' else 1, 'min': s.y0, 'max': s.y1 }) pts = [] #[[p.date,p.value] for p in ser.datapoints.filter(date__gte=start).order_by('date')] name = s.name if name is None or name == '': name = ser.name sop = {'name': name, 'id': 'series_%d' % ser.id, 'type': s.type, 'yAxis': s.axis-1, 'data': pts} if not s.color is None and len(s.color)>0: sop['color'] = s.color if s.type == 'scatter': sop['tooltip'] = {'valueSuffix': ' '+ser.unit, 'headerFormat': '<small>{point.key}</small><br/><table>', 'pointFormat': '<tr><td style="color:{series.color}">{series.name}</td>\ <td style = "text-align: right">: <b>{point.y}</b></td></tr>'} else: sop['tooltip'] = {'valueSuffix': ' ' + ser.unit} if s.type == 'column' and s.stack is not None: sop['stacking'] = s.stack if s.type == 'area' and s.series2: sop['type'] = 'arearange' sop['fillOpacity'] = 0.3 allseries.append(sop) options['series'] = allseries for band in chart.plotband_set.all(): if band.orientation == 'h': ax = options['yAxis'][band.axis-1] lo = float(band.low) hi = float(band.high) every = max(1,float(band.repetition)) b = [] for i in range(20): if lo < ymax: b.append({'color': band.style.fillcolor, 'borderWidth': band.style.borderwidth, 'borderColor': band.style.bordercolor, 'from': lo, 'to': hi, 'label': {'text':band.label}}) lo += every hi += every else: ax = options['xAxis'] lo = parse(band.low) hi = parse(band.high) def parserep(r): from dateutil.relativedelta import relativedelta pattern = r'(?P<rep>\d*)(?P<how>[hdwmy])' match = re.match(pattern, r,re.IGNORECASE) if match: rep = match.group('rep') or 1 how = match.group('how') if how == 'h': delta = relativedelta(hours=int(rep)) elif how == 'd': delta = relativedelta(days=int(rep)) elif how == 'w': delta = relativedelta(weeks=int(rep)) elif how == 'm': delta = relativedelta(months=int(rep)) elif how == 'y': delta = relativedelta(years=int(rep)) return delta return None delta = parserep(band.repetition) b = [] for i in range(20): if aware(lo) < aware(tmax): b.append({'color': band.style.fillcolor, 'borderWidth': band.style.borderwidth, 'borderColor': band.style.bordercolor, 'from': lo, 'to': hi, 'label': {'text':band.label}}) if delta: lo += delta hi += delta if not 'plotBands' in ax: ax['plotBands'] = [] ax['plotBands'].extend(b) for line in chart.plotline_set.all(): pass jop = json.dumps(options,default=date_handler) # remove quotes around date stuff jop = re.sub(r'\"(Date\.UTC\([\d,]+\))\"',r'\1', jop) return jop