def process(station_id): url = 'https://www.vigicrues.gouv.fr/services/observations.json/index.php?CdStationHydro=%(station_id)s&GrdSerie=H&FormatSortie=simple' % { 'station_id': station_id } print(url) r = requests.get(url) t = r.text.encode(r.encoding) j = json.loads(t) if isinstance(j, list) and len(j) > 0 and j[0].has_key('error_msg'): print('Error: %s' % j[0]['error_msg']) return values = map( lambda timestampvalue: [ datetime.datetime.fromtimestamp(timestampvalue[0] / 1000), timestampvalue[1] ], json.loads(t)['Serie']['ObssHydro']) print(station_id, 'H', len(values)) if len(values) > 0: saveValues('vigicrues', station_id, values) url = 'https://www.vigicrues.gouv.fr/services/observations.json/index.php?CdStationHydro=%(station_id)s&GrdSerie=Q&FormatSortie=simple' % { 'station_id': station_id } print(url) r = requests.get(url) t = r.text.encode(r.encoding) values = map( lambda timestampvalue: [ datetime.datetime.fromtimestamp(timestampvalue[0] / 1000), timestampvalue[1] ], json.loads(t)['Serie']['ObssHydro']) print(station_id, 'Q', len(values)) if len(values) > 0: saveValues('vigicrues', '%s-q' % station_id, values)
def main(): for station in loadStations('chmino'): # Get measures in HTML form tag = station['senal_id'] url = 'http://saih.chminosil.es/index.php?url=/datos/graficas_numeros/tag:%s&historia=0' % tag print(url) r = requests.get(url, cookies={'lang': 'es'}) html = r.text.encode(r.encoding) # Find date / values table tree = etree.HTML(html) trs = tree.xpath('//th[.="Fecha"]/parent::tr/following::tr') # Parse each value of the table values = [] for tr in trs: fecha, valor = tr.xpath('td') d = datetime.datetime.strptime(fecha.text, '%d/%m/%Y %H:%M') if valor.text.strip() == '': # ignore empty values continue v = float(valor.text.replace(',', '.')) values.append((d, v)) # Save data if we got something if len(values) > 0: values.sort( key=lambda dv: dv[0]) # Values must be sorted for saveValue() saveValues('chmino', '%s' % tag, values)
def process(station): direct_url = station['direct_url'] values = list(getData(direct_url)) if len(values) > 0: values.sort(key=lambda x: x[0] ) # values must be sorted for saveValues algorithm saveValues('chduero', station['senal_id'], values)
def main(): for station in loadStations('chguadiana'): values = list(getData(station)) if len(values) > 0: values.sort(key=lambda x: x[0] ) # values must be sorted for saveValues algorithm saveValues('chguadiana', '%(type)s_%(station_id)s' % station, values)
def process(station, epoch): url = 'http://aca-web.gencat.cat/sentilo-catalog-web/admin/sensor/lastObs/AFORAMENT-EST.%(component)s.%(sensor)s/?ts=%(timestamp_ms)d' % { 'component': station['station_id'], 'sensor': station['senal_id'], 'timestamp_ms': epoch * 1000 } print(url) r = requests.get(url) values = map(parsePoint, r.json()) if len(values) > 0: saveValues('gencat', station['senal_id'], values)
def scrap(): already_done = [] for name, code1, code2, river, value, when in listStations(): if value.strip() == '--': continue if code2 in already_done: continue already_done.append(code2) value = float(value) when = datetime.datetime.strptime(when, '%d/%m/%Y %H:%M') print(code2) saveValues('arpal', code1, [(when, value)])
def main(): pat = re.compile(r"<tr><td class='_lineas' >([^<]*)<..td><td class=.._lineas.. style=..text-align:right;..>([^<]*)<..td><..tr>") for station in loadStations('chebro'): if station['tag']=='': continue print(station['id']) date_start_s = datetime.datetime.strftime(datetime.datetime.now()-datetime.timedelta(days=1),'%Y-%m-%d-%H-%M') date_end_s = datetime.datetime.strftime(datetime.datetime.now()-datetime.timedelta(days=-1),'%Y-%m-%d-%H-%M') url = 'http://195.55.247.237/saihebro/views/elements/graficas/ajax.php?url=/sedh/ajax_obtener_datos_numericos/fecha_ini:%s/fecha_fin:%s/intervalo_hora:0/intervalo_15m:1/fecha:%s/_senal:%s'%(date_start_s,date_end_s,date_end_s,station['tag']) print(url) r = requests.get(url) points = pat.findall(r.text) values = list(map(lambda p: (datetime.datetime.strptime(p[0],'%d\\/%m\\/%Y %H:%M'),float(p[1].replace('.','').replace(',','.'))),filter(lambda p: p[1]!='',points))) values.sort(key=lambda x:x[0]) # values must be sorted for saveValues algorithm if len(values)>0: saveValues('chebro','%s'%station['id'],values)
def process(station): station_id = station['id'] nb = station['nb'] letter = station['letter'] url = 'http://www.chguadalquivir.es/saih/saihhist2.aspx?s1=%s_%d%s&dia=1' % ( station_id, nb, letter) print(url) r = requests.get(url) t = etree.XML(r.text.encode(r.encoding)) if t.find('s1') == None or t.find('x') == None: print('Error with %s' % url) return s1 = map(float, t.find('s1').text.split(';')) x = map(lambda d: datetime.datetime.fromtimestamp(float(d)), t.find('x').text.split(';')[:-1]) values = zip(x, s1) if len(values) > 0: saveValues('chguadalquivir', station_id, values)
def main(): for station in loadStations('chj'): print(station['id']) ps = 24 * 60 / 5 + 3 url = 'http://saih.chj.es/chj/saih/stats/datosGrafico?v=%s&t=ultimos5minutales&%d=30' % ( station['var'], ps) print(url) r = requests.get(url) values = list( filter( lambda dv: dv[1] != None, map( lambda dv: (datetime.datetime.strptime( dv[0], '%d/%m/%Y %H:%M'), dv[1]), r.json()[1]))) if len(values) > 0: values.sort(key=lambda x: x[0] ) # values must be sorted for saveValues algorithm saveValues('chj', station['id'], values)
def process(station): # Get measure chart page url = 'http://www.redhidrosurmedioambiente.es/saih/mapa/tiempo/real/grafica/%s' % station[ 'senal_id'] print(url) r = requests.get(url) html = r.text.encode(r.encoding) # Look for chart data in the javacript, and parse it i = html.find('var lineChartData = {') if i == -1: i = html.find('var barChartData = {') if i == -1: raise Exception('data not found') j = html.find('};', i) if j == -1: raise Exception('data not found') chartdata = demjson.decode(html[i + 19:j + 1]) # 19: ok for both bar or line # Parse datetimes ts = map(lambda s: datetime.datetime.strptime(s, '%d/%m/%y %H:%M'), chartdata['labels']) # Find the right data set (other data sets are alerting levels) if len(chartdata['datasets']) == 1: # only one dataset, use it vs = map(float, chartdata['datasets'][0]['data']) else: # several datasets, select the right one for d in chartdata['datasets']: lbl = d['label'] if lbl.startswith('NIVEL') or lbl.startswith( u'Precipitación horaria'): vs = map(float, d['data']) break # Merge datetimes and values values = zip(ts, vs) # Save values if we got something if len(values) > 0: saveValues('hidrosur', station['senal_id'], values)
def main(): flow4level_updated = False stations = loadStations('chcantabrico') for station in stations: url = 'https://www.chcantabrico.es/evolucion-de-niveles/-/descarga/csv/nivel/%s' % station['id'] print(url) r = requests.get(url) csv = r.text.encode(r.encoding) lines = csv.split('\n')[2:] values = map(parseLine,filter(lambda line: line.strip()!='',lines)) if len(values)>0: saveValues('chcantabrico','nivel_%s'%station['id'],values) flow4level = getFlow4Level(station['url']) len_bf = len(station['flow4level']) station['flow4level'].update(flow4level) #update `station` dict in place with new value(s) if len(station['flow4level'])>len_bf: flow4level_updated = True if flow4level_updated: print('New value got for flow4level') json.dump(stations,open('stations_chcantabrico.json','w'))
def process(code_station, minfreq): lasts = filter(lambda x: not (x is None), (getLastUpdateTime('rdbrmc', 'pluie_%s' % code_station), getLastUpdateTime('rdbrmc', 'cote_%s' % code_station), getLastUpdateTime('rdbrmc', 'debit_%s' % code_station))) if len(lasts) > 0: last_upd = min(lasts) if minfreq == -1: minfreq = 5 #print 'Must not be updated before',last_upd+datetime.timedelta(minutes=minfreq) if last_upd + datetime.timedelta( minutes=minfreq) > datetime.datetime.now(): print( 'Skip %s: %s > %s' % (code_station, last_upd + datetime.timedelta(minutes=minfreq), datetime.datetime.now())) return else: print('%s: no previous values' % code_station) url = 'http://www.rdbrmc.com/hydroreel2/station.php?codestation=%d' % code_station print(url) r = requests.get(url) t = r.text.encode(r.encoding) value_pluie = findValue(cumul_horare_re, t) value_cote = findValue(cotes_re, t) value_debit = findValue(debits_re, t) if value_pluie != None: saveValues('rdbrmc', 'pluie_%s' % code_station, [value_pluie]) if value_cote != None: saveValues('rdbrmc', 'cote_%s' % code_station, [value_cote]) if value_debit != None: saveValues('rdbrmc', 'debit_%s' % code_station, [value_debit])
def process(senal_id): values = list(getLastValues(senal_id)) values.sort( key=lambda x: x[0]) # values must be sorted for saveValues algorithm if len(values) > 0: saveValues('chtajo', senal_id, values)
def main(): stations = loadStations('arpapiemonte') for station in stations: d = getData(station['url']) if len(d)>0: saveValues('arpapiemonte',station['station_id'],filter(lambda dv:dv[1]!=None,map(lambda dv: (datetime.datetime.fromtimestamp(dv[0]/1000.0),dv[1]),d['data'])))