def process(station_id):
url = 'https://www.vigicrues.gouv.fr/services/observations.json/index.php?CdStationHydro=%(station_id)s&GrdSerie=H&FormatSortie=simple' % {
'station_id': station_id
}
print(url)
r = requests.get(url)
t = r.text.encode(r.encoding)
j = json.loads(t)
if isinstance(j, list) and len(j) > 0 and j[0].has_key('error_msg'):
print('Error: %s' % j[0]['error_msg'])
return
values = map(
lambda timestampvalue: [
datetime.datetime.fromtimestamp(timestampvalue[0] / 1000),
timestampvalue[1]
],
json.loads(t)['Serie']['ObssHydro'])
print(station_id, 'H', len(values))
if len(values) > 0:
saveValues('vigicrues', station_id, values)
url = 'https://www.vigicrues.gouv.fr/services/observations.json/index.php?CdStationHydro=%(station_id)s&GrdSerie=Q&FormatSortie=simple' % {
'station_id': station_id
}
print(url)
r = requests.get(url)
t = r.text.encode(r.encoding)
values = map(
lambda timestampvalue: [
datetime.datetime.fromtimestamp(timestampvalue[0] / 1000),
timestampvalue[1]
],
json.loads(t)['Serie']['ObssHydro'])
print(station_id, 'Q', len(values))
if len(values) > 0:
saveValues('vigicrues', '%s-q' % station_id, values)
def main():
for station in loadStations('chmino'):
# Get measures in HTML form
tag = station['senal_id']
url = 'http://saih.chminosil.es/index.php?url=/datos/graficas_numeros/tag:%s&historia=0' % tag
print(url)
r = requests.get(url, cookies={'lang': 'es'})
html = r.text.encode(r.encoding)
# Find date / values table
tree = etree.HTML(html)
trs = tree.xpath('//th[.="Fecha"]/parent::tr/following::tr')
# Parse each value of the table
values = []
for tr in trs:
fecha, valor = tr.xpath('td')
d = datetime.datetime.strptime(fecha.text, '%d/%m/%Y %H:%M')
if valor.text.strip() == '': # ignore empty values
continue
v = float(valor.text.replace(',', '.'))
values.append((d, v))
# Save data if we got something
if len(values) > 0:
values.sort(
key=lambda dv: dv[0]) # Values must be sorted for saveValue()
saveValues('chmino', '%s' % tag, values)
def process(station):
direct_url = station['direct_url']
values = list(getData(direct_url))
if len(values) > 0:
values.sort(key=lambda x: x[0]
) # values must be sorted for saveValues algorithm
saveValues('chduero', station['senal_id'], values)
def main():
for station in loadStations('chguadiana'):
values = list(getData(station))
if len(values) > 0:
values.sort(key=lambda x: x[0]
) # values must be sorted for saveValues algorithm
saveValues('chguadiana', '%(type)s_%(station_id)s' % station,
values)
def process(station, epoch):
url = 'http://aca-web.gencat.cat/sentilo-catalog-web/admin/sensor/lastObs/AFORAMENT-EST.%(component)s.%(sensor)s/?ts=%(timestamp_ms)d' % {
'component': station['station_id'],
'sensor': station['senal_id'],
'timestamp_ms': epoch * 1000
}
print(url)
r = requests.get(url)
values = map(parsePoint, r.json())
if len(values) > 0:
saveValues('gencat', station['senal_id'], values)
def scrap():
already_done = []
for name, code1, code2, river, value, when in listStations():
if value.strip() == '--':
continue
if code2 in already_done:
continue
already_done.append(code2)
value = float(value)
when = datetime.datetime.strptime(when, '%d/%m/%Y %H:%M')
print(code2)
saveValues('arpal', code1, [(when, value)])
def main():
pat = re.compile(r"<tr><td class='_lineas' >([^<]*)<..td><td class=.._lineas.. style=..text-align:right;..>([^<]*)<..td><..tr>")
for station in loadStations('chebro'):
if station['tag']=='':
continue
print(station['id'])
date_start_s = datetime.datetime.strftime(datetime.datetime.now()-datetime.timedelta(days=1),'%Y-%m-%d-%H-%M')
date_end_s = datetime.datetime.strftime(datetime.datetime.now()-datetime.timedelta(days=-1),'%Y-%m-%d-%H-%M')
url = 'http://195.55.247.237/saihebro/views/elements/graficas/ajax.php?url=/sedh/ajax_obtener_datos_numericos/fecha_ini:%s/fecha_fin:%s/intervalo_hora:0/intervalo_15m:1/fecha:%s/_senal:%s'%(date_start_s,date_end_s,date_end_s,station['tag'])
print(url)
r = requests.get(url)
points = pat.findall(r.text)
values = list(map(lambda p: (datetime.datetime.strptime(p[0],'%d\\/%m\\/%Y %H:%M'),float(p[1].replace('.','').replace(',','.'))),filter(lambda p: p[1]!='',points)))
values.sort(key=lambda x:x[0]) # values must be sorted for saveValues algorithm
if len(values)>0:
saveValues('chebro','%s'%station['id'],values)
def process(station):
station_id = station['id']
nb = station['nb']
letter = station['letter']
url = 'http://www.chguadalquivir.es/saih/saihhist2.aspx?s1=%s_%d%s&dia=1' % (
station_id, nb, letter)
print(url)
r = requests.get(url)
t = etree.XML(r.text.encode(r.encoding))
if t.find('s1') == None or t.find('x') == None:
print('Error with %s' % url)
return
s1 = map(float, t.find('s1').text.split(';'))
x = map(lambda d: datetime.datetime.fromtimestamp(float(d)),
t.find('x').text.split(';')[:-1])
values = zip(x, s1)
if len(values) > 0:
saveValues('chguadalquivir', station_id, values)
def main():
for station in loadStations('chj'):
print(station['id'])
ps = 24 * 60 / 5 + 3
url = 'http://saih.chj.es/chj/saih/stats/datosGrafico?v=%s&t=ultimos5minutales&%d=30' % (
station['var'], ps)
print(url)
r = requests.get(url)
values = list(
filter(
lambda dv: dv[1] != None,
map(
lambda dv: (datetime.datetime.strptime(
dv[0], '%d/%m/%Y %H:%M'), dv[1]),
r.json()[1])))
if len(values) > 0:
values.sort(key=lambda x: x[0]
) # values must be sorted for saveValues algorithm
saveValues('chj', station['id'], values)
def process(station):
# Get measure chart page
url = 'http://www.redhidrosurmedioambiente.es/saih/mapa/tiempo/real/grafica/%s' % station[
'senal_id']
print(url)
r = requests.get(url)
html = r.text.encode(r.encoding)
# Look for chart data in the javacript, and parse it
i = html.find('var lineChartData = {')
if i == -1:
i = html.find('var barChartData = {')
if i == -1:
raise Exception('data not found')
j = html.find('};', i)
if j == -1:
raise Exception('data not found')
chartdata = demjson.decode(html[i + 19:j +
1]) # 19: ok for both bar or line
# Parse datetimes
ts = map(lambda s: datetime.datetime.strptime(s, '%d/%m/%y %H:%M'),
chartdata['labels'])
# Find the right data set (other data sets are alerting levels)
if len(chartdata['datasets']) == 1: # only one dataset, use it
vs = map(float, chartdata['datasets'][0]['data'])
else: # several datasets, select the right one
for d in chartdata['datasets']:
lbl = d['label']
if lbl.startswith('NIVEL') or lbl.startswith(
u'Precipitación horaria'):
vs = map(float, d['data'])
break
# Merge datetimes and values
values = zip(ts, vs)
# Save values if we got something
if len(values) > 0:
saveValues('hidrosur', station['senal_id'], values)
def main():
flow4level_updated = False
stations = loadStations('chcantabrico')
for station in stations:
url = 'https://www.chcantabrico.es/evolucion-de-niveles/-/descarga/csv/nivel/%s' % station['id']
print(url)
r = requests.get(url)
csv = r.text.encode(r.encoding)
lines = csv.split('\n')[2:]
values = map(parseLine,filter(lambda line: line.strip()!='',lines))
if len(values)>0:
saveValues('chcantabrico','nivel_%s'%station['id'],values)
flow4level = getFlow4Level(station['url'])
len_bf = len(station['flow4level'])
station['flow4level'].update(flow4level) #update `station` dict in place with new value(s)
if len(station['flow4level'])>len_bf:
flow4level_updated = True
if flow4level_updated:
print('New value got for flow4level')
json.dump(stations,open('stations_chcantabrico.json','w'))
def process(code_station, minfreq):
lasts = filter(lambda x: not (x is None),
(getLastUpdateTime('rdbrmc', 'pluie_%s' % code_station),
getLastUpdateTime('rdbrmc', 'cote_%s' % code_station),
getLastUpdateTime('rdbrmc', 'debit_%s' % code_station)))
if len(lasts) > 0:
last_upd = min(lasts)
if minfreq == -1:
minfreq = 5
#print 'Must not be updated before',last_upd+datetime.timedelta(minutes=minfreq)
if last_upd + datetime.timedelta(
minutes=minfreq) > datetime.datetime.now():
print(
'Skip %s: %s > %s' %
(code_station, last_upd + datetime.timedelta(minutes=minfreq),
datetime.datetime.now()))
return
else:
print('%s: no previous values' % code_station)
url = 'http://www.rdbrmc.com/hydroreel2/station.php?codestation=%d' % code_station
print(url)
r = requests.get(url)
t = r.text.encode(r.encoding)
value_pluie = findValue(cumul_horare_re, t)
value_cote = findValue(cotes_re, t)
value_debit = findValue(debits_re, t)
if value_pluie != None:
saveValues('rdbrmc', 'pluie_%s' % code_station, [value_pluie])
if value_cote != None:
saveValues('rdbrmc', 'cote_%s' % code_station, [value_cote])
if value_debit != None:
saveValues('rdbrmc', 'debit_%s' % code_station, [value_debit])
def process(senal_id):
values = list(getLastValues(senal_id))
values.sort(
key=lambda x: x[0]) # values must be sorted for saveValues algorithm
if len(values) > 0:
saveValues('chtajo', senal_id, values)
def main():
stations = loadStations('arpapiemonte')
for station in stations:
d = getData(station['url'])
if len(d)>0:
saveValues('arpapiemonte',station['station_id'],filter(lambda dv:dv[1]!=None,map(lambda dv: (datetime.datetime.fromtimestamp(dv[0]/1000.0),dv[1]),d['data'])))
