def main():
for station in loadStations('chmino'):
# Get measures in HTML form
tag = station['senal_id']
url = 'http://saih.chminosil.es/index.php?url=/datos/graficas_numeros/tag:%s&historia=0' % tag
print(url)
r = requests.get(url, cookies={'lang': 'es'})
html = r.text.encode(r.encoding)
# Find date / values table
tree = etree.HTML(html)
trs = tree.xpath('//th[.="Fecha"]/parent::tr/following::tr')
# Parse each value of the table
values = []
for tr in trs:
fecha, valor = tr.xpath('td')
d = datetime.datetime.strptime(fecha.text, '%d/%m/%Y %H:%M')
if valor.text.strip() == '': # ignore empty values
continue
v = float(valor.text.replace(',', '.'))
values.append((d, v))
# Save data if we got something
if len(values) > 0:
values.sort(
key=lambda dv: dv[0]) # Values must be sorted for saveValue()
saveValues('chmino', '%s' % tag, values)
def main():
for station in loadStations('chguadiana'):
values = list(getData(station))
if len(values) > 0:
values.sort(key=lambda x: x[0]
) # values must be sorted for saveValues algorithm
saveValues('chguadiana', '%(type)s_%(station_id)s' % station,
values)
def main():
stations = loadStations('vigicrues')
for station in stations:
station_id = station['id'][:-2]
print(station_id)
lat, lon = getLatLonFromSandreId(station_id)
if lat != None and lon != None:
station['lat'] = lat
station['lon'] = lon
f = open('stations_vigicrues.json', 'w')
json.dump(stations, f)
f.close()
def main():
pat = re.compile(r"<tr><td class='_lineas' >([^<]*)<..td><td class=.._lineas.. style=..text-align:right;..>([^<]*)<..td><..tr>")
for station in loadStations('chebro'):
if station['tag']=='':
continue
print(station['id'])
date_start_s = datetime.datetime.strftime(datetime.datetime.now()-datetime.timedelta(days=1),'%Y-%m-%d-%H-%M')
date_end_s = datetime.datetime.strftime(datetime.datetime.now()-datetime.timedelta(days=-1),'%Y-%m-%d-%H-%M')
url = 'http://195.55.247.237/saihebro/views/elements/graficas/ajax.php?url=/sedh/ajax_obtener_datos_numericos/fecha_ini:%s/fecha_fin:%s/intervalo_hora:0/intervalo_15m:1/fecha:%s/_senal:%s'%(date_start_s,date_end_s,date_end_s,station['tag'])
print(url)
r = requests.get(url)
points = pat.findall(r.text)
values = list(map(lambda p: (datetime.datetime.strptime(p[0],'%d\\/%m\\/%Y %H:%M'),float(p[1].replace('.','').replace(',','.'))),filter(lambda p: p[1]!='',points)))
values.sort(key=lambda x:x[0]) # values must be sorted for saveValues algorithm
if len(values)>0:
saveValues('chebro','%s'%station['id'],values)
def main():
for station in loadStations('chj'):
print(station['id'])
ps = 24 * 60 / 5 + 3
url = 'http://saih.chj.es/chj/saih/stats/datosGrafico?v=%s&t=ultimos5minutales&%d=30' % (
station['var'], ps)
print(url)
r = requests.get(url)
values = list(
filter(
lambda dv: dv[1] != None,
map(
lambda dv: (datetime.datetime.strptime(
dv[0], '%d/%m/%Y %H:%M'), dv[1]),
r.json()[1])))
if len(values) > 0:
values.sort(key=lambda x: x[0]
) # values must be sorted for saveValues algorithm
saveValues('chj', station['id'], values)
def main():
flow4level_updated = False
stations = loadStations('chcantabrico')
for station in stations:
url = 'https://www.chcantabrico.es/evolucion-de-niveles/-/descarga/csv/nivel/%s' % station['id']
print(url)
r = requests.get(url)
csv = r.text.encode(r.encoding)
lines = csv.split('\n')[2:]
values = map(parseLine,filter(lambda line: line.strip()!='',lines))
if len(values)>0:
saveValues('chcantabrico','nivel_%s'%station['id'],values)
flow4level = getFlow4Level(station['url'])
len_bf = len(station['flow4level'])
station['flow4level'].update(flow4level) #update `station` dict in place with new value(s)
if len(station['flow4level'])>len_bf:
flow4level_updated = True
if flow4level_updated:
print('New value got for flow4level')
json.dump(stations,open('stations_chcantabrico.json','w'))
def main():
for station in loadStations('chtajo'):
print('%s: %s %s' % (station['senal_id'], station['name'],
station['desc'])).encode('utf8')
process(station['senal_id'])
def main():
stations = loadStations('arpapiemonte')
for station in stations:
d = getData(station['url'])
if len(d)>0:
saveValues('arpapiemonte',station['station_id'],filter(lambda dv:dv[1]!=None,map(lambda dv: (datetime.datetime.fromtimestamp(dv[0]/1000.0),dv[1]),d['data'])))
def main():
stations = loadStations('gencat')
print('Loaded %d stations' % len(stations))
epoch = int(time.time())
for station in stations:
process(station, epoch)
def main():
stations = loadStations('hidrosur')
for station in stations:
process(station)
else:
print('%s: no previous values' % code_station)
url = 'http://www.rdbrmc.com/hydroreel2/station.php?codestation=%d' % code_station
print(url)
r = requests.get(url)
t = r.text.encode(r.encoding)
value_pluie = findValue(cumul_horare_re, t)
value_cote = findValue(cotes_re, t)
value_debit = findValue(debits_re, t)
if value_pluie != None:
saveValues('rdbrmc', 'pluie_%s' % code_station, [value_pluie])
if value_cote != None:
saveValues('rdbrmc', 'cote_%s' % code_station, [value_cote])
if value_debit != None:
saveValues('rdbrmc', 'debit_%s' % code_station, [value_debit])
if __name__ == '__main__':
freq4id = {}
for station in loadStations('rdbrmc'):
station_id = station['id']
if station_id in stations_ids_to_ignore:
continue
if station_id not in freq4id:
freq4id[station_id] = station['freq']
else:
if station['freq'] < freq4id[station_id]:
freq4id = station['freq']
for station_id, minfreq in freq4id.iteritems():
process(station_id, minfreq)
if len(values) > 0:
saveValues('vigicrues', station_id, values)
url = 'https://www.vigicrues.gouv.fr/services/observations.json/index.php?CdStationHydro=%(station_id)s&GrdSerie=Q&FormatSortie=simple' % {
'station_id': station_id
}
print(url)
r = requests.get(url)
t = r.text.encode(r.encoding)
values = map(
lambda timestampvalue: [
datetime.datetime.fromtimestamp(timestampvalue[0] / 1000),
timestampvalue[1]
],
json.loads(t)['Serie']['ObssHydro'])
print(station_id, 'Q', len(values))
if len(values) > 0:
saveValues('vigicrues', '%s-q' % station_id, values)
if __name__ == '__main__':
if len(sys.argv) == 2 and sys.argv[1] in ('-h', '--help'):
print('Usage: %s [station_id]' % sys.argv[0])
elif len(sys.argv) == 2:
process(sys.argv[1])
else:
for station in loadStations('vigicrues'):
try:
process(station['id'])
except Exception, e:
print('Error: %s' % str(e))
def main():
stations = loadStations('chduero')
for station in stations:
print('%s %s' %
(station['name'].encode('utf8'), station['desc'].encode('utf8')))
process(station)
def main():
stations = loadStations('chguadalquivir')
for station in stations:
process(station)