def read_individually_downloaded_sw_file(file_name):
with open(file_name) as f:
reader = csv.reader(f)
rows = [r for r in reader]
details = rows[13][0]
# print details
site_id, site_info, lat, lng, elev = re.match('Site (\d+) (.+) Lat:(.*) Long:(.*) Elev:(.*)', details).groups()
lat = float(lat)
lng = float(lng)
elev = float(elev)
headers = rows[15]
date_i = 0; assert headers[date_i] == "Datetime"
level_i = 1; assert headers[level_i] == "Water Level (m) Mean"
flow_i = 3; assert headers[flow_i] == "Discharge (Ml/d) Mean"
old_flow_i = 5; assert headers[old_flow_i] == "Discharge (Ml/d) Mean"
start_i = 16
while rows[start_i][level_i] == '':
start_i += 1
end_i = len(rows)
while len(rows[end_i-1]) == 0:
end_i -= 1
# print "start_i-end-i", start_i, '-', end_i
dates = numpy.array([datetime.datetime.strptime(r[date_i], "%d/%m/%Y %H:%M:%S") for r in rows[start_i:end_i]])
levels = utils.interpolate( numpy.array([utils.as_float(r[level_i]) for r in rows[start_i:end_i]]) )
flows = utils.interpolate( numpy.array([utils.as_float(r[flow_i]) for r in rows[start_i:end_i]]) )
return site_id, site_info, lat, lng, elev, dates, levels, flows
def read_bulk_downloaded_sw_file(dir_name, site_id, data_types):
# get details of all sites
site_details = {}
with open(os.path.join(dir_name, "Site Details.csv")) as f:
reader = csv.DictReader(f)
for row in reader:
site_details[row["Site Id"]] = row
data_types_values = {}
for data_type in data_types:
file_name = os.path.join(dir_name, site_id+"."+data_type+".csv")
with open(file_name) as f:
reader = csv.reader(f)
rows = [r for r in reader]
headers = rows[2]
date_i = 0; assert headers[date_i] == "Date"
data_i = 1; assert headers[data_i] == "Mean"
start_i = 3
dates = numpy.array([datetime.datetime.strptime(r[date_i], "%H:%M:%S %d/%m/%Y") for r in rows[start_i:]])
data = utils.interpolate( numpy.array([utils.as_float(r[data_i]) for r in rows[start_i:]]) )
data_types_values[data_type] ={
"type": data_type,
"dates": dates,
"data": data
}
return site_details[site_id], data_types_values
def get_bom_climate(zipped_sites_dir, chosen_id):
bom_re = re.compile("(\d{6})_(\d{3}).zip")
zipped_files = [f for f in os.listdir(zipped_sites_dir) if bom_re.match(f)]
climate_data = {}
for zipped_f in zipped_files:
site_id = bom_re.match(zipped_f).group(1)
obs_code = bom_re.match(zipped_f).group(2)
obs_type = bom_obs_types[obs_code]
if site_id == chosen_id:
archive = zipfile.ZipFile(os.path.join(zipped_sites_dir, zipped_f))
csvfile = filter(lambda filename: filename.endswith(".csv"), archive.namelist())[0]
reader = csv.DictReader(archive.open(csvfile))
raw_rows = [row for row in reader]
# dates = [row["Year"]+"-"+row["Month"]+"-"+row["Day"] for row in raw_rows]
dates = numpy.array(
[datetime.datetime(int(row["Year"]), int(row["Month"]), int(row["Day"])) for row in raw_rows]
)
values = numpy.array([utils.as_float(row[obs_type]) for row in raw_rows])
climate_data[obs_code] = {"type": obs_type, "dates": dates, "values": values}
return climate_data