def process_file(_filename, _path, force, zone):
"""
This function just gets the data row by row and creates a list of arrays
consistent for all different csv files that will comply with the common
process_data function in commonlib
:param _filename:
:param _path:
:return:
"""
log = logging.getLogger(__name__)
def initialize_series():
return dict(WaterCold=[], Electricity=[])
with open(path.join(_path, _filename), 'r') as f:
rows = []
prev_cons = 0
x = 0
for line in f:
if x == 0:
x += 1 # from first line we take only the consumption
prev_cons = float(line.split("|")[2])
continue
rows.append(line)
usernames = {}
meter_data = {}
used_meters = [] # to create a new empty series for each new meter
for row in rows:
i = find_nth(row, "|", 3)
line = row[0:i].split('|')
meter_id = line[0]
if meter_id not in used_meters:
used_meters.append(meter_id)
series = initialize_series() # new meter! Init new series!
_dt = line[1]
cons = line[2]
if not cons:
cons = 0
cons = float(cons)
consumption = cons - prev_cons
prev_cons = cons # keep it to subtract it on next iteration
consumption /= 1000
_type = "WaterCold"
dt = datetime.strptime(_dt, "%d-%m-%Y %H:%M:%S")
#series[_type].append((dt, consumption))
"""
meter_data = dict of dicts of arrays
"""
try: # find previously inserted value
_dict = meter_data[meter_id]
_dict[_type].append((dt, consumption))
#print "append for %s value %s (%s)" % (_type, consumption, dt)
except KeyError: # add new meter data
series[_type].append((dt, consumption))
meter_data[meter_id] = series
#print "create for %s value %s (%s)" % (_type, consumption, dt)
# when we create a HH we need a new username
usernames[meter_id] = meter_id
z_names = [zone]
process_data(meter_data, usernames, force, z_names, {})
def process_file(_filename, _path, force):
"""
This function just gets the data row by row and creates a list of arrays
consistent for all different csv files that will comply with the common
process_data function in commonlib
:param _filename:
:param _path:
:return:
"""
log = logging.getLogger(__name__)
def initialize_series():
return dict(WaterCold=[], Electricity=[])
## "U" for universal-newline mode.
with open(path.join(_path, _filename), 'rU') as f:
usernames = {}
data = csv.reader(f, encoding="utf-8")
meter_data = {}
used_meters = [] # to create a new empty series for each new meter
x = 0
for row in data:
if x == 0:
x += 1 # skip first row
continue
meter_id = row[0]
if meter_id not in used_meters:
used_meters.append(meter_id)
series = initialize_series() # new meter! Init new series!
_dt = row[1]
_tm = row[2]
consumption = row[3]
if not consumption:
consumption = 0
consumption = float(consumption)
_type = "WaterCold"
str_dt = "%s %s" % (_dt, _tm)
dt = datetime.strptime(str_dt, "%d/%m/%Y %H:%M")
"""
meter_data = dict of dicts of arrays
"""
try:
_dict = meter_data[meter_id]
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
_dict[_type].append(t)
except KeyError: # add new meter data
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
series[_type].append(t)
meter_data[meter_id] = series
# when we create a HH we need a new username
username = "******" + meter_id
usernames[meter_id] = username
z_names = ["UK water"]
process_data(meter_data, usernames, force, z_names, {})
def process_file(raw_data, _path, force, z_dict):
"""
This function just gets the data row by row and creates a list of arrays
consistent for all different csv files that will comply with the common
process_data function in commonlib
:param _filename:
:param _path:
:return:
"""
log = logging.getLogger(__name__)
def initialize_series():
return dict(WaterCold=[], Electricity=[])
z_names = []
usernames = {}
meter_data = {}
used_meters = [] # to create a new empty series for each new meter
curr_data = {} # to calculate and store periodic values from total
for row in raw_data:
meter_id = row[0]
consumption = float(row[2])
_dt = row[1]
dt = datetime.strptime(_dt, "%d-%m-%Y %H:%M:%S")
try:
_dict = curr_data[meter_id]
_dict[dt] = consumption
except KeyError:
_dict = {dt: consumption}
curr_data[meter_id] = _dict
# Now we will try and create period values by subtracting
# the previous value for each consumption value and finally
# from the last one the yesterday's total (old_cons).
keys = curr_data.keys()
for m_id in keys:
values = curr_data[m_id].keys()
timestamps = sorted(values)
_dict = curr_data[m_id]
for c in range(len(timestamps)-1, 0, -1): # 0: stops at 1
val1 = _dict[timestamps[c]]
val2 = _dict[timestamps[c-1]]
cons = val1 - val2
_dict[timestamps[c]] = cons
_dict[timestamps[0]] = 0 # first recorded consumption will be 0
for meter_id in keys:
if meter_id not in used_meters:
used_meters.append(meter_id)
series = initialize_series() # new meter! Init new series!
timestamps = curr_data[meter_id].keys()
timestamps = sorted(timestamps)
for ts in timestamps:
dt = ts
consumption = curr_data[meter_id][dt]
if not consumption:
consumption = 0
consumption = float(consumption) / 1000.0 # make it cubic metres
_type = "WaterCold"
"""
meter_data = dict of dicts of arrays
"""
try:
_dict = meter_data[meter_id]
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
_dict[_type].append(t)
except KeyError: # add new meter data
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
series[_type].append(t)
meter_data[meter_id] = series
# when we create a HH we need a new username
username = "******" + meter_id
usernames[meter_id] = username
keys = z_dict.keys()
for k in keys:
z = z_dict[k]
if z not in z_names:
z_names.append(z)
# print meter_data
process_data(meter_data, usernames, force, z_names, z_dict)
def process_file(_filename, _path, old_cons):
"""
This function just gets the data row by row and creates a list of arrays
consistent for all different csv files that will comply with the common
process_data function in commonlib
:param _filename:
:param _path:
:return:
"""
log = logging.getLogger(__name__)
def initialize_series():
return dict(WaterCold=[], Electricity=[])
with open(path.join(_path, _filename), 'r') as f:
usernames = {}
data = csv.reader(f, encoding="utf-8")
meter_data = {}
fifteen_data = {}
x = 0
used_meters = [] # to create a new empty series for each new meter
for row in data:
if x == 0:
x += 1 # skip first row
continue
meter_id = row[0]
if meter_id == "8173467":
pass
if not meter_id:
meter_id = row[1]
if meter_id not in used_meters:
used_meters.append(meter_id)
series = initialize_series() # new meter! Init new series!
if meter_id:
_date = row[3]
try:
_time = row[4]
except IndexError:
_time = "02:05 AM"
consumption = row[2]
if not consumption:
consumption = 0
consumption = float(consumption)
if consumption > 99999999:
continue # probably false reading
try:
cons = old_cons[meter_id]
consumption -= float(cons)
consumption *= 100.0 # to get litres
consumption /= 1000.0
except (KeyError, ValueError) as e:
log.debug("UK: Consumption value for meter %s and date %s "
"not inserted because %s"
% (meter_id, _date, repr(e)))
consumption = 0
_type = "WaterCold"
_dt = "%s %s" % (_date, _time)
try:
dt = datetime.strptime(_dt, "%d/%m/%Y %H:%M")
except ValueError:
# convert 12h to 24h time
dt = datetime.strptime(_dt, "%m/%d/%Y %I:%M %p")
#series[_type].append((dt, consumption))
"""
meter_data = dict of dicts of arrays
"""
try:
_dict = meter_data[meter_id]
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
_dict[_type].append(t)
except KeyError: # add new meter data
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
series[_type].append(t)
meter_data[meter_id] = series
# when we create a HH we need a new username
username = "******" + meter_id
usernames[meter_id] = username
z_names = ["UK water"]
process_data(meter_data, usernames, False, z_names, {})
def process_file(data):
"""
This function just gets the data row by row and creates a list of arrays
consistent for all different csv files that will comply with the common
process_data function in commonlib
:param _filename:
:param _path:
:return:
"""
log = logging.getLogger(__name__)
def initialize_series():
return dict(WaterCold=[], Electricity=[])
usernames = {}
meter_data = {}
used_meters = [] # to create a new empty series for each new meter
for row in data:
meter_id = row[0]
if meter_id not in used_meters:
used_meters.append(meter_id)
series = initialize_series() # new meter! Init new series!
_dt = row[1]
try:
consumption = float(row[3])
if consumption < 0:
continue
except ValueError:
continue
if 'Electricity' in meter_id:
_type = "Electricity"
elif 'Water' in meter_id:
_type = "WaterCold"
else:
log.debug("Consumption type not found! Skipping this row!")
continue
dt = datetime.strptime(_dt, "%Y/%m/%d %H:%M")
dt_aware = make_aware(dt, timezone('UTC'))
gr_tz = timezone("Europe/Athens")
athens_dt = gr_tz.normalize(dt_aware.astimezone(gr_tz))
dt = make_naive(athens_dt, gr_tz) # athens time in naive format
# series[_type].append((dt, consumption))
"""
meter_data = dict of dicts of arrays
"""
beg = meter_id.rfind("_") + 1
end = meter_id.rfind("/")
serial_no = meter_id[beg:end]
""" because meter was swapped, the meter id was changed.
we need to append new data to old meter id
"""
if serial_no == "005E4F":
serial_no = "006047"
try: # find previously inserted value
_dict = meter_data[serial_no]
_dict[_type].append((dt, consumption))
# print "append for %s value %s (%s)" % (_type, consumption, dt)
except KeyError: # add new meter data
series[_type].append((dt, consumption))
meter_data[serial_no] = series
# print "create for %s value %s (%s)" % (_type, consumption, dt)
# when we create a HH we need a new username
username = serial_no
usernames[serial_no] = "GR%s" % username
z_names = ["Greece electric-water"]
process_data(meter_data, usernames, True, z_names, {})
def process_file(data, force):
"""
This function just gets the data row by row and creates a list of arrays
consistent for all different csv files that will comply with the common
process_data function in commonlib
:param meter readings in list of lists:
:return:
"""
log = logging.getLogger(__name__)
def initialize_series():
return dict(WaterCold=[], Electricity=[])
used_meters = [] # to create a new empty series for each new meter
usernames = {}
meter_data = {}
for row in data:
meter_id = row[0]
if meter_id not in used_meters:
used_meters.append(meter_id)
series = initialize_series() # new meter! Init new series!
_dt = row[1]
try:
consumption = float(row[3])
if consumption < 0:
continue
except ValueError:
continue
if 'Electricity' in meter_id:
_type = "Electricity"
elif 'Water' in meter_id:
_type = "WaterCold"
else:
log.debug("Consumption type not found! Skipping this row!")
continue
dt = datetime.strptime(_dt, "%Y/%m/%d %H:%M")
dt_aware = make_aware(dt, timezone('UTC'))
gr_tz = timezone("Europe/Athens")
athens_dt = gr_tz.normalize(dt_aware.astimezone(gr_tz))
dt = make_naive(athens_dt, gr_tz) # athens time in naive format
#series[_type].append((dt, consumption))
"""
meter_data = dict of dicts of arrays
"""
beg = meter_id.rfind("_") + 1
end = meter_id.rfind("/")
serial_no = meter_id[beg:end]
# TODO! Find a better way to handle meter swaps
""" because meter was swapped, the meter id was changed.
we need to append new data to old meter id
"""
if serial_no == "005E4F":
serial_no = "006047"
try: # find previously inserted value
_dict = meter_data[serial_no]
_dict[_type].append((dt, consumption))
#print "append for %s value %s (%s)" % (_type, consumption, dt)
except KeyError: # add new meter data
series[_type].append((dt, consumption))
meter_data[serial_no] = series
#print "create for %s value %s (%s)" % (_type, consumption, dt)
# when we create a HH we need a new username
username = serial_no
usernames[serial_no] = "GR%s" % username
z_names = ["Greece electric-water"]
process_data(meter_data, usernames, force, z_names, {})
def process_file(raw_data, _path, force, z_dict):
"""
This function just gets the data row by row and creates a list of arrays
consistent for all different csv files that will comply with the common
process_data function in commonlib
:param _filename:
:param _path:
:return:
"""
log = logging.getLogger(__name__)
def initialize_series():
return dict(WaterCold=[], Electricity=[])
z_names = []
usernames = {}
meter_data = {}
used_meters = [] # to create a new empty series for each new meter
curr_data = {} # to calculate and store periodic values from total
for row in raw_data:
meter_id = row[0]
consumption = float(row[2])
_dt = row[1]
dt = datetime.strptime(_dt, "%d-%m-%Y %H:%M:%S")
try:
_dict = curr_data[meter_id]
_dict[dt] = consumption
except KeyError:
_dict = {dt: consumption}
curr_data[meter_id] = _dict
# Now we will try and create period values by subtracting
# the previous value for each consumption value and finally
# from the last one the yesterday's total (old_cons).
keys = curr_data.keys()
for m_id in keys:
values = curr_data[m_id].keys()
timestamps = sorted(values)
_dict = curr_data[m_id]
for c in range(len(timestamps)-1, 0, -1): # 0: stops at 1
val1 = _dict[timestamps[c]]
val2 = _dict[timestamps[c-1]]
cons = val1 - val2
_dict[timestamps[c]] = cons
_dict[timestamps[0]] = 0 # first recorded consumption will be 0
for meter_id in keys:
if meter_id not in used_meters:
used_meters.append(meter_id)
series = initialize_series() # new meter! Init new series!
timestamps = curr_data[meter_id].keys()
for ts in timestamps:
dt = ts
consumption = curr_data[meter_id][dt]
if not consumption:
consumption = 0
consumption = float(consumption) / 1000.0 # make it cubic metres
_type = "WaterCold"
"""
meter_data = dict of dicts of arrays
"""
try:
_dict = meter_data[meter_id]
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
_dict[_type].append(t)
except KeyError: # add new meter data
_tuples = create_15_mins(dt, consumption)
for t in _tuples:
series[_type].append(t)
meter_data[meter_id] = series
# when we create a HH we need a new username
username = "******" + meter_id
usernames[meter_id] = username
keys = z_dict.keys()
for k in keys:
z = z_dict[k]
if z not in z_names:
z_names.append(z)
process_data(meter_data, usernames, force, z_names, z_dict)