def test_get_first_five_rows(self):
"""Make sure we get our first five rows back correctly."""
import_record = ImportRecord.objects.create()
expected_raw_columns = ['tax id', 'name', 'etc.']
expected_raw_rows = [
['02023', '12 Jefferson St.', 'etc.'],
['12433', '23 Washington St.', 'etc.'],
['04422', '4 Adams St.', 'etc.'],
]
expected = [
dict(zip(expected_raw_columns, row)) for row in expected_raw_rows
]
expected_saved_format = '\n'.join(
[ROW_DELIMITER.join(row) for row in expected_raw_rows])
import_file = ImportFile.objects.create(
import_record=import_record,
cached_first_row=ROW_DELIMITER.join(expected_raw_columns),
cached_second_to_fifth_row=expected_saved_format)
# Just make sure we were saved correctly
self.assertEqual(import_file.cached_second_to_fifth_row,
expected_saved_format)
url = reverse_lazy("api:v2:import_files-first-five-rows",
args=[import_file.pk])
resp = self.client.get(url, content_type='application/json')
body = json.loads(resp.content)
self.assertEqual(body.get('first_five_rows', []), expected)
def test_get_first_five_rows_with_newlines(self):
import_record = ImportRecord.objects.create()
expected_raw_columns = ['id', 'name', 'etc']
expected_raw_rows = [
['1', 'test', 'new\nline'],
['2', 'test', 'single'],
]
expected = [
dict(zip(expected_raw_columns, row)) for row in expected_raw_rows
]
expected_saved_format = "1%stest%snew\nline\n2%stest%ssingle".replace(
'%s', ROW_DELIMITER)
import_file = ImportFile.objects.create(
import_record=import_record,
cached_first_row=ROW_DELIMITER.join(expected_raw_columns),
cached_second_to_fifth_row=expected_saved_format)
# Just make sure we were saved correctly
self.assertEqual(import_file.cached_second_to_fifth_row,
expected_saved_format)
url = reverse_lazy("api:v2:import_files-first-five-rows",
args=[import_file.pk])
resp = self.client.get(url, content_type='application/json')
body = json.loads(resp.content)
self.assertEqual(body.get('first_five_rows', []), expected)
def test_get_first_five_rows_with_newlines(self):
import_record = ImportRecord.objects.create()
expected_raw_columns = ['id', 'name', 'etc']
expected_raw_rows = [
['1', 'test', 'new\nline'],
['2', 'test', 'single'],
]
expected = [
dict(zip(expected_raw_columns, row)) for row in expected_raw_rows
]
expected_saved_format = "1%stest%snew\nline\n2%stest%ssingle".replace(
'%s', ROW_DELIMITER)
import_file = ImportFile.objects.create(
import_record=import_record,
cached_first_row=ROW_DELIMITER.join(expected_raw_columns),
cached_second_to_fifth_row=expected_saved_format
)
# Just make sure we were saved correctly
self.assertEqual(
import_file.cached_second_to_fifth_row, expected_saved_format
)
url = reverse_lazy("apiv2:import_files-first-five-rows", args=[import_file.pk])
resp = self.client.get(
url, content_type='application/json'
)
body = json.loads(resp.content)
self.assertEqual(body.get('first_five_rows', []), expected)
def cache_first_rows(self):
self.file.seek(0)
counter = 0
csv_reader = csv.reader(self.local_file)
NUM_LINES_TO_CAPTURE = 6
for row in csv_reader:
counter += 1
if counter <= NUM_LINES_TO_CAPTURE:
if counter == 1:
self.cached_first_row = ROW_DELIMITER.join(row)
self.cached_second_to_fifth_row = ""
else:
self.cached_second_to_fifth_row += "%s\n" % ROW_DELIMITER.join(row)
self.num_rows = counter
if self.has_header_row:
self.num_rows = self.num_rows - 1
self.num_columns = len(self.first_row_columns)
def cache_first_rows(self):
self.file.seek(0)
counter = 0
csv_reader = csv.reader(self.local_file)
NUM_LINES_TO_CAPTURE = 6
for row in csv_reader:
counter += 1
if counter <= NUM_LINES_TO_CAPTURE:
if counter == 1:
self.cached_first_row = ROW_DELIMITER.join(row)
self.cached_second_to_fifth_row = ''
else:
self.cached_second_to_fifth_row += '%s\n' % ROW_DELIMITER.join(row)
self.num_rows = counter
if self.has_header_row:
self.num_rows = self.num_rows - 1
self.num_columns = len(self.first_row_columns)
def test_get_first_five_rows(self):
"""Make sure we get our first five rows back correctly."""
import_record = ImportRecord.objects.create()
expected_raw_columns = ['tax id', 'name', 'etc.']
expected_raw_rows = [
['02023', '12 Jefferson St.', 'etc.'],
['12433', '23 Washington St.', 'etc.'],
['04422', '4 Adams St.', 'etc.'],
]
expected = [
dict(zip(expected_raw_columns, row)) for row in expected_raw_rows
]
expected_saved_format = '\n'.join([
ROW_DELIMITER.join(row) for row
in expected_raw_rows
])
import_file = ImportFile.objects.create(
import_record=import_record,
cached_first_row=ROW_DELIMITER.join(expected_raw_columns),
cached_second_to_fifth_row=expected_saved_format
)
# Just make sure we were saved correctly
self.assertEqual(
import_file.cached_second_to_fifth_row, expected_saved_format
)
url = reverse_lazy("apiv2:import_files-first-five-rows", args=[import_file.pk])
resp = self.client.get(
url, content_type='application/json'
)
body = json.loads(resp.content)
self.assertEqual(body.get('first_five_rows', []), expected)
def test_get_raw_column_names(self):
"""Make sure we get column names back in a format we expect."""
import_record = ImportRecord.objects.create()
expected_raw_columns = ['tax id', 'name', 'etc.']
expected_saved_format = ROW_DELIMITER.join(expected_raw_columns)
import_file = ImportFile.objects.create(
import_record=import_record,
cached_first_row=expected_saved_format)
# Just make sure we were saved correctly
self.assertEqual(import_file.cached_first_row, expected_saved_format)
url = reverse_lazy("api:v2:import_files-raw-column-names",
args=[import_file.pk])
resp = self.client.get(url, content_type='application/json')
body = json.loads(resp.content)
self.assertEqual(body.get('raw_columns', []), expected_raw_columns)
def test_get_raw_column_names(self):
"""Make sure we get column names back in a format we expect."""
import_record = ImportRecord.objects.create()
expected_raw_columns = ['tax id', 'name', 'etc.']
expected_saved_format = ROW_DELIMITER.join(expected_raw_columns)
import_file = ImportFile.objects.create(
import_record=import_record,
cached_first_row=expected_saved_format
)
# Just make sure we were saved correctly
self.assertEqual(import_file.cached_first_row, expected_saved_format)
url = reverse_lazy("apiv2:import_files-raw-column-names", args=[import_file.pk])
resp = self.client.get(
url, content_type='application/json'
)
body = json.loads(resp.content)
self.assertEqual(body.get('raw_columns', []), expected_raw_columns)
def first_five_rows_helper(headers, raw_data):
save_format = '\n'.join([ROW_DELIMITER.join(row) for row in raw_data])
expected = [dict(zip(headers, row)) for row in raw_data]
return save_format, expected
def create_models(data, import_file, cycle):
"""
Create a PropertyState and a Meter. Then, create TimeSeries models for each meter
reading in data.
:param data: dict, building data from a Green Button XML file from xml_importer.building_data
:param import_file: ImportFile, reference to Green Button XML file
:param cycle: Cycle, the cycle from which the property view will be attached
:returns: PropertyState
"""
# cache data on import_file; this is a proof of concept and we
# only have two example files available so we hardcode the only
# heading present.
# NL: Yuck, not sure that this makes much sense here, or anywhere in this method
import_file.cached_first_row = ROW_DELIMITER.join(["address"])
import_file.cached_second_to_fifth_row = ROW_DELIMITER.join(
[data['address']])
import_file.save()
property_state = PropertyState()
property_state.import_file = import_file
property_state.organization = import_file.import_record.super_organization
property_state.address_line_1 = data['address']
property_state.source_type = GREEN_BUTTON_BS # TODO: Green Button Fix -- prob can be removed
property_state.save()
pv = property_state.promote(cycle)
# create meter for this dataset (each dataset is a single energy type)
e_type = energy_type(data['service_category'])
e_type_string = next(pair[1] for pair in Meter.ENERGY_TYPES
if pair[0] == e_type)
m_name = "gb_{0}[{1}]".format(str(property_state.id), e_type_string)
m_energy_units = energy_units(data['meter']['uom'])
meter = Meter.objects.create(name=m_name,
energy_type=e_type,
energy_units=m_energy_units,
property_view=pv)
meter.save()
# now time series data for the meter
for reading in data['interval']['readings']:
# how to deal with timezones?
start_time = int(reading['start_time'])
duration = int(reading['duration'])
begin_time = datetime.fromtimestamp(start_time,
tz=timezone.get_current_timezone())
end_time = datetime.fromtimestamp(start_time + duration,
tz=timezone.get_current_timezone())
value = reading['value']
cost = reading['cost']
new_ts = TimeSeries.objects.create(
begin_time=begin_time,
end_time=end_time,
reading=value,
cost=cost,
meter=meter,
)
new_ts.save()
return pv
def create_models(data, import_file):
"""
Create a BuildingSnapshot, a CanonicalBuilding, and a Meter. Then, create
TimeSeries models for each meter reading in data.
:param data: dictionary of building data from a Green Button XML file
in the form returned by xml_importer.building_data
:param import_file: ImportFile referencing the original xml file; needed
for linking to BuildingSnapshot and for determining super_organization
:returns: the created CanonicalBuilding
"""
# cache data on import_file; this is a proof of concept and we
# only have two example files available so we hardcode the only
# heading present.
import_file.cached_first_row = ROW_DELIMITER.join(["address"])
import_file.cached_second_to_fifth_row = ROW_DELIMITER.join(
[data['address']]
)
import_file.save()
raw_bs = BuildingSnapshot()
raw_bs.import_file = import_file
# We require a save to get our PK
# We save here to set our initial source PKs.
raw_bs.save()
super_org = import_file.import_record.super_organization
raw_bs.super_organization = super_org
set_initial_sources(raw_bs)
raw_bs.address_line_1 = data['address']
raw_bs.source_type = GREEN_BUTTON_BS
raw_bs.save()
# create canonical building
cb = CanonicalBuilding.objects.create(canonical_snapshot=raw_bs)
raw_bs.canonical_building = cb
raw_bs.save()
# log building creation
AuditLog.objects.create(
organization=import_file.import_record.super_organization,
user=import_file.import_record.owner,
content_object=cb,
action="create_building",
action_note="Created building",
)
# create meter for this dataset (each dataset is a single energy type)
e_type = energy_type(data['service_category'])
e_type_string = next(
pair[1] for pair in seed.models.ENERGY_TYPES if pair[0] == e_type
)
m_name = "gb_{0}[{1}]".format(str(raw_bs.id), e_type_string)
m_energy_units = energy_units(data['meter']['uom'])
meter = Meter.objects.create(
name=m_name, energy_type=e_type, energy_units=m_energy_units
)
meter.building_snapshot.add(raw_bs)
meter.save()
# now time series data for the meter
for reading in data['interval']['readings']:
start_time = int(reading['start_time'])
duration = int(reading['duration'])
begin_time = datetime.fromtimestamp(start_time)
end_time = datetime.fromtimestamp(start_time + duration)
value = reading['value']
cost = reading['cost']
new_ts = TimeSeries.objects.create(
begin_time=begin_time,
end_time=end_time,
reading=value,
cost=cost
)
new_ts.meter = meter
new_ts.save()
return cb
def create_models(data, import_file):
"""
Create a BuildingSnapshot, a CanonicalBuilding, and a Meter. Then, create
TimeSeries models for each meter reading in data.
:param data: dictionary of building data from a Green Button XML file
in the form returned by xml_importer.building_data
:param import_file: ImportFile referencing the original xml file; needed
for linking to BuildingSnapshot and for determining super_organization
:returns: the created CanonicalBuilding
"""
# cache data on import_file; this is a proof of concept and we
# only have two example files available so we hardcode the only
# heading present.
import_file.cached_first_row = ROW_DELIMITER.join(["address"])
import_file.cached_second_to_fifth_row = ROW_DELIMITER.join(
[data['address']])
import_file.save()
raw_bs = BuildingSnapshot()
raw_bs.import_file = import_file
# We require a save to get our PK
# We save here to set our initial source PKs.
raw_bs.save()
super_org = import_file.import_record.super_organization
raw_bs.super_organization = super_org
raw_bs.address_line_1 = data['address']
raw_bs.source_type = GREEN_BUTTON_BS
raw_bs.save()
# create canonical building
cb = CanonicalBuilding.objects.create(canonical_snapshot=raw_bs)
raw_bs.canonical_building = cb
raw_bs.save()
# log building creation
AuditLog.objects.create(
organization=import_file.import_record.super_organization,
user=import_file.import_record.owner,
content_object=cb,
action="create_building",
action_note="Created building",
)
# create meter for this dataset (each dataset is a single energy type)
e_type = energy_type(data['service_category'])
e_type_string = next(pair[1] for pair in seed.models.ENERGY_TYPES
if pair[0] == e_type)
m_name = "gb_{0}[{1}]".format(str(raw_bs.id), e_type_string)
m_energy_units = energy_units(data['meter']['uom'])
meter = Meter.objects.create(name=m_name,
energy_type=e_type,
energy_units=m_energy_units)
meter.building_snapshot.add(raw_bs)
meter.save()
# now time series data for the meter
for reading in data['interval']['readings']:
start_time = int(reading['start_time'])
duration = int(reading['duration'])
begin_time = datetime.fromtimestamp(start_time,
tz=timezone.get_current_timezone())
end_time = datetime.fromtimestamp(start_time + duration,
tz=timezone.get_current_timezone())
value = reading['value']
cost = reading['cost']
new_ts = TimeSeries.objects.create(begin_time=begin_time,
end_time=end_time,
reading=value,
cost=cost)
new_ts.meter = meter
new_ts.save()
return cb