class AnalysisPropertyViewViewSet(viewsets.ViewSet):
serializer_class = AnalysisPropertyViewSerializer
model = AnalysisPropertyView
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field(True)])
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def list(self, request, analysis_pk):
organization_id = int(request.query_params.get('organization_id', 0))
try:
views_queryset = AnalysisPropertyView.objects.filter(
analysis=analysis_pk,
analysis__organization_id=organization_id)
except AnalysisPropertyView.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
"Requested analysis doesn't exist in this organization."
},
status=HTTP_409_CONFLICT)
serialized_views = []
for view in views_queryset:
serialized_view = AnalysisPropertyViewSerializer(view).data
serialized_views.append(serialized_view)
return JsonResponse({'status': 'success', 'views': serialized_views})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field(True)])
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def retrieve(self, request, analysis_pk, pk):
organization_id = int(request.query_params.get('organization_id', 0))
try:
view = AnalysisPropertyView.objects.get(
id=pk,
analysis=analysis_pk,
analysis__organization_id=organization_id)
except AnalysisPropertyView.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
"Requested analysis property view doesn't exist in this organization and/or analysis."
},
status=HTTP_409_CONFLICT)
return JsonResponse({
'status': 'success',
'view': AnalysisPropertyViewSerializer(view).data
})
class MeasureViewSet(viewsets.ReadOnlyModelViewSet, OrgMixin):
"""
list:
Return a list of all measures
retrieve:
Return a measure by a unique id
"""
serializer_class = MeasureSerializer
parser_classes = (
JSONParser,
FormParser,
)
renderer_classes = (JSONRenderer, )
pagination_class = None
def get_queryset(self):
org_id = self.get_organization(self.request)
return Measure.objects.filter(organization_id=org_id)
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=no_body)
@has_perm_class('can_modify_data')
@action(detail=False, methods=['POST'])
def reset(self, request):
"""
Reset all the measures back to the defaults (as provided by BuildingSync)
"""
organization_id = request.query_params.get('organization_id', None)
if not organization_id:
return JsonResponse(
{
'status': 'error',
'message': 'organization_id not provided'
},
status=status.HTTP_400_BAD_REQUEST)
Measure.populate_measures(organization_id)
data = dict(measures=list(
Measure.objects.filter(
organization_id=organization_id).order_by('id').values()))
data['status'] = 'success'
return JsonResponse(data)
class GeocodeViewSet(viewsets.ViewSet, OrgMixin):
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'property_view_ids': ['integer'],
'taxlot_view_ids': ['integer'],
},
description='IDs by inventory type for records to be geocoded.'
)
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=False, methods=['POST'])
def geocode_by_ids(self, request):
"""
Submit a request to geocode property and tax lot records.
"""
body = dict(request.data)
org_id = self.get_organization(request)
property_view_ids = body.get('property_view_ids')
taxlot_view_ids = body.get('taxlot_view_ids')
if property_view_ids:
property_views = PropertyView.objects.filter(
id__in=property_view_ids,
cycle__organization_id=org_id
)
properties = PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id'))
)
geocode_buildings(properties)
if taxlot_view_ids:
taxlot_views = TaxLotView.objects.filter(
id__in=taxlot_view_ids,
cycle__organization_id=org_id
)
taxlots = TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id'))
)
geocode_buildings(taxlots)
return JsonResponse({'status': 'success'})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'property_view_ids': ['integer'],
'taxlot_view_ids': ['integer'],
},
description='IDs by inventory type for records to be used in building a geocoding summary.'
)
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_view_data')
@action(detail=False, methods=['POST'])
def confidence_summary(self, request):
"""
Generate a summary of geocoding confidence values for property and
tax lot records.
"""
body = dict(request.data)
org_id = self.get_organization(request)
property_view_ids = body.get('property_view_ids')
taxlot_view_ids = body.get('taxlot_view_ids')
result = {}
if property_view_ids:
property_views = PropertyView.objects.filter(
id__in=property_view_ids,
cycle__organization_id=org_id
)
result["properties"] = {
'not_geocoded': PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id')),
geocoding_confidence__isnull=True
).count(),
'high_confidence': PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id')),
geocoding_confidence__startswith='High'
).count(),
'low_confidence': PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id')),
geocoding_confidence__startswith='Low'
).count(),
'manual': PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id')),
geocoding_confidence='Manually geocoded (N/A)'
).count(),
'missing_address_components': PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id')),
geocoding_confidence='Missing address components (N/A)'
).count(),
}
if taxlot_view_ids:
taxlot_views = TaxLotView.objects.filter(
id__in=taxlot_view_ids,
cycle__organization_id=org_id
)
result["tax_lots"] = {
'not_geocoded': TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id')),
geocoding_confidence__isnull=True
).count(),
'high_confidence': TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id')),
geocoding_confidence__startswith='High'
).count(),
'low_confidence': TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id')),
geocoding_confidence__startswith='Low'
).count(),
'manual': TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id')),
geocoding_confidence='Manually geocoded (N/A)'
).count(),
'missing_address_components': TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id')),
geocoding_confidence='Missing address components (N/A)'
).count(),
}
return result
class TaxLotPropertyViewSet(GenericViewSet):
"""
The TaxLotProperty field is used to return the properties and tax lots from the join table.
This method presently only works with the CSV, but should eventually be extended to be the
viewset for any tax lot / property join API call.
"""
renderer_classes = (JSONRenderer, )
serializer_class = TaxLotPropertySerializer
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field("cycle_id", True, "Cycle ID"),
AutoSchemaHelper.query_string_field(
"inventory_type", False,
"Either 'taxlots' or 'properties' and defaults to 'properties'."
),
],
request_body=AutoSchemaHelper.schema_factory(
{
'ids': ['integer'],
'filename': 'string',
'export_type': 'string',
'profile_id': 'integer'
},
description='- ids: (View) IDs for records to be exported\n'
'- filename: desired filename including extension (defaulting to \'ExportedData.{export_type}\')\n'
'- export_types: \'csv\', \'geojson\', \'xlsx\' (defaulting to \'csv\')\n'
'- profile_id: Column List Profile ID to use for customizing fields included in export'
),
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=False, methods=['POST'])
def export(self, request):
"""
Download a collection of the TaxLot and Properties in multiple formats.
"""
cycle_pk = request.query_params.get('cycle_id', None)
if not cycle_pk:
return JsonResponse({
'status':
'error',
'message':
'Must pass in cycle_id as query parameter'
})
org_id = request.query_params['organization_id']
if 'profile_id' not in request.data:
profile_id = None
else:
if request.data['profile_id'] == 'None' or request.data[
'profile_id'] == '':
profile_id = None
else:
profile_id = request.data['profile_id']
# get the class to operate on and the relationships
view_klass_str = request.query_params.get('inventory_type',
'properties')
view_klass = INVENTORY_MODELS[view_klass_str]
# Set the first column to be the ID
column_name_mappings = OrderedDict([('id', 'ID')])
column_ids, add_column_name_mappings, columns_from_database = ColumnListProfile.return_columns(
org_id, profile_id, view_klass_str)
column_name_mappings.update(add_column_name_mappings)
select_related = ['state', 'cycle']
ids = request.data.get('ids', [])
filter_str = {'cycle': cycle_pk}
if hasattr(view_klass, 'property'):
select_related.append('property')
prefetch_related = ['labels']
filter_str = {'property__organization_id': org_id}
if ids:
filter_str['id__in'] = ids
# always export the labels and notes
column_name_mappings['property_notes'] = 'Property Notes'
column_name_mappings['property_labels'] = 'Property Labels'
elif hasattr(view_klass, 'taxlot'):
select_related.append('taxlot')
prefetch_related = ['labels']
filter_str = {'taxlot__organization_id': org_id}
if ids:
filter_str['id__in'] = ids
# always export the labels and notes
column_name_mappings['taxlot_notes'] = 'Tax Lot Notes'
column_name_mappings['taxlot_labels'] = 'Tax Lot Labels'
model_views = view_klass.objects.select_related(
*select_related).prefetch_related(*prefetch_related).filter(
**filter_str).order_by('id')
# get the data in a dict which includes the related data
data = TaxLotProperty.get_related(model_views, column_ids,
columns_from_database)
# add labels and notes
for i, record in enumerate(model_views):
label_string = []
note_string = []
for label in list(record.labels.all().order_by('name')):
label_string.append(label.name)
for note in list(record.notes.all().order_by('created')):
note_string.append(note.created.astimezone().strftime(
"%Y-%m-%d %I:%M:%S %p") + "\n" + note.text)
if hasattr(record, 'property'):
data[i]['property_labels'] = ','.join(label_string)
data[i]['property_notes'] = '\n----------\n'.join(note_string)
elif hasattr(record, 'taxlot'):
data[i]['taxlot_labels'] = ','.join(label_string)
data[i]['taxlot_notes'] = '\n----------\n'.join(note_string)
# force the data into the same order as the IDs
if ids:
order_dict = {obj_id: index for index, obj_id in enumerate(ids)}
if view_klass_str == 'properties':
view_id_str = 'property_view_id'
else:
view_id_str = 'taxlot_view_id'
data.sort(key=lambda inventory_obj: order_dict[inventory_obj[
view_id_str]])
export_type = request.data.get('export_type', 'csv')
filename = request.data.get('filename', f"ExportedData.{export_type}")
if export_type == "csv":
return self._csv_response(filename, data, column_name_mappings)
elif export_type == "geojson":
return self._json_response(filename, data, column_name_mappings)
elif export_type == "xlsx":
return self._spreadsheet_response(filename, data,
column_name_mappings)
def _csv_response(self, filename, data, column_name_mappings):
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}"'.format(
filename)
writer = csv.writer(response)
# check the first item in the header and make sure that it isn't ID (it can be id, or iD).
# excel doesn't like the first item to be ID in a CSV
header = list(column_name_mappings.values())
if header[0] == 'ID':
header[0] = 'id'
writer.writerow(header)
# iterate over the results to preserve column order and write row.
for datum in data:
row = []
for column in column_name_mappings:
row_result = datum.get(column, None)
# Try grabbing the value out of the related field if not found yet.
if row_result is None and datum.get('related'):
row_result = datum['related'][0].get(column, None)
# Convert quantities (this is typically handled in the JSON Encoder, but that isn't here).
if isinstance(row_result, ureg.Quantity):
row_result = row_result.magnitude
elif isinstance(row_result, datetime.datetime):
row_result = row_result.strftime("%Y-%m-%d %H:%M:%S")
elif isinstance(row_result, datetime.date):
row_result = row_result.strftime("%Y-%m-%d")
row.append(row_result)
writer.writerow(row)
return response
def _spreadsheet_response(self, filename, data, column_name_mappings):
response = HttpResponse(
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
response['Content-Disposition'] = 'attachment; filename="{}"'.format(
filename)
scenario_keys = ('id', 'name', 'description',
'annual_site_energy_savings',
'annual_source_energy_savings', 'annual_cost_savings',
'analysis_state', 'analysis_state_message',
'annual_electricity_savings',
'annual_natural_gas_savings', 'annual_site_energy',
'annual_source_energy', 'annual_natural_gas_energy',
'annual_electricity_energy', 'annual_peak_demand',
'annual_site_energy_use_intensity',
'annual_source_energy_use_intensity')
scenario_key_mappings = {
'annual_site_energy_savings': 'annual_site_energy_savings_mmbtu',
'annual_source_energy_savings':
'annual_source_energy_savings_mmbtu',
'annual_cost_savings': 'annual_cost_savings_dollars',
'annual_site_energy': 'annual_site_energy_kbtu',
'annual_site_energy_use_intensity':
'annual_site_energy_use_intensity_kbtu_ft2',
'annual_source_energy': 'annual_source_energy_kbtu',
'annual_source_energy_use_intensity':
'annual_source_energy_use_intensity_kbtu_ft2',
'annual_natural_gas_energy': 'annual_natural_gas_energy_mmbtu',
'annual_electricity_energy': 'annual_electricity_energy_mmbtu',
'annual_peak_demand': 'annual_peak_demand_kw',
'annual_electricity_savings': 'annual_electricity_savings_kbtu',
'annual_natural_gas_savings': 'annual_natural_gas_savings_kbtu'
}
property_measure_keys = ('id', 'property_measure_name', 'measure_id',
'cost_mv', 'cost_total_first',
'cost_installation', 'cost_material',
'cost_capital_replacement',
'cost_residual_value')
measure_keys = ('name', 'display_name', 'category',
'category_display_name')
# find measures and scenarios
for i, record in enumerate(data):
measures = PropertyMeasure.objects.filter(
property_state_id=record['property_state_id'])
record['measures'] = measures
scenarios = Scenario.objects.filter(
property_state_id=record['property_state_id'])
record['scenarios'] = scenarios
output = io.BytesIO()
wb = xlsxwriter.Workbook(output, {'remove_timezone': True})
# add tabs
ws1 = wb.add_worksheet('Properties')
ws2 = wb.add_worksheet('Measures')
ws3 = wb.add_worksheet('Scenarios')
ws4 = wb.add_worksheet('Scenario Measure Join Table')
ws5 = wb.add_worksheet('Meter Readings')
bold = wb.add_format({'bold': True})
row = 0
row2 = 0
col2 = 0
row3 = 0
col3 = 0
row4 = 0
row5 = 0
for index, val in enumerate(list(column_name_mappings.values())):
# Do not write the first element as ID, this causes weird issues with Excel.
if index == 0 and val == 'ID':
ws1.write(row, index, 'id', bold)
else:
ws1.write(row, index, val, bold)
# iterate over the results to preserve column order and write row.
add_m_headers = True
add_s_headers = True
for datum in data:
row += 1
id = None
for index, column in enumerate(column_name_mappings):
if column == 'id':
id = datum.get(column, None)
row_result = datum.get(column, None)
# Try grabbing the value out of the related field if not found yet.
if row_result is None and datum.get('related'):
row_result = datum['related'][0].get(column, None)
# Convert quantities (this is typically handled in the JSON Encoder, but that isn't here).
if isinstance(row_result, ureg.Quantity):
row_result = row_result.magnitude
elif isinstance(row_result, datetime.datetime):
row_result = row_result.strftime("%Y-%m-%d %H:%M:%S")
elif isinstance(row_result, datetime.date):
row_result = row_result.strftime("%Y-%m-%d")
ws1.write(row, index, row_result)
# measures
for index, m in enumerate(datum['measures']):
if add_m_headers:
# grab headers
for key in property_measure_keys:
ws2.write(row2, col2, key, bold)
col2 += 1
for key in measure_keys:
ws2.write(row2, col2, 'measure ' + key, bold)
col2 += 1
add_m_headers = False
row2 += 1
col2 = 0
for key in property_measure_keys:
ws2.write(row2, col2, getattr(m, key))
col2 += 1
for key in measure_keys:
ws2.write(row2, col2, getattr(m.measure, key))
col2 += 1
# scenarios (and join table)
# join table
ws4.write('A1', 'property_id', bold)
ws4.write('B1', 'scenario_id', bold)
ws4.write('C1', 'measure_id', bold)
for index, s in enumerate(datum['scenarios']):
scenario_id = s.id
if add_s_headers:
# grab headers
for key in scenario_keys:
# double check scenario_key_mappings in case a different header is desired
if key in scenario_key_mappings.keys():
key = scenario_key_mappings[key]
ws3.write(row3, col3, key, bold)
col3 += 1
add_s_headers = False
row3 += 1
col3 = 0
for key in scenario_keys:
ws3.write(row3, col3, getattr(s, key))
col3 += 1
for sm in s.measures.all():
row4 += 1
ws4.write(row4, 0, id)
ws4.write(row4, 1, scenario_id)
ws4.write(row4, 2, sm.id)
# scenario meter readings
ws5.write('A1', 'scenario_id', bold)
ws5.write('B1', 'meter_id', bold)
ws5.write('C1', 'type', bold)
ws5.write('D1', 'start_time', bold)
ws5.write('E1', 'end_time', bold)
ws5.write('F1', 'reading', bold)
ws5.write('G1', 'units', bold)
ws5.write('H1', 'is_virtual', bold)
# datetime formatting
date_format = wb.add_format({'num_format': 'yyyy-mm-dd hh:mm:ss'})
for index, s in enumerate(datum['scenarios']):
scenario_id = s.id
# retrieve meters
meters = Meter.objects.filter(scenario_id=scenario_id)
for m in meters:
# retrieve readings
readings = MeterReading.objects.filter(
meter_id=m.id).order_by('start_time')
for r in readings:
row5 += 1
ws5.write(row5, 0, scenario_id)
ws5.write(row5, 1, m.id)
the_type = next((item for item in Meter.ENERGY_TYPES
if item[0] == m.type), None)
the_type = the_type[1] if the_type is not None else None
ws5.write(
row5, 2, the_type
) # use energy type enum to determine reading type
ws5.write_datetime(row5, 3, r.start_time, date_format)
ws5.write_datetime(row5, 4, r.end_time, date_format)
ws5.write(row5, 5,
r.reading) # this is now a float field
ws5.write(row5, 6, r.source_unit)
ws5.write(row5, 7, m.is_virtual)
wb.close()
# xlsx_data contains the Excel file
xlsx_data = output.getvalue()
response.write(xlsx_data)
return response
def _json_response(self, filename, data, column_name_mappings):
polygon_fields = [
"bounding_box", "centroid", "property_footprint",
"taxlot_footprint", "long_lat"
]
features = []
# extract related records
related_records = self._extract_related(data)
# append related_records to data
complete_data = data + related_records
for datum in complete_data:
feature = {"type": "Feature", "properties": {}}
for key, value in datum.items():
if value is None:
continue
if isinstance(value, ureg.Quantity):
value = value.magnitude
elif isinstance(value, datetime.datetime):
value = value.strftime("%Y-%m-%d %H:%M:%S")
elif isinstance(value, datetime.date):
value = value.strftime("%Y-%m-%d")
if value and any(k in key for k in polygon_fields):
"""
If object is a polygon and is populated, add the 'geometry'
key-value-pair in the appropriate GeoJSON format.
When the first geometry is added, the correct format is
established. When/If a second geometry is added, this is
appended alongside the previous geometry.
"""
individual_geometry = {}
# long_lat
if key == 'long_lat':
coordinates = self._serialized_point(value)
# point
individual_geometry = {
"coordinates": coordinates,
"type": "Point"
}
else:
# polygons
coordinates = self._serialized_coordinates(value)
individual_geometry = {
"coordinates": [coordinates],
"type": "Polygon"
}
if feature.get("geometry", None) is None:
feature["geometry"] = {
"type": "GeometryCollection",
"geometries": [individual_geometry]
}
else:
feature["geometry"]["geometries"].append(
individual_geometry)
else:
"""
Non-polygon data
"""
display_key = column_name_mappings.get(key, key)
feature["properties"][display_key] = value
# # store point geometry in case you need it
# if display_key == "Longitude":
# point_geometry[0] = value
# if display_key == "Latitude":
# point_geometry[1] = value
"""
Before appending feature, ensure that if there is no geometry recorded.
Note that the GeoJson will not render if no lat/lng
"""
# add style information
if feature["properties"].get("property_state_id") is not None:
feature["properties"]["stroke"] = "#185189" # buildings color
elif feature["properties"].get("taxlot_state_id") is not None:
feature["properties"]["stroke"] = "#10A0A0" # buildings color
feature["properties"]["marker-color"] = "#E74C3C"
# feature["properties"]["stroke-width"] = 3
feature["properties"]["fill-opacity"] = 0
# append feature
features.append(feature)
response_dict = {
"type": "FeatureCollection",
"crs": {
"type": "EPSG",
"properties": {
"code": 4326
}
},
"features": features
}
response = JsonResponse(response_dict)
response['Content-Disposition'] = 'attachment; filename="{}"'.format(
filename)
return response
def _serialized_coordinates(self, polygon_wkt):
string_coord_pairs = polygon_wkt.lstrip('POLYGON (').rstrip(')').split(
', ')
coordinates = []
for coord_pair in string_coord_pairs:
float_coords = [float(coord) for coord in coord_pair.split(' ')]
coordinates.append(float_coords)
return coordinates
def _serialized_point(self, point_wkt):
string_coords = point_wkt.lstrip('POINT (').rstrip(')').split(', ')
coordinates = []
for coord in string_coords[0].split(' '):
coordinates.append(float(coord))
return coordinates
def _extract_related(self, data):
# extract all related records into a separate array
related = []
# figure out if we are dealing with properties or taxlots
if data[0].get("property_state_id", None) is not None:
is_property = True
elif data[0].get("taxlot_state_id", None) is not None:
is_property = False
for datum in data:
if datum.get("related", None) is not None:
for record in datum["related"]:
related.append(record)
# make array unique
if is_property:
unique = [dict(p) for p in set(tuple(i.items()) for i in related)]
else:
unique = [dict(p) for p in set(tuple(i.items()) for i in related)]
return unique
class DatasetViewSet(viewsets.ViewSet):
raise_exception = True
@swagger_auto_schema_org_query_param
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_view_data')
def list(self, request):
"""
Retrieves all datasets for the user's organization.
"""
org_id = request.query_params.get('organization_id', None)
org = Organization.objects.get(pk=org_id)
datasets = []
for d in ImportRecord.objects.filter(super_organization=org):
importfiles = [obj_to_dict(f) for f in d.files]
dataset = obj_to_dict(d)
dataset['importfiles'] = importfiles
if d.last_modified_by:
dataset['last_modified_by'] = d.last_modified_by.email
dataset['updated_at'] = convert_to_js_timestamp(d.updated_at)
datasets.append(dataset)
return JsonResponse({
'status': 'success',
'datasets': datasets,
})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field()
],
request_body=AutoSchemaHelper.schema_factory(
{'dataset': 'string'},
description='The new name for this dataset'))
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
def update(self, request, pk=None):
"""
Updates the name of a dataset (ImportRecord).
"""
organization_id = request.query_params.get('organization_id', None)
if organization_id is None:
return JsonResponse({
'status':
'error',
'message':
'Missing organization_id query parameter'
})
try:
organization_id = int(organization_id)
except ValueError:
return JsonResponse({
'status': 'error',
'message': 'Bad (non-numeric) organization_id'
})
dataset_id = pk
name = request.data['dataset']
# check if user has access to the dataset
d = ImportRecord.objects.filter(super_organization_id=organization_id,
pk=dataset_id)
if not d.exists():
return JsonResponse(
{
'status': 'error',
'message':
'user does not have permission to update dataset',
},
status=status.HTTP_400_BAD_REQUEST)
d = d[0]
d.name = name
d.save()
return JsonResponse({
'status': 'success',
})
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_view_data')
def retrieve(self, request, pk=None):
"""
Retrieves a dataset (ImportRecord).
"""
organization_id = request.query_params.get('organization_id', None)
if organization_id is None:
return JsonResponse(
{
'status': 'error',
'message': 'Missing organization_id query parameter'
},
status=status.HTTP_400_BAD_REQUEST)
try:
organization_id = int(organization_id)
except ValueError:
return JsonResponse(
{
'status': 'error',
'message': 'Bad (non-numeric) organization_id'
},
status=status.HTTP_400_BAD_REQUEST)
valid_orgs = OrganizationUser.objects.filter(
user_id=request.user.id).values_list(
'organization_id', flat=True).order_by('organization_id')
if organization_id not in valid_orgs:
return JsonResponse(
{
'status': 'error',
'message':
'Cannot access datasets for this organization id',
},
status=status.HTTP_403_FORBIDDEN)
# check if dataset exists
try:
d = ImportRecord.objects.get(pk=pk)
except ImportRecord.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'dataset with id {} does not exist'.format(pk)
},
status=status.HTTP_404_NOT_FOUND)
if d.super_organization_id != organization_id:
return JsonResponse(
{
'status':
'error',
'message':
'Organization ID mismatch between dataset and organization'
},
status=status.HTTP_400_BAD_REQUEST)
dataset = obj_to_dict(d)
importfiles = []
for f in d.files:
importfile = obj_to_dict(f)
if not f.uploaded_filename:
importfile['name'] = f.filename_only
else:
importfile['name'] = f.uploaded_filename
importfiles.append(importfile)
dataset['importfiles'] = importfiles
if d.last_modified_by:
dataset['last_modified_by'] = d.last_modified_by.email
dataset['updated_at'] = convert_to_js_timestamp(d.updated_at)
return JsonResponse({
'status': 'success',
'dataset': dataset,
})
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def destroy(self, request, pk=None):
"""
Deletes a dataset (ImportRecord).
"""
organization_id = int(request.query_params.get('organization_id',
None))
dataset_id = pk
# check if user has access to the dataset
d = ImportRecord.objects.filter(super_organization_id=organization_id,
pk=dataset_id)
if not d.exists():
return JsonResponse(
{
'status':
'error',
'message':
'user does not have permission to deactivate dataset',
},
status=status.HTTP_403_FORBIDDEN)
d = d[0]
d.delete()
return JsonResponse({'status': 'success'})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{'name': 'string'},
description='Name of the dataset to be created'),
responses={
200:
AutoSchemaHelper.schema_factory({
'id': 'integer',
'name': 'string',
})
})
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
def create(self, request):
"""
Creates a new empty dataset (ImportRecord).
"""
body = request.data
org_id = int(request.query_params.get('organization_id', None))
try:
org = Organization.objects.get(pk=org_id)
except Organization.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'organization_id not provided'
},
status=status.HTTP_400_BAD_REQUEST)
record = ImportRecord.objects.create(name=body['name'],
app='seed',
start_time=timezone.now(),
created_at=timezone.now(),
last_modified_by=request.user,
super_organization=org,
owner=request.user)
return JsonResponse({
'status': 'success',
'id': record.pk,
'name': record.name
})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
responses={
200: AutoSchemaHelper.schema_factory({
'datasets_count': 'integer',
})
})
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_viewer')
@require_organization_id_class
@action(detail=False, methods=['GET'])
def count(self, request):
"""
Retrieves the number of datasets for an org.
"""
org_id = int(request.query_params.get('organization_id', None))
datasets_count = ImportRecord.objects.filter(
super_organization_id=org_id).count()
return JsonResponse({
'status': 'success',
'datasets_count': datasets_count
})
class UploadViewSet(viewsets.ViewSet):
"""
Endpoint to upload data files to, if uploading to local file storage.
Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES``
Returns::
{
'import_record': True,
'import_file_id': The ID of the newly-uploaded ImportFile
}
"""
parser_classes = (FormParser, MultiPartParser)
@swagger_auto_schema(
request_body=no_body,
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.upload_file_field(name='file',
required=True,
description='File to Upload'),
AutoSchemaHelper.form_integer_field(
name='import_record',
required=True,
description=
'the dataset ID you want to associate this file with.'),
AutoSchemaHelper.form_string_field(
name='source_type',
required=True,
description=
'the type of file (e.g. "Portfolio Raw" or "Assessed Raw")'),
AutoSchemaHelper.form_string_field(
name='source_program_version',
required=False,
description=
'the version of the file as related to the source_type'),
])
@has_perm_class('can_modify_data')
@api_endpoint_class
@ajax_request_class
def create(self, request):
"""
Upload a new file to an import_record. This is a multipart/form upload.
"""
if len(request.FILES) == 0:
return JsonResponse({
'success':
False,
'message':
"Must pass file in as a Multipart/Form post"
})
# Fineuploader requires the field to be qqfile it appears... so why not support both? ugh.
if 'qqfile' in request.data:
the_file = request.data['qqfile']
else:
the_file = request.data['file']
filename = the_file.name
path = os.path.join(settings.MEDIA_ROOT, "uploads", filename)
# Get a unique filename using the get_available_name method in FileSystemStorage
s = FileSystemStorage()
path = s.get_available_name(path)
# verify the directory exists
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# save the file
with open(path, 'wb+') as temp_file:
for chunk in the_file.chunks():
temp_file.write(chunk)
org_id = request.query_params.get('organization_id', None)
import_record_pk = request.POST.get('import_record',
request.GET.get('import_record'))
try:
record = ImportRecord.objects.get(pk=import_record_pk,
super_organization_id=org_id)
except ImportRecord.DoesNotExist:
# clean up the uploaded file
os.unlink(path)
return JsonResponse({
'success':
False,
'message':
"Import Record %s not found" % import_record_pk
})
source_type = request.POST.get('source_type',
request.GET.get('source_type'))
# Add Program & Version fields (empty string if not given)
kw_fields = {
field: request.POST.get(field, request.GET.get(field, ''))
for field in ['source_program', 'source_program_version']
}
f = ImportFile.objects.create(import_record=record,
uploaded_filename=filename,
file=path,
source_type=source_type,
**kw_fields)
return JsonResponse({'success': True, "import_file_id": f.pk})
@staticmethod
def _get_pint_var_from_pm_value_object(pm_value):
units = pint.UnitRegistry()
if '@uom' in pm_value and '#text' in pm_value:
# this is the correct expected path for unit-based attributes
string_value = pm_value['#text']
try:
float_value = float(string_value)
except ValueError:
return {
'success':
False,
'message':
'Could not cast value to float: \"%s\"' % string_value
}
original_unit_string = pm_value['@uom']
if original_unit_string == 'kBtu':
pint_val = float_value * units.kBTU
elif original_unit_string == 'kBtu/ft²':
pint_val = float_value * units.kBTU / units.sq_ft
elif original_unit_string == 'Metric Tons CO2e':
pint_val = float_value * units.metric_ton
elif original_unit_string == 'kgCO2e/ft²':
pint_val = float_value * units.kilogram / units.sq_ft
else:
return {
'success':
False,
'message':
'Unsupported units string: \"%s\"' % original_unit_string
}
return {'success': True, 'pint_value': pint_val}
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'import_record_id':
'integer',
'properties': [{
'address_1': 'string',
'city': 'string',
'state_province': 'string',
'postal_code': 'string',
'county': 'string',
'country': 'string',
'property_name': 'string',
'property_id': 'integer',
'year_built': 'integer',
}],
},
required=['import_record_id', 'properties'],
description='An object containing meta data for a property'))
@has_perm_class('can_modify_data')
@api_endpoint_class
@ajax_request_class
@action(detail=False, methods=['POST'], parser_classes=(JSONParser, ))
def create_from_pm_import(self, request):
"""
Create an import_record from a PM import request.
TODO: The properties key here is going to be an enormous amount of XML data at times, need to change this
This allows the PM import workflow to be treated essentially the same as a standard file upload
The process comprises the following steps:
* Get a unique file name for this portfolio manager import
"""
doing_pint = False
if 'properties' not in request.data:
return JsonResponse(
{
'success': False,
'message': "Must pass properties in the request body."
},
status=status.HTTP_400_BAD_REQUEST)
# base file name (will be appended with a random string to ensure uniqueness if multiple on the same day)
today_date = datetime.datetime.today().strftime('%Y-%m-%d')
file_name = "pm_import_%s.csv" % today_date
# create a folder to keep pm_import files
path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports",
file_name)
# Get a unique filename using the get_available_name method in FileSystemStorage
s = FileSystemStorage()
path = s.get_available_name(path)
# verify the directory exists
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# This list should cover the core keys coming from PM, ensuring that they map easily
# We will also look for keys not in this list and just map them to themselves
# pm_key_to_column_heading_map = {
# 'address_1': 'Address',
# 'city': 'City',
# 'state_province': 'State',
# 'postal_code': 'Zip',
# 'county': 'County',
# 'country': 'Country',
# 'property_name': 'Property Name',
# 'property_id': 'Property ID',
# 'year_built': 'Year Built',
# }
# so now it looks like we *don't* need to override these, but instead we should leave all the headers as-is
# I'm going to leave this in here for right now, but if it turns out that we don't need it after testing,
# then I'll remove it entirely
pm_key_to_column_heading_map = {}
# We will also create a list of values that are used in PM export to indicate a value wasn't available
# When we import them into SEED here we will be sure to not write those values
pm_flagged_bad_string_values = [
'Not Available',
'Unable to Check (not enough data)',
'No Current Year Ending Date',
]
# We will make a pass through the first property to get the list of unexpected keys
for pm_property in request.data['properties']:
for pm_key_name, _ in pm_property.items():
if pm_key_name not in pm_key_to_column_heading_map:
pm_key_to_column_heading_map[pm_key_name] = pm_key_name
break
# Create the header row of the csv file first
rows = []
header_row = []
for _, csv_header in pm_key_to_column_heading_map.items():
header_row.append(csv_header)
rows.append(header_row)
num_properties = len(request.data['properties'])
property_num = 0
last_time = datetime.datetime.now()
_log.debug("About to try to import %s properties from ESPM" %
num_properties)
_log.debug("Starting at %s" % last_time)
# Create a single row for each building
for pm_property in request.data['properties']:
# report some helpful info every 20 properties
property_num += 1
if property_num % 20 == 0:
new_time = datetime.datetime.now()
_log.debug("On property number %s; current time: %s" %
(property_num, new_time))
this_row = []
# Loop through all known PM variables
for pm_variable, _ in pm_key_to_column_heading_map.items():
# Initialize this to False for each pm_variable we will search through
added = False
# Check if this PM export has this variable in it
if pm_variable in pm_property:
# If so, create a convenience variable to store it
this_pm_variable = pm_property[pm_variable]
# Next we need to check type. If it is a string, we will add it here to avoid parsing numerics
# However, we need to be sure to not add the flagged bad strings.
# However, a flagged value *could* be a value property name, and we would want to allow that
if isinstance(this_pm_variable, basestring):
if pm_variable == 'property_name':
this_row.append(this_pm_variable)
added = True
elif pm_variable == 'property_notes':
sanitized_string = this_pm_variable.replace(
'\n', ' ')
this_row.append(sanitized_string)
added = True
elif this_pm_variable not in pm_flagged_bad_string_values:
this_row.append(this_pm_variable)
added = True
# If it isn't a string, it should be a dictionary, storing numeric data and units, etc.
else:
# As long as it is a valid dictionary, try to get a meaningful value out of it
if this_pm_variable and '#text' in this_pm_variable and this_pm_variable[
'#text'] != 'Not Available':
# Coerce the value into a proper set of Pint units for us
if doing_pint:
new_var = UploadViewSet._get_pint_var_from_pm_value_object(
this_pm_variable)
if new_var['success']:
pint_value = new_var['pint_value']
this_row.append(pint_value.magnitude)
added = True
# TODO: What to do with the pint_value.units here?
else:
this_row.append(
float(this_pm_variable['#text']))
added = True
# And finally, if we haven't set the added flag, give the csv column a blank value
if not added:
this_row.append('')
# Then add this property row of data
rows.append(this_row)
# Then write the actual data out as csv
with open(path, 'w', encoding='utf-8') as csv_file:
pm_csv_writer = csv.writer(csv_file)
for row_num, row in enumerate(rows):
pm_csv_writer.writerow(row)
# Look up the import record (data set)
org_id = request.data['organization_id']
import_record_pk = request.data['import_record_id']
try:
record = ImportRecord.objects.get(pk=import_record_pk,
super_organization_id=org_id)
except ImportRecord.DoesNotExist:
# clean up the uploaded file
os.unlink(path)
return JsonResponse({
'success':
False,
'message':
"Import Record %s not found" % import_record_pk
})
# Create a new import file object in the database
f = ImportFile.objects.create(
import_record=record,
uploaded_filename=file_name,
file=path,
source_type=SEED_DATA_SOURCES[PORTFOLIO_RAW],
**{
'source_program': 'PortfolioManager',
'source_program_version': '1.0'
})
# Return the newly created import file ID
return JsonResponse({'success': True, 'import_file_id': f.pk})
class ColumnViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet,
OrgCreateUpdateMixin):
"""
create:
Create a new Column within a specified org or user's currently activated org.
update:
Update a column and modify which dataset it belongs to.
delete:
Deletes a single column.
"""
raise_exception = True
serializer_class = ColumnSerializer
renderer_classes = (JSONRenderer, )
model = Column
pagination_class = None
parser_classes = (JSONParser, FormParser)
def get_queryset(self):
# check if the request is properties or taxlots
org_id = self.get_organization(self.request)
return Column.objects.filter(organization_id=org_id)
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(required=False),
AutoSchemaHelper.query_string_field(
name='inventory_type',
required=False,
description=
'Which inventory type is being matched (for related fields and naming)'
'\nDefault: "property"'),
AutoSchemaHelper.query_boolean_field(
name='only_used',
required=False,
description=
'Determine whether or not to show only the used fields '
'(i.e. only columns that have been mapped)'
'\nDefault: "false"'),
AutoSchemaHelper.query_boolean_field(
name='display_units',
required=False,
description='If true, any columns that have units will have them'
' added as a suffix to the display_name'
'\nDefault: "false"'),
], )
@api_endpoint_class
@ajax_request_class
def list(self, request):
"""
Retrieves all columns for the user's organization including the raw database columns. Will
return all the columns across both the Property and Tax Lot tables. The related field will
be true if the column came from the other table that is not the 'inventory_type' (which
defaults to Property)
"""
organization_id = self.get_organization(self.request)
inventory_type = request.query_params.get('inventory_type', 'property')
only_used = json.loads(request.query_params.get('only_used', 'false'))
columns = Column.retrieve_all(organization_id, inventory_type,
only_used)
organization = Organization.objects.get(pk=organization_id)
if json.loads(request.query_params.get('display_units', 'false')):
columns = [add_pint_unit_suffix(organization, x) for x in columns]
return JsonResponse({
'status': 'success',
'columns': columns,
})
@swagger_auto_schema_org_query_param
@ajax_request_class
def retrieve(self, request, pk=None):
"""
This API endpoint retrieves a Column
"""
organization_id = self.get_organization(self.request)
# check if column exists for the organization
try:
c = Column.objects.get(pk=pk)
except Column.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'column with id {} does not exist'.format(pk)
},
status=status.HTTP_404_NOT_FOUND)
if c.organization.id != organization_id:
return JsonResponse(
{
'status':
'error',
'message':
'Organization ID mismatch between column and organization'
},
status=status.HTTP_400_BAD_REQUEST)
return JsonResponse({
'status': 'success',
'column': ColumnSerializer(c).data
})
@ajax_request_class
@has_perm_class('can_modify_data')
def update(self, request, pk=None):
organization_id = request.query_params.get('organization_id', None)
request.data['shared_field_type'] = request.data['sharedFieldType']
del request.data['sharedFieldType']
# Ensure ComStock uniqueness across properties and taxlots together
if request.data['comstock_mapping'] is not None:
Column.objects.filter(organization_id=organization_id, comstock_mapping=request.data['comstock_mapping']) \
.update(comstock_mapping=None)
return super(ColumnViewSet, self).update(request, pk)
@ajax_request_class
@has_perm_class('can_modify_data')
def destroy(self, request, pk=None):
org_id = self.get_organization(request)
try:
column = Column.objects.get(id=pk, organization_id=org_id)
except Column.DoesNotExist:
return JsonResponse(
{
'success':
False,
'message':
'Cannot find column in org=%s with pk=%s' % (org_id, pk)
},
status=status.HTTP_404_NOT_FOUND)
if not column.is_extra_data:
return JsonResponse(
{
'success': False,
'message': 'Only extra_data columns can be deleted'
},
status=status.HTTP_400_BAD_REQUEST)
# Delete key from jsonb data
if column.table_name == 'PropertyState':
states = PropertyState.objects.filter(
organization_id=org_id,
data_state=DATA_STATE_MATCHING,
extra_data__has_key=column.column_name)
state_count = states.count()
# far faster than iterating states, popping the key, and saving, but doesn't update the hash
# with connection.cursor() as cursor:
# cursor.execute("UPDATE seed_propertystate "
# "SET extra_data = extra_data - %s "
# "WHERE organization_id = %s AND data_state = %s",
# [column.column_name, org_id, DATA_STATE_MATCHING])
elif column.table_name == 'TaxLotState':
states = TaxLotState.objects.filter(
organization_id=org_id,
data_state=DATA_STATE_MATCHING,
extra_data__has_key=column.column_name)
state_count = states.count()
# far faster than iterating states, popping the key, and saving, but doesn't update the hash
# with connection.cursor() as cursor:
# cursor.execute("UPDATE seed_taxlotstate "
# "SET extra_data = extra_data - %s "
# "WHERE organization_id = %s AND data_state = %s",
# [column.column_name, org_id, DATA_STATE_MATCHING])
else:
return JsonResponse(
{
'success':
False,
'message':
'Unexpected table_name \'%s\' for column with pk=%s' %
(column.table_name, pk)
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Pop the key and update the hash
for state in states:
state.extra_data.pop(column.column_name)
state.save()
# Delete all mappings from raw column names to the mapped column, then delete the mapped column
ColumnMapping.objects.filter(column_mapped=column).delete()
column.delete()
table_display_name = column.table_name if state_count == 1 else column.table_name + 's'
return JsonResponse(
{
'success':
True,
'message':
'Removed \'%s\' from %s %s' %
(column.column_name, state_count, table_display_name)
},
status=status.HTTP_200_OK)
@swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory({
'new_column_name':
'string',
'overwrite':
'boolean'
}))
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def rename(self, request, pk=None):
"""
This API endpoint renames a Column
"""
org_id = self.get_organization(request)
try:
column = Column.objects.get(id=pk, organization_id=org_id)
except Column.DoesNotExist:
return JsonResponse(
{
'success':
False,
'message':
'Cannot find column in org=%s with pk=%s' % (org_id, pk)
},
status=status.HTTP_404_NOT_FOUND)
new_column_name = request.data.get('new_column_name', None)
overwrite = request.data.get('overwrite', False)
if not new_column_name:
return JsonResponse(
{
'success':
False,
'message':
'You must specify the name of the new column as "new_column_name"'
},
status=status.HTTP_400_BAD_REQUEST)
result = column.rename_column(new_column_name, overwrite)
if not result[0]:
return JsonResponse(
{
'success':
False,
'message':
'Unable to rename column with message: "%s"' % result[1]
},
status=status.HTTP_400_BAD_REQUEST)
else:
return JsonResponse({'success': True, 'message': result[1]})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_string_field(
'inventory_type',
required=True,
description='Inventory Type, either "property" or "taxlot"')
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_viewer')
@action(detail=False, methods=['GET'])
def mappable(self, request):
"""
List only inventory columns that are mappable
"""
organization_id = int(request.query_params.get('organization_id'))
inventory_type = request.query_params.get('inventory_type')
if inventory_type not in ['property', 'taxlot']:
return JsonResponse({
'status':
'error',
'message':
'Query param `inventory_type` must be "property" or "taxlot"'
})
columns = Column.retrieve_mapping_columns(organization_id,
inventory_type)
return JsonResponse({'status': 'success', 'columns': columns})
class PortfolioManagerViewSet(GenericViewSet):
"""
This viewset contains two API views: /template_list/ and /report/ that are used to interface SEED with ESPM
"""
serializer_class = PortfolioManagerSerializer
@swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory(
{
'username': '******',
'password': '******'
},
description='ESPM account credentials.',
required=['username', 'password']),
responses={
200:
AutoSchemaHelper.schema_factory({
'status':
'string',
'templates': [{
'template_information_1': 'string',
'template_information_2': 'integer',
'[other keys...]': 'string'
}],
'message':
'string'
}),
})
@action(detail=False, methods=['POST'])
def template_list(self, request):
"""
This API view makes a request to ESPM for the list of available report templates, including root templates and
child data requests.
---
This API responds with a JSON object with two keys: status, which will be a string -
either error or success. If successful, a second key, templates, will hold the list of templates from ESPM. If
not successful, a second key, message, will include an error description that can be presented on the UI.
"""
if 'username' not in request.data:
return JsonResponse(
{
'status':
'error',
'message':
'Invalid call to PM worker: missing username for PM account'
},
status=status.HTTP_400_BAD_REQUEST)
if 'password' not in request.data:
return JsonResponse(
{
'status':
'error',
'message':
'Invalid call to PM worker: missing password for PM account'
},
status=status.HTTP_400_BAD_REQUEST)
username = request.data['username']
password = request.data['password']
pm = PortfolioManagerImport(username, password)
try:
possible_templates = pm.get_list_of_report_templates()
except PMExcept as pme:
return JsonResponse({
'status': 'error',
'message': str(pme)
},
status=status.HTTP_400_BAD_REQUEST)
except Exception as e:
return JsonResponse({
'status': 'error',
'message': e
},
status=status.HTTP_400_BAD_REQUEST)
return JsonResponse({
'status': 'success',
'templates': possible_templates
})
@swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory(
{
'username': '******',
'password': '******',
'template': {
'[copy information from template_list]': 'string'
}
},
description='ESPM account credentials.',
required=['username', 'password']),
responses={
200:
AutoSchemaHelper.schema_factory({
'status':
'string',
'properties': [{
'properties_information_1': 'string',
'properties_information_2': 'integer',
'[other keys...]': 'string'
}],
'message':
'string'
}),
})
@action(detail=False, methods=['POST'])
def report(self, request):
"""
This API view makes a request to ESPM to generate and download a report based on a specific template.
---
This API responds with a JSON object with two keys: status, which will be a string -
either error or success. If successful, a second key, properties, will hold the list of properties found in
this generated report. If not successful, a second key, message, will include an error description that can be
presented on the UI.
"""
if 'username' not in request.data:
return JsonResponse(
{
'status':
'error',
'message':
'Invalid call to PM worker: missing username for PM account'
},
status=status.HTTP_400_BAD_REQUEST)
if 'password' not in request.data:
return JsonResponse(
{
'status':
'error',
'message':
'Invalid call to PM worker: missing password for PM account'
},
status=status.HTTP_400_BAD_REQUEST)
if 'template' not in request.data:
return JsonResponse(
{
'status':
'error',
'message':
'Invalid call to PM worker: missing template for PM account'
},
status=status.HTTP_400_BAD_REQUEST)
username = request.data['username']
password = request.data['password']
template = request.data['template']
pm = PortfolioManagerImport(username, password)
try:
try:
if 'z_seed_child_row' not in template:
return JsonResponse(
{
'status':
'error',
'message':
'Invalid template formulation during portfolio manager data import'
},
status=status.HTTP_400_BAD_REQUEST)
if template['z_seed_child_row']:
content = pm.generate_and_download_child_data_request_report(
template)
else:
content = pm.generate_and_download_template_report(
template)
except PMExcept as pme:
return JsonResponse({
'status': 'error',
'message': str(pme)
},
status=status.HTTP_400_BAD_REQUEST)
try:
content_object = xmltodict.parse(content,
dict_constructor=dict)
except Exception: # catch all because xmltodict doesn't specify a class of Exceptions
return JsonResponse(
{
'status': 'error',
'message': 'Malformed XML from template download'
},
status=status.HTTP_400_BAD_REQUEST)
try:
possible_properties = content_object['report'][
'informationAndMetrics']['row']
if isinstance(possible_properties, list):
properties = possible_properties
elif isinstance(possible_properties, dict):
properties = [possible_properties]
else: # OrderedDict hints that a 'preview' report was generated, anything else is an unhandled case
return JsonResponse(
{
'status':
'error',
'message':
'Property list was not a list...was a preview report template used on accident?'
},
status=status.HTTP_400_BAD_REQUEST)
except KeyError:
return JsonResponse(
{
'status':
'error',
'message':
'Processed template successfully, but missing keys -- is the report empty on Portfolio Manager?'
},
status=status.HTTP_400_BAD_REQUEST)
return JsonResponse({
'status': 'success',
'properties': properties
})
except Exception as e:
return JsonResponse({
'status': 'error',
'message': e
},
status=status.HTTP_400_BAD_REQUEST)
return d.get(severity)
class EmailAndIDSerializer(serializers.Serializer):
email = serializers.CharField(max_length=100)
user_id = serializers.IntegerField()
class ListUsersResponseSerializer(serializers.Serializer):
users = EmailAndIDSerializer(many=True)
# this is used for swagger docs for some views below
user_response_schema = AutoSchemaHelper.schema_factory({
'first_name': 'string',
'last_name': 'string',
'email': 'string',
'api_key': 'string',
})
class UserViewSet(viewsets.ViewSet):
raise_exception = True
def validate_request_user(self, pk, request):
try:
user = User.objects.get(pk=pk)
except ObjectDoesNotExist:
return False, JsonResponse(
{'status': 'error', 'message': "Could not find user with pk = " + str(pk)},
status=status.HTTP_404_NOT_FOUND)
if not user == request.user:
class UserViewSet(viewsets.ViewSet):
raise_exception = True
def validate_request_user(self, pk, request):
try:
user = User.objects.get(pk=pk)
except ObjectDoesNotExist:
return False, JsonResponse(
{'status': 'error', 'message': "Could not find user with pk = " + str(pk)},
status=status.HTTP_404_NOT_FOUND)
if not user == request.user:
return False, JsonResponse(
{'status': 'error', 'message': "Cannot access user with pk = " + str(pk)},
status=status.HTTP_403_FORBIDDEN)
return True, user
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'org_name': 'string',
'first_name': 'string',
'last_name': 'string',
'role': 'string',
'email': 'string',
},
required=['first_name', 'last_name', 'role', 'email'],
description='An object containing meta data for a new user:\n'
'-org_name: New organization name if creating a new organization for this user\n'
'-first_name: First name of new user\n'
'-last_name: Last name of new user\n'
'-role: one of owner, member, or viewer\n'
'-email: Email address of the new user'
),
responses={
200: AutoSchemaHelper.schema_factory(
{
'status': 'string',
'message': 'string',
'org': 'string',
'org_created': 'boolean',
'username': '******',
'user_id': 'string',
},
description='Properties:\n'
'-org: name of new org (or existing org)\n'
'-org_created: true if new org created\n'
'-username: username of new user\n'
'-user_id: user id (pk) of new user'
)
}
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_owner')
def create(self, request):
"""
Creates a new SEED user. One of 'organization_id' or 'org_name' is needed.
Sends invitation email to the new user.
"""
body = request.data
org_name = body.get('org_name')
org_id = request.query_params.get('organization_id', None)
if (org_name and org_id) or (not org_name and not org_id):
return JsonResponse({
'status': 'error',
'message': 'Choose either an existing org or provide a new one'
}, status=status.HTTP_409_CONFLICT)
first_name = body['first_name']
last_name = body['last_name']
email = body['email']
username = body['email']
user, created = User.objects.get_or_create(username=username.lower())
if org_id:
org = Organization.objects.get(pk=org_id)
org_created = False
else:
org, _, _ = create_organization(user, org_name)
org_created = True
# Add the user to the org. If this is the org's first user,
# the user becomes the owner/admin automatically.
# see Organization.add_member()
if not org.is_member(user):
org.add_member(user)
if body.get('role'):
# check if this is a dict, if so, grab the value out of 'value'
role = body['role']
try:
_get_role_from_js(role)
except Exception:
return JsonResponse({'status': 'error', 'message': 'valid arguments for role are [viewer, member, '
'owner]'},
status=status.HTTP_400_BAD_REQUEST)
OrganizationUser.objects.filter(
organization_id=org.pk,
user_id=user.pk
).update(role_level=_get_role_from_js(role))
if created:
user.set_unusable_password()
user.email = email
user.first_name = first_name
user.last_name = last_name
user.save()
try:
domain = request.get_host()
except Exception:
domain = 'seed-platform.org'
invite_to_seed(
domain, user.email, default_token_generator.make_token(user), user.pk, first_name
)
return JsonResponse({
'status': 'success',
'message': user.email,
'org': org.name,
'org_created': org_created,
'username': user.username,
'user_id': user.id
})
@swagger_auto_schema(
responses={
200: ListUsersResponseSerializer,
}
)
@ajax_request_class
@has_perm_class('requires_superuser')
def list(self, request):
"""
Retrieves all users' email addresses and IDs.
Only usable by superusers.
"""
users = []
for user in User.objects.only('id', 'email'):
users.append({'email': user.email, 'user_id': user.id})
return JsonResponse({'users': users})
@ajax_request_class
@api_endpoint_class
@action(detail=False, methods=['GET'])
def current(self, request):
"""
Returns the id (primary key) for the current user to allow it
to be passed to other user related endpoints
---
type:
pk:
description: Primary key for the current user
required: true
type: string
"""
return JsonResponse({'pk': request.user.id})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'role': 'string',
},
description='new role for user',
),
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_owner')
@action(detail=True, methods=['PUT'])
def role(self, request, pk=None):
"""
Updates a user's role within an organization.
"""
body = request.data
role = _get_role_from_js(body['role'])
user_id = pk
organization_id = request.query_params.get('organization_id', None)
is_last_member = not OrganizationUser.objects.filter(
organization_id=organization_id,
).exclude(user_id=user_id).exists()
if is_last_member:
return JsonResponse({
'status': 'error',
'message': 'an organization must have at least one member'
}, status=status.HTTP_409_CONFLICT)
is_last_owner = not OrganizationUser.objects.filter(
organization_id=organization_id,
role_level=ROLE_OWNER,
).exclude(user_id=user_id).exists()
if is_last_owner:
return JsonResponse({
'status': 'error',
'message': 'an organization must have at least one owner level member'
}, status=status.HTTP_409_CONFLICT)
OrganizationUser.objects.filter(
user_id=user_id,
organization_id=organization_id
).update(role_level=role)
return JsonResponse({'status': 'success'})
@swagger_auto_schema(
responses={
200: user_response_schema,
}
)
@api_endpoint_class
@ajax_request_class
def retrieve(self, request, pk=None):
"""
Retrieves the a user's first_name, last_name, email
and api key if exists by user ID (pk).
"""
ok, content = self.validate_request_user(pk, request)
if ok:
user = content
else:
return content
return JsonResponse({
'status': 'success',
'first_name': user.first_name,
'last_name': user.last_name,
'email': user.email,
'api_key': user.api_key,
})
@ajax_request_class
@action(detail=True, methods=['POST'])
def generate_api_key(self, request, pk=None):
"""
Generates a new API key
---
parameter_strategy: replace
parameters:
- name: pk
description: User ID / primary key
type: integer
required: true
paramType: path
type:
status:
description: success or error
type: string
required: true
api_key:
description: the new API key for this user
type: string
required: true
"""
ok, content = self.validate_request_user(pk, request)
if ok:
user = content
else:
return content
user.generate_key()
return {
'status': 'success',
'api_key': User.objects.get(pk=pk).api_key
}
@swagger_auto_schema(
request_body=AutoSchemaHelper.schema_factory({
'first_name': 'string',
'last_name': 'string',
'email': 'string'
}),
description='An object containing meta data for a updated user: \n'
'- Required - first_name, last_name, email',
responses={
200: user_response_schema,
}
)
@api_endpoint_class
@ajax_request_class
def update(self, request, pk=None):
"""
Updates the user's first name, last name, and email
"""
body = request.data
ok, content = self.validate_request_user(pk, request)
if ok:
user = content
else:
return content
json_user = body
user.first_name = json_user.get('first_name')
user.last_name = json_user.get('last_name')
user.email = json_user.get('email')
user.username = json_user.get('email')
user.save()
return JsonResponse({
'status': 'success',
'first_name': user.first_name,
'last_name': user.last_name,
'email': user.email,
'api_key': user.api_key,
})
@swagger_auto_schema(
request_body=AutoSchemaHelper.schema_factory(
{
'current_password': '******',
'password_1': 'string',
'password_2': 'string',
},
description='Fill in the current and new matching passwords'),
)
@ajax_request_class
@action(detail=True, methods=['PUT'])
def set_password(self, request, pk=None):
"""
sets/updates a user's password, follows the min requirement of
django password validation settings in config/settings/common.py
"""
body = request.data
ok, content = self.validate_request_user(pk, request)
if ok:
user = content
else:
return content
current_password = body.get('current_password')
p1 = body.get('password_1')
p2 = body.get('password_2')
if not user.check_password(current_password):
return JsonResponse({'status': 'error', 'message': 'current password is not valid'},
status=status.HTTP_400_BAD_REQUEST)
if p1 is None or p1 != p2:
return JsonResponse({'status': 'error', 'message': 'entered password do not match'},
status=status.HTTP_400_BAD_REQUEST)
try:
validate_password(p2)
except ValidationError as e:
return JsonResponse({'status': 'error', 'message': e.messages[0]},
status=status.HTTP_400_BAD_REQUEST)
user.set_password(p1)
user.save()
return JsonResponse({'status': 'success'})
@ajax_request_class
def get_actions(self, request):
"""returns all actions"""
return {
'status': 'success',
'actions': list(PERMS.keys()),
}
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'actions': ['string'],
},
description='A list of actions to check: examples include (requires_parent_org_owner, '
'requires_owner, requires_member, requires_viewer, '
'requires_superuser, can_create_sub_org, can_remove_org)'
),
responses={
200: AutoSchemaHelper.schema_factory({
'auth': {
'action_name': 'boolean'
}
})
}
)
@ajax_request_class
@action(detail=True, methods=['POST'])
def is_authorized(self, request, pk=None):
"""
Checks the auth for a given action, if user is the owner of the parent
org then True is returned for each action
"""
ok, content = self.validate_request_user(pk, request)
if ok:
user = content
else:
return content
actions, org, error, message = self._parse_is_authenticated_params(request)
if error:
return JsonResponse({
'status': 'error',
'message': message
}, status=status.HTTP_400_BAD_REQUEST)
# If the only action requested is 'requires_superuser' no need to check an org affiliation
if len(actions) == 1 and actions[0] == 'requires_superuser':
return JsonResponse(
{'status': 'success', 'auth': {'requires_superuser': user.is_superuser}})
auth = self._try_parent_org_auth(user, org, actions)
if auth:
return JsonResponse({'status': 'success', 'auth': auth})
try:
ou = OrganizationUser.objects.get(user=user, organization=org)
except OrganizationUser.DoesNotExist:
return JsonResponse({'status': 'error', 'message': 'user does not exist'})
auth = {action: PERMS[action](ou) for action in actions}
return JsonResponse({'status': 'success', 'auth': auth})
def _parse_is_authenticated_params(self, request):
"""checks if the org exists and if the actions are present
:param request: the request
:returns: tuple (actions, org, error, message)
"""
error = False
message = ""
body = request.data
if not body.get('actions'):
message = 'no actions to check'
error = True
org_id = request.query_params.get('organization_id', None)
if org_id == '':
message = 'organization id is undefined'
error = True
org = None
else:
try:
org = Organization.objects.get(pk=org_id)
except Organization.DoesNotExist:
message = 'organization does not exist'
error = True
org = None
return body.get('actions'), org, error, message
def _try_parent_org_auth(self, user, organization, actions):
"""checks the parent org for permissions, if the user is not the owner of
the parent org, then None is returned.
:param user: the request user
:param organization: org to check its parent
:param actions: list of str actions to check
:returns: a dict of action permission resolutions or None
"""
try:
ou = OrganizationUser.objects.get(
user=user,
organization=organization.parent_org,
role_level=ROLE_OWNER
)
except OrganizationUser.DoesNotExist:
return None
return {
action: PERMS['requires_owner'](ou) for action in actions
}
@swagger_auto_schema(
responses={
200: AutoSchemaHelper.schema_factory({
'show_shared_buildings': 'boolean'
})
}
)
@ajax_request_class
@action(detail=True, methods=['GET'])
def shared_buildings(self, request, pk=None):
"""
Get the request user's ``show_shared_buildings`` attr
"""
ok, content = self.validate_request_user(pk, request)
if ok:
user = content
else:
return content
return JsonResponse({
'status': 'success',
'show_shared_buildings': user.show_shared_buildings,
})
@swagger_auto_schema_org_query_param
@ajax_request_class
@action(detail=True, methods=['PUT'])
def default_organization(self, request, pk=None):
"""
Sets the user's default organization
"""
ok, content = self.validate_request_user(pk, request)
if ok:
user = content
else:
return content
user.default_organization_id = request.query_params.get('organization_id', None)
user.save()
return {'status': 'success'}
@has_perm_class('requires_superuser')
@ajax_request_class
@action(detail=True, methods=['PUT'])
def deactivate(self, request, pk=None):
"""
Deactivates a user
"""
try:
user_id = pk
user = SEEDUser.objects.get(
id=user_id
)
user.deactivate_user()
return JsonResponse({
'status': 'successfully deactivated',
'data': user.email
})
except Exception as e:
return JsonResponse({
'status': 'error',
'data': str(e),
}, status=HTTP_400_BAD_REQUEST)
CreateModelMixin,
)
from seed.decorators import ajax_request_class
from seed.lib.superperms.orgs.decorators import has_perm_class
from seed.lib.superperms.orgs.permissions import SEEDOrgPermissions
from seed.models.data_quality import DataQualityCheck, Rule
from seed.serializers.rules import RuleSerializer
from seed.utils.api import api_endpoint_class
from seed.utils.api_schema import AutoSchemaHelper
from seed.utils.viewsets import UpdateWithoutPatchModelMixin
_log = logging.getLogger(__name__)
nested_org_id_path_field = AutoSchemaHelper.base_field(
"nested_organization_id", "IN_PATH",
"Organization ID - identifier used to specify a DataQualityCheck and its Rules",
True, "TYPE_INTEGER")
@method_decorator(
name='list',
decorator=swagger_auto_schema(manual_parameters=[nested_org_id_path_field])
)
@method_decorator(
name='update',
decorator=swagger_auto_schema(manual_parameters=[nested_org_id_path_field])
)
@method_decorator(
name='destroy',
decorator=swagger_auto_schema(manual_parameters=[nested_org_id_path_field])
)
class ImportFileViewSet(viewsets.ViewSet):
raise_exception = True
queryset = ImportFile.objects.all()
@api_endpoint_class
@ajax_request_class
def retrieve(self, request, pk=None):
"""
Retrieves details about an ImportFile.
"""
import_file_id = pk
orgs = request.user.orgs.all()
try:
import_file = ImportFile.objects.get(pk=import_file_id)
d = ImportRecord.objects.filter(super_organization__in=orgs,
pk=import_file.import_record_id)
except ObjectDoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not access an import file with this ID'
},
status=status.HTTP_403_FORBIDDEN)
# check if user has access to the import file
if not d.exists():
return JsonResponse(
{
'status': 'error',
'message': 'Could not locate import file with this ID',
'import_file': {},
},
status=status.HTTP_400_BAD_REQUEST)
f = obj_to_dict(import_file)
f['name'] = import_file.filename_only
if not import_file.uploaded_filename:
f['uploaded_filename'] = import_file.filename
f['dataset'] = obj_to_dict(import_file.import_record)
return JsonResponse({
'status': 'success',
'import_file': f,
})
@swagger_auto_schema_org_query_param
@has_perm_class('requires_viewer')
@api_endpoint_class
@ajax_request_class
@action(detail=True, methods=['GET'])
def first_five_rows(self, request, pk=None):
"""
Retrieves the first five rows of an ImportFile.
"""
org_id = request.query_params.get('organization_id', None)
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
if import_file.cached_second_to_fifth_row is None:
return JsonResponse(
{
'status':
'error',
'message':
'Internal problem occurred, import file first five rows not cached'
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
'''
import_file.cached_second_to_fifth_row is a field that contains the first
5 lines of data from the file, split on newlines, delimited by
ROW_DELIMITER. This becomes an issue when fields have newlines in them,
so the following is to handle newlines in the fields.
In the case of only one data column there will be no ROW_DELIMITER.
'''
header = import_file.cached_first_row.split(ROW_DELIMITER)
data = import_file.cached_second_to_fifth_row
return JsonResponse({
'status':
'success',
'first_five_rows':
convert_first_five_rows_to_list(header, data)
})
@swagger_auto_schema_org_query_param
@has_perm_class('requires_viewer')
@api_endpoint_class
@ajax_request_class
@action(detail=True, methods=['GET'])
def raw_column_names(self, request, pk=None):
"""
Retrieves a list of all column names from an ImportFile.
"""
org_id = request.query_params.get('organization_id', None)
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
return JsonResponse({
'status': 'success',
'raw_columns': import_file.first_row_columns
})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
],
responses={200: MappingResultsResponseSerializer})
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'], url_path='mapping_results')
def mapping_results(self, request, pk=None):
"""
Retrieves a paginated list of Properties and Tax Lots for an import file after mapping.
"""
import_file_id = pk
org_id = request.query_params.get('organization_id', None)
org = Organization.objects.get(pk=org_id)
try:
# get the field names that were in the mapping
import_file = ImportFile.objects.get(
pk=import_file_id, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
# List of the only fields to show
field_names = import_file.get_cached_mapped_columns
# set of fields
fields = {
'PropertyState': ['id', 'extra_data', 'lot_number'],
'TaxLotState': ['id', 'extra_data']
}
columns_from_db = Column.retrieve_all(org_id)
property_column_name_mapping = {}
taxlot_column_name_mapping = {}
for field_name in field_names:
# find the field from the columns in the database
for column in columns_from_db:
if column['table_name'] == 'PropertyState' and \
field_name[0] == 'PropertyState' and \
field_name[1] == column['column_name']:
property_column_name_mapping[
column['column_name']] = column['name']
if not column['is_extra_data']:
fields['PropertyState'].append(
field_name[1]) # save to the raw db fields
continue
elif column['table_name'] == 'TaxLotState' and \
field_name[0] == 'TaxLotState' and \
field_name[1] == column['column_name']:
taxlot_column_name_mapping[
column['column_name']] = column['name']
if not column['is_extra_data']:
fields['TaxLotState'].append(
field_name[1]) # save to the raw db fields
continue
inventory_type = request.data.get('inventory_type', 'all')
result = {'status': 'success'}
if inventory_type == 'properties' or inventory_type == 'all':
properties = PropertyState.objects.filter(
import_file_id=import_file_id,
data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING],
merge_state__in=[
MERGE_STATE_UNKNOWN, MERGE_STATE_NEW
]).only(*fields['PropertyState']).order_by('id')
property_results = []
for prop in properties:
prop_dict = TaxLotProperty.model_to_dict_with_mapping(
prop,
property_column_name_mapping,
fields=fields['PropertyState'],
exclude=['extra_data'])
prop_dict.update(
TaxLotProperty.extra_data_to_dict_with_mapping(
prop.extra_data,
property_column_name_mapping,
fields=prop.extra_data.keys(),
).items())
prop_dict = apply_display_unit_preferences(org, prop_dict)
property_results.append(prop_dict)
result['properties'] = property_results
if inventory_type == 'taxlots' or inventory_type == 'all':
tax_lots = TaxLotState.objects.filter(
import_file_id=import_file_id,
data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING],
merge_state__in=[MERGE_STATE_UNKNOWN, MERGE_STATE_NEW
]).only(*fields['TaxLotState']).order_by('id')
tax_lot_results = []
for tax_lot in tax_lots:
tax_lot_dict = TaxLotProperty.model_to_dict_with_mapping(
tax_lot,
taxlot_column_name_mapping,
fields=fields['TaxLotState'],
exclude=['extra_data'])
tax_lot_dict.update(
TaxLotProperty.extra_data_to_dict_with_mapping(
tax_lot.extra_data,
taxlot_column_name_mapping,
fields=tax_lot.extra_data.keys(),
).items())
tax_lot_dict = apply_display_unit_preferences(
org, tax_lot_dict)
tax_lot_results.append(tax_lot_dict)
result['tax_lots'] = tax_lot_results
return result
@staticmethod
def has_coparent(state_id, inventory_type, fields=None):
"""
Return the coparent of the current state id based on the inventory type. If fields
are given (as a list), then it will only return the fields specified of the state model
object as a dictionary.
:param state_id: int, ID of PropertyState or TaxLotState
:param inventory_type: string, either properties | taxlots
:param fields: list, either None or list of fields to return
:return: dict or state object, If fields is not None then will return state_object
"""
state_model = PropertyState if inventory_type == 'properties' else TaxLotState
# TODO: convert coparent to instance method, not class method
audit_entry, audit_count = state_model.coparent(state_id)
if audit_count == 0:
return False
if audit_count > 1:
return JsonResponse(
{
'status':
'error',
'message':
'Internal problem occurred, more than one merge record found'
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return audit_entry[0]
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
],
request_body=AutoSchemaHelper.schema_factory({
'remap':
'boolean',
'mark_as_done':
'boolean',
}))
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def map(self, request, pk=None):
"""
Starts a background task to convert imported raw data into
PropertyState and TaxLotState, using user's column mappings.
"""
body = request.data
remap = body.get('remap', False)
mark_as_done = body.get('mark_as_done', True)
org_id = request.query_params.get('organization_id', None)
import_file = ImportFile.objects.filter(
pk=pk, import_record__super_organization_id=org_id)
if not import_file.exists():
return {
'status': 'error',
'message': 'ImportFile {} does not exist'.format(pk)
}
# return remap_data(import_file_id)
return JsonResponse(map_data(pk, remap, mark_as_done))
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def start_system_matching_and_geocoding(self, request, pk=None):
"""
Starts a background task to attempt automatic matching between buildings
in an ImportFile with other existing buildings within the same org.
"""
org_id = request.query_params.get('organization_id', None)
try:
ImportFile.objects.get(pk=pk,
import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
return geocode_and_match_buildings_task(pk)
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def start_data_quality_checks(self, request, pk=None):
"""
Starts a background task to attempt automatic matching between buildings
in an ImportFile with other existing buildings within the same org.
"""
org_id = request.query_params.get('organization_id', None)
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
return_value = do_checks(org_id, None, None, import_file.pk)
# step 5: create a new model instance
return JsonResponse({
'progress_key': return_value['progress_key'],
'progress': return_value,
})
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def validate_use_cases(self, request, pk=None):
"""
Starts a background task to call BuildingSync's use case validation
tool.
"""
org_id = request.query_params.get('organization_id', None)
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
return task_validate_use_cases(import_file.pk)
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
],
request_body=AutoSchemaHelper.schema_factory(
{'cycle_id': 'string'}))
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def start_save_data(self, request, pk=None):
"""
Starts a background task to import raw data from an ImportFile
into PropertyState objects as extra_data. If the cycle_id is set to
year_ending then the cycle ID will be set to the year_ending column for each
record in the uploaded file. Note that the year_ending flag is not yet enabled.
"""
body = request.data
import_file_id = pk
if not import_file_id:
return JsonResponse(
{
'status': 'error',
'message': 'must pass file_id of file to save'
},
status=status.HTTP_400_BAD_REQUEST)
org_id = request.query_params.get('organization_id', None)
try:
import_file = ImportFile.objects.get(
pk=import_file_id, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
cycle_id = body.get('cycle_id')
if not cycle_id:
return JsonResponse(
{
'status': 'error',
'message':
'must pass cycle_id of the cycle to save the data'
},
status=status.HTTP_400_BAD_REQUEST)
elif cycle_id == 'year_ending':
_log.error("NOT CONFIGURED FOR YEAR ENDING OPTION AT THE MOMENT")
return JsonResponse(
{
'status': 'error',
'message':
'SEED is unable to parse year_ending at the moment'
},
status=status.HTTP_400_BAD_REQUEST)
else:
# find the cycle
cycle = Cycle.objects.get(id=cycle_id)
if cycle:
# assign the cycle id to the import file object
import_file.cycle = cycle
import_file.save()
else:
return JsonResponse(
{
'status': 'error',
'message': 'cycle_id was invalid'
},
status=status.HTTP_400_BAD_REQUEST)
return JsonResponse(task_save_raw(import_file.id))
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def mapping_done(self, request, pk=None):
"""
Tell the backend that the mapping is complete.
"""
import_file_id = pk
if not import_file_id:
return JsonResponse(
{
'status': 'error',
'message': 'must pass import_file_id'
},
status=status.HTTP_400_BAD_REQUEST)
org_id = request.query_params.get('organization_id', None)
try:
import_file = ImportFile.objects.get(
pk=import_file_id, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'no import file with given id'
},
status=status.HTTP_404_NOT_FOUND)
import_file.mapping_done = True
import_file.save()
return JsonResponse({'status': 'success', 'message': ''})
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def matching_and_geocoding_results(self, request, pk=None):
"""
Retrieves the number of matched and unmatched properties & tax lots for
a given ImportFile record. Specifically for new imports
"""
org_id = request.query_params.get('organization_id', None)
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
# property views associated with this imported file (including merges)
properties_new = []
properties_matched = list(
PropertyState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
merge_state=MERGE_STATE_MERGED,
).values_list('id', flat=True))
# Check audit log in case PropertyStates are listed as "new" but were merged into a different property
properties = list(
PropertyState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
merge_state=MERGE_STATE_NEW,
))
for state in properties:
audit_creation_id = PropertyAuditLog.objects.only('id').exclude(
import_filename=None).get(state_id=state.id,
name='Import Creation')
if PropertyAuditLog.objects.exclude(
record_type=AUDIT_USER_EDIT).filter(
parent1_id=audit_creation_id).exists():
properties_matched.append(state.id)
else:
properties_new.append(state.id)
tax_lots_new = []
tax_lots_matched = list(
TaxLotState.objects.only('id').filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
merge_state=MERGE_STATE_MERGED,
).values_list('id', flat=True))
# Check audit log in case TaxLotStates are listed as "new" but were merged into a different tax lot
taxlots = list(
TaxLotState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
merge_state=MERGE_STATE_NEW,
))
for state in taxlots:
audit_creation_id = TaxLotAuditLog.objects.only('id').exclude(
import_filename=None).get(state_id=state.id,
name='Import Creation')
if TaxLotAuditLog.objects.exclude(
record_type=AUDIT_USER_EDIT).filter(
parent1_id=audit_creation_id).exists():
tax_lots_matched.append(state.id)
else:
tax_lots_new.append(state.id)
# Construct Geocode Results
property_geocode_results = {
'high_confidence':
len(
PropertyState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence__startswith='High')),
'low_confidence':
len(
PropertyState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence__startswith='Low')),
'manual':
len(
PropertyState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence='Manually geocoded (N/A)')),
'missing_address_components':
len(
PropertyState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence='Missing address components (N/A)')),
}
tax_lot_geocode_results = {
'high_confidence':
len(
TaxLotState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence__startswith='High')),
'low_confidence':
len(
TaxLotState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence__startswith='Low')),
'manual':
len(
TaxLotState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence='Manually geocoded (N/A)')),
'missing_address_components':
len(
TaxLotState.objects.filter(
import_file__pk=import_file.pk,
data_state=DATA_STATE_MATCHING,
geocoding_confidence='Missing address components (N/A)')),
}
# merge in any of the matching results from the JSON field
return {
'status':
'success',
'import_file_records':
import_file.matching_results_data.get('import_file_records', None),
'properties': {
'initial_incoming':
import_file.matching_results_data.get(
'property_initial_incoming', None),
'duplicates_against_existing':
import_file.matching_results_data.get(
'property_duplicates_against_existing', None),
'duplicates_within_file':
import_file.matching_results_data.get(
'property_duplicates_within_file', None),
'merges_against_existing':
import_file.matching_results_data.get(
'property_merges_against_existing', None),
'merges_between_existing':
import_file.matching_results_data.get(
'property_merges_between_existing', None),
'merges_within_file':
import_file.matching_results_data.get(
'property_merges_within_file', None),
'new':
import_file.matching_results_data.get('property_new', None),
'geocoded_high_confidence':
property_geocode_results.get('high_confidence'),
'geocoded_low_confidence':
property_geocode_results.get('low_confidence'),
'geocoded_manually':
property_geocode_results.get('manual'),
'geocode_not_possible':
property_geocode_results.get('missing_address_components'),
},
'tax_lots': {
'initial_incoming':
import_file.matching_results_data.get(
'tax_lot_initial_incoming', None),
'duplicates_against_existing':
import_file.matching_results_data.get(
'tax_lot_duplicates_against_existing', None),
'duplicates_within_file':
import_file.matching_results_data.get(
'tax_lot_duplicates_within_file', None),
'merges_against_existing':
import_file.matching_results_data.get(
'tax_lot_merges_against_existing', None),
'merges_between_existing':
import_file.matching_results_data.get(
'tax_lot_merges_between_existing', None),
'merges_within_file':
import_file.matching_results_data.get(
'tax_lot_merges_within_file', None),
'new':
import_file.matching_results_data.get('tax_lot_new', None),
'geocoded_high_confidence':
tax_lot_geocode_results.get('high_confidence'),
'geocoded_low_confidence':
tax_lot_geocode_results.get('low_confidence'),
'geocoded_manually':
tax_lot_geocode_results.get('manual'),
'geocode_not_possible':
tax_lot_geocode_results.get('missing_address_components'),
}
}
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def mapping_suggestions(self, request, pk):
"""
Returns suggested mappings from an uploaded file's headers to known
data fields.
"""
organization_id = request.query_params.get('organization_id', None)
result = {'status': 'success'}
membership = OrganizationUser.objects.select_related('organization') \
.get(organization_id=organization_id, user=request.user)
organization = membership.organization
# For now, each organization holds their own mappings. This is non-ideal, but it is the
# way it is for now. In order to move to parent_org holding, then we need to be able to
# dynamically match columns based on the names and not the db id (or support many-to-many).
# parent_org = organization.get_parent()
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=organization.pk)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
# Get a list of the database fields in a list, these are the db columns and the extra_data columns
property_columns = Column.retrieve_mapping_columns(
organization.pk, 'property')
taxlot_columns = Column.retrieve_mapping_columns(
organization.pk, 'taxlot')
# If this is a portfolio manager file, then load in the PM mappings and if the column_mappings
# are not in the original mappings, default to PM
if import_file.from_portfolio_manager:
pm_mappings = simple_mapper.get_pm_mapping(
import_file.first_row_columns, resolve_duplicates=True)
suggested_mappings = mapper.build_column_mapping(
import_file.first_row_columns,
Column.retrieve_all_by_tuple(organization_id),
previous_mapping=get_column_mapping,
map_args=[organization],
default_mappings=pm_mappings,
thresh=80)
elif import_file.from_buildingsync:
bsync_mappings = xml_mapper.build_column_mapping()
suggested_mappings = mapper.build_column_mapping(
import_file.first_row_columns,
Column.retrieve_all_by_tuple(organization_id),
previous_mapping=get_column_mapping,
map_args=[organization],
default_mappings=bsync_mappings,
thresh=80)
else:
# All other input types
suggested_mappings = mapper.build_column_mapping(
import_file.first_row_columns,
Column.retrieve_all_by_tuple(organization.pk),
previous_mapping=get_column_mapping,
map_args=[organization],
thresh=
80 # percentage match that we require. 80% is random value for now.
)
# replace None with empty string for column names and PropertyState for tables
# TODO #239: Move this fix to build_column_mapping
for m in suggested_mappings:
table, destination_field, _confidence = suggested_mappings[m]
if destination_field is None:
suggested_mappings[m][1] = ''
# Fix the table name, eventually move this to the build_column_mapping
for m in suggested_mappings:
table, _destination_field, _confidence = suggested_mappings[m]
# Do not return the campus, created, updated fields... that is force them to be in the property state
if not table or table == 'Property':
suggested_mappings[m][0] = 'PropertyState'
elif table == 'TaxLot':
suggested_mappings[m][0] = 'TaxLotState'
result['suggested_column_mappings'] = suggested_mappings
result['property_columns'] = property_columns
result['taxlot_columns'] = taxlot_columns
return JsonResponse(result)
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
def destroy(self, request, pk):
"""
Deletes an import file
"""
organization_id = int(request.query_params.get('organization_id',
None))
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=organization_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
# check if the import record exists for the file and organization
d = ImportRecord.objects.filter(super_organization_id=organization_id,
pk=import_file.import_record.pk)
if not d.exists():
return JsonResponse(
{
'status': 'error',
'message': 'user does not have permission to delete file',
},
status=status.HTTP_403_FORBIDDEN)
# This does not actually delete the object because it is a NonDeletableModel
import_file.delete()
return JsonResponse({'status': 'success'})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field(
'view_id', required=True, description='ID for property view')
])
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def greenbutton_meters_preview(self, request, pk):
"""
Returns validated type units and proposed imports
"""
org_id = request.query_params.get('organization_id')
view_id = request.query_params.get('view_id')
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
parser = reader.GreenButtonParser(import_file.local_file)
raw_meter_data = list(parser.data)
try:
property_id = PropertyView.objects.get(
pk=view_id, cycle__organization_id=org_id).property_id
except PropertyView.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'Could not find property with pk=' + str(view_id)
},
status=status.HTTP_400_BAD_REQUEST)
meters_parser = MetersParser(org_id,
raw_meter_data,
source_type=Meter.GREENBUTTON,
property_id=property_id)
result = {}
result["validated_type_units"] = meters_parser.validated_type_units()
result["proposed_imports"] = meters_parser.proposed_imports
import_file.matching_results_data['property_id'] = property_id
import_file.save()
return result
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()])
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def pm_meters_preview(self, request, pk):
"""
Returns validated type units, proposed imports and unlinkable PM ids
"""
org_id = request.query_params.get('organization_id')
try:
import_file = ImportFile.objects.get(
pk=pk, import_record__super_organization_id=org_id)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'Could not find import file with pk=' + str(pk)
},
status=status.HTTP_400_BAD_REQUEST)
parser = reader.MCMParser(import_file.local_file,
sheet_name='Meter Entries')
raw_meter_data = list(parser.data)
meters_parser = MetersParser(org_id, raw_meter_data)
result = {}
result["validated_type_units"] = meters_parser.validated_type_units()
result["proposed_imports"] = meters_parser.proposed_imports
result["unlinkable_pm_ids"] = meters_parser.unlinkable_pm_ids
return result
StatusLabel as Label, )
from seed.serializers.labels import (
LabelSerializer, )
from seed.utils.api import drf_api_endpoint
from django.utils.decorators import method_decorator
from seed.utils.api_schema import AutoSchemaHelper
from seed.utils.labels import filter_labels_for_inv_type
from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet
@method_decorator(
name='retrieve',
decorator=swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
)
]),
)
@method_decorator(
name='list',
decorator=swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
)
]),
)
@method_decorator(
class DataQualityCheckViewSet(viewsets.ViewSet):
"""
Handles Data Quality API operations within Inventory backend.
(1) Post, wait, get…
(2) Respond with what changed
"""
# Remove lookup_field once data_quality_check_id is used and "pk" can be used
lookup_field = 'organization_id'
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.base_field(
"organization_id", "IN_PATH",
"Organization ID - identifier used to specify a DataQualityCheck",
True, "TYPE_INTEGER")
],
request_body=AutoSchemaHelper.schema_factory(
{
'property_view_ids': ['integer'],
'taxlot_view_ids': ['integer'],
},
description='An object containing IDs of the records to perform'
' data quality checks on. Should contain two keys- '
'property_view_ids and taxlot_view_ids, each of '
'which is an array of appropriate IDs.',
),
responses={
200:
AutoSchemaHelper.schema_factory({
'num_properties': 'integer',
'num_taxlots': 'integer',
'progress_key': 'string',
'progress': {},
})
})
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['POST'])
def start(self, request, organization_id):
"""
This API endpoint will create a new data_quality check process in the background,
on potentially a subset of properties/taxlots, and return back a query key
"""
body = request.data
property_view_ids = body['property_view_ids']
taxlot_view_ids = body['taxlot_view_ids']
property_state_ids = PropertyView.objects.filter(
id__in=property_view_ids).values_list('state_id', flat=True)
taxlot_state_ids = TaxLotView.objects.filter(
id__in=taxlot_view_ids).values_list('state_id', flat=True)
# For now, organization_id is the only key currently used to identify DataQualityChecks
return_value = do_checks(organization_id, property_state_ids,
taxlot_state_ids)
return JsonResponse({
'num_properties': len(property_state_ids),
'num_taxlots': len(taxlot_state_ids),
# TODO #239: Deprecate progress_key from here and just use the 'progess.progress_key'
'progress_key': return_value['progress_key'],
'progress': return_value,
})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field("run_id", True,
"Import file ID or cache key"),
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=False, methods=['GET'])
def results_csv(self, request):
"""
Download a CSV of the results from a data quality run based on either the ID that was
given during the creation of the data quality task or the ID of the
import file which had it's records checked.
Note that it is not related to objects in the database, since the results
are stored in redis!
"""
run_id = request.query_params.get('run_id')
if run_id is None:
return JsonResponse(
{
'status': 'error',
'message':
'must include Import file ID or cache key as run_id'
},
status=status.HTTP_400_BAD_REQUEST)
data_quality_results = get_cache_raw(
DataQualityCheck.cache_key(run_id))
response = HttpResponse(content_type='text/csv')
response[
'Content-Disposition'] = 'attachment; filename="Data Quality Check Results.csv"'
writer = csv.writer(response)
if data_quality_results is None:
writer.writerow(['Error'])
writer.writerow(['data quality results not found'])
return response
writer.writerow([
'Table', 'Address Line 1', 'PM Property ID', 'Tax Lot ID',
'Custom ID', 'Field', 'Applied Label', 'Condition',
'Error Message', 'Severity'
])
for row in data_quality_results:
for result in row['data_quality_results']:
writer.writerow([
row['data_quality_results'][0]['table_name'],
row['address_line_1'],
row['pm_property_id'] if 'pm_property_id' in row else None,
row['jurisdiction_tax_lot_id']
if 'jurisdiction_tax_lot_id' in row else None,
row['custom_id_1'],
result['formatted_field'],
result.get('label', None),
result['condition'],
# the detailed_message field can have units which has superscripts/subscripts, so unidecode it!
unidecode(result['detailed_message']),
result['severity']
])
return response
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field("run_id", True,
"Import file ID or cache key"),
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=False, methods=['GET'])
def results(self, request):
"""
Return the results of a data quality run based on either the ID that was
given during the creation of the data quality task or the ID of the
import file which had it's records checked.
Note that it is not related to objects in the database, since the results
are stored in redis!
"""
data_quality_id = request.query_params['run_id']
data_quality_results = get_cache_raw(
DataQualityCheck.cache_key(data_quality_id))
return JsonResponse({'data': data_quality_results})
class OrganizationViewSet(viewsets.ViewSet):
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['DELETE'])
def columns(self, request, pk=None):
"""
Delete all columns for an organization. This method is typically not recommended if there
are data in the inventory as it will invalidate all extra_data fields. This also removes
all the column mappings that existed.
---
parameters:
- name: pk
description: The organization_id
required: true
paramType: path
type:
status:
description: success or error
type: string
required: true
column_mappings_deleted_count:
description: Number of column_mappings that were deleted
type: integer
required: true
columns_deleted_count:
description: Number of columns that were deleted
type: integer
required: true
"""
try:
org = Organization.objects.get(pk=pk)
c_count, cm_count = Column.delete_all(org)
return JsonResponse({
'status': 'success',
'column_mappings_deleted_count': cm_count,
'columns_deleted_count': c_count,
})
except Organization.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
'organization with with id {} does not exist'.format(pk)
},
status=status.HTTP_404_NOT_FOUND)
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_integer_field('import_file_id',
required=True,
description='Import file id'),
openapi.Parameter('id',
openapi.IN_PATH,
type=openapi.TYPE_INTEGER,
description='Organization id'),
],
request_body=SaveColumnMappingsRequestPayloadSerializer,
responses={200: 'success response'})
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['POST'])
def column_mappings(self, request, pk=None):
"""
Saves the mappings between the raw headers of an ImportFile and the
destination fields in the `to_table_name` model which should be either
PropertyState or TaxLotState
Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES``
"""
import_file_id = request.query_params.get('import_file_id')
if import_file_id is None:
return JsonResponse(
{
'status': 'error',
'message': 'Query param `import_file_id` is required'
},
status=status.HTTP_400_BAD_REQUEST)
try:
_ = ImportFile.objects.get(pk=import_file_id)
organization = Organization.objects.get(pk=pk)
except ImportFile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'No import file found'
},
status=status.HTTP_404_NOT_FOUND)
except Organization.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'No organization found'
},
status=status.HTTP_404_NOT_FOUND)
result = Column.create_mappings(request.data.get('mappings',
[]), organization,
request.user, import_file_id)
if result:
return JsonResponse({'status': 'success'})
else:
return JsonResponse({'status': 'error'})
def _start_whole_org_match_merge_link(self,
org_id,
state_class_name,
proposed_columns=[]):
identifier = randint(100, 100000)
result_key = _get_match_merge_link_key(identifier)
set_cache_raw(result_key, {})
progress_data = ProgressData(func_name='org_match_merge_link',
unique_id=identifier)
progress_data.delete()
whole_org_match_merge_link.apply_async(
args=(org_id, state_class_name, proposed_columns),
link=cache_match_merge_link_result.s(identifier,
progress_data.key))
return progress_data.key
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_boolean_field(
'brief',
required=False,
description='If true, only return high-level organization details')
])
@api_endpoint_class
@ajax_request_class
def list(self, request):
"""
Retrieves all orgs the user has access to.
"""
# if brief==true only return high-level organization details
brief = json.loads(request.query_params.get('brief', 'false'))
if brief:
if request.user.is_superuser:
qs = Organization.objects.only('id', 'name', 'parent_org_id')
else:
qs = request.user.orgs.only('id', 'name', 'parent_org_id')
orgs = _dict_org_brief(request, qs)
if len(orgs) == 0:
return JsonResponse(
{
'status':
'error',
'message':
'Your SEED account is not associated with any organizations. '
'Please contact a SEED administrator.'
},
status=status.HTTP_401_UNAUTHORIZED)
else:
return JsonResponse({'organizations': orgs})
else:
if request.user.is_superuser:
qs = Organization.objects.all()
else:
qs = request.user.orgs.all()
orgs = _dict_org(request, qs)
if len(orgs) == 0:
return JsonResponse(
{
'status':
'error',
'message':
'Your SEED account is not associated with any organizations. '
'Please contact a SEED administrator.'
},
status=status.HTTP_401_UNAUTHORIZED)
else:
return JsonResponse({'organizations': orgs})
@method_decorator(permission_required('seed.can_access_admin'))
@api_endpoint_class
@ajax_request_class
def destroy(self, request, pk=None):
"""
Starts a background task to delete an organization and all related data.
"""
return JsonResponse(tasks.delete_organization(pk))
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def retrieve(self, request, pk=None):
"""
Retrieves a single organization by id.
"""
org_id = pk
if org_id is None:
return JsonResponse(
{
'status': 'error',
'message': 'no organization_id sent'
},
status=status.HTTP_400_BAD_REQUEST)
try:
org = Organization.objects.get(pk=org_id)
except Organization.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message': 'organization does not exist'
},
status=status.HTTP_404_NOT_FOUND)
if (not request.user.is_superuser
and not OrganizationUser.objects.filter(
user=request.user,
organization=org,
role_level__in=[ROLE_OWNER, ROLE_MEMBER, ROLE_VIEWER
]).exists()):
# TODO: better permission and return 401 or 403
return JsonResponse(
{
'status': 'error',
'message': 'user is not the owner of the org'
},
status=status.HTTP_403_FORBIDDEN)
return JsonResponse({
'status': 'success',
'organization': _dict_org(request, [org])[0],
})
@swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory(
{
'organization_name': 'string',
'user_id': 'integer',
},
required=['organization_name', 'user_id'],
description='Properties:\n'
'- organization_name: The new organization name\n'
'- user_id: The user ID (primary key) to be used as the owner of the new organization'
))
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_parent_org_owner')
def create(self, request):
"""
Creates a new organization.
"""
body = request.data
user = User.objects.get(pk=body['user_id'])
org_name = body['organization_name']
if Organization.objects.filter(name=org_name).exists():
return JsonResponse(
{
'status': 'error',
'message': 'organization name already exists'
},
status=status.HTTP_409_CONFLICT)
org, _, _ = create_organization(user, org_name, org_name)
return JsonResponse({
'status': 'success',
'message': 'organization created',
'organization': _dict_org(request, [org])[0]
})
@api_endpoint_class
@ajax_request_class
@method_decorator(permission_required('seed.can_access_admin'))
@action(detail=True, methods=['DELETE'])
def inventory(self, request, pk=None):
"""
Starts a background task to delete all properties & taxlots
in an org.
"""
return JsonResponse(tasks.delete_organization_inventory(pk))
@swagger_auto_schema(
request_body=SaveSettingsSerializer, )
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_owner')
@action(detail=True, methods=['PUT'])
def save_settings(self, request, pk=None):
"""
Saves an organization's settings: name, query threshold, shared fields, etc
"""
body = request.data
org = Organization.objects.get(pk=pk)
posted_org = body.get('organization', None)
if posted_org is None:
return JsonResponse(
{
'status': 'error',
'message': 'malformed request'
},
status=status.HTTP_400_BAD_REQUEST)
desired_threshold = posted_org.get('query_threshold', None)
if desired_threshold is not None:
org.query_threshold = desired_threshold
desired_name = posted_org.get('name', None)
if desired_name is not None:
org.name = desired_name
def is_valid_choice(choice_tuples, s):
"""choice_tuples is std model ((value, label), ...)"""
return (s is not None) and (s in [
choice[0] for choice in choice_tuples
])
def warn_bad_pint_spec(kind, unit_string):
if unit_string is not None:
_log.warn("got bad {0} unit string {1} for org {2}".format(
kind, unit_string, org.name))
def warn_bad_units(kind, unit_string):
_log.warn("got bad {0} unit string {1} for org {2}".format(
kind, unit_string, org.name))
desired_display_units_eui = posted_org.get('display_units_eui')
if is_valid_choice(Organization.MEASUREMENT_CHOICES_EUI,
desired_display_units_eui):
org.display_units_eui = desired_display_units_eui
else:
warn_bad_pint_spec('eui', desired_display_units_eui)
desired_display_units_area = posted_org.get('display_units_area')
if is_valid_choice(Organization.MEASUREMENT_CHOICES_AREA,
desired_display_units_area):
org.display_units_area = desired_display_units_area
else:
warn_bad_pint_spec('area', desired_display_units_area)
desired_display_significant_figures = posted_org.get(
'display_significant_figures')
if isinstance(
desired_display_significant_figures,
int) and desired_display_significant_figures >= 0: # noqa
org.display_significant_figures = desired_display_significant_figures
elif desired_display_significant_figures is not None:
_log.warn("got bad sig figs {0} for org {1}".format(
desired_display_significant_figures, org.name))
desired_display_meter_units = posted_org.get('display_meter_units')
if desired_display_meter_units:
org.display_meter_units = desired_display_meter_units
desired_thermal_conversion_assumption = posted_org.get(
'thermal_conversion_assumption')
if is_valid_choice(Organization.THERMAL_CONVERSION_ASSUMPTION_CHOICES,
desired_thermal_conversion_assumption):
org.thermal_conversion_assumption = desired_thermal_conversion_assumption
# Update MapQuest API Key if it's been changed
mapquest_api_key = posted_org.get('mapquest_api_key', '')
if mapquest_api_key != org.mapquest_api_key:
org.mapquest_api_key = mapquest_api_key
comstock_enabled = posted_org.get('comstock_enabled', False)
if comstock_enabled != org.comstock_enabled:
org.comstock_enabled = comstock_enabled
org.save()
# Update the selected exportable fields.
new_public_column_names = posted_org.get('public_fields', None)
if new_public_column_names is not None:
old_public_columns = Column.objects.filter(
organization=org, shared_field_type=Column.SHARED_PUBLIC)
# turn off sharing in the old_pub_fields
for col in old_public_columns:
col.shared_field_type = Column.SHARED_NONE
col.save()
# for now just iterate over this to grab the new columns.
for col in new_public_column_names:
new_col = Column.objects.filter(organization=org, id=col['id'])
if len(new_col) == 1:
new_col = new_col.first()
new_col.shared_field_type = Column.SHARED_PUBLIC
new_col.save()
return JsonResponse({'status': 'success'})
@has_perm_class('requires_member')
@api_endpoint_class
@ajax_request_class
@action(detail=True, methods=['GET'])
def query_threshold(self, request, pk=None):
"""
Returns the "query_threshold" for an org. Searches from
members of sibling orgs must return at least this many buildings
from orgs they do not belong to, or else buildings from orgs they
don't belong to will be removed from the results.
"""
org = Organization.objects.get(pk=pk)
return JsonResponse({
'status': 'success',
'query_threshold': org.query_threshold
})
@swagger_auto_schema(responses={200: SharedFieldsReturnSerializer})
@has_perm_class('requires_member')
@api_endpoint_class
@ajax_request_class
@action(detail=True, methods=['GET'])
def shared_fields(self, request, pk=None):
"""
Retrieves all fields marked as shared for the organization. Will only return used fields.
"""
result = {'status': 'success', 'public_fields': []}
columns = Column.retrieve_all(pk, 'property', True)
for c in columns:
if c['sharedFieldType'] == 'Public':
new_column = {
'table_name': c['table_name'],
'name': c['name'],
'column_name': c['column_name'],
# this is the field name in the db. The other name can have tax_
'display_name': c['display_name']
}
result['public_fields'].append(new_column)
return JsonResponse(result)
@swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory(
{
'sub_org_name': 'string',
'sub_org_owner_email': 'string',
},
required=['sub_org_name', 'sub_org_owner_email'],
description='Properties:\n'
'- sub_org_name: Name of the new sub organization\n'
'- sub_org_owner_email: Email of the owner of the sub organization, which must already exist',
))
@has_perm_class('requires_member')
@api_endpoint_class
@ajax_request_class
@action(detail=True, methods=['POST'])
def sub_org(self, request, pk=None):
"""
Creates a child org of a parent org.
"""
body = request.data
org = Organization.objects.get(pk=pk)
email = body['sub_org_owner_email'].lower()
try:
user = User.objects.get(username=email)
except User.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
'User with email address (%s) does not exist' % email
},
status=status.HTTP_400_BAD_REQUEST)
created, mess_or_org, _ = create_suborganization(
user, org, body['sub_org_name'], ROLE_OWNER)
if created:
return JsonResponse({
'status': 'success',
'organization_id': mess_or_org.pk
})
else:
return JsonResponse({
'status': 'error',
'message': mess_or_org
},
status=status.HTTP_409_CONFLICT)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def matching_criteria_columns(self, request, pk=None):
"""
Retrieve all matching criteria columns for an org.
"""
try:
org = Organization.objects.get(pk=pk)
except ObjectDoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'Could not retrieve organization at pk = ' + str(pk)
},
status=status.HTTP_404_NOT_FOUND)
matching_criteria_column_names = dict(
org.column_set.filter(
is_matching_criteria=True).values('table_name').annotate(
column_names=ArrayAgg('column_name')).values_list(
'table_name', 'column_names'))
return JsonResponse(matching_criteria_column_names)
@swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory(
{'inventory_type': 'string'},
required=['inventory_type'],
description='Properties:\n'
'- inventory_type: either "properties" or "taxlots"'))
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['POST'])
def match_merge_link(self, request, pk=None):
"""
Run match_merge_link for an org.
"""
inventory_type = request.data.get('inventory_type', None)
if inventory_type not in ['properties', 'taxlots']:
return JsonResponse(
{
'status':
'error',
'message':
'Provided inventory type should either be "properties" or "taxlots".'
},
status=status.HTTP_404_NOT_FOUND)
try:
org = Organization.objects.get(pk=pk)
except ObjectDoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'Could not retrieve organization at pk = ' + str(pk)
},
status=status.HTTP_404_NOT_FOUND)
state_class_name = 'PropertyState' if inventory_type == 'properties' else 'TaxLotState'
progress_key = self._start_whole_org_match_merge_link(
org.id, state_class_name)
return JsonResponse({'progress_key': progress_key})
@swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory(
{
'inventory_type': 'string',
'add': ['string'],
'remove': ['string'],
},
required=['inventory_type'],
description='Properties:\n'
'- inventory_type: either "properties" or "taxlots"\n'
'- add: list of column names\n'
'- remove: list of column names'))
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['POST'])
def match_merge_link_preview(self, request, pk=None):
"""
Run match_merge_link preview for an org and record type.
"""
inventory_type = request.data.get('inventory_type', None)
if inventory_type not in ['properties', 'taxlots']:
return JsonResponse(
{
'status':
'error',
'message':
'Provided inventory type should either be "properties" or "taxlots".'
},
status=status.HTTP_404_NOT_FOUND)
try:
org = Organization.objects.get(pk=pk)
except ObjectDoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'Could not retrieve organization at pk = ' + str(pk)
},
status=status.HTTP_404_NOT_FOUND)
state_class_name = 'PropertyState' if inventory_type == 'properties' else 'TaxLotState'
current_columns = matching_criteria_column_names(
org.id, state_class_name)
add = set(request.data.get('add', []))
remove = set(request.data.get('remove', []))
provided_columns = Column.objects.filter(
column_name__in=add.union(remove),
organization_id=org.id,
table_name=state_class_name)
if provided_columns.count() != (len(add) + len(remove)):
return JsonResponse(
{
'status': 'error',
'message': 'Invalid column names provided.'
},
status=status.HTTP_404_NOT_FOUND)
proposed_columns = current_columns.union(add).difference(remove)
progress_key = self._start_whole_org_match_merge_link(
org.id, state_class_name, list(proposed_columns))
return JsonResponse({'progress_key': progress_key})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_integer_field(
'match_merge_link_id',
required=True,
description='ID of match merge link')
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def match_merge_link_result(self, request, pk=None):
try:
Organization.objects.get(pk=pk)
except ObjectDoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'Could not retrieve organization at pk = ' + str(pk)
},
status=status.HTTP_404_NOT_FOUND)
identifier = request.query_params['match_merge_link_id']
result_key = _get_match_merge_link_key(identifier)
# using unsafe serialization b/c the result might not be a dict
return JsonResponse(get_cache_raw(result_key), safe=False)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def geocoding_columns(self, request, pk=None):
"""
Retrieve all geocoding columns for an org.
"""
try:
org = Organization.objects.get(pk=pk)
except ObjectDoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'Could not retrieve organization at pk = ' + str(pk)
},
status=status.HTTP_404_NOT_FOUND)
geocoding_columns_qs = org.column_set.\
filter(geocoding_order__gt=0).\
order_by('geocoding_order').\
values('table_name', 'column_name')
geocoding_columns = {
'PropertyState': [],
'TaxLotState': [],
}
for col in geocoding_columns_qs:
geocoding_columns[col['table_name']].append(col['column_name'])
return JsonResponse(geocoding_columns)
def get_cycles(self, start, end, organization_id):
if not isinstance(start, type(end)):
raise TypeError('start and end not same types')
# if of type int or convertable assume they are cycle ids
try:
start = int(start)
end = int(end)
except ValueError as error: # noqa
# assume string is JS date
if isinstance(start, basestring):
start_datetime = dateutil.parser.parse(start)
end_datetime = dateutil.parser.parse(end)
else:
raise Exception('Date is not a string')
# get date times from cycles
if isinstance(start, int):
cycle = Cycle.objects.get(pk=start,
organization_id=organization_id)
start_datetime = cycle.start
if start == end:
end_datetime = cycle.end
else:
end_datetime = Cycle.objects.get(
pk=end, organization_id=organization_id).end
return Cycle.objects.filter(
start__gte=start_datetime,
end__lte=end_datetime,
organization_id=organization_id).order_by('start')
def get_data(self, property_view, x_var, y_var):
result = None
state = property_view.state
if getattr(state, x_var, None) and getattr(state, y_var, None):
result = {
"id": property_view.property_id,
"x": getattr(state, x_var),
"y": getattr(state, y_var),
}
return result
def get_raw_report_data(self, organization_id, cycles, x_var, y_var,
campus_only):
all_property_views = PropertyView.objects.select_related(
'property',
'state').filter(property__organization_id=organization_id,
cycle_id__in=cycles)
organization = Organization.objects.get(pk=organization_id)
results = []
for cycle in cycles:
property_views = all_property_views.filter(cycle_id=cycle)
count_total = []
count_with_data = []
data = []
for property_view in property_views:
property_pk = property_view.property_id
if property_view.property.campus and campus_only:
count_total.append(property_pk)
result = self.get_data(property_view, x_var, y_var)
if result:
result['yr_e'] = cycle.end.strftime('%Y')
data.append(result)
count_with_data.append(property_pk)
elif not property_view.property.campus:
count_total.append(property_pk)
result = self.get_data(property_view, x_var, y_var)
if result:
result['yr_e'] = cycle.end.strftime('%Y')
de_unitted_result = apply_display_unit_preferences(
organization, result)
data.append(de_unitted_result)
count_with_data.append(property_pk)
result = {
"cycle_id": cycle.pk,
"chart_data": data,
"property_counts": {
"yr_e": cycle.end.strftime('%Y'),
"num_properties": len(count_total),
"num_properties_w-data": len(count_with_data),
},
}
results.append(result)
return results
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_string_field(
'x_var', required=True, description='Raw column name for x axis'),
AutoSchemaHelper.query_string_field(
'y_var', required=True, description='Raw column name for y axis'),
AutoSchemaHelper.query_string_field(
'start',
required=True,
description='Start time, in the format "2018-12-31T23:53:00-08:00"'
),
AutoSchemaHelper.query_string_field(
'end',
required=True,
description='End time, in the format "2018-12-31T23:53:00-08:00"'),
AutoSchemaHelper.query_string_field(
'campus_only',
required=False,
description='If true, includes campuses'),
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def report(self, request, pk=None):
"""
Retrieve a summary report for charting x vs y
"""
campus_only = json.loads(
request.query_params.get('campus_only', 'false'))
params = {}
missing_params = []
error = ''
for param in ['x_var', 'y_var', 'start', 'end']:
val = request.query_params.get(param, None)
if not val:
missing_params.append(param)
else:
params[param] = val
if missing_params:
error = "{} Missing params: {}".format(error,
", ".join(missing_params))
if error:
status_code = status.HTTP_400_BAD_REQUEST
result = {'status': 'error', 'message': error}
else:
cycles = self.get_cycles(params['start'], params['end'], pk)
data = self.get_raw_report_data(pk, cycles, params['x_var'],
params['y_var'], campus_only)
for datum in data:
if datum['property_counts']['num_properties_w-data'] != 0:
break
property_counts = []
chart_data = []
for datum in data:
property_counts.append(datum['property_counts'])
chart_data.extend(datum['chart_data'])
data = {
'property_counts': property_counts,
'chart_data': chart_data,
}
result = {'status': 'success', 'data': data}
status_code = status.HTTP_200_OK
return Response(result, status=status_code)
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_string_field(
'x_var', required=True, description='Raw column name for x axis'),
AutoSchemaHelper.query_string_field(
'y_var',
required=True,
description=
'Raw column name for y axis, must be one of: "gross_floor_area", "use_description", "year_built"'
),
AutoSchemaHelper.query_string_field(
'start',
required=True,
description='Start time, in the format "2018-12-31T23:53:00-08:00"'
),
AutoSchemaHelper.query_string_field(
'end',
required=True,
description='End time, in the format "2018-12-31T23:53:00-08:00"'),
AutoSchemaHelper.query_string_field(
'campus_only',
required=False,
description='If true, includes campuses'),
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def report_aggregated(self, request, pk=None):
"""
Retrieve a summary report for charting x vs y aggregated by y_var
"""
campus_only = json.loads(
request.query_params.get('campus_only', 'false'))
valid_y_values = ['gross_floor_area', 'use_description', 'year_built']
params = {}
missing_params = []
empty = True
error = ''
for param in ['x_var', 'y_var', 'start', 'end']:
val = request.query_params.get(param, None)
if not val:
missing_params.append(param)
elif param == 'y_var' and val not in valid_y_values:
error = "{} {} is not a valid value for {}.".format(
error, val, param)
else:
params[param] = val
if missing_params:
error = "{} Missing params: {}".format(error,
", ".join(missing_params))
if error:
status_code = status.HTTP_400_BAD_REQUEST
result = {'status': 'error', 'message': error}
else:
cycles = self.get_cycles(params['start'], params['end'], pk)
x_var = params['x_var']
y_var = params['y_var']
data = self.get_raw_report_data(pk, cycles, x_var, y_var,
campus_only)
for datum in data:
if datum['property_counts']['num_properties_w-data'] != 0:
empty = False
break
if empty:
result = {'status': 'error', 'message': 'No data found'}
status_code = status.HTTP_404_NOT_FOUND
if not empty or not error:
chart_data = []
property_counts = []
for datum in data:
buildings = datum['chart_data']
yr_e = datum['property_counts']['yr_e']
chart_data.extend(self.aggregate_data(yr_e, y_var, buildings)),
property_counts.append(datum['property_counts'])
# Send back to client
aggregated_data = {
'chart_data': chart_data,
'property_counts': property_counts
}
result = {
'status': 'success',
'aggregated_data': aggregated_data,
}
status_code = status.HTTP_200_OK
return Response(result, status=status_code)
def aggregate_data(self, yr_e, y_var, buildings):
aggregation_method = {
'use_description': self.aggregate_use_description,
'year_built': self.aggregate_year_built,
'gross_floor_area': self.aggregate_gross_floor_area,
}
return aggregation_method[y_var](yr_e, buildings)
def aggregate_use_description(self, yr_e, buildings):
# Group buildings in this year_ending group into uses
chart_data = []
grouped_uses = defaultdict(list)
for b in buildings:
grouped_uses[str(b['y']).lower()].append(b)
# Now iterate over use groups to make each chart item
for use, buildings_in_uses in grouped_uses.items():
chart_data.append({
'x': median([b['x'] for b in buildings_in_uses]),
'y': use.capitalize(),
'yr_e': yr_e
})
return chart_data
def aggregate_year_built(self, yr_e, buildings):
# Group buildings in this year_ending group into decades
chart_data = []
grouped_decades = defaultdict(list)
for b in buildings:
grouped_decades['%s0' % str(b['y'])[:-1]].append(b)
# Now iterate over decade groups to make each chart item
for decade, buildings_in_decade in grouped_decades.items():
chart_data.append({
'x':
median([b['x'] for b in buildings_in_decade]),
'y':
'%s-%s' % (decade, '%s9' % str(decade)[:-1]), # 1990-1999
'yr_e':
yr_e
})
return chart_data
def aggregate_gross_floor_area(self, yr_e, buildings):
chart_data = []
y_display_map = {
0: '0-99k',
100000: '100-199k',
200000: '200k-299k',
300000: '300k-399k',
400000: '400-499k',
500000: '500-599k',
600000: '600-699k',
700000: '700-799k',
800000: '800-899k',
900000: '900-999k',
1000000: 'over 1,000k',
}
max_bin = max(y_display_map)
# Group buildings in this year_ending group into ranges
grouped_ranges = defaultdict(list)
for b in buildings:
area = b['y']
# make sure anything greater than the biggest bin gets put in
# the biggest bin
range_bin = min(max_bin, round_down_hundred_thousand(area))
grouped_ranges[range_bin].append(b)
# Now iterate over range groups to make each chart item
for range_floor, buildings_in_range in grouped_ranges.items():
chart_data.append({
'x': median([b['x'] for b in buildings_in_range]),
'y': y_display_map[range_floor],
'yr_e': yr_e
})
return chart_data
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_string_field(
'x_var', required=True, description='Raw column name for x axis'),
AutoSchemaHelper.query_string_field('x_label',
required=True,
description='Label for x axis'),
AutoSchemaHelper.query_string_field(
'y_var', required=True, description='Raw column name for y axis'),
AutoSchemaHelper.query_string_field('y_label',
required=True,
description='Label for y axis'),
AutoSchemaHelper.query_string_field(
'start',
required=True,
description='Start time, in the format "2018-12-31T23:53:00-08:00"'
),
AutoSchemaHelper.query_string_field(
'end',
required=True,
description='End time, in the format "2018-12-31T23:53:00-08:00"'),
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['GET'])
def report_export(self, request, pk=None):
"""
Export a report as a spreadsheet
"""
params = {}
missing_params = []
error = ''
for param in ['x_var', 'x_label', 'y_var', 'y_label', 'start', 'end']:
val = request.query_params.get(param, None)
if not val:
missing_params.append(param)
else:
params[param] = val
if missing_params:
error = "{} Missing params: {}".format(error,
", ".join(missing_params))
if error:
status_code = status.HTTP_400_BAD_REQUEST
result = {'status': 'error', 'message': error}
return Response(result, status=status_code)
response = HttpResponse(
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
response['Content-Disposition'] = 'attachment; filename="report-data"'
# Create WB
output = BytesIO()
wb = Workbook(output, {'remove_timezone': True})
# Create sheets
count_sheet = wb.add_worksheet('Counts')
base_sheet = wb.add_worksheet('Raw')
agg_sheet = wb.add_worksheet('Agg')
# Enable bold format and establish starting cells
bold = wb.add_format({'bold': True})
data_row_start = 0
data_col_start = 0
# Write all headers across all sheets
count_sheet.write(data_row_start, data_col_start, 'Year Ending', bold)
count_sheet.write(data_row_start, data_col_start + 1,
'Properties with Data', bold)
count_sheet.write(data_row_start, data_col_start + 2,
'Total Properties', bold)
base_sheet.write(data_row_start, data_col_start, 'ID', bold)
base_sheet.write(data_row_start, data_col_start + 1,
request.query_params.get('x_label'), bold)
base_sheet.write(data_row_start, data_col_start + 2,
request.query_params.get('y_label'), bold)
base_sheet.write(data_row_start, data_col_start + 3, 'Year Ending',
bold)
agg_sheet.write(data_row_start, data_col_start,
request.query_params.get('x_label'), bold)
agg_sheet.write(data_row_start, data_col_start + 1,
request.query_params.get('y_label'), bold)
agg_sheet.write(data_row_start, data_col_start + 2, 'Year Ending',
bold)
# Gather base data
cycles = self.get_cycles(params['start'], params['end'], pk)
data = self.get_raw_report_data(pk, cycles, params['x_var'],
params['y_var'], False)
base_row = data_row_start + 1
agg_row = data_row_start + 1
count_row = data_row_start + 1
for cycle_results in data:
total_count = cycle_results['property_counts']['num_properties']
with_data_count = cycle_results['property_counts'][
'num_properties_w-data']
yr_e = cycle_results['property_counts']['yr_e']
# Write Counts
count_sheet.write(count_row, data_col_start, yr_e)
count_sheet.write(count_row, data_col_start + 1, with_data_count)
count_sheet.write(count_row, data_col_start + 2, total_count)
count_row += 1
# Write Base/Raw Data
data_rows = cycle_results['chart_data']
for datum in data_rows:
base_sheet.write(base_row, data_col_start, datum.get('id'))
base_sheet.write(base_row, data_col_start + 1, datum.get('x'))
base_sheet.write(base_row, data_col_start + 2, datum.get('y'))
base_sheet.write(base_row, data_col_start + 3,
datum.get('yr_e'))
base_row += 1
# Gather and write Agg data
for agg_datum in self.aggregate_data(yr_e, params['y_var'],
data_rows):
agg_sheet.write(agg_row, data_col_start, agg_datum.get('x'))
agg_sheet.write(agg_row, data_col_start + 1,
agg_datum.get('y'))
agg_sheet.write(agg_row, data_col_start + 2,
agg_datum.get('yr_e'))
agg_row += 1
wb.close()
xlsx_data = output.getvalue()
response.write(xlsx_data)
return response
@has_perm_class('requires_member')
@ajax_request_class
@action(detail=True, methods=['GET'])
def geocode_api_key_exists(self, request, pk=None):
"""
Returns true if the organization has a mapquest api key
"""
org = Organization.objects.get(id=pk)
if org.mapquest_api_key:
return True
else:
return False
class LabelInventoryViewSet(APIView):
"""API endpoint for viewing and creating labels.
Returns::
[
{
'id': Label's primary key
'name': Name given to label
'color': Color of label,
'organization_id': Id of organization label belongs to,
'is_applied': Will be empty array if not applied to property/taxlots
}
]
---
"""
renderer_classes = (JSONRenderer, )
parser_classes = (JSONParser, )
inventory_models = {'property': PropertyView, 'taxlot': TaxLotView}
errors = {
'disjoint':
ErrorState(
status.HTTP_422_UNPROCESSABLE_ENTITY,
'add_label_ids and remove_label_ids cannot contain elements in common'
),
'missing_org':
ErrorState(status.HTTP_422_UNPROCESSABLE_ENTITY,
'missing organization_id'),
'missing_inventory_ids':
ErrorState(status.HTTP_422_UNPROCESSABLE_ENTITY,
'inventory_ids cannot be undefined or empty')
}
@property
def models(self):
"""
Exposes Django's internal models for join table.
Used for bulk_create operations.
"""
return {
'property': apps.get_model('seed', 'PropertyView_labels'),
'taxlot': apps.get_model('seed', 'TaxLotView_labels')
}
def get_queryset(self, inventory_type, organization_id):
Model = self.models[inventory_type]
return Model.objects.filter(
statuslabel__super_organization_id=organization_id)
def get_label_desc(self, add_label_ids, remove_label_ids):
return Label.objects.filter(pk__in=add_label_ids +
remove_label_ids).values(
'id', 'color', 'name')
def get_inventory_id(self, q, inventory_type):
return getattr(q, "{}view_id".format(inventory_type))
def exclude(self, qs, inventory_type, label_ids):
exclude = {label: [] for label in label_ids}
for q in qs:
if q.statuslabel_id in label_ids:
inventory_id = self.get_inventory_id(q, inventory_type)
exclude[q.statuslabel_id].append(inventory_id)
return exclude
def filter_by_inventory(self, qs, inventory_type, inventory_ids):
if inventory_ids:
filterdict = {
"{}view__pk__in".format(inventory_type): inventory_ids
}
qs = qs.filter(**filterdict)
return qs
def label_factory(self, inventory_type, label_id, inventory_id):
Model = self.models[inventory_type]
# Ensure the the label org and inventory org are the same
inventory_parent_org_id = getattr(Model, "{}view".format(inventory_type)).get_queryset().get(pk=inventory_id)\
.cycle.organization.get_parent().id
label_super_org_id = Model.statuslabel.get_queryset().get(
pk=label_id).super_organization_id
if inventory_parent_org_id == label_super_org_id:
create_dict = {
'statuslabel_id': label_id,
"{}view_id".format(inventory_type): inventory_id
}
return Model(**create_dict)
else:
raise IntegrityError(
'Label with super_organization_id={} cannot be applied to a record with parent '
'organization_id={}.'.format(label_super_org_id,
inventory_parent_org_id))
def add_labels(self, qs, inventory_type, inventory_ids, add_label_ids):
added = []
if add_label_ids:
model = self.models[inventory_type]
exclude = self.exclude(qs, inventory_type, add_label_ids)
new_inventory_labels = [
self.label_factory(inventory_type, label_id, pk)
for label_id in add_label_ids for pk in inventory_ids
if pk not in exclude[label_id]
]
model.objects.bulk_create(new_inventory_labels)
added = [
self.get_inventory_id(m, inventory_type)
for m in new_inventory_labels
]
return added
def remove_labels(self, qs, inventory_type, remove_label_ids):
removed = []
if remove_label_ids:
rqs = qs.filter(statuslabel_id__in=remove_label_ids)
removed = [self.get_inventory_id(q, inventory_type) for q in rqs]
rqs.delete()
return removed
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'add_label_ids': ['integer'],
'remove_label_ids': ['integer'],
'inventory_ids': ['integer'],
},
required=['inventory_ids'],
description='Properties:\n'
'- add_label_ids: label ids to add to the inventories\n'
'- remove_label_ids: label ids to remove from the inventories\n'
'- inventory_ids: List of inventory IDs'))
@has_perm_class('can_modify_data')
def put(self, request, inventory_type):
"""
Updates label assignments to inventory items.
Returns::
{
'status': {string} 'success' or 'error'
'message': {string} Error message if error
'num_updated': {integer} Number of properties/taxlots updated
'labels': [ List of labels affected.
{
'color': {string}
'id': {int}
'label': {'string'}
'name': {string}
}...
]
}
"""
add_label_ids = request.data.get('add_label_ids', [])
remove_label_ids = request.data.get('remove_label_ids', [])
inventory_ids = request.data.get('inventory_ids', [])
organization_id = request.query_params['organization_id']
error = None
# ensure add_label_ids and remove_label_ids are different
if not set(add_label_ids).isdisjoint(remove_label_ids):
error = self.errors['disjoint']
elif not organization_id:
error = self.errors['missing_org']
elif len(inventory_ids) == 0:
error = self.errors['missing_inventory_ids']
if error:
result = {'status': 'error', 'message': str(error)}
status_code = error.status_code
else:
qs = self.get_queryset(inventory_type, organization_id)
qs = self.filter_by_inventory(qs, inventory_type, inventory_ids)
removed = self.remove_labels(qs, inventory_type, remove_label_ids)
added = self.add_labels(qs, inventory_type, inventory_ids,
add_label_ids)
num_updated = len(set(added).union(removed))
labels = self.get_label_desc(add_label_ids, remove_label_ids)
result = {
'status': 'success',
'num_updated': num_updated,
'labels': labels
}
status_code = status.HTTP_200_OK
return response.Response(result, status=status_code)
class UbidViewSet(viewsets.ViewSet, OrgMixin):
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'property_view_ids': ['integer'],
'taxlot_view_ids': ['integer'],
},
description=
'IDs by inventory type for records to have their UBID decoded.'))
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=False, methods=['POST'])
def decode_by_ids(self, request):
"""
Submit a request to decode UBIDs for property and tax lot records.
"""
body = dict(request.data)
org_id = self.get_organization(request)
property_view_ids = body.get('property_view_ids')
taxlot_view_ids = body.get('taxlot_view_ids')
if property_view_ids:
property_views = PropertyView.objects.filter(
id__in=property_view_ids, cycle__organization_id=org_id)
properties = PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id')))
decode_unique_ids(properties)
if taxlot_view_ids:
taxlot_views = TaxLotView.objects.filter(
id__in=taxlot_view_ids, cycle__organization_id=org_id)
taxlots = TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id')))
decode_unique_ids(taxlots)
return JsonResponse({'status': 'success'})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'property_view_ids': ['integer'],
'taxlot_view_ids': ['integer'],
},
description=
'IDs by inventory type for records to be used in building a UBID decoding summary.'
))
@ajax_request_class
@has_perm_class('can_view_data')
@action(detail=False, methods=['POST'])
def decode_results(self, request):
"""
Generate a summary of populated, unpopulated, and decoded UBIDs for
property and tax lot records.
"""
body = dict(request.data)
org_id = self.get_organization(request)
ubid_unpopulated = 0
ubid_successfully_decoded = 0
ubid_not_decoded = 0
ulid_unpopulated = 0
ulid_successfully_decoded = 0
ulid_not_decoded = 0
property_view_ids = body.get('property_view_ids')
taxlot_view_ids = body.get('taxlot_view_ids')
if property_view_ids:
property_views = PropertyView.objects.filter(
id__in=property_view_ids, cycle__organization_id=org_id)
property_states = PropertyState.objects.filter(
id__in=Subquery(property_views.values('state_id')))
ubid_unpopulated = property_states.filter(
ubid__isnull=True).count()
ubid_successfully_decoded = property_states.filter(
ubid__isnull=False,
bounding_box__isnull=False,
centroid__isnull=False).count()
# for ubid_not_decoded, bounding_box could be populated from a GeoJSON import
ubid_not_decoded = property_states.filter(
ubid__isnull=False, centroid__isnull=True).count()
if taxlot_view_ids:
taxlot_views = TaxLotView.objects.filter(
id__in=taxlot_view_ids, cycle__organization_id=org_id)
taxlot_states = TaxLotState.objects.filter(
id__in=Subquery(taxlot_views.values('state_id')))
ulid_unpopulated = taxlot_states.filter(ulid__isnull=True).count()
ulid_successfully_decoded = taxlot_states.filter(
ulid__isnull=False,
bounding_box__isnull=False,
centroid__isnull=False).count()
# for ulid_not_decoded, bounding_box could be populated from a GeoJSON import
ulid_not_decoded = taxlot_states.filter(
ulid__isnull=False, centroid__isnull=True).count()
result = {
"ubid_unpopulated": ubid_unpopulated,
"ubid_successfully_decoded": ubid_successfully_decoded,
"ubid_not_decoded": ubid_not_decoded,
"ulid_unpopulated": ulid_unpopulated,
"ulid_successfully_decoded": ulid_successfully_decoded,
"ulid_not_decoded": ulid_not_decoded,
}
return result
class ColumnMappingProfileViewSet(OrgMixin, ViewSet):
permission_classes = (SEEDOrgPermissions, )
# req by SEEDOrgPermissions, but currently not used by any methods.
queryset = ColumnMappingProfile.objects.none()
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
)
],
request_body=AutoSchemaHelper.schema_factory(
{'profile_type': ['string']},
description=
"Possible Types: 'Normal', 'BuildingSync Default', BuildingSync Custom'"
))
@api_endpoint_class
@action(detail=False,
methods=['POST']) # POST in order to provide array/list
def filter(self, request):
"""
Retrieves all profiles for an organization.
"""
try:
profile_types = request.data.get('profile_type', [])
profile_types = [
ColumnMappingProfile.get_profile_type(pt)
for pt in profile_types
]
filter_params = {
'organizations__pk': self.get_organization(request, True).id
}
if profile_types:
filter_params['profile_type__in'] = profile_types
profiles = ColumnMappingProfile.objects.filter(**filter_params)
data = [ColumnMappingProfileSerializer(p).data for p in profiles]
return JsonResponse({
'status': 'success',
'data': data,
})
except Exception as e:
raise e
return JsonResponse({
'status': 'error',
'data': str(e),
},
status=HTTP_400_BAD_REQUEST)
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
)
],
request_body=AutoSchemaHelper.schema_factory(
{
'name':
'string',
'mappings': [{
'to_field': 'string',
'from_field': 'string',
'from_units': 'string',
'to_table_name': 'string',
}]
},
description="Optional 'name' or 'mappings'.\n" +
mappings_description))
@api_endpoint_class
def update(self, request, pk=None):
"""
Updates a profile given appropriate request data. The body should contain
only valid fields for ColumnMappingProfile objects.
"""
org_id = self.get_organization(request, True).id
try:
profile = ColumnMappingProfile.objects.get(
organizations__pk=org_id, pk=pk)
except ColumnMappingProfile.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'data': 'No profile with given id'
},
status=HTTP_400_BAD_REQUEST)
if profile.profile_type == ColumnMappingProfile.BUILDINGSYNC_DEFAULT:
return JsonResponse(
{
'status': 'error',
'data': 'Default BuildingSync profile are not editable'
},
status=HTTP_400_BAD_REQUEST)
updated_name, updated_mappings = request.data.get(
'name'), request.data.get('mappings')
# update the name
if updated_name is not None:
profile.name = updated_name
# update the mappings according to the profile type
if updated_mappings is not None:
if profile.profile_type == ColumnMappingProfile.BUILDINGSYNC_CUSTOM:
# only allow these updates to the mappings
# - changing the to_field or from_units
# - removing mappings
original_mappings_dict = {
m['from_field']: m.copy()
for m in profile.mappings
}
final_mappings = []
for updated_mapping in updated_mappings:
from_field = updated_mapping['from_field']
original_mapping = original_mappings_dict.get(from_field)
if original_mapping is not None:
original_mapping['to_field'] = updated_mapping[
'to_field']
original_mapping['from_units'] = updated_mapping[
'from_units']
final_mappings.append(original_mapping)
del original_mappings_dict[from_field]
profile.mappings = final_mappings
elif updated_mappings:
# indiscriminantly update the mappings
profile.mappings = updated_mappings
profile.save()
return JsonResponse({
'status':
'success',
'data':
ColumnMappingProfileSerializer(profile).data,
})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
)
],
request_body=AutoSchemaHelper.schema_factory(
{
'name':
'string',
'mappings': [{
'to_field': 'string',
'from_field': 'string',
'from_units': 'string',
'to_table_name': 'string',
}]
},
description=mappings_description,
required=['name', 'mappings']))
@api_endpoint_class
def create(self, request, pk=None):
"""
Creates a new profile given appropriate request data. The body should
contain only valid fields for ColumnMappingProfile objects.
"""
org_id = self.get_organization(request, True).id
try:
profile_data = request.data
profile_data['organizations'] = [org_id]
ser_profile = ColumnMappingProfileSerializer(data=profile_data)
if ser_profile.is_valid():
profile = ser_profile.save()
response_status = 'success'
response_data = ColumnMappingProfileSerializer(profile).data
response_code = HTTP_200_OK
else:
response_status = 'error'
response_data = ser_profile.errors
response_code = HTTP_400_BAD_REQUEST
return JsonResponse(
{
'status': response_status,
'data': response_data,
},
status=response_code)
except Exception as e:
return JsonResponse({
'status': 'error',
'data': str(e),
},
status=HTTP_400_BAD_REQUEST)
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
)
])
@api_endpoint_class
def destroy(self, request, pk=None):
"""
Deletes a specific profile.
"""
org_id = self.get_organization(request, True).id
try:
profile = ColumnMappingProfile.objects.get(
organizations__pk=org_id, pk=pk)
except Exception as e:
return JsonResponse({
'status': 'error',
'data': str(e),
},
status=HTTP_400_BAD_REQUEST)
if profile.profile_type == ColumnMappingProfile.BUILDINGSYNC_DEFAULT:
return JsonResponse(
{
'status': 'error',
'data': 'Not allowed to edit default BuildingSync profiles'
},
status=HTTP_400_BAD_REQUEST)
else:
profile.delete()
return JsonResponse({
'status': 'success',
'data': "Successfully deleted",
})
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
)
],
request_body=AutoSchemaHelper.schema_factory(
{'headers': ['string']},
description=
"Raw headers - the exact headers for columns in an import file.",
required=['headers']))
@api_endpoint_class
@action(detail=False, methods=['POST'])
def suggestions(self, request):
"""
Retrieves suggestions given raw column headers.
"""
try:
org_id = self.get_organization(request, True).id
raw_headers = request.data.get('headers', [])
suggested_mappings = mapper.build_column_mapping(
raw_headers,
Column.retrieve_all_by_tuple(org_id),
previous_mapping=None,
map_args=None,
thresh=
80 # percentage match that we require. 80% is random value for now.
)
# replace None with empty string for column names and PropertyState for tables
# TODO #239: Move this fix to build_column_mapping
for m in suggested_mappings:
table, destination_field, _confidence = suggested_mappings[m]
if destination_field is None:
suggested_mappings[m][1] = ''
# Fix the table name, eventually move this to the build_column_mapping
for m in suggested_mappings:
table, _destination_field, _confidence = suggested_mappings[m]
# Do not return the campus, created, updated fields... that is force them to be in the property state
if not table or table == 'Property':
suggested_mappings[m][0] = 'PropertyState'
elif table == 'TaxLot':
suggested_mappings[m][0] = 'TaxLotState'
return JsonResponse({
'status': 'success',
'data': suggested_mappings,
})
except Exception as e:
return JsonResponse({
'status': 'error',
'data': str(e),
},
status=HTTP_400_BAD_REQUEST)
class AnalysisViewSet(viewsets.ViewSet):
serializer_class = AnalysisSerializer
model = Analysis
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
],
request_body=CreateAnalysisSerializer,
)
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def create(self, request):
serializer = CreateAnalysisSerializer(data=request.data)
if not serializer.is_valid():
return JsonResponse({
'status': 'error',
'message': 'Bad request',
'errors': serializer.errors
})
analysis = serializer.save(
user_id=request.user.id,
organization_id=request.query_params['organization_id'])
pipeline = AnalysisPipeline.factory(analysis)
try:
progress_data = pipeline.prepare_analysis(
serializer.validated_data['property_view_ids'])
return JsonResponse({
'status': 'success',
'progress_key': progress_data['progress_key'],
'progress': progress_data,
})
except AnalysisPipelineException as e:
return JsonResponse({
'status': 'error',
'message': str(e)
},
status=HTTP_409_CONFLICT)
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(True),
AutoSchemaHelper.query_integer_field('property_id', False,
'Property ID')
])
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def list(self, request):
organization_id = request.query_params.get('organization_id', None)
property_id = request.query_params.get('property_id', None)
analyses = []
if property_id is not None:
analyses_queryset = (Analysis.objects.filter(
organization=organization_id,
analysispropertyview__property=property_id).distinct().
order_by('id'))
else:
analyses_queryset = (Analysis.objects.filter(
organization=organization_id).order_by('id'))
for analysis in analyses_queryset:
property_view_info = analysis.get_property_view_info(property_id)
serialized_analysis = AnalysisSerializer(analysis).data
serialized_analysis.update(property_view_info)
analyses.append(serialized_analysis)
return JsonResponse({'status': 'success', 'analyses': analyses})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field(True)])
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def retrieve(self, request, pk):
organization_id = int(request.query_params.get('organization_id', 0))
try:
analysis = Analysis.objects.get(id=pk,
organization_id=organization_id)
except Analysis.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
"Requested analysis doesn't exist in this organization."
},
status=HTTP_409_CONFLICT)
serialized_analysis = AnalysisSerializer(analysis).data
property_view_info = analysis.get_property_view_info()
serialized_analysis.update(property_view_info)
return JsonResponse({
'status': 'success',
'analysis': serialized_analysis
})
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()])
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['post'])
def start(self, request, pk):
organization_id = int(request.query_params.get('organization_id', 0))
try:
analysis = Analysis.objects.get(id=pk,
organization_id=organization_id)
pipeline = AnalysisPipeline.factory(analysis)
progress_data = pipeline.start_analysis()
return JsonResponse({
'status': 'success',
'progress_key': progress_data['progress_key'],
'progress': progress_data,
})
except Analysis.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
'Requested analysis doesn\'t exist in this organization.'
},
status=HTTP_409_CONFLICT)
except AnalysisPipelineException as e:
return JsonResponse({
'status': 'error',
'message': str(e)
},
status=HTTP_409_CONFLICT)
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()])
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['post'])
def stop(self, request, pk):
organization_id = int(request.query_params.get('organization_id', 0))
try:
analysis = Analysis.objects.get(id=pk,
organization_id=organization_id)
pipeline = AnalysisPipeline.factory(analysis)
pipeline.stop()
return JsonResponse({
'status': 'success',
})
except Analysis.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
'Requested analysis doesn\'t exist in this organization.'
},
status=HTTP_409_CONFLICT)
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()])
@require_organization_id_class
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
def destroy(self, request, pk):
organization_id = int(request.query_params.get('organization_id', 0))
try:
analysis = Analysis.objects.get(id=pk,
organization_id=organization_id)
pipeline = AnalysisPipeline.factory(analysis)
pipeline.delete()
return JsonResponse({
'status': 'success',
})
except Analysis.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
'Requested analysis doesn\'t exist in this organization.'
},
status=HTTP_409_CONFLICT)
class ColumnListProfileViewSet(OrgValidateMixin,
SEEDOrgNoPatchOrOrgCreateModelViewSet):
"""
API endpoint for returning Column List Profiles
create:
Create a new list profile. The list of columns is an array of column primary keys. If using Swagger, then
this will be enters as a list with returns between each primary key.
JSON POST Example:
{
"name": "some new name 3",
"profile_location": "List View Profile",
"inventory_type": "Tax Lot",
"columns": [
{"id": 1, "pinned": false, "order": 10},
{"id": 5, "pinned": true, "order": 14},
{"id": 7, "pinned": true, "order": 14}
]
}
"""
serializer_class = ColumnListProfileSerializer
model = ColumnListProfile
filter_backends = (ColumnListProfileFilterBackend, )
pagination_class = None
# force_parent = True # Ideally the column list profiles would inherit from the parent,
# but not yet.
# Overridden to augment with protected ComStock list profile if enabled
def retrieve(self, request, *args, **kwargs):
org_id = self.get_organization(self.request)
try:
org = Organization.objects.get(pk=org_id)
except Organization.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'organization with id %s does not exist' % org_id
},
status=status.HTTP_404_NOT_FOUND)
if not org.comstock_enabled or kwargs['pk'] != 'null':
return super(ColumnListProfileViewSet,
self).retrieve(request, args, kwargs)
result = {
'status': 'success',
'data': {
'id': None,
'name': 'ComStock',
'profile_location': VIEW_LOCATION_TYPES[VIEW_LIST][1],
'inventory_type':
VIEW_LIST_INVENTORY_TYPE[VIEW_LIST_PROPERTY][1],
'columns': self.list_comstock_columns(org_id)
}
}
return JsonResponse(result, status=status.HTTP_200_OK)
# Overridden to augment with protected ComStock list profile if enabled
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(
required=False,
description=
"Optional org id which overrides the users (default) current org id"
),
AutoSchemaHelper.query_string_field(
name='inventory_type',
required=True,
description="'Property' or 'Tax Lot' for filtering."),
AutoSchemaHelper.query_string_field(
name='profile_location',
required=True,
description=
"'List View Profile' or 'Detail View Profile' for filtering."),
])
def list(self, request, *args, **kwargs):
org_id = self.get_organization(self.request)
try:
org = Organization.objects.get(pk=org_id)
except Organization.DoesNotExist:
return JsonResponse(
{
'status': 'error',
'message':
'organization with id %s does not exist' % org_id
},
status=status.HTTP_404_NOT_FOUND)
inventory_type = request.query_params.get('inventory_type')
profile_location = request.query_params.get('profile_location')
if not org.comstock_enabled or inventory_type == 'Tax Lot' or profile_location == 'Detail View Profile':
return super(ColumnListProfileViewSet,
self).list(request, args, kwargs)
queryset = self.filter_queryset(self.get_queryset())
page = self.paginate_queryset(queryset)
if page is not None:
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
results = list(queryset)
base_profiles = self.get_serializer(results, many=True).data
# Add ComStock columns
base_profiles.append({
"id": None,
"name": "ComStock",
"profile_location": profile_location,
"inventory_type": inventory_type,
"columns": self.list_comstock_columns(org_id)
})
return Response(base_profiles)
@staticmethod
def list_comstock_columns(org_id):
comstock_columns = Column.objects.filter(organization_id=org_id, comstock_mapping__isnull=False) \
.order_by('comstock_mapping')
results = []
for index, column in enumerate(comstock_columns):
results.append({
"id": column.id,
"pinned": False,
"order": index + 1,
"column_name": column.column_name,
"table_name": column.table_name,
"comstock_mapping": column.comstock_mapping
})
return results
class TaxlotViewSet(viewsets.ViewSet, OrgMixin, ProfileIdMixin):
renderer_classes = (JSONRenderer,)
serializer_class = TaxLotSerializer
parser_classes = (JSONParser,)
_organization = None
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field(required=True)],
request_body=AutoSchemaHelper.schema_factory(
{
'selected': ['integer'],
},
description='IDs for taxlots to be checked for which labels are applied.'
)
)
@action(detail=False, methods=['POST'])
def labels(self, request):
"""
Returns a list of all labels where the is_applied field
in the response pertains to the labels applied to taxlot_view
"""
labels = StatusLabel.objects.filter(
super_organization=self.get_parent_org(self.request)
).order_by("name").distinct()
super_organization = self.get_organization(request)
# TODO: refactor to avoid passing request here
return get_labels(request, labels, super_organization, 'taxlot_view')
def _get_filtered_results(self, request, profile_id):
page = request.query_params.get('page', 1)
per_page = request.query_params.get('per_page', 1)
org_id = request.query_params.get('organization_id', None)
cycle_id = request.query_params.get('cycle')
# check if there is a query paramater for the profile_id. If so, then use that one
profile_id = request.query_params.get('profile_id', profile_id)
if not org_id:
return JsonResponse(
{'status': 'error', 'message': 'Need to pass organization_id as query parameter'},
status=status.HTTP_400_BAD_REQUEST)
if cycle_id:
cycle = Cycle.objects.get(organization_id=org_id, pk=cycle_id)
else:
cycle = Cycle.objects.filter(organization_id=org_id).order_by('name')
if cycle:
cycle = cycle.first()
else:
return JsonResponse({
'status': 'error',
'message': 'Could not locate cycle',
'pagination': {
'total': 0
},
'cycle_id': None,
'results': []
})
# Return taxlot views limited to the 'taxlot_view_ids' list. Otherwise, if selected is empty, return all
if 'taxlot_view_ids' in request.data and request.data['taxlot_view_ids']:
taxlot_views_list = TaxLotView.objects.select_related('taxlot', 'state', 'cycle') \
.filter(id__in=request.data['taxlot_view_ids'], taxlot__organization_id=org_id,
cycle=cycle) \
.order_by('id')
else:
taxlot_views_list = TaxLotView.objects.select_related('taxlot', 'state', 'cycle') \
.filter(taxlot__organization_id=org_id, cycle=cycle) \
.order_by('id')
paginator = Paginator(taxlot_views_list, per_page)
try:
taxlot_views = paginator.page(page)
page = int(page)
except PageNotAnInteger:
taxlot_views = paginator.page(1)
page = 1
except EmptyPage:
taxlot_views = paginator.page(paginator.num_pages)
page = paginator.num_pages
org = Organization.objects.get(pk=org_id)
# Retrieve all the columns that are in the db for this organization
columns_from_database = Column.retrieve_all(org_id, 'taxlot', False)
# This uses an old method of returning the show_columns. There is a new method that
# is preferred in v2.1 API with the ProfileIdMixin.
if profile_id is None:
show_columns = None
elif profile_id == -1:
show_columns = list(Column.objects.filter(
organization_id=org_id
).values_list('id', flat=True))
else:
try:
profile = ColumnListProfile.objects.get(
organization=org,
id=profile_id,
profile_location=VIEW_LIST,
inventory_type=VIEW_LIST_TAXLOT
)
show_columns = list(ColumnListProfileColumn.objects.filter(
column_list_profile_id=profile.id
).values_list('column_id', flat=True))
except ColumnListProfile.DoesNotExist:
show_columns = None
related_results = TaxLotProperty.get_related(taxlot_views, show_columns,
columns_from_database)
# collapse units here so we're only doing the last page; we're already a
# realized list by now and not a lazy queryset
unit_collapsed_results = [apply_display_unit_preferences(org, x) for x in related_results]
response = {
'pagination': {
'page': page,
'start': paginator.page(page).start_index(),
'end': paginator.page(page).end_index(),
'num_pages': paginator.num_pages,
'has_next': paginator.page(page).has_next(),
'has_previous': paginator.page(page).has_previous(),
'total': paginator.count
},
'cycle_id': cycle.id,
'results': unit_collapsed_results
}
return JsonResponse(response)
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field(
'cycle',
required=True,
description='The ID of the cycle to get taxlots'
),
AutoSchemaHelper.query_integer_field(
'page',
required=False,
description='The current page of taxlots to return'
),
AutoSchemaHelper.query_integer_field(
'per_page',
required=False,
description='The number of items per page to return'
),
AutoSchemaHelper.query_integer_field(
'profile_id',
required=False,
description='The ID of the column profile to use'
)
]
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_viewer')
def list(self, request):
"""
List all the properties
"""
return self._get_filtered_results(request, profile_id=-1)
@swagger_auto_schema(
request_body=AutoSchemaHelper.schema_factory(
{
'organization_id': 'integer',
'profile_id': 'integer',
'cycle_ids': ['integer'],
},
required=['organization_id', 'cycle_ids'],
description='Properties:\n'
'- organization_id: ID of organization\n'
'- profile_id: Either an id of a list settings profile, '
'or undefined\n'
'- cycle_ids: The IDs of the cycle to get taxlots'
)
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_viewer')
@action(detail=False, methods=['POST'])
def filter_by_cycle(self, request):
"""
List all the taxlots with all columns
"""
# NOTE: we are using a POST http method b/c swagger and django handle
# arrays differently in query parameters. ie this is just simpler
org_id = request.data.get('organization_id', None)
profile_id = request.data.get('profile_id', -1)
cycle_ids = request.data.get('cycle_ids', [])
if not org_id:
return JsonResponse(
{'status': 'error', 'message': 'Need to pass organization_id as query parameter'},
status=status.HTTP_400_BAD_REQUEST)
response = taxlots_across_cycles(org_id, profile_id, cycle_ids)
return JsonResponse(response)
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field(
'cycle',
required=True,
description='The ID of the cycle to get tax lots'
),
AutoSchemaHelper.query_integer_field(
'page',
required=False,
description='The current page of taxlots to return'
),
AutoSchemaHelper.query_integer_field(
'per_page',
required=False,
description='The number of items per page to return'
),
],
request_body=AutoSchemaHelper.schema_factory(
{
'profile_id': 'integer',
'taxlot_view_ids': ['integer'],
},
required=['profile_id'],
description='Properties:\n'
'- profile_id: Either an id of a list settings profile, or undefined\n'
'- taxlot_view_ids: List of taxlot view ids'
)
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_viewer')
@action(detail=False, methods=['POST'])
def filter(self, request):
"""
List all the tax lots
"""
if 'profile_id' not in request.data:
profile_id = None
else:
if request.data['profile_id'] == 'None':
profile_id = None
else:
profile_id = request.data['profile_id']
return self._get_filtered_results(request, profile_id=profile_id)
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
],
request_body=AutoSchemaHelper.schema_factory(
{
'taxlot_view_ids': ['integer']
},
required=['taxlot_view_ids'],
description='Properties:\n'
'- taxlot_view_ids: Array containing tax lot state ids to merge'
)
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=False, methods=['POST'])
def merge(self, request):
"""
Merge multiple tax lot records into a single new record, and run this
new record through a match and merge round within it's current Cycle.
"""
body = request.data
organization_id = int(request.query_params.get('organization_id', None))
taxlot_view_ids = body.get('taxlot_view_ids', [])
taxlot_states = TaxLotView.objects.filter(
id__in=taxlot_view_ids,
cycle__organization_id=organization_id
).values('id', 'state_id')
# get the state ids in order according to the given view ids
taxlot_states_dict = {t['id']: t['state_id'] for t in taxlot_states}
taxlot_state_ids = [
taxlot_states_dict[view_id]
for view_id in taxlot_view_ids if view_id in taxlot_states_dict
]
if len(taxlot_state_ids) != len(taxlot_view_ids):
return {
'status': 'error',
'message': 'All records not found.'
}
# Check the number of taxlot_state_ids to merge
if len(taxlot_state_ids) < 2:
return JsonResponse({
'status': 'error',
'message': 'At least two ids are necessary to merge'
}, status=status.HTTP_400_BAD_REQUEST)
merged_state = merge_taxlots(taxlot_state_ids, organization_id, 'Manual Match')
merge_count, link_count, view_id = match_merge_link(merged_state.taxlotview_set.first().id, 'TaxLotState')
result = {
'status': 'success'
}
result.update({
'match_merged_count': merge_count,
'match_link_count': link_count,
})
return result
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()]
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def unmerge(self, request, pk=None):
"""
Unmerge a taxlot view into two taxlot views
"""
try:
old_view = TaxLotView.objects.select_related(
'taxlot', 'cycle', 'state'
).get(
id=pk,
taxlot__organization_id=self.request.GET['organization_id']
)
except TaxLotView.DoesNotExist:
return {
'status': 'error',
'message': 'taxlot view with id {} does not exist'.format(pk)
}
# Duplicate pairing
paired_view_ids = list(TaxLotProperty.objects.filter(taxlot_view_id=old_view.id)
.order_by('property_view_id').values_list('property_view_id',
flat=True))
# Capture previous associated labels
label_ids = list(old_view.labels.all().values_list('id', flat=True))
notes = old_view.notes.all()
for note in notes:
note.taxlot_view = None
merged_state = old_view.state
if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED:
return {
'status': 'error',
'message': 'taxlot view with id {} is not a merged taxlot view'.format(pk)
}
log = TaxLotAuditLog.objects.select_related('parent_state1', 'parent_state2').filter(
state=merged_state
).order_by('-id').first()
if log.parent_state1 is None or log.parent_state2 is None:
return {
'status': 'error',
'message': 'taxlot view with id {} must have two parent states'.format(pk)
}
state1 = log.parent_state1
state2 = log.parent_state2
cycle_id = old_view.cycle_id
# Clone the taxlot record twice
old_taxlot = old_view.taxlot
new_taxlot = old_taxlot
new_taxlot.id = None
new_taxlot.save()
new_taxlot_2 = TaxLot.objects.get(pk=new_taxlot.pk)
new_taxlot_2.id = None
new_taxlot_2.save()
# If the canonical TaxLot is NOT associated to another -View
if not TaxLotView.objects.filter(taxlot_id=old_view.taxlot_id).exclude(pk=old_view.id).exists():
TaxLot.objects.get(pk=old_view.taxlot_id).delete()
# Create the views
new_view1 = TaxLotView(
cycle_id=cycle_id,
taxlot_id=new_taxlot.id,
state=state1
)
new_view2 = TaxLotView(
cycle_id=cycle_id,
taxlot_id=new_taxlot_2.id,
state=state2
)
# Mark the merged state as deleted
merged_state.merge_state = MERGE_STATE_DELETE
merged_state.save()
# Change the merge_state of the individual states
if log.parent1.name in ['Import Creation',
'Manual Edit'] and log.parent1.import_filename is not None:
# State belongs to a new record
state1.merge_state = MERGE_STATE_NEW
else:
state1.merge_state = MERGE_STATE_MERGED
if log.parent2.name in ['Import Creation',
'Manual Edit'] and log.parent2.import_filename is not None:
# State belongs to a new record
state2.merge_state = MERGE_STATE_NEW
else:
state2.merge_state = MERGE_STATE_MERGED
# In most cases data_state will already be 3 (DATA_STATE_MATCHING), but if one of the parents was a
# de-duplicated record then data_state will be 0. This step ensures that the new states will be 3.
state1.data_state = DATA_STATE_MATCHING
state2.data_state = DATA_STATE_MATCHING
state1.save()
state2.save()
# Delete the audit log entry for the merge
log.delete()
old_view.delete()
new_view1.save()
new_view2.save()
# Asssociate labels
label_objs = StatusLabel.objects.filter(pk__in=label_ids)
new_view1.labels.set(label_objs)
new_view2.labels.set(label_objs)
# Duplicate notes to the new views
for note in notes:
created = note.created
updated = note.updated
note.id = None
note.taxlot_view = new_view1
note.save()
ids = [note.id]
note.id = None
note.taxlot_view = new_view2
note.save()
ids.append(note.id)
# Correct the created and updated times to match the original note
Note.objects.filter(id__in=ids).update(created=created, updated=updated)
for paired_view_id in paired_view_ids:
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view1.id,
property_view_id=paired_view_id).save()
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view2.id,
property_view_id=paired_view_id).save()
return {
'status': 'success',
'view_id': new_view1.id
}
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()]
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['GET'])
def links(self, request, pk=None):
"""
Get taxlot details for each linked taxlot across org cycles
"""
organization_id = request.query_params.get('organization_id', None)
base_view = TaxLotView.objects.select_related('cycle').filter(
pk=pk,
cycle__organization_id=organization_id
)
if base_view.exists():
result = {'data': []}
linked_views = TaxLotView.objects.select_related('cycle').filter(
taxlot_id=base_view.get().taxlot_id,
cycle__organization_id=organization_id
).order_by('-cycle__start')
for linked_view in linked_views:
state_data = TaxLotStateSerializer(linked_view.state).data
state_data['cycle_id'] = linked_view.cycle.id
state_data['view_id'] = linked_view.id
result['data'].append(state_data)
return JsonResponse(result, status=status.HTTP_200_OK)
else:
result = {
'status': 'error',
'message': 'property view with id {} does not exist in given organization'.format(pk)
}
return JsonResponse(result)
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()]
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['POST'])
def match_merge_link(self, request, pk=None):
"""
Runs match merge link for an individual taxlot.
Note that this method can return a view_id of None if the given -View
was not involved in a merge.
"""
org_id = request.query_params.get('organization_id', None)
taxlot_view = TaxLotView.objects.get(
pk=pk,
cycle__organization_id=org_id
)
merge_count, link_count, view_id = match_merge_link(taxlot_view.pk, 'TaxLotState')
result = {
'view_id': view_id,
'match_merged_count': merge_count,
'match_link_count': link_count,
}
return JsonResponse(result)
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field(
'property_id',
required=True,
description='The property id to pair up with this taxlot'
)
]
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['PUT'])
def pair(self, request, pk=None):
"""
Pair a property to this taxlot
"""
organization_id = int(request.query_params.get('organization_id'))
property_id = int(request.query_params.get('property_id'))
taxlot_id = int(pk)
return pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, True)
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field(
'property_id',
required=True,
description='The property id to unpair from this taxlot'
)
]
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=True, methods=['PUT'])
def unpair(self, request, pk=None):
"""
Unpair a property from this taxlot
"""
organization_id = int(request.query_params.get('organization_id'))
property_id = int(request.query_params.get('property_id'))
taxlot_id = int(pk)
return pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, False)
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=AutoSchemaHelper.schema_factory(
{
'taxlot_view_ids': ['integer']
},
required=['taxlot_view_ids'],
description='A list of taxlot view ids to delete'
)
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
@action(detail=False, methods=['DELETE'])
def batch_delete(self, request):
"""
Batch delete several tax lots
"""
org_id = request.query_params.get('organization_id', None)
taxlot_view_ids = request.data.get('taxlot_view_ids', [])
taxlot_state_ids = TaxLotView.objects.filter(
id__in=taxlot_view_ids,
cycle__organization_id=org_id
).values_list('state_id', flat=True)
resp = TaxLotState.objects.filter(pk__in=Subquery(taxlot_state_ids)).delete()
if resp[0] == 0:
return JsonResponse({'status': 'warning', 'message': 'No action was taken'})
return JsonResponse({'status': 'success', 'taxlots': resp[1]['seed.TaxLotState']})
def _get_taxlot_view(self, taxlot_pk):
try:
taxlot_view = TaxLotView.objects.select_related(
'taxlot', 'cycle', 'state'
).get(
id=taxlot_pk,
taxlot__organization_id=self.request.GET['organization_id']
)
result = {
'status': 'success',
'taxlot_view': taxlot_view
}
except TaxLotView.DoesNotExist:
result = {
'status': 'error',
'message': 'taxlot view with id {} does not exist'.format(taxlot_pk)
}
return result
def get_history(self, taxlot_view):
"""Return history in reverse order"""
# access the history from the property state
history, master = taxlot_view.state.history()
# convert the history and master states to StateSerializers
master['state'] = TaxLotStateSerializer(master['state_data']).data
del master['state_data']
del master['state_id']
for h in history:
h['state'] = TaxLotStateSerializer(h['state_data']).data
del h['state_data']
del h['state_id']
return history, master
def _get_properties(self, taxlot_view_pk):
property_view_pks = TaxLotProperty.objects.filter(
taxlot_view_id=taxlot_view_pk
).values_list('property_view_id', flat=True)
property_views = PropertyView.objects.filter(
pk__in=property_view_pks
).select_related('cycle', 'state')
properties = []
for property_view in property_views:
properties.append(PropertyViewSerializer(property_view).data)
return properties
@swagger_auto_schema_org_query_param
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_view_data')
def retrieve(self, request, pk):
"""
Get taxlot details
"""
result = self._get_taxlot_view(pk)
if result.get('status', None) != 'error':
taxlot_view = result.pop('taxlot_view')
result.update(TaxLotViewSerializer(taxlot_view).data)
# remove TaxLotView id from result
result.pop('id')
result['state'] = TaxLotStateSerializer(taxlot_view.state).data
result['properties'] = self._get_properties(taxlot_view.pk)
result['history'], master = self.get_history(taxlot_view)
result = update_result_with_master(result, master)
return JsonResponse(result, status=status.HTTP_200_OK)
else:
return JsonResponse(result, status=status.HTTP_404_NOT_FOUND)
@swagger_auto_schema(
manual_parameters=[AutoSchemaHelper.query_org_id_field()],
request_body=UpdateTaxLotPayloadSerializer,
)
@api_endpoint_class
@ajax_request_class
@has_perm_class('can_modify_data')
def update(self, request, pk):
"""
Update a taxlot and run the updated record through a match and merge
round within it's current Cycle.
"""
data = request.data
result = self._get_taxlot_view(pk)
if result.get('status', 'error') != 'error':
taxlot_view = result.pop('taxlot_view')
taxlot_state_data = TaxLotStateSerializer(taxlot_view.state).data
# get the taxlot state information from the request
new_taxlot_state_data = data['state']
# set empty strings to None
for key, val in new_taxlot_state_data.items():
if val == '':
new_taxlot_state_data[key] = None
changed_fields, previous_data = get_changed_fields(taxlot_state_data, new_taxlot_state_data)
if not changed_fields:
result.update(
{'status': 'success', 'message': 'Records are identical'}
)
return JsonResponse(result, status=status.HTTP_204_NO_CONTENT)
else:
# Not sure why we are going through the pain of logging this all right now... need to
# reevaluate this.
log = TaxLotAuditLog.objects.select_related().filter(
state=taxlot_view.state
).order_by('-id').first()
# if checks above pass, create an exact copy of the current state for historical purposes
if log.name == 'Import Creation':
# Add new state by removing the existing ID.
taxlot_state_data.pop('id')
# Remove the import_file_id for the first edit of a new record
# If the import file has been deleted and this value remains the serializer won't be valid
taxlot_state_data.pop('import_file')
new_taxlot_state_serializer = TaxLotStateSerializer(
data=taxlot_state_data
)
if new_taxlot_state_serializer.is_valid():
# create the new property state, and perform an initial save / moving relationships
new_state = new_taxlot_state_serializer.save()
# then assign this state to the property view and save the whole view
taxlot_view.state = new_state
taxlot_view.save()
TaxLotAuditLog.objects.create(organization=log.organization,
parent1=log,
parent2=None,
parent_state1=log.state,
parent_state2=None,
state=new_state,
name='Manual Edit',
description=None,
import_filename=log.import_filename,
record_type=AUDIT_USER_EDIT)
result.update(
{'state': new_taxlot_state_serializer.data}
)
# save the property view so that the datetime gets updated on the property.
taxlot_view.save()
else:
result.update({
'status': 'error',
'message': 'Invalid update data with errors: {}'.format(
new_taxlot_state_serializer.errors)}
)
return JsonResponse(result, status=status.HTTP_422_UNPROCESSABLE_ENTITY)
# redo assignment of this variable in case this was an initial edit
taxlot_state_data = TaxLotStateSerializer(taxlot_view.state).data
if 'extra_data' in new_taxlot_state_data:
taxlot_state_data['extra_data'].update(
new_taxlot_state_data['extra_data']
)
taxlot_state_data.update(
{k: v for k, v in new_taxlot_state_data.items() if k != 'extra_data'}
)
log = TaxLotAuditLog.objects.select_related().filter(
state=taxlot_view.state
).order_by('-id').first()
if log.name in ['Manual Edit', 'Manual Match', 'System Match', 'Merge current state in migration']:
# Convert this to using the serializer to save the data. This will override the
# previous values in the state object.
# Note: We should be able to use partial update here and pass in the changed
# fields instead of the entire state_data.
updated_taxlot_state_serializer = TaxLotStateSerializer(
taxlot_view.state,
data=taxlot_state_data
)
if updated_taxlot_state_serializer.is_valid():
# create the new property state, and perform an initial save / moving
# relationships
updated_taxlot_state_serializer.save()
result.update(
{'state': updated_taxlot_state_serializer.data}
)
# save the property view so that the datetime gets updated on the property.
taxlot_view.save()
Note.create_from_edit(request.user.id, taxlot_view, new_taxlot_state_data, previous_data)
merge_count, link_count, view_id = match_merge_link(taxlot_view.id, 'TaxLotState')
result.update({
'view_id': view_id,
'match_merged_count': merge_count,
'match_link_count': link_count,
})
return JsonResponse(result, status=status.HTTP_200_OK)
else:
result.update({
'status': 'error',
'message': 'Invalid update data with errors: {}'.format(
updated_taxlot_state_serializer.errors)}
)
return JsonResponse(result, status=status.HTTP_422_UNPROCESSABLE_ENTITY)
else:
result = {
'status': 'error',
'message': 'Unrecognized audit log name: ' + log.name
}
return JsonResponse(result, status=status.HTTP_204_NO_CONTENT)
else:
return JsonResponse(result, status=status.HTTP_404_NOT_FOUND)
