class DataQualityCheckViewSet(viewsets.ViewSet): """ Handles Data Quality API operations within Inventory backend. (1) Post, wait, get… (2) Respond with what changed """ # Remove lookup_field once data_quality_check_id is used and "pk" can be used lookup_field = 'organization_id' @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.base_field( "organization_id", "IN_PATH", "Organization ID - identifier used to specify a DataQualityCheck", True, "TYPE_INTEGER") ], request_body=AutoSchemaHelper.schema_factory( { 'property_view_ids': ['integer'], 'taxlot_view_ids': ['integer'], }, description='An object containing IDs of the records to perform' ' data quality checks on. Should contain two keys- ' 'property_view_ids and taxlot_view_ids, each of ' 'which is an array of appropriate IDs.', ), responses={ 200: AutoSchemaHelper.schema_factory({ 'num_properties': 'integer', 'num_taxlots': 'integer', 'progress_key': 'string', 'progress': {}, }) }) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['POST']) def start(self, request, organization_id): """ This API endpoint will create a new data_quality check process in the background, on potentially a subset of properties/taxlots, and return back a query key """ body = request.data property_view_ids = body['property_view_ids'] taxlot_view_ids = body['taxlot_view_ids'] property_state_ids = PropertyView.objects.filter( id__in=property_view_ids).values_list('state_id', flat=True) taxlot_state_ids = TaxLotView.objects.filter( id__in=taxlot_view_ids).values_list('state_id', flat=True) # For now, organization_id is the only key currently used to identify DataQualityChecks return_value = do_checks(organization_id, property_state_ids, taxlot_state_ids) return JsonResponse({ 'num_properties': len(property_state_ids), 'num_taxlots': len(taxlot_state_ids), # TODO #239: Deprecate progress_key from here and just use the 'progess.progress_key' 'progress_key': return_value['progress_key'], 'progress': return_value, }) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field("run_id", True, "Import file ID or cache key"), ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=False, methods=['GET']) def results_csv(self, request): """ Download a CSV of the results from a data quality run based on either the ID that was given during the creation of the data quality task or the ID of the import file which had it's records checked. Note that it is not related to objects in the database, since the results are stored in redis! """ run_id = request.query_params.get('run_id') if run_id is None: return JsonResponse( { 'status': 'error', 'message': 'must include Import file ID or cache key as run_id' }, status=status.HTTP_400_BAD_REQUEST) data_quality_results = get_cache_raw( DataQualityCheck.cache_key(run_id)) response = HttpResponse(content_type='text/csv') response[ 'Content-Disposition'] = 'attachment; filename="Data Quality Check Results.csv"' writer = csv.writer(response) if data_quality_results is None: writer.writerow(['Error']) writer.writerow(['data quality results not found']) return response writer.writerow([ 'Table', 'Address Line 1', 'PM Property ID', 'Tax Lot ID', 'Custom ID', 'Field', 'Applied Label', 'Condition', 'Error Message', 'Severity' ]) for row in data_quality_results: for result in row['data_quality_results']: writer.writerow([ row['data_quality_results'][0]['table_name'], row['address_line_1'], row['pm_property_id'] if 'pm_property_id' in row else None, row['jurisdiction_tax_lot_id'] if 'jurisdiction_tax_lot_id' in row else None, row['custom_id_1'], result['formatted_field'], result.get('label', None), result['condition'], # the detailed_message field can have units which has superscripts/subscripts, so unidecode it! unidecode(result['detailed_message']), result['severity'] ]) return response @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field("run_id", True, "Import file ID or cache key"), ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=False, methods=['GET']) def results(self, request): """ Return the results of a data quality run based on either the ID that was given during the creation of the data quality task or the ID of the import file which had it's records checked. Note that it is not related to objects in the database, since the results are stored in redis! """ data_quality_id = request.query_params['run_id'] data_quality_results = get_cache_raw( DataQualityCheck.cache_key(data_quality_id)) return JsonResponse({'data': data_quality_results})
class TaxLotPropertyViewSet(GenericViewSet): """ The TaxLotProperty field is used to return the properties and tax lots from the join table. This method presently only works with the CSV, but should eventually be extended to be the viewset for any tax lot / property join API call. """ renderer_classes = (JSONRenderer, ) serializer_class = TaxLotPropertySerializer @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field("cycle_id", True, "Cycle ID"), AutoSchemaHelper.query_string_field( "inventory_type", False, "Either 'taxlots' or 'properties' and defaults to 'properties'." ), ], request_body=AutoSchemaHelper.schema_factory( { 'ids': ['integer'], 'filename': 'string', 'export_type': 'string', 'profile_id': 'integer' }, description='- ids: (View) IDs for records to be exported\n' '- filename: desired filename including extension (defaulting to \'ExportedData.{export_type}\')\n' '- export_types: \'csv\', \'geojson\', \'xlsx\' (defaulting to \'csv\')\n' '- profile_id: Column List Profile ID to use for customizing fields included in export' ), ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=False, methods=['POST']) def export(self, request): """ Download a collection of the TaxLot and Properties in multiple formats. """ cycle_pk = request.query_params.get('cycle_id', None) if not cycle_pk: return JsonResponse({ 'status': 'error', 'message': 'Must pass in cycle_id as query parameter' }) org_id = request.query_params['organization_id'] if 'profile_id' not in request.data: profile_id = None else: if request.data['profile_id'] == 'None' or request.data[ 'profile_id'] == '': profile_id = None else: profile_id = request.data['profile_id'] # get the class to operate on and the relationships view_klass_str = request.query_params.get('inventory_type', 'properties') view_klass = INVENTORY_MODELS[view_klass_str] # Set the first column to be the ID column_name_mappings = OrderedDict([('id', 'ID')]) column_ids, add_column_name_mappings, columns_from_database = ColumnListProfile.return_columns( org_id, profile_id, view_klass_str) column_name_mappings.update(add_column_name_mappings) select_related = ['state', 'cycle'] ids = request.data.get('ids', []) filter_str = {'cycle': cycle_pk} if hasattr(view_klass, 'property'): select_related.append('property') prefetch_related = ['labels'] filter_str = {'property__organization_id': org_id} if ids: filter_str['id__in'] = ids # always export the labels and notes column_name_mappings['property_notes'] = 'Property Notes' column_name_mappings['property_labels'] = 'Property Labels' elif hasattr(view_klass, 'taxlot'): select_related.append('taxlot') prefetch_related = ['labels'] filter_str = {'taxlot__organization_id': org_id} if ids: filter_str['id__in'] = ids # always export the labels and notes column_name_mappings['taxlot_notes'] = 'Tax Lot Notes' column_name_mappings['taxlot_labels'] = 'Tax Lot Labels' model_views = view_klass.objects.select_related( *select_related).prefetch_related(*prefetch_related).filter( **filter_str).order_by('id') # get the data in a dict which includes the related data data = TaxLotProperty.get_related(model_views, column_ids, columns_from_database) # add labels and notes for i, record in enumerate(model_views): label_string = [] note_string = [] for label in list(record.labels.all().order_by('name')): label_string.append(label.name) for note in list(record.notes.all().order_by('created')): note_string.append(note.created.astimezone().strftime( "%Y-%m-%d %I:%M:%S %p") + "\n" + note.text) if hasattr(record, 'property'): data[i]['property_labels'] = ','.join(label_string) data[i]['property_notes'] = '\n----------\n'.join(note_string) elif hasattr(record, 'taxlot'): data[i]['taxlot_labels'] = ','.join(label_string) data[i]['taxlot_notes'] = '\n----------\n'.join(note_string) # force the data into the same order as the IDs if ids: order_dict = {obj_id: index for index, obj_id in enumerate(ids)} if view_klass_str == 'properties': view_id_str = 'property_view_id' else: view_id_str = 'taxlot_view_id' data.sort(key=lambda inventory_obj: order_dict[inventory_obj[ view_id_str]]) export_type = request.data.get('export_type', 'csv') filename = request.data.get('filename', f"ExportedData.{export_type}") if export_type == "csv": return self._csv_response(filename, data, column_name_mappings) elif export_type == "geojson": return self._json_response(filename, data, column_name_mappings) elif export_type == "xlsx": return self._spreadsheet_response(filename, data, column_name_mappings) def _csv_response(self, filename, data, column_name_mappings): response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="{}"'.format( filename) writer = csv.writer(response) # check the first item in the header and make sure that it isn't ID (it can be id, or iD). # excel doesn't like the first item to be ID in a CSV header = list(column_name_mappings.values()) if header[0] == 'ID': header[0] = 'id' writer.writerow(header) # iterate over the results to preserve column order and write row. for datum in data: row = [] for column in column_name_mappings: row_result = datum.get(column, None) # Try grabbing the value out of the related field if not found yet. if row_result is None and datum.get('related'): row_result = datum['related'][0].get(column, None) # Convert quantities (this is typically handled in the JSON Encoder, but that isn't here). if isinstance(row_result, ureg.Quantity): row_result = row_result.magnitude elif isinstance(row_result, datetime.datetime): row_result = row_result.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(row_result, datetime.date): row_result = row_result.strftime("%Y-%m-%d") row.append(row_result) writer.writerow(row) return response def _spreadsheet_response(self, filename, data, column_name_mappings): response = HttpResponse( content_type= 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) response['Content-Disposition'] = 'attachment; filename="{}"'.format( filename) scenario_keys = ('id', 'name', 'description', 'annual_site_energy_savings', 'annual_source_energy_savings', 'annual_cost_savings', 'analysis_state', 'analysis_state_message', 'annual_electricity_savings', 'annual_natural_gas_savings', 'annual_site_energy', 'annual_source_energy', 'annual_natural_gas_energy', 'annual_electricity_energy', 'annual_peak_demand', 'annual_site_energy_use_intensity', 'annual_source_energy_use_intensity') scenario_key_mappings = { 'annual_site_energy_savings': 'annual_site_energy_savings_mmbtu', 'annual_source_energy_savings': 'annual_source_energy_savings_mmbtu', 'annual_cost_savings': 'annual_cost_savings_dollars', 'annual_site_energy': 'annual_site_energy_kbtu', 'annual_site_energy_use_intensity': 'annual_site_energy_use_intensity_kbtu_ft2', 'annual_source_energy': 'annual_source_energy_kbtu', 'annual_source_energy_use_intensity': 'annual_source_energy_use_intensity_kbtu_ft2', 'annual_natural_gas_energy': 'annual_natural_gas_energy_mmbtu', 'annual_electricity_energy': 'annual_electricity_energy_mmbtu', 'annual_peak_demand': 'annual_peak_demand_kw', 'annual_electricity_savings': 'annual_electricity_savings_kbtu', 'annual_natural_gas_savings': 'annual_natural_gas_savings_kbtu' } property_measure_keys = ('id', 'property_measure_name', 'measure_id', 'cost_mv', 'cost_total_first', 'cost_installation', 'cost_material', 'cost_capital_replacement', 'cost_residual_value') measure_keys = ('name', 'display_name', 'category', 'category_display_name') # find measures and scenarios for i, record in enumerate(data): measures = PropertyMeasure.objects.filter( property_state_id=record['property_state_id']) record['measures'] = measures scenarios = Scenario.objects.filter( property_state_id=record['property_state_id']) record['scenarios'] = scenarios output = io.BytesIO() wb = xlsxwriter.Workbook(output, {'remove_timezone': True}) # add tabs ws1 = wb.add_worksheet('Properties') ws2 = wb.add_worksheet('Measures') ws3 = wb.add_worksheet('Scenarios') ws4 = wb.add_worksheet('Scenario Measure Join Table') ws5 = wb.add_worksheet('Meter Readings') bold = wb.add_format({'bold': True}) row = 0 row2 = 0 col2 = 0 row3 = 0 col3 = 0 row4 = 0 row5 = 0 for index, val in enumerate(list(column_name_mappings.values())): # Do not write the first element as ID, this causes weird issues with Excel. if index == 0 and val == 'ID': ws1.write(row, index, 'id', bold) else: ws1.write(row, index, val, bold) # iterate over the results to preserve column order and write row. add_m_headers = True add_s_headers = True for datum in data: row += 1 id = None for index, column in enumerate(column_name_mappings): if column == 'id': id = datum.get(column, None) row_result = datum.get(column, None) # Try grabbing the value out of the related field if not found yet. if row_result is None and datum.get('related'): row_result = datum['related'][0].get(column, None) # Convert quantities (this is typically handled in the JSON Encoder, but that isn't here). if isinstance(row_result, ureg.Quantity): row_result = row_result.magnitude elif isinstance(row_result, datetime.datetime): row_result = row_result.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(row_result, datetime.date): row_result = row_result.strftime("%Y-%m-%d") ws1.write(row, index, row_result) # measures for index, m in enumerate(datum['measures']): if add_m_headers: # grab headers for key in property_measure_keys: ws2.write(row2, col2, key, bold) col2 += 1 for key in measure_keys: ws2.write(row2, col2, 'measure ' + key, bold) col2 += 1 add_m_headers = False row2 += 1 col2 = 0 for key in property_measure_keys: ws2.write(row2, col2, getattr(m, key)) col2 += 1 for key in measure_keys: ws2.write(row2, col2, getattr(m.measure, key)) col2 += 1 # scenarios (and join table) # join table ws4.write('A1', 'property_id', bold) ws4.write('B1', 'scenario_id', bold) ws4.write('C1', 'measure_id', bold) for index, s in enumerate(datum['scenarios']): scenario_id = s.id if add_s_headers: # grab headers for key in scenario_keys: # double check scenario_key_mappings in case a different header is desired if key in scenario_key_mappings.keys(): key = scenario_key_mappings[key] ws3.write(row3, col3, key, bold) col3 += 1 add_s_headers = False row3 += 1 col3 = 0 for key in scenario_keys: ws3.write(row3, col3, getattr(s, key)) col3 += 1 for sm in s.measures.all(): row4 += 1 ws4.write(row4, 0, id) ws4.write(row4, 1, scenario_id) ws4.write(row4, 2, sm.id) # scenario meter readings ws5.write('A1', 'scenario_id', bold) ws5.write('B1', 'meter_id', bold) ws5.write('C1', 'type', bold) ws5.write('D1', 'start_time', bold) ws5.write('E1', 'end_time', bold) ws5.write('F1', 'reading', bold) ws5.write('G1', 'units', bold) ws5.write('H1', 'is_virtual', bold) # datetime formatting date_format = wb.add_format({'num_format': 'yyyy-mm-dd hh:mm:ss'}) for index, s in enumerate(datum['scenarios']): scenario_id = s.id # retrieve meters meters = Meter.objects.filter(scenario_id=scenario_id) for m in meters: # retrieve readings readings = MeterReading.objects.filter( meter_id=m.id).order_by('start_time') for r in readings: row5 += 1 ws5.write(row5, 0, scenario_id) ws5.write(row5, 1, m.id) the_type = next((item for item in Meter.ENERGY_TYPES if item[0] == m.type), None) the_type = the_type[1] if the_type is not None else None ws5.write( row5, 2, the_type ) # use energy type enum to determine reading type ws5.write_datetime(row5, 3, r.start_time, date_format) ws5.write_datetime(row5, 4, r.end_time, date_format) ws5.write(row5, 5, r.reading) # this is now a float field ws5.write(row5, 6, r.source_unit) ws5.write(row5, 7, m.is_virtual) wb.close() # xlsx_data contains the Excel file xlsx_data = output.getvalue() response.write(xlsx_data) return response def _json_response(self, filename, data, column_name_mappings): polygon_fields = [ "bounding_box", "centroid", "property_footprint", "taxlot_footprint", "long_lat" ] features = [] # extract related records related_records = self._extract_related(data) # append related_records to data complete_data = data + related_records for datum in complete_data: feature = {"type": "Feature", "properties": {}} for key, value in datum.items(): if value is None: continue if isinstance(value, ureg.Quantity): value = value.magnitude elif isinstance(value, datetime.datetime): value = value.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(value, datetime.date): value = value.strftime("%Y-%m-%d") if value and any(k in key for k in polygon_fields): """ If object is a polygon and is populated, add the 'geometry' key-value-pair in the appropriate GeoJSON format. When the first geometry is added, the correct format is established. When/If a second geometry is added, this is appended alongside the previous geometry. """ individual_geometry = {} # long_lat if key == 'long_lat': coordinates = self._serialized_point(value) # point individual_geometry = { "coordinates": coordinates, "type": "Point" } else: # polygons coordinates = self._serialized_coordinates(value) individual_geometry = { "coordinates": [coordinates], "type": "Polygon" } if feature.get("geometry", None) is None: feature["geometry"] = { "type": "GeometryCollection", "geometries": [individual_geometry] } else: feature["geometry"]["geometries"].append( individual_geometry) else: """ Non-polygon data """ display_key = column_name_mappings.get(key, key) feature["properties"][display_key] = value # # store point geometry in case you need it # if display_key == "Longitude": # point_geometry[0] = value # if display_key == "Latitude": # point_geometry[1] = value """ Before appending feature, ensure that if there is no geometry recorded. Note that the GeoJson will not render if no lat/lng """ # add style information if feature["properties"].get("property_state_id") is not None: feature["properties"]["stroke"] = "#185189" # buildings color elif feature["properties"].get("taxlot_state_id") is not None: feature["properties"]["stroke"] = "#10A0A0" # buildings color feature["properties"]["marker-color"] = "#E74C3C" # feature["properties"]["stroke-width"] = 3 feature["properties"]["fill-opacity"] = 0 # append feature features.append(feature) response_dict = { "type": "FeatureCollection", "crs": { "type": "EPSG", "properties": { "code": 4326 } }, "features": features } response = JsonResponse(response_dict) response['Content-Disposition'] = 'attachment; filename="{}"'.format( filename) return response def _serialized_coordinates(self, polygon_wkt): string_coord_pairs = polygon_wkt.lstrip('POLYGON (').rstrip(')').split( ', ') coordinates = [] for coord_pair in string_coord_pairs: float_coords = [float(coord) for coord in coord_pair.split(' ')] coordinates.append(float_coords) return coordinates def _serialized_point(self, point_wkt): string_coords = point_wkt.lstrip('POINT (').rstrip(')').split(', ') coordinates = [] for coord in string_coords[0].split(' '): coordinates.append(float(coord)) return coordinates def _extract_related(self, data): # extract all related records into a separate array related = [] # figure out if we are dealing with properties or taxlots if data[0].get("property_state_id", None) is not None: is_property = True elif data[0].get("taxlot_state_id", None) is not None: is_property = False for datum in data: if datum.get("related", None) is not None: for record in datum["related"]: related.append(record) # make array unique if is_property: unique = [dict(p) for p in set(tuple(i.items()) for i in related)] else: unique = [dict(p) for p in set(tuple(i.items()) for i in related)] return unique
class AnalysisViewSet(viewsets.ViewSet): serializer_class = AnalysisSerializer model = Analysis @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=CreateAnalysisSerializer, ) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def create(self, request): serializer = CreateAnalysisSerializer(data=request.data) if not serializer.is_valid(): return JsonResponse({ 'status': 'error', 'message': 'Bad request', 'errors': serializer.errors }) analysis = serializer.save( user_id=request.user.id, organization_id=request.query_params['organization_id']) pipeline = AnalysisPipeline.factory(analysis) try: progress_data = pipeline.prepare_analysis( serializer.validated_data['property_view_ids']) return JsonResponse({ 'status': 'success', 'progress_key': progress_data['progress_key'], 'progress': progress_data, }) except AnalysisPipelineException as e: return JsonResponse({ 'status': 'error', 'message': str(e) }, status=HTTP_409_CONFLICT) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(True), AutoSchemaHelper.query_integer_field('property_id', False, 'Property ID') ]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def list(self, request): organization_id = request.query_params.get('organization_id', None) property_id = request.query_params.get('property_id', None) analyses = [] if property_id is not None: analyses_queryset = (Analysis.objects.filter( organization=organization_id, analysispropertyview__property=property_id).distinct(). order_by('id')) else: analyses_queryset = (Analysis.objects.filter( organization=organization_id).order_by('id')) for analysis in analyses_queryset: property_view_info = analysis.get_property_view_info(property_id) serialized_analysis = AnalysisSerializer(analysis).data serialized_analysis.update(property_view_info) analyses.append(serialized_analysis) return JsonResponse({'status': 'success', 'analyses': analyses}) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field(True)]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def retrieve(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': "Requested analysis doesn't exist in this organization." }, status=HTTP_409_CONFLICT) serialized_analysis = AnalysisSerializer(analysis).data property_view_info = analysis.get_property_view_info() serialized_analysis.update(property_view_info) return JsonResponse({ 'status': 'success', 'analysis': serialized_analysis }) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['post']) def start(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) pipeline = AnalysisPipeline.factory(analysis) progress_data = pipeline.start_analysis() return JsonResponse({ 'status': 'success', 'progress_key': progress_data['progress_key'], 'progress': progress_data, }) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Requested analysis doesn\'t exist in this organization.' }, status=HTTP_409_CONFLICT) except AnalysisPipelineException as e: return JsonResponse({ 'status': 'error', 'message': str(e) }, status=HTTP_409_CONFLICT) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['post']) def stop(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) pipeline = AnalysisPipeline.factory(analysis) pipeline.stop() return JsonResponse({ 'status': 'success', }) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Requested analysis doesn\'t exist in this organization.' }, status=HTTP_409_CONFLICT) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def destroy(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) pipeline = AnalysisPipeline.factory(analysis) pipeline.delete() return JsonResponse({ 'status': 'success', }) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Requested analysis doesn\'t exist in this organization.' }, status=HTTP_409_CONFLICT)
class OrganizationViewSet(viewsets.ViewSet): @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['DELETE']) def columns(self, request, pk=None): """ Delete all columns for an organization. This method is typically not recommended if there are data in the inventory as it will invalidate all extra_data fields. This also removes all the column mappings that existed. --- parameters: - name: pk description: The organization_id required: true paramType: path type: status: description: success or error type: string required: true column_mappings_deleted_count: description: Number of column_mappings that were deleted type: integer required: true columns_deleted_count: description: Number of columns that were deleted type: integer required: true """ try: org = Organization.objects.get(pk=pk) c_count, cm_count = Column.delete_all(org) return JsonResponse({ 'status': 'success', 'column_mappings_deleted_count': cm_count, 'columns_deleted_count': c_count, }) except Organization.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'organization with with id {} does not exist'.format(pk) }, status=status.HTTP_404_NOT_FOUND) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_integer_field('import_file_id', required=True, description='Import file id'), openapi.Parameter('id', openapi.IN_PATH, type=openapi.TYPE_INTEGER, description='Organization id'), ], request_body=SaveColumnMappingsRequestPayloadSerializer, responses={200: 'success response'}) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['POST']) def column_mappings(self, request, pk=None): """ Saves the mappings between the raw headers of an ImportFile and the destination fields in the `to_table_name` model which should be either PropertyState or TaxLotState Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES`` """ import_file_id = request.query_params.get('import_file_id') if import_file_id is None: return JsonResponse( { 'status': 'error', 'message': 'Query param `import_file_id` is required' }, status=status.HTTP_400_BAD_REQUEST) try: _ = ImportFile.objects.get(pk=import_file_id) organization = Organization.objects.get(pk=pk) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'No import file found' }, status=status.HTTP_404_NOT_FOUND) except Organization.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'No organization found' }, status=status.HTTP_404_NOT_FOUND) result = Column.create_mappings(request.data.get('mappings', []), organization, request.user, import_file_id) if result: return JsonResponse({'status': 'success'}) else: return JsonResponse({'status': 'error'}) def _start_whole_org_match_merge_link(self, org_id, state_class_name, proposed_columns=[]): identifier = randint(100, 100000) result_key = _get_match_merge_link_key(identifier) set_cache_raw(result_key, {}) progress_data = ProgressData(func_name='org_match_merge_link', unique_id=identifier) progress_data.delete() whole_org_match_merge_link.apply_async( args=(org_id, state_class_name, proposed_columns), link=cache_match_merge_link_result.s(identifier, progress_data.key)) return progress_data.key @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_boolean_field( 'brief', required=False, description='If true, only return high-level organization details') ]) @api_endpoint_class @ajax_request_class def list(self, request): """ Retrieves all orgs the user has access to. """ # if brief==true only return high-level organization details brief = json.loads(request.query_params.get('brief', 'false')) if brief: if request.user.is_superuser: qs = Organization.objects.only('id', 'name', 'parent_org_id') else: qs = request.user.orgs.only('id', 'name', 'parent_org_id') orgs = _dict_org_brief(request, qs) if len(orgs) == 0: return JsonResponse( { 'status': 'error', 'message': 'Your SEED account is not associated with any organizations. ' 'Please contact a SEED administrator.' }, status=status.HTTP_401_UNAUTHORIZED) else: return JsonResponse({'organizations': orgs}) else: if request.user.is_superuser: qs = Organization.objects.all() else: qs = request.user.orgs.all() orgs = _dict_org(request, qs) if len(orgs) == 0: return JsonResponse( { 'status': 'error', 'message': 'Your SEED account is not associated with any organizations. ' 'Please contact a SEED administrator.' }, status=status.HTTP_401_UNAUTHORIZED) else: return JsonResponse({'organizations': orgs}) @method_decorator(permission_required('seed.can_access_admin')) @api_endpoint_class @ajax_request_class def destroy(self, request, pk=None): """ Starts a background task to delete an organization and all related data. """ return JsonResponse(tasks.delete_organization(pk)) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def retrieve(self, request, pk=None): """ Retrieves a single organization by id. """ org_id = pk if org_id is None: return JsonResponse( { 'status': 'error', 'message': 'no organization_id sent' }, status=status.HTTP_400_BAD_REQUEST) try: org = Organization.objects.get(pk=org_id) except Organization.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'organization does not exist' }, status=status.HTTP_404_NOT_FOUND) if (not request.user.is_superuser and not OrganizationUser.objects.filter( user=request.user, organization=org, role_level__in=[ROLE_OWNER, ROLE_MEMBER, ROLE_VIEWER ]).exists()): # TODO: better permission and return 401 or 403 return JsonResponse( { 'status': 'error', 'message': 'user is not the owner of the org' }, status=status.HTTP_403_FORBIDDEN) return JsonResponse({ 'status': 'success', 'organization': _dict_org(request, [org])[0], }) @swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory( { 'organization_name': 'string', 'user_id': 'integer', }, required=['organization_name', 'user_id'], description='Properties:\n' '- organization_name: The new organization name\n' '- user_id: The user ID (primary key) to be used as the owner of the new organization' )) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') def create(self, request): """ Creates a new organization. """ body = request.data user = User.objects.get(pk=body['user_id']) org_name = body['organization_name'] if Organization.objects.filter(name=org_name).exists(): return JsonResponse( { 'status': 'error', 'message': 'organization name already exists' }, status=status.HTTP_409_CONFLICT) org, _, _ = create_organization(user, org_name, org_name) return JsonResponse({ 'status': 'success', 'message': 'organization created', 'organization': _dict_org(request, [org])[0] }) @api_endpoint_class @ajax_request_class @method_decorator(permission_required('seed.can_access_admin')) @action(detail=True, methods=['DELETE']) def inventory(self, request, pk=None): """ Starts a background task to delete all properties & taxlots in an org. """ return JsonResponse(tasks.delete_organization_inventory(pk)) @swagger_auto_schema( request_body=SaveSettingsSerializer, ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_owner') @action(detail=True, methods=['PUT']) def save_settings(self, request, pk=None): """ Saves an organization's settings: name, query threshold, shared fields, etc """ body = request.data org = Organization.objects.get(pk=pk) posted_org = body.get('organization', None) if posted_org is None: return JsonResponse( { 'status': 'error', 'message': 'malformed request' }, status=status.HTTP_400_BAD_REQUEST) desired_threshold = posted_org.get('query_threshold', None) if desired_threshold is not None: org.query_threshold = desired_threshold desired_name = posted_org.get('name', None) if desired_name is not None: org.name = desired_name def is_valid_choice(choice_tuples, s): """choice_tuples is std model ((value, label), ...)""" return (s is not None) and (s in [ choice[0] for choice in choice_tuples ]) def warn_bad_pint_spec(kind, unit_string): if unit_string is not None: _log.warn("got bad {0} unit string {1} for org {2}".format( kind, unit_string, org.name)) def warn_bad_units(kind, unit_string): _log.warn("got bad {0} unit string {1} for org {2}".format( kind, unit_string, org.name)) desired_display_units_eui = posted_org.get('display_units_eui') if is_valid_choice(Organization.MEASUREMENT_CHOICES_EUI, desired_display_units_eui): org.display_units_eui = desired_display_units_eui else: warn_bad_pint_spec('eui', desired_display_units_eui) desired_display_units_area = posted_org.get('display_units_area') if is_valid_choice(Organization.MEASUREMENT_CHOICES_AREA, desired_display_units_area): org.display_units_area = desired_display_units_area else: warn_bad_pint_spec('area', desired_display_units_area) desired_display_significant_figures = posted_org.get( 'display_significant_figures') if isinstance( desired_display_significant_figures, int) and desired_display_significant_figures >= 0: # noqa org.display_significant_figures = desired_display_significant_figures elif desired_display_significant_figures is not None: _log.warn("got bad sig figs {0} for org {1}".format( desired_display_significant_figures, org.name)) desired_display_meter_units = posted_org.get('display_meter_units') if desired_display_meter_units: org.display_meter_units = desired_display_meter_units desired_thermal_conversion_assumption = posted_org.get( 'thermal_conversion_assumption') if is_valid_choice(Organization.THERMAL_CONVERSION_ASSUMPTION_CHOICES, desired_thermal_conversion_assumption): org.thermal_conversion_assumption = desired_thermal_conversion_assumption # Update MapQuest API Key if it's been changed mapquest_api_key = posted_org.get('mapquest_api_key', '') if mapquest_api_key != org.mapquest_api_key: org.mapquest_api_key = mapquest_api_key comstock_enabled = posted_org.get('comstock_enabled', False) if comstock_enabled != org.comstock_enabled: org.comstock_enabled = comstock_enabled org.save() # Update the selected exportable fields. new_public_column_names = posted_org.get('public_fields', None) if new_public_column_names is not None: old_public_columns = Column.objects.filter( organization=org, shared_field_type=Column.SHARED_PUBLIC) # turn off sharing in the old_pub_fields for col in old_public_columns: col.shared_field_type = Column.SHARED_NONE col.save() # for now just iterate over this to grab the new columns. for col in new_public_column_names: new_col = Column.objects.filter(organization=org, id=col['id']) if len(new_col) == 1: new_col = new_col.first() new_col.shared_field_type = Column.SHARED_PUBLIC new_col.save() return JsonResponse({'status': 'success'}) @has_perm_class('requires_member') @api_endpoint_class @ajax_request_class @action(detail=True, methods=['GET']) def query_threshold(self, request, pk=None): """ Returns the "query_threshold" for an org. Searches from members of sibling orgs must return at least this many buildings from orgs they do not belong to, or else buildings from orgs they don't belong to will be removed from the results. """ org = Organization.objects.get(pk=pk) return JsonResponse({ 'status': 'success', 'query_threshold': org.query_threshold }) @swagger_auto_schema(responses={200: SharedFieldsReturnSerializer}) @has_perm_class('requires_member') @api_endpoint_class @ajax_request_class @action(detail=True, methods=['GET']) def shared_fields(self, request, pk=None): """ Retrieves all fields marked as shared for the organization. Will only return used fields. """ result = {'status': 'success', 'public_fields': []} columns = Column.retrieve_all(pk, 'property', True) for c in columns: if c['sharedFieldType'] == 'Public': new_column = { 'table_name': c['table_name'], 'name': c['name'], 'column_name': c['column_name'], # this is the field name in the db. The other name can have tax_ 'display_name': c['display_name'] } result['public_fields'].append(new_column) return JsonResponse(result) @swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory( { 'sub_org_name': 'string', 'sub_org_owner_email': 'string', }, required=['sub_org_name', 'sub_org_owner_email'], description='Properties:\n' '- sub_org_name: Name of the new sub organization\n' '- sub_org_owner_email: Email of the owner of the sub organization, which must already exist', )) @has_perm_class('requires_member') @api_endpoint_class @ajax_request_class @action(detail=True, methods=['POST']) def sub_org(self, request, pk=None): """ Creates a child org of a parent org. """ body = request.data org = Organization.objects.get(pk=pk) email = body['sub_org_owner_email'].lower() try: user = User.objects.get(username=email) except User.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'User with email address (%s) does not exist' % email }, status=status.HTTP_400_BAD_REQUEST) created, mess_or_org, _ = create_suborganization( user, org, body['sub_org_name'], ROLE_OWNER) if created: return JsonResponse({ 'status': 'success', 'organization_id': mess_or_org.pk }) else: return JsonResponse({ 'status': 'error', 'message': mess_or_org }, status=status.HTTP_409_CONFLICT) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def matching_criteria_columns(self, request, pk=None): """ Retrieve all matching criteria columns for an org. """ try: org = Organization.objects.get(pk=pk) except ObjectDoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not retrieve organization at pk = ' + str(pk) }, status=status.HTTP_404_NOT_FOUND) matching_criteria_column_names = dict( org.column_set.filter( is_matching_criteria=True).values('table_name').annotate( column_names=ArrayAgg('column_name')).values_list( 'table_name', 'column_names')) return JsonResponse(matching_criteria_column_names) @swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory( {'inventory_type': 'string'}, required=['inventory_type'], description='Properties:\n' '- inventory_type: either "properties" or "taxlots"')) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['POST']) def match_merge_link(self, request, pk=None): """ Run match_merge_link for an org. """ inventory_type = request.data.get('inventory_type', None) if inventory_type not in ['properties', 'taxlots']: return JsonResponse( { 'status': 'error', 'message': 'Provided inventory type should either be "properties" or "taxlots".' }, status=status.HTTP_404_NOT_FOUND) try: org = Organization.objects.get(pk=pk) except ObjectDoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not retrieve organization at pk = ' + str(pk) }, status=status.HTTP_404_NOT_FOUND) state_class_name = 'PropertyState' if inventory_type == 'properties' else 'TaxLotState' progress_key = self._start_whole_org_match_merge_link( org.id, state_class_name) return JsonResponse({'progress_key': progress_key}) @swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory( { 'inventory_type': 'string', 'add': ['string'], 'remove': ['string'], }, required=['inventory_type'], description='Properties:\n' '- inventory_type: either "properties" or "taxlots"\n' '- add: list of column names\n' '- remove: list of column names')) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['POST']) def match_merge_link_preview(self, request, pk=None): """ Run match_merge_link preview for an org and record type. """ inventory_type = request.data.get('inventory_type', None) if inventory_type not in ['properties', 'taxlots']: return JsonResponse( { 'status': 'error', 'message': 'Provided inventory type should either be "properties" or "taxlots".' }, status=status.HTTP_404_NOT_FOUND) try: org = Organization.objects.get(pk=pk) except ObjectDoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not retrieve organization at pk = ' + str(pk) }, status=status.HTTP_404_NOT_FOUND) state_class_name = 'PropertyState' if inventory_type == 'properties' else 'TaxLotState' current_columns = matching_criteria_column_names( org.id, state_class_name) add = set(request.data.get('add', [])) remove = set(request.data.get('remove', [])) provided_columns = Column.objects.filter( column_name__in=add.union(remove), organization_id=org.id, table_name=state_class_name) if provided_columns.count() != (len(add) + len(remove)): return JsonResponse( { 'status': 'error', 'message': 'Invalid column names provided.' }, status=status.HTTP_404_NOT_FOUND) proposed_columns = current_columns.union(add).difference(remove) progress_key = self._start_whole_org_match_merge_link( org.id, state_class_name, list(proposed_columns)) return JsonResponse({'progress_key': progress_key}) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_integer_field( 'match_merge_link_id', required=True, description='ID of match merge link') ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def match_merge_link_result(self, request, pk=None): try: Organization.objects.get(pk=pk) except ObjectDoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not retrieve organization at pk = ' + str(pk) }, status=status.HTTP_404_NOT_FOUND) identifier = request.query_params['match_merge_link_id'] result_key = _get_match_merge_link_key(identifier) # using unsafe serialization b/c the result might not be a dict return JsonResponse(get_cache_raw(result_key), safe=False) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def geocoding_columns(self, request, pk=None): """ Retrieve all geocoding columns for an org. """ try: org = Organization.objects.get(pk=pk) except ObjectDoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not retrieve organization at pk = ' + str(pk) }, status=status.HTTP_404_NOT_FOUND) geocoding_columns_qs = org.column_set.\ filter(geocoding_order__gt=0).\ order_by('geocoding_order').\ values('table_name', 'column_name') geocoding_columns = { 'PropertyState': [], 'TaxLotState': [], } for col in geocoding_columns_qs: geocoding_columns[col['table_name']].append(col['column_name']) return JsonResponse(geocoding_columns) def get_cycles(self, start, end, organization_id): if not isinstance(start, type(end)): raise TypeError('start and end not same types') # if of type int or convertable assume they are cycle ids try: start = int(start) end = int(end) except ValueError as error: # noqa # assume string is JS date if isinstance(start, basestring): start_datetime = dateutil.parser.parse(start) end_datetime = dateutil.parser.parse(end) else: raise Exception('Date is not a string') # get date times from cycles if isinstance(start, int): cycle = Cycle.objects.get(pk=start, organization_id=organization_id) start_datetime = cycle.start if start == end: end_datetime = cycle.end else: end_datetime = Cycle.objects.get( pk=end, organization_id=organization_id).end return Cycle.objects.filter( start__gte=start_datetime, end__lte=end_datetime, organization_id=organization_id).order_by('start') def get_data(self, property_view, x_var, y_var): result = None state = property_view.state if getattr(state, x_var, None) and getattr(state, y_var, None): result = { "id": property_view.property_id, "x": getattr(state, x_var), "y": getattr(state, y_var), } return result def get_raw_report_data(self, organization_id, cycles, x_var, y_var, campus_only): all_property_views = PropertyView.objects.select_related( 'property', 'state').filter(property__organization_id=organization_id, cycle_id__in=cycles) organization = Organization.objects.get(pk=organization_id) results = [] for cycle in cycles: property_views = all_property_views.filter(cycle_id=cycle) count_total = [] count_with_data = [] data = [] for property_view in property_views: property_pk = property_view.property_id if property_view.property.campus and campus_only: count_total.append(property_pk) result = self.get_data(property_view, x_var, y_var) if result: result['yr_e'] = cycle.end.strftime('%Y') data.append(result) count_with_data.append(property_pk) elif not property_view.property.campus: count_total.append(property_pk) result = self.get_data(property_view, x_var, y_var) if result: result['yr_e'] = cycle.end.strftime('%Y') de_unitted_result = apply_display_unit_preferences( organization, result) data.append(de_unitted_result) count_with_data.append(property_pk) result = { "cycle_id": cycle.pk, "chart_data": data, "property_counts": { "yr_e": cycle.end.strftime('%Y'), "num_properties": len(count_total), "num_properties_w-data": len(count_with_data), }, } results.append(result) return results @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_string_field( 'x_var', required=True, description='Raw column name for x axis'), AutoSchemaHelper.query_string_field( 'y_var', required=True, description='Raw column name for y axis'), AutoSchemaHelper.query_string_field( 'start', required=True, description='Start time, in the format "2018-12-31T23:53:00-08:00"' ), AutoSchemaHelper.query_string_field( 'end', required=True, description='End time, in the format "2018-12-31T23:53:00-08:00"'), AutoSchemaHelper.query_string_field( 'campus_only', required=False, description='If true, includes campuses'), ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def report(self, request, pk=None): """ Retrieve a summary report for charting x vs y """ campus_only = json.loads( request.query_params.get('campus_only', 'false')) params = {} missing_params = [] error = '' for param in ['x_var', 'y_var', 'start', 'end']: val = request.query_params.get(param, None) if not val: missing_params.append(param) else: params[param] = val if missing_params: error = "{} Missing params: {}".format(error, ", ".join(missing_params)) if error: status_code = status.HTTP_400_BAD_REQUEST result = {'status': 'error', 'message': error} else: cycles = self.get_cycles(params['start'], params['end'], pk) data = self.get_raw_report_data(pk, cycles, params['x_var'], params['y_var'], campus_only) for datum in data: if datum['property_counts']['num_properties_w-data'] != 0: break property_counts = [] chart_data = [] for datum in data: property_counts.append(datum['property_counts']) chart_data.extend(datum['chart_data']) data = { 'property_counts': property_counts, 'chart_data': chart_data, } result = {'status': 'success', 'data': data} status_code = status.HTTP_200_OK return Response(result, status=status_code) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_string_field( 'x_var', required=True, description='Raw column name for x axis'), AutoSchemaHelper.query_string_field( 'y_var', required=True, description= 'Raw column name for y axis, must be one of: "gross_floor_area", "use_description", "year_built"' ), AutoSchemaHelper.query_string_field( 'start', required=True, description='Start time, in the format "2018-12-31T23:53:00-08:00"' ), AutoSchemaHelper.query_string_field( 'end', required=True, description='End time, in the format "2018-12-31T23:53:00-08:00"'), AutoSchemaHelper.query_string_field( 'campus_only', required=False, description='If true, includes campuses'), ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def report_aggregated(self, request, pk=None): """ Retrieve a summary report for charting x vs y aggregated by y_var """ campus_only = json.loads( request.query_params.get('campus_only', 'false')) valid_y_values = ['gross_floor_area', 'use_description', 'year_built'] params = {} missing_params = [] empty = True error = '' for param in ['x_var', 'y_var', 'start', 'end']: val = request.query_params.get(param, None) if not val: missing_params.append(param) elif param == 'y_var' and val not in valid_y_values: error = "{} {} is not a valid value for {}.".format( error, val, param) else: params[param] = val if missing_params: error = "{} Missing params: {}".format(error, ", ".join(missing_params)) if error: status_code = status.HTTP_400_BAD_REQUEST result = {'status': 'error', 'message': error} else: cycles = self.get_cycles(params['start'], params['end'], pk) x_var = params['x_var'] y_var = params['y_var'] data = self.get_raw_report_data(pk, cycles, x_var, y_var, campus_only) for datum in data: if datum['property_counts']['num_properties_w-data'] != 0: empty = False break if empty: result = {'status': 'error', 'message': 'No data found'} status_code = status.HTTP_404_NOT_FOUND if not empty or not error: chart_data = [] property_counts = [] for datum in data: buildings = datum['chart_data'] yr_e = datum['property_counts']['yr_e'] chart_data.extend(self.aggregate_data(yr_e, y_var, buildings)), property_counts.append(datum['property_counts']) # Send back to client aggregated_data = { 'chart_data': chart_data, 'property_counts': property_counts } result = { 'status': 'success', 'aggregated_data': aggregated_data, } status_code = status.HTTP_200_OK return Response(result, status=status_code) def aggregate_data(self, yr_e, y_var, buildings): aggregation_method = { 'use_description': self.aggregate_use_description, 'year_built': self.aggregate_year_built, 'gross_floor_area': self.aggregate_gross_floor_area, } return aggregation_method[y_var](yr_e, buildings) def aggregate_use_description(self, yr_e, buildings): # Group buildings in this year_ending group into uses chart_data = [] grouped_uses = defaultdict(list) for b in buildings: grouped_uses[str(b['y']).lower()].append(b) # Now iterate over use groups to make each chart item for use, buildings_in_uses in grouped_uses.items(): chart_data.append({ 'x': median([b['x'] for b in buildings_in_uses]), 'y': use.capitalize(), 'yr_e': yr_e }) return chart_data def aggregate_year_built(self, yr_e, buildings): # Group buildings in this year_ending group into decades chart_data = [] grouped_decades = defaultdict(list) for b in buildings: grouped_decades['%s0' % str(b['y'])[:-1]].append(b) # Now iterate over decade groups to make each chart item for decade, buildings_in_decade in grouped_decades.items(): chart_data.append({ 'x': median([b['x'] for b in buildings_in_decade]), 'y': '%s-%s' % (decade, '%s9' % str(decade)[:-1]), # 1990-1999 'yr_e': yr_e }) return chart_data def aggregate_gross_floor_area(self, yr_e, buildings): chart_data = [] y_display_map = { 0: '0-99k', 100000: '100-199k', 200000: '200k-299k', 300000: '300k-399k', 400000: '400-499k', 500000: '500-599k', 600000: '600-699k', 700000: '700-799k', 800000: '800-899k', 900000: '900-999k', 1000000: 'over 1,000k', } max_bin = max(y_display_map) # Group buildings in this year_ending group into ranges grouped_ranges = defaultdict(list) for b in buildings: area = b['y'] # make sure anything greater than the biggest bin gets put in # the biggest bin range_bin = min(max_bin, round_down_hundred_thousand(area)) grouped_ranges[range_bin].append(b) # Now iterate over range groups to make each chart item for range_floor, buildings_in_range in grouped_ranges.items(): chart_data.append({ 'x': median([b['x'] for b in buildings_in_range]), 'y': y_display_map[range_floor], 'yr_e': yr_e }) return chart_data @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_string_field( 'x_var', required=True, description='Raw column name for x axis'), AutoSchemaHelper.query_string_field('x_label', required=True, description='Label for x axis'), AutoSchemaHelper.query_string_field( 'y_var', required=True, description='Raw column name for y axis'), AutoSchemaHelper.query_string_field('y_label', required=True, description='Label for y axis'), AutoSchemaHelper.query_string_field( 'start', required=True, description='Start time, in the format "2018-12-31T23:53:00-08:00"' ), AutoSchemaHelper.query_string_field( 'end', required=True, description='End time, in the format "2018-12-31T23:53:00-08:00"'), ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def report_export(self, request, pk=None): """ Export a report as a spreadsheet """ params = {} missing_params = [] error = '' for param in ['x_var', 'x_label', 'y_var', 'y_label', 'start', 'end']: val = request.query_params.get(param, None) if not val: missing_params.append(param) else: params[param] = val if missing_params: error = "{} Missing params: {}".format(error, ", ".join(missing_params)) if error: status_code = status.HTTP_400_BAD_REQUEST result = {'status': 'error', 'message': error} return Response(result, status=status_code) response = HttpResponse( content_type= 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) response['Content-Disposition'] = 'attachment; filename="report-data"' # Create WB output = BytesIO() wb = Workbook(output, {'remove_timezone': True}) # Create sheets count_sheet = wb.add_worksheet('Counts') base_sheet = wb.add_worksheet('Raw') agg_sheet = wb.add_worksheet('Agg') # Enable bold format and establish starting cells bold = wb.add_format({'bold': True}) data_row_start = 0 data_col_start = 0 # Write all headers across all sheets count_sheet.write(data_row_start, data_col_start, 'Year Ending', bold) count_sheet.write(data_row_start, data_col_start + 1, 'Properties with Data', bold) count_sheet.write(data_row_start, data_col_start + 2, 'Total Properties', bold) base_sheet.write(data_row_start, data_col_start, 'ID', bold) base_sheet.write(data_row_start, data_col_start + 1, request.query_params.get('x_label'), bold) base_sheet.write(data_row_start, data_col_start + 2, request.query_params.get('y_label'), bold) base_sheet.write(data_row_start, data_col_start + 3, 'Year Ending', bold) agg_sheet.write(data_row_start, data_col_start, request.query_params.get('x_label'), bold) agg_sheet.write(data_row_start, data_col_start + 1, request.query_params.get('y_label'), bold) agg_sheet.write(data_row_start, data_col_start + 2, 'Year Ending', bold) # Gather base data cycles = self.get_cycles(params['start'], params['end'], pk) data = self.get_raw_report_data(pk, cycles, params['x_var'], params['y_var'], False) base_row = data_row_start + 1 agg_row = data_row_start + 1 count_row = data_row_start + 1 for cycle_results in data: total_count = cycle_results['property_counts']['num_properties'] with_data_count = cycle_results['property_counts'][ 'num_properties_w-data'] yr_e = cycle_results['property_counts']['yr_e'] # Write Counts count_sheet.write(count_row, data_col_start, yr_e) count_sheet.write(count_row, data_col_start + 1, with_data_count) count_sheet.write(count_row, data_col_start + 2, total_count) count_row += 1 # Write Base/Raw Data data_rows = cycle_results['chart_data'] for datum in data_rows: base_sheet.write(base_row, data_col_start, datum.get('id')) base_sheet.write(base_row, data_col_start + 1, datum.get('x')) base_sheet.write(base_row, data_col_start + 2, datum.get('y')) base_sheet.write(base_row, data_col_start + 3, datum.get('yr_e')) base_row += 1 # Gather and write Agg data for agg_datum in self.aggregate_data(yr_e, params['y_var'], data_rows): agg_sheet.write(agg_row, data_col_start, agg_datum.get('x')) agg_sheet.write(agg_row, data_col_start + 1, agg_datum.get('y')) agg_sheet.write(agg_row, data_col_start + 2, agg_datum.get('yr_e')) agg_row += 1 wb.close() xlsx_data = output.getvalue() response.write(xlsx_data) return response @has_perm_class('requires_member') @ajax_request_class @action(detail=True, methods=['GET']) def geocode_api_key_exists(self, request, pk=None): """ Returns true if the organization has a mapquest api key """ org = Organization.objects.get(id=pk) if org.mapquest_api_key: return True else: return False
class ImportFileViewSet(viewsets.ViewSet): raise_exception = True queryset = ImportFile.objects.all() @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): """ Retrieves details about an ImportFile. """ import_file_id = pk orgs = request.user.orgs.all() try: import_file = ImportFile.objects.get(pk=import_file_id) d = ImportRecord.objects.filter(super_organization__in=orgs, pk=import_file.import_record_id) except ObjectDoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not access an import file with this ID' }, status=status.HTTP_403_FORBIDDEN) # check if user has access to the import file if not d.exists(): return JsonResponse( { 'status': 'error', 'message': 'Could not locate import file with this ID', 'import_file': {}, }, status=status.HTTP_400_BAD_REQUEST) f = obj_to_dict(import_file) f['name'] = import_file.filename_only if not import_file.uploaded_filename: f['uploaded_filename'] = import_file.filename f['dataset'] = obj_to_dict(import_file.import_record) return JsonResponse({ 'status': 'success', 'import_file': f, }) @swagger_auto_schema_org_query_param @has_perm_class('requires_viewer') @api_endpoint_class @ajax_request_class @action(detail=True, methods=['GET']) def first_five_rows(self, request, pk=None): """ Retrieves the first five rows of an ImportFile. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) if import_file.cached_second_to_fifth_row is None: return JsonResponse( { 'status': 'error', 'message': 'Internal problem occurred, import file first five rows not cached' }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) ''' import_file.cached_second_to_fifth_row is a field that contains the first 5 lines of data from the file, split on newlines, delimited by ROW_DELIMITER. This becomes an issue when fields have newlines in them, so the following is to handle newlines in the fields. In the case of only one data column there will be no ROW_DELIMITER. ''' header = import_file.cached_first_row.split(ROW_DELIMITER) data = import_file.cached_second_to_fifth_row return JsonResponse({ 'status': 'success', 'first_five_rows': convert_first_five_rows_to_list(header, data) }) @swagger_auto_schema_org_query_param @has_perm_class('requires_viewer') @api_endpoint_class @ajax_request_class @action(detail=True, methods=['GET']) def raw_column_names(self, request, pk=None): """ Retrieves a list of all column names from an ImportFile. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return JsonResponse({ 'status': 'success', 'raw_columns': import_file.first_row_columns }) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], responses={200: MappingResultsResponseSerializer}) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST'], url_path='mapping_results') def mapping_results(self, request, pk=None): """ Retrieves a paginated list of Properties and Tax Lots for an import file after mapping. """ import_file_id = pk org_id = request.query_params.get('organization_id', None) org = Organization.objects.get(pk=org_id) try: # get the field names that were in the mapping import_file = ImportFile.objects.get( pk=import_file_id, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # List of the only fields to show field_names = import_file.get_cached_mapped_columns # set of fields fields = { 'PropertyState': ['id', 'extra_data', 'lot_number'], 'TaxLotState': ['id', 'extra_data'] } columns_from_db = Column.retrieve_all(org_id) property_column_name_mapping = {} taxlot_column_name_mapping = {} for field_name in field_names: # find the field from the columns in the database for column in columns_from_db: if column['table_name'] == 'PropertyState' and \ field_name[0] == 'PropertyState' and \ field_name[1] == column['column_name']: property_column_name_mapping[ column['column_name']] = column['name'] if not column['is_extra_data']: fields['PropertyState'].append( field_name[1]) # save to the raw db fields continue elif column['table_name'] == 'TaxLotState' and \ field_name[0] == 'TaxLotState' and \ field_name[1] == column['column_name']: taxlot_column_name_mapping[ column['column_name']] = column['name'] if not column['is_extra_data']: fields['TaxLotState'].append( field_name[1]) # save to the raw db fields continue inventory_type = request.data.get('inventory_type', 'all') result = {'status': 'success'} if inventory_type == 'properties' or inventory_type == 'all': properties = PropertyState.objects.filter( import_file_id=import_file_id, data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], merge_state__in=[ MERGE_STATE_UNKNOWN, MERGE_STATE_NEW ]).only(*fields['PropertyState']).order_by('id') property_results = [] for prop in properties: prop_dict = TaxLotProperty.model_to_dict_with_mapping( prop, property_column_name_mapping, fields=fields['PropertyState'], exclude=['extra_data']) prop_dict.update( TaxLotProperty.extra_data_to_dict_with_mapping( prop.extra_data, property_column_name_mapping, fields=prop.extra_data.keys(), ).items()) prop_dict = apply_display_unit_preferences(org, prop_dict) property_results.append(prop_dict) result['properties'] = property_results if inventory_type == 'taxlots' or inventory_type == 'all': tax_lots = TaxLotState.objects.filter( import_file_id=import_file_id, data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], merge_state__in=[MERGE_STATE_UNKNOWN, MERGE_STATE_NEW ]).only(*fields['TaxLotState']).order_by('id') tax_lot_results = [] for tax_lot in tax_lots: tax_lot_dict = TaxLotProperty.model_to_dict_with_mapping( tax_lot, taxlot_column_name_mapping, fields=fields['TaxLotState'], exclude=['extra_data']) tax_lot_dict.update( TaxLotProperty.extra_data_to_dict_with_mapping( tax_lot.extra_data, taxlot_column_name_mapping, fields=tax_lot.extra_data.keys(), ).items()) tax_lot_dict = apply_display_unit_preferences( org, tax_lot_dict) tax_lot_results.append(tax_lot_dict) result['tax_lots'] = tax_lot_results return result @staticmethod def has_coparent(state_id, inventory_type, fields=None): """ Return the coparent of the current state id based on the inventory type. If fields are given (as a list), then it will only return the fields specified of the state model object as a dictionary. :param state_id: int, ID of PropertyState or TaxLotState :param inventory_type: string, either properties | taxlots :param fields: list, either None or list of fields to return :return: dict or state object, If fields is not None then will return state_object """ state_model = PropertyState if inventory_type == 'properties' else TaxLotState # TODO: convert coparent to instance method, not class method audit_entry, audit_count = state_model.coparent(state_id) if audit_count == 0: return False if audit_count > 1: return JsonResponse( { 'status': 'error', 'message': 'Internal problem occurred, more than one merge record found' }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return audit_entry[0] @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=AutoSchemaHelper.schema_factory({ 'remap': 'boolean', 'mark_as_done': 'boolean', })) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def map(self, request, pk=None): """ Starts a background task to convert imported raw data into PropertyState and TaxLotState, using user's column mappings. """ body = request.data remap = body.get('remap', False) mark_as_done = body.get('mark_as_done', True) org_id = request.query_params.get('organization_id', None) import_file = ImportFile.objects.filter( pk=pk, import_record__super_organization_id=org_id) if not import_file.exists(): return { 'status': 'error', 'message': 'ImportFile {} does not exist'.format(pk) } # return remap_data(import_file_id) return JsonResponse(map_data(pk, remap, mark_as_done)) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def start_system_matching_and_geocoding(self, request, pk=None): """ Starts a background task to attempt automatic matching between buildings in an ImportFile with other existing buildings within the same org. """ org_id = request.query_params.get('organization_id', None) try: ImportFile.objects.get(pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return geocode_and_match_buildings_task(pk) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def start_data_quality_checks(self, request, pk=None): """ Starts a background task to attempt automatic matching between buildings in an ImportFile with other existing buildings within the same org. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return_value = do_checks(org_id, None, None, import_file.pk) # step 5: create a new model instance return JsonResponse({ 'progress_key': return_value['progress_key'], 'progress': return_value, }) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def validate_use_cases(self, request, pk=None): """ Starts a background task to call BuildingSync's use case validation tool. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return task_validate_use_cases(import_file.pk) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=AutoSchemaHelper.schema_factory( {'cycle_id': 'string'})) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def start_save_data(self, request, pk=None): """ Starts a background task to import raw data from an ImportFile into PropertyState objects as extra_data. If the cycle_id is set to year_ending then the cycle ID will be set to the year_ending column for each record in the uploaded file. Note that the year_ending flag is not yet enabled. """ body = request.data import_file_id = pk if not import_file_id: return JsonResponse( { 'status': 'error', 'message': 'must pass file_id of file to save' }, status=status.HTTP_400_BAD_REQUEST) org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=import_file_id, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) cycle_id = body.get('cycle_id') if not cycle_id: return JsonResponse( { 'status': 'error', 'message': 'must pass cycle_id of the cycle to save the data' }, status=status.HTTP_400_BAD_REQUEST) elif cycle_id == 'year_ending': _log.error("NOT CONFIGURED FOR YEAR ENDING OPTION AT THE MOMENT") return JsonResponse( { 'status': 'error', 'message': 'SEED is unable to parse year_ending at the moment' }, status=status.HTTP_400_BAD_REQUEST) else: # find the cycle cycle = Cycle.objects.get(id=cycle_id) if cycle: # assign the cycle id to the import file object import_file.cycle = cycle import_file.save() else: return JsonResponse( { 'status': 'error', 'message': 'cycle_id was invalid' }, status=status.HTTP_400_BAD_REQUEST) return JsonResponse(task_save_raw(import_file.id)) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def mapping_done(self, request, pk=None): """ Tell the backend that the mapping is complete. """ import_file_id = pk if not import_file_id: return JsonResponse( { 'status': 'error', 'message': 'must pass import_file_id' }, status=status.HTTP_400_BAD_REQUEST) org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=import_file_id, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'no import file with given id' }, status=status.HTTP_404_NOT_FOUND) import_file.mapping_done = True import_file.save() return JsonResponse({'status': 'success', 'message': ''}) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def matching_and_geocoding_results(self, request, pk=None): """ Retrieves the number of matched and unmatched properties & tax lots for a given ImportFile record. Specifically for new imports """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # property views associated with this imported file (including merges) properties_new = [] properties_matched = list( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED, ).values_list('id', flat=True)) # Check audit log in case PropertyStates are listed as "new" but were merged into a different property properties = list( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_NEW, )) for state in properties: audit_creation_id = PropertyAuditLog.objects.only('id').exclude( import_filename=None).get(state_id=state.id, name='Import Creation') if PropertyAuditLog.objects.exclude( record_type=AUDIT_USER_EDIT).filter( parent1_id=audit_creation_id).exists(): properties_matched.append(state.id) else: properties_new.append(state.id) tax_lots_new = [] tax_lots_matched = list( TaxLotState.objects.only('id').filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED, ).values_list('id', flat=True)) # Check audit log in case TaxLotStates are listed as "new" but were merged into a different tax lot taxlots = list( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_NEW, )) for state in taxlots: audit_creation_id = TaxLotAuditLog.objects.only('id').exclude( import_filename=None).get(state_id=state.id, name='Import Creation') if TaxLotAuditLog.objects.exclude( record_type=AUDIT_USER_EDIT).filter( parent1_id=audit_creation_id).exists(): tax_lots_matched.append(state.id) else: tax_lots_new.append(state.id) # Construct Geocode Results property_geocode_results = { 'high_confidence': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='High')), 'low_confidence': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='Low')), 'manual': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Manually geocoded (N/A)')), 'missing_address_components': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Missing address components (N/A)')), } tax_lot_geocode_results = { 'high_confidence': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='High')), 'low_confidence': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='Low')), 'manual': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Manually geocoded (N/A)')), 'missing_address_components': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Missing address components (N/A)')), } # merge in any of the matching results from the JSON field return { 'status': 'success', 'import_file_records': import_file.matching_results_data.get('import_file_records', None), 'properties': { 'initial_incoming': import_file.matching_results_data.get( 'property_initial_incoming', None), 'duplicates_against_existing': import_file.matching_results_data.get( 'property_duplicates_against_existing', None), 'duplicates_within_file': import_file.matching_results_data.get( 'property_duplicates_within_file', None), 'merges_against_existing': import_file.matching_results_data.get( 'property_merges_against_existing', None), 'merges_between_existing': import_file.matching_results_data.get( 'property_merges_between_existing', None), 'merges_within_file': import_file.matching_results_data.get( 'property_merges_within_file', None), 'new': import_file.matching_results_data.get('property_new', None), 'geocoded_high_confidence': property_geocode_results.get('high_confidence'), 'geocoded_low_confidence': property_geocode_results.get('low_confidence'), 'geocoded_manually': property_geocode_results.get('manual'), 'geocode_not_possible': property_geocode_results.get('missing_address_components'), }, 'tax_lots': { 'initial_incoming': import_file.matching_results_data.get( 'tax_lot_initial_incoming', None), 'duplicates_against_existing': import_file.matching_results_data.get( 'tax_lot_duplicates_against_existing', None), 'duplicates_within_file': import_file.matching_results_data.get( 'tax_lot_duplicates_within_file', None), 'merges_against_existing': import_file.matching_results_data.get( 'tax_lot_merges_against_existing', None), 'merges_between_existing': import_file.matching_results_data.get( 'tax_lot_merges_between_existing', None), 'merges_within_file': import_file.matching_results_data.get( 'tax_lot_merges_within_file', None), 'new': import_file.matching_results_data.get('tax_lot_new', None), 'geocoded_high_confidence': tax_lot_geocode_results.get('high_confidence'), 'geocoded_low_confidence': tax_lot_geocode_results.get('low_confidence'), 'geocoded_manually': tax_lot_geocode_results.get('manual'), 'geocode_not_possible': tax_lot_geocode_results.get('missing_address_components'), } } @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def mapping_suggestions(self, request, pk): """ Returns suggested mappings from an uploaded file's headers to known data fields. """ organization_id = request.query_params.get('organization_id', None) result = {'status': 'success'} membership = OrganizationUser.objects.select_related('organization') \ .get(organization_id=organization_id, user=request.user) organization = membership.organization # For now, each organization holds their own mappings. This is non-ideal, but it is the # way it is for now. In order to move to parent_org holding, then we need to be able to # dynamically match columns based on the names and not the db id (or support many-to-many). # parent_org = organization.get_parent() try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=organization.pk) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # Get a list of the database fields in a list, these are the db columns and the extra_data columns property_columns = Column.retrieve_mapping_columns( organization.pk, 'property') taxlot_columns = Column.retrieve_mapping_columns( organization.pk, 'taxlot') # If this is a portfolio manager file, then load in the PM mappings and if the column_mappings # are not in the original mappings, default to PM if import_file.from_portfolio_manager: pm_mappings = simple_mapper.get_pm_mapping( import_file.first_row_columns, resolve_duplicates=True) suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization_id), previous_mapping=get_column_mapping, map_args=[organization], default_mappings=pm_mappings, thresh=80) elif import_file.from_buildingsync: bsync_mappings = xml_mapper.build_column_mapping() suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization_id), previous_mapping=get_column_mapping, map_args=[organization], default_mappings=bsync_mappings, thresh=80) else: # All other input types suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization.pk), previous_mapping=get_column_mapping, map_args=[organization], thresh= 80 # percentage match that we require. 80% is random value for now. ) # replace None with empty string for column names and PropertyState for tables # TODO #239: Move this fix to build_column_mapping for m in suggested_mappings: table, destination_field, _confidence = suggested_mappings[m] if destination_field is None: suggested_mappings[m][1] = '' # Fix the table name, eventually move this to the build_column_mapping for m in suggested_mappings: table, _destination_field, _confidence = suggested_mappings[m] # Do not return the campus, created, updated fields... that is force them to be in the property state if not table or table == 'Property': suggested_mappings[m][0] = 'PropertyState' elif table == 'TaxLot': suggested_mappings[m][0] = 'TaxLotState' result['suggested_column_mappings'] = suggested_mappings result['property_columns'] = property_columns result['taxlot_columns'] = taxlot_columns return JsonResponse(result) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def destroy(self, request, pk): """ Deletes an import file """ organization_id = int(request.query_params.get('organization_id', None)) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=organization_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # check if the import record exists for the file and organization d = ImportRecord.objects.filter(super_organization_id=organization_id, pk=import_file.import_record.pk) if not d.exists(): return JsonResponse( { 'status': 'error', 'message': 'user does not have permission to delete file', }, status=status.HTTP_403_FORBIDDEN) # This does not actually delete the object because it is a NonDeletableModel import_file.delete() return JsonResponse({'status': 'success'}) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'view_id', required=True, description='ID for property view') ]) @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def greenbutton_meters_preview(self, request, pk): """ Returns validated type units and proposed imports """ org_id = request.query_params.get('organization_id') view_id = request.query_params.get('view_id') try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) parser = reader.GreenButtonParser(import_file.local_file) raw_meter_data = list(parser.data) try: property_id = PropertyView.objects.get( pk=view_id, cycle__organization_id=org_id).property_id except PropertyView.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find property with pk=' + str(view_id) }, status=status.HTTP_400_BAD_REQUEST) meters_parser = MetersParser(org_id, raw_meter_data, source_type=Meter.GREENBUTTON, property_id=property_id) result = {} result["validated_type_units"] = meters_parser.validated_type_units() result["proposed_imports"] = meters_parser.proposed_imports import_file.matching_results_data['property_id'] = property_id import_file.save() return result @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def pm_meters_preview(self, request, pk): """ Returns validated type units, proposed imports and unlinkable PM ids """ org_id = request.query_params.get('organization_id') try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) parser = reader.MCMParser(import_file.local_file, sheet_name='Meter Entries') raw_meter_data = list(parser.data) meters_parser = MetersParser(org_id, raw_meter_data) result = {} result["validated_type_units"] = meters_parser.validated_type_units() result["proposed_imports"] = meters_parser.proposed_imports result["unlinkable_pm_ids"] = meters_parser.unlinkable_pm_ids return result
class TaxlotViewSet(viewsets.ViewSet, OrgMixin, ProfileIdMixin): renderer_classes = (JSONRenderer,) serializer_class = TaxLotSerializer parser_classes = (JSONParser,) _organization = None @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field(required=True)], request_body=AutoSchemaHelper.schema_factory( { 'selected': ['integer'], }, description='IDs for taxlots to be checked for which labels are applied.' ) ) @action(detail=False, methods=['POST']) def labels(self, request): """ Returns a list of all labels where the is_applied field in the response pertains to the labels applied to taxlot_view """ labels = StatusLabel.objects.filter( super_organization=self.get_parent_org(self.request) ).order_by("name").distinct() super_organization = self.get_organization(request) # TODO: refactor to avoid passing request here return get_labels(request, labels, super_organization, 'taxlot_view') def _get_filtered_results(self, request, profile_id): page = request.query_params.get('page', 1) per_page = request.query_params.get('per_page', 1) org_id = request.query_params.get('organization_id', None) cycle_id = request.query_params.get('cycle') # check if there is a query paramater for the profile_id. If so, then use that one profile_id = request.query_params.get('profile_id', profile_id) if not org_id: return JsonResponse( {'status': 'error', 'message': 'Need to pass organization_id as query parameter'}, status=status.HTTP_400_BAD_REQUEST) if cycle_id: cycle = Cycle.objects.get(organization_id=org_id, pk=cycle_id) else: cycle = Cycle.objects.filter(organization_id=org_id).order_by('name') if cycle: cycle = cycle.first() else: return JsonResponse({ 'status': 'error', 'message': 'Could not locate cycle', 'pagination': { 'total': 0 }, 'cycle_id': None, 'results': [] }) # Return taxlot views limited to the 'taxlot_view_ids' list. Otherwise, if selected is empty, return all if 'taxlot_view_ids' in request.data and request.data['taxlot_view_ids']: taxlot_views_list = TaxLotView.objects.select_related('taxlot', 'state', 'cycle') \ .filter(id__in=request.data['taxlot_view_ids'], taxlot__organization_id=org_id, cycle=cycle) \ .order_by('id') else: taxlot_views_list = TaxLotView.objects.select_related('taxlot', 'state', 'cycle') \ .filter(taxlot__organization_id=org_id, cycle=cycle) \ .order_by('id') paginator = Paginator(taxlot_views_list, per_page) try: taxlot_views = paginator.page(page) page = int(page) except PageNotAnInteger: taxlot_views = paginator.page(1) page = 1 except EmptyPage: taxlot_views = paginator.page(paginator.num_pages) page = paginator.num_pages org = Organization.objects.get(pk=org_id) # Retrieve all the columns that are in the db for this organization columns_from_database = Column.retrieve_all(org_id, 'taxlot', False) # This uses an old method of returning the show_columns. There is a new method that # is preferred in v2.1 API with the ProfileIdMixin. if profile_id is None: show_columns = None elif profile_id == -1: show_columns = list(Column.objects.filter( organization_id=org_id ).values_list('id', flat=True)) else: try: profile = ColumnListProfile.objects.get( organization=org, id=profile_id, profile_location=VIEW_LIST, inventory_type=VIEW_LIST_TAXLOT ) show_columns = list(ColumnListProfileColumn.objects.filter( column_list_profile_id=profile.id ).values_list('column_id', flat=True)) except ColumnListProfile.DoesNotExist: show_columns = None related_results = TaxLotProperty.get_related(taxlot_views, show_columns, columns_from_database) # collapse units here so we're only doing the last page; we're already a # realized list by now and not a lazy queryset unit_collapsed_results = [apply_display_unit_preferences(org, x) for x in related_results] response = { 'pagination': { 'page': page, 'start': paginator.page(page).start_index(), 'end': paginator.page(page).end_index(), 'num_pages': paginator.num_pages, 'has_next': paginator.page(page).has_next(), 'has_previous': paginator.page(page).has_previous(), 'total': paginator.count }, 'cycle_id': cycle.id, 'results': unit_collapsed_results } return JsonResponse(response) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'cycle', required=True, description='The ID of the cycle to get taxlots' ), AutoSchemaHelper.query_integer_field( 'page', required=False, description='The current page of taxlots to return' ), AutoSchemaHelper.query_integer_field( 'per_page', required=False, description='The number of items per page to return' ), AutoSchemaHelper.query_integer_field( 'profile_id', required=False, description='The ID of the column profile to use' ) ] ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') def list(self, request): """ List all the properties """ return self._get_filtered_results(request, profile_id=-1) @swagger_auto_schema( request_body=AutoSchemaHelper.schema_factory( { 'organization_id': 'integer', 'profile_id': 'integer', 'cycle_ids': ['integer'], }, required=['organization_id', 'cycle_ids'], description='Properties:\n' '- organization_id: ID of organization\n' '- profile_id: Either an id of a list settings profile, ' 'or undefined\n' '- cycle_ids: The IDs of the cycle to get taxlots' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @action(detail=False, methods=['POST']) def filter_by_cycle(self, request): """ List all the taxlots with all columns """ # NOTE: we are using a POST http method b/c swagger and django handle # arrays differently in query parameters. ie this is just simpler org_id = request.data.get('organization_id', None) profile_id = request.data.get('profile_id', -1) cycle_ids = request.data.get('cycle_ids', []) if not org_id: return JsonResponse( {'status': 'error', 'message': 'Need to pass organization_id as query parameter'}, status=status.HTTP_400_BAD_REQUEST) response = taxlots_across_cycles(org_id, profile_id, cycle_ids) return JsonResponse(response) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'cycle', required=True, description='The ID of the cycle to get tax lots' ), AutoSchemaHelper.query_integer_field( 'page', required=False, description='The current page of taxlots to return' ), AutoSchemaHelper.query_integer_field( 'per_page', required=False, description='The number of items per page to return' ), ], request_body=AutoSchemaHelper.schema_factory( { 'profile_id': 'integer', 'taxlot_view_ids': ['integer'], }, required=['profile_id'], description='Properties:\n' '- profile_id: Either an id of a list settings profile, or undefined\n' '- taxlot_view_ids: List of taxlot view ids' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @action(detail=False, methods=['POST']) def filter(self, request): """ List all the tax lots """ if 'profile_id' not in request.data: profile_id = None else: if request.data['profile_id'] == 'None': profile_id = None else: profile_id = request.data['profile_id'] return self._get_filtered_results(request, profile_id=profile_id) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=AutoSchemaHelper.schema_factory( { 'taxlot_view_ids': ['integer'] }, required=['taxlot_view_ids'], description='Properties:\n' '- taxlot_view_ids: Array containing tax lot state ids to merge' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def merge(self, request): """ Merge multiple tax lot records into a single new record, and run this new record through a match and merge round within it's current Cycle. """ body = request.data organization_id = int(request.query_params.get('organization_id', None)) taxlot_view_ids = body.get('taxlot_view_ids', []) taxlot_states = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=organization_id ).values('id', 'state_id') # get the state ids in order according to the given view ids taxlot_states_dict = {t['id']: t['state_id'] for t in taxlot_states} taxlot_state_ids = [ taxlot_states_dict[view_id] for view_id in taxlot_view_ids if view_id in taxlot_states_dict ] if len(taxlot_state_ids) != len(taxlot_view_ids): return { 'status': 'error', 'message': 'All records not found.' } # Check the number of taxlot_state_ids to merge if len(taxlot_state_ids) < 2: return JsonResponse({ 'status': 'error', 'message': 'At least two ids are necessary to merge' }, status=status.HTTP_400_BAD_REQUEST) merged_state = merge_taxlots(taxlot_state_ids, organization_id, 'Manual Match') merge_count, link_count, view_id = match_merge_link(merged_state.taxlotview_set.first().id, 'TaxLotState') result = { 'status': 'success' } result.update({ 'match_merged_count': merge_count, 'match_link_count': link_count, }) return result @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def unmerge(self, request, pk=None): """ Unmerge a taxlot view into two taxlot views """ try: old_view = TaxLotView.objects.select_related( 'taxlot', 'cycle', 'state' ).get( id=pk, taxlot__organization_id=self.request.GET['organization_id'] ) except TaxLotView.DoesNotExist: return { 'status': 'error', 'message': 'taxlot view with id {} does not exist'.format(pk) } # Duplicate pairing paired_view_ids = list(TaxLotProperty.objects.filter(taxlot_view_id=old_view.id) .order_by('property_view_id').values_list('property_view_id', flat=True)) # Capture previous associated labels label_ids = list(old_view.labels.all().values_list('id', flat=True)) notes = old_view.notes.all() for note in notes: note.taxlot_view = None merged_state = old_view.state if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED: return { 'status': 'error', 'message': 'taxlot view with id {} is not a merged taxlot view'.format(pk) } log = TaxLotAuditLog.objects.select_related('parent_state1', 'parent_state2').filter( state=merged_state ).order_by('-id').first() if log.parent_state1 is None or log.parent_state2 is None: return { 'status': 'error', 'message': 'taxlot view with id {} must have two parent states'.format(pk) } state1 = log.parent_state1 state2 = log.parent_state2 cycle_id = old_view.cycle_id # Clone the taxlot record twice old_taxlot = old_view.taxlot new_taxlot = old_taxlot new_taxlot.id = None new_taxlot.save() new_taxlot_2 = TaxLot.objects.get(pk=new_taxlot.pk) new_taxlot_2.id = None new_taxlot_2.save() # If the canonical TaxLot is NOT associated to another -View if not TaxLotView.objects.filter(taxlot_id=old_view.taxlot_id).exclude(pk=old_view.id).exists(): TaxLot.objects.get(pk=old_view.taxlot_id).delete() # Create the views new_view1 = TaxLotView( cycle_id=cycle_id, taxlot_id=new_taxlot.id, state=state1 ) new_view2 = TaxLotView( cycle_id=cycle_id, taxlot_id=new_taxlot_2.id, state=state2 ) # Mark the merged state as deleted merged_state.merge_state = MERGE_STATE_DELETE merged_state.save() # Change the merge_state of the individual states if log.parent1.name in ['Import Creation', 'Manual Edit'] and log.parent1.import_filename is not None: # State belongs to a new record state1.merge_state = MERGE_STATE_NEW else: state1.merge_state = MERGE_STATE_MERGED if log.parent2.name in ['Import Creation', 'Manual Edit'] and log.parent2.import_filename is not None: # State belongs to a new record state2.merge_state = MERGE_STATE_NEW else: state2.merge_state = MERGE_STATE_MERGED # In most cases data_state will already be 3 (DATA_STATE_MATCHING), but if one of the parents was a # de-duplicated record then data_state will be 0. This step ensures that the new states will be 3. state1.data_state = DATA_STATE_MATCHING state2.data_state = DATA_STATE_MATCHING state1.save() state2.save() # Delete the audit log entry for the merge log.delete() old_view.delete() new_view1.save() new_view2.save() # Asssociate labels label_objs = StatusLabel.objects.filter(pk__in=label_ids) new_view1.labels.set(label_objs) new_view2.labels.set(label_objs) # Duplicate notes to the new views for note in notes: created = note.created updated = note.updated note.id = None note.taxlot_view = new_view1 note.save() ids = [note.id] note.id = None note.taxlot_view = new_view2 note.save() ids.append(note.id) # Correct the created and updated times to match the original note Note.objects.filter(id__in=ids).update(created=created, updated=updated) for paired_view_id in paired_view_ids: TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view1.id, property_view_id=paired_view_id).save() TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view2.id, property_view_id=paired_view_id).save() return { 'status': 'success', 'view_id': new_view1.id } @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['GET']) def links(self, request, pk=None): """ Get taxlot details for each linked taxlot across org cycles """ organization_id = request.query_params.get('organization_id', None) base_view = TaxLotView.objects.select_related('cycle').filter( pk=pk, cycle__organization_id=organization_id ) if base_view.exists(): result = {'data': []} linked_views = TaxLotView.objects.select_related('cycle').filter( taxlot_id=base_view.get().taxlot_id, cycle__organization_id=organization_id ).order_by('-cycle__start') for linked_view in linked_views: state_data = TaxLotStateSerializer(linked_view.state).data state_data['cycle_id'] = linked_view.cycle.id state_data['view_id'] = linked_view.id result['data'].append(state_data) return JsonResponse(result, status=status.HTTP_200_OK) else: result = { 'status': 'error', 'message': 'property view with id {} does not exist in given organization'.format(pk) } return JsonResponse(result) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def match_merge_link(self, request, pk=None): """ Runs match merge link for an individual taxlot. Note that this method can return a view_id of None if the given -View was not involved in a merge. """ org_id = request.query_params.get('organization_id', None) taxlot_view = TaxLotView.objects.get( pk=pk, cycle__organization_id=org_id ) merge_count, link_count, view_id = match_merge_link(taxlot_view.pk, 'TaxLotState') result = { 'view_id': view_id, 'match_merged_count': merge_count, 'match_link_count': link_count, } return JsonResponse(result) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'property_id', required=True, description='The property id to pair up with this taxlot' ) ] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['PUT']) def pair(self, request, pk=None): """ Pair a property to this taxlot """ organization_id = int(request.query_params.get('organization_id')) property_id = int(request.query_params.get('property_id')) taxlot_id = int(pk) return pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, True) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'property_id', required=True, description='The property id to unpair from this taxlot' ) ] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['PUT']) def unpair(self, request, pk=None): """ Unpair a property from this taxlot """ organization_id = int(request.query_params.get('organization_id')) property_id = int(request.query_params.get('property_id')) taxlot_id = int(pk) return pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, False) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'taxlot_view_ids': ['integer'] }, required=['taxlot_view_ids'], description='A list of taxlot view ids to delete' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['DELETE']) def batch_delete(self, request): """ Batch delete several tax lots """ org_id = request.query_params.get('organization_id', None) taxlot_view_ids = request.data.get('taxlot_view_ids', []) taxlot_state_ids = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=org_id ).values_list('state_id', flat=True) resp = TaxLotState.objects.filter(pk__in=Subquery(taxlot_state_ids)).delete() if resp[0] == 0: return JsonResponse({'status': 'warning', 'message': 'No action was taken'}) return JsonResponse({'status': 'success', 'taxlots': resp[1]['seed.TaxLotState']}) def _get_taxlot_view(self, taxlot_pk): try: taxlot_view = TaxLotView.objects.select_related( 'taxlot', 'cycle', 'state' ).get( id=taxlot_pk, taxlot__organization_id=self.request.GET['organization_id'] ) result = { 'status': 'success', 'taxlot_view': taxlot_view } except TaxLotView.DoesNotExist: result = { 'status': 'error', 'message': 'taxlot view with id {} does not exist'.format(taxlot_pk) } return result def get_history(self, taxlot_view): """Return history in reverse order""" # access the history from the property state history, master = taxlot_view.state.history() # convert the history and master states to StateSerializers master['state'] = TaxLotStateSerializer(master['state_data']).data del master['state_data'] del master['state_id'] for h in history: h['state'] = TaxLotStateSerializer(h['state_data']).data del h['state_data'] del h['state_id'] return history, master def _get_properties(self, taxlot_view_pk): property_view_pks = TaxLotProperty.objects.filter( taxlot_view_id=taxlot_view_pk ).values_list('property_view_id', flat=True) property_views = PropertyView.objects.filter( pk__in=property_view_pks ).select_related('cycle', 'state') properties = [] for property_view in property_views: properties.append(PropertyViewSerializer(property_view).data) return properties @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_view_data') def retrieve(self, request, pk): """ Get taxlot details """ result = self._get_taxlot_view(pk) if result.get('status', None) != 'error': taxlot_view = result.pop('taxlot_view') result.update(TaxLotViewSerializer(taxlot_view).data) # remove TaxLotView id from result result.pop('id') result['state'] = TaxLotStateSerializer(taxlot_view.state).data result['properties'] = self._get_properties(taxlot_view.pk) result['history'], master = self.get_history(taxlot_view) result = update_result_with_master(result, master) return JsonResponse(result, status=status.HTTP_200_OK) else: return JsonResponse(result, status=status.HTTP_404_NOT_FOUND) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=UpdateTaxLotPayloadSerializer, ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def update(self, request, pk): """ Update a taxlot and run the updated record through a match and merge round within it's current Cycle. """ data = request.data result = self._get_taxlot_view(pk) if result.get('status', 'error') != 'error': taxlot_view = result.pop('taxlot_view') taxlot_state_data = TaxLotStateSerializer(taxlot_view.state).data # get the taxlot state information from the request new_taxlot_state_data = data['state'] # set empty strings to None for key, val in new_taxlot_state_data.items(): if val == '': new_taxlot_state_data[key] = None changed_fields, previous_data = get_changed_fields(taxlot_state_data, new_taxlot_state_data) if not changed_fields: result.update( {'status': 'success', 'message': 'Records are identical'} ) return JsonResponse(result, status=status.HTTP_204_NO_CONTENT) else: # Not sure why we are going through the pain of logging this all right now... need to # reevaluate this. log = TaxLotAuditLog.objects.select_related().filter( state=taxlot_view.state ).order_by('-id').first() # if checks above pass, create an exact copy of the current state for historical purposes if log.name == 'Import Creation': # Add new state by removing the existing ID. taxlot_state_data.pop('id') # Remove the import_file_id for the first edit of a new record # If the import file has been deleted and this value remains the serializer won't be valid taxlot_state_data.pop('import_file') new_taxlot_state_serializer = TaxLotStateSerializer( data=taxlot_state_data ) if new_taxlot_state_serializer.is_valid(): # create the new property state, and perform an initial save / moving relationships new_state = new_taxlot_state_serializer.save() # then assign this state to the property view and save the whole view taxlot_view.state = new_state taxlot_view.save() TaxLotAuditLog.objects.create(organization=log.organization, parent1=log, parent2=None, parent_state1=log.state, parent_state2=None, state=new_state, name='Manual Edit', description=None, import_filename=log.import_filename, record_type=AUDIT_USER_EDIT) result.update( {'state': new_taxlot_state_serializer.data} ) # save the property view so that the datetime gets updated on the property. taxlot_view.save() else: result.update({ 'status': 'error', 'message': 'Invalid update data with errors: {}'.format( new_taxlot_state_serializer.errors)} ) return JsonResponse(result, status=status.HTTP_422_UNPROCESSABLE_ENTITY) # redo assignment of this variable in case this was an initial edit taxlot_state_data = TaxLotStateSerializer(taxlot_view.state).data if 'extra_data' in new_taxlot_state_data: taxlot_state_data['extra_data'].update( new_taxlot_state_data['extra_data'] ) taxlot_state_data.update( {k: v for k, v in new_taxlot_state_data.items() if k != 'extra_data'} ) log = TaxLotAuditLog.objects.select_related().filter( state=taxlot_view.state ).order_by('-id').first() if log.name in ['Manual Edit', 'Manual Match', 'System Match', 'Merge current state in migration']: # Convert this to using the serializer to save the data. This will override the # previous values in the state object. # Note: We should be able to use partial update here and pass in the changed # fields instead of the entire state_data. updated_taxlot_state_serializer = TaxLotStateSerializer( taxlot_view.state, data=taxlot_state_data ) if updated_taxlot_state_serializer.is_valid(): # create the new property state, and perform an initial save / moving # relationships updated_taxlot_state_serializer.save() result.update( {'state': updated_taxlot_state_serializer.data} ) # save the property view so that the datetime gets updated on the property. taxlot_view.save() Note.create_from_edit(request.user.id, taxlot_view, new_taxlot_state_data, previous_data) merge_count, link_count, view_id = match_merge_link(taxlot_view.id, 'TaxLotState') result.update({ 'view_id': view_id, 'match_merged_count': merge_count, 'match_link_count': link_count, }) return JsonResponse(result, status=status.HTTP_200_OK) else: result.update({ 'status': 'error', 'message': 'Invalid update data with errors: {}'.format( updated_taxlot_state_serializer.errors)} ) return JsonResponse(result, status=status.HTTP_422_UNPROCESSABLE_ENTITY) else: result = { 'status': 'error', 'message': 'Unrecognized audit log name: ' + log.name } return JsonResponse(result, status=status.HTTP_204_NO_CONTENT) else: return JsonResponse(result, status=status.HTTP_404_NOT_FOUND)