class AnalysisPropertyViewViewSet(viewsets.ViewSet): serializer_class = AnalysisPropertyViewSerializer model = AnalysisPropertyView @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field(True)]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def list(self, request, analysis_pk): organization_id = int(request.query_params.get('organization_id', 0)) try: views_queryset = AnalysisPropertyView.objects.filter( analysis=analysis_pk, analysis__organization_id=organization_id) except AnalysisPropertyView.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': "Requested analysis doesn't exist in this organization." }, status=HTTP_409_CONFLICT) serialized_views = [] for view in views_queryset: serialized_view = AnalysisPropertyViewSerializer(view).data serialized_views.append(serialized_view) return JsonResponse({'status': 'success', 'views': serialized_views}) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field(True)]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def retrieve(self, request, analysis_pk, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: view = AnalysisPropertyView.objects.get( id=pk, analysis=analysis_pk, analysis__organization_id=organization_id) except AnalysisPropertyView.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': "Requested analysis property view doesn't exist in this organization and/or analysis." }, status=HTTP_409_CONFLICT) return JsonResponse({ 'status': 'success', 'view': AnalysisPropertyViewSerializer(view).data })
class MeasureViewSet(viewsets.ReadOnlyModelViewSet, OrgMixin): """ list: Return a list of all measures retrieve: Return a measure by a unique id """ serializer_class = MeasureSerializer parser_classes = ( JSONParser, FormParser, ) renderer_classes = (JSONRenderer, ) pagination_class = None def get_queryset(self): org_id = self.get_organization(self.request) return Measure.objects.filter(organization_id=org_id) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=no_body) @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def reset(self, request): """ Reset all the measures back to the defaults (as provided by BuildingSync) """ organization_id = request.query_params.get('organization_id', None) if not organization_id: return JsonResponse( { 'status': 'error', 'message': 'organization_id not provided' }, status=status.HTTP_400_BAD_REQUEST) Measure.populate_measures(organization_id) data = dict(measures=list( Measure.objects.filter( organization_id=organization_id).order_by('id').values())) data['status'] = 'success' return JsonResponse(data)
class ColumnMappingProfileViewSet(OrgMixin, ViewSet): permission_classes = (SEEDOrgPermissions, ) # req by SEEDOrgPermissions, but currently not used by any methods. queryset = ColumnMappingProfile.objects.none() @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ) ], request_body=AutoSchemaHelper.schema_factory( {'profile_type': ['string']}, description= "Possible Types: 'Normal', 'BuildingSync Default', BuildingSync Custom'" )) @api_endpoint_class @action(detail=False, methods=['POST']) # POST in order to provide array/list def filter(self, request): """ Retrieves all profiles for an organization. """ try: profile_types = request.data.get('profile_type', []) profile_types = [ ColumnMappingProfile.get_profile_type(pt) for pt in profile_types ] filter_params = { 'organizations__pk': self.get_organization(request, True).id } if profile_types: filter_params['profile_type__in'] = profile_types profiles = ColumnMappingProfile.objects.filter(**filter_params) data = [ColumnMappingProfileSerializer(p).data for p in profiles] return JsonResponse({ 'status': 'success', 'data': data, }) except Exception as e: raise e return JsonResponse({ 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ) ], request_body=AutoSchemaHelper.schema_factory( { 'name': 'string', 'mappings': [{ 'to_field': 'string', 'from_field': 'string', 'from_units': 'string', 'to_table_name': 'string', }] }, description="Optional 'name' or 'mappings'.\n" + mappings_description)) @api_endpoint_class def update(self, request, pk=None): """ Updates a profile given appropriate request data. The body should contain only valid fields for ColumnMappingProfile objects. """ org_id = self.get_organization(request, True).id try: profile = ColumnMappingProfile.objects.get( organizations__pk=org_id, pk=pk) except ColumnMappingProfile.DoesNotExist: return JsonResponse( { 'status': 'error', 'data': 'No profile with given id' }, status=HTTP_400_BAD_REQUEST) if profile.profile_type == ColumnMappingProfile.BUILDINGSYNC_DEFAULT: return JsonResponse( { 'status': 'error', 'data': 'Default BuildingSync profile are not editable' }, status=HTTP_400_BAD_REQUEST) updated_name, updated_mappings = request.data.get( 'name'), request.data.get('mappings') # update the name if updated_name is not None: profile.name = updated_name # update the mappings according to the profile type if updated_mappings is not None: if profile.profile_type == ColumnMappingProfile.BUILDINGSYNC_CUSTOM: # only allow these updates to the mappings # - changing the to_field or from_units # - removing mappings original_mappings_dict = { m['from_field']: m.copy() for m in profile.mappings } final_mappings = [] for updated_mapping in updated_mappings: from_field = updated_mapping['from_field'] original_mapping = original_mappings_dict.get(from_field) if original_mapping is not None: original_mapping['to_field'] = updated_mapping[ 'to_field'] original_mapping['from_units'] = updated_mapping[ 'from_units'] final_mappings.append(original_mapping) del original_mappings_dict[from_field] profile.mappings = final_mappings elif updated_mappings: # indiscriminantly update the mappings profile.mappings = updated_mappings profile.save() return JsonResponse({ 'status': 'success', 'data': ColumnMappingProfileSerializer(profile).data, }) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ) ], request_body=AutoSchemaHelper.schema_factory( { 'name': 'string', 'mappings': [{ 'to_field': 'string', 'from_field': 'string', 'from_units': 'string', 'to_table_name': 'string', }] }, description=mappings_description, required=['name', 'mappings'])) @api_endpoint_class def create(self, request, pk=None): """ Creates a new profile given appropriate request data. The body should contain only valid fields for ColumnMappingProfile objects. """ org_id = self.get_organization(request, True).id try: profile_data = request.data profile_data['organizations'] = [org_id] ser_profile = ColumnMappingProfileSerializer(data=profile_data) if ser_profile.is_valid(): profile = ser_profile.save() response_status = 'success' response_data = ColumnMappingProfileSerializer(profile).data response_code = HTTP_200_OK else: response_status = 'error' response_data = ser_profile.errors response_code = HTTP_400_BAD_REQUEST return JsonResponse( { 'status': response_status, 'data': response_data, }, status=response_code) except Exception as e: return JsonResponse({ 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ) ]) @api_endpoint_class def destroy(self, request, pk=None): """ Deletes a specific profile. """ org_id = self.get_organization(request, True).id try: profile = ColumnMappingProfile.objects.get( organizations__pk=org_id, pk=pk) except Exception as e: return JsonResponse({ 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST) if profile.profile_type == ColumnMappingProfile.BUILDINGSYNC_DEFAULT: return JsonResponse( { 'status': 'error', 'data': 'Not allowed to edit default BuildingSync profiles' }, status=HTTP_400_BAD_REQUEST) else: profile.delete() return JsonResponse({ 'status': 'success', 'data': "Successfully deleted", }) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ) ], request_body=AutoSchemaHelper.schema_factory( {'headers': ['string']}, description= "Raw headers - the exact headers for columns in an import file.", required=['headers'])) @api_endpoint_class @action(detail=False, methods=['POST']) def suggestions(self, request): """ Retrieves suggestions given raw column headers. """ try: org_id = self.get_organization(request, True).id raw_headers = request.data.get('headers', []) suggested_mappings = mapper.build_column_mapping( raw_headers, Column.retrieve_all_by_tuple(org_id), previous_mapping=None, map_args=None, thresh= 80 # percentage match that we require. 80% is random value for now. ) # replace None with empty string for column names and PropertyState for tables # TODO #239: Move this fix to build_column_mapping for m in suggested_mappings: table, destination_field, _confidence = suggested_mappings[m] if destination_field is None: suggested_mappings[m][1] = '' # Fix the table name, eventually move this to the build_column_mapping for m in suggested_mappings: table, _destination_field, _confidence = suggested_mappings[m] # Do not return the campus, created, updated fields... that is force them to be in the property state if not table or table == 'Property': suggested_mappings[m][0] = 'PropertyState' elif table == 'TaxLot': suggested_mappings[m][0] = 'TaxLotState' return JsonResponse({ 'status': 'success', 'data': suggested_mappings, }) except Exception as e: return JsonResponse({ 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST)
class LabelInventoryViewSet(APIView): """API endpoint for viewing and creating labels. Returns:: [ { 'id': Label's primary key 'name': Name given to label 'color': Color of label, 'organization_id': Id of organization label belongs to, 'is_applied': Will be empty array if not applied to property/taxlots } ] --- """ renderer_classes = (JSONRenderer, ) parser_classes = (JSONParser, ) inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} errors = { 'disjoint': ErrorState( status.HTTP_422_UNPROCESSABLE_ENTITY, 'add_label_ids and remove_label_ids cannot contain elements in common' ), 'missing_org': ErrorState(status.HTTP_422_UNPROCESSABLE_ENTITY, 'missing organization_id'), 'missing_inventory_ids': ErrorState(status.HTTP_422_UNPROCESSABLE_ENTITY, 'inventory_ids cannot be undefined or empty') } @property def models(self): """ Exposes Django's internal models for join table. Used for bulk_create operations. """ return { 'property': apps.get_model('seed', 'PropertyView_labels'), 'taxlot': apps.get_model('seed', 'TaxLotView_labels') } def get_queryset(self, inventory_type, organization_id): Model = self.models[inventory_type] return Model.objects.filter( statuslabel__super_organization_id=organization_id) def get_label_desc(self, add_label_ids, remove_label_ids): return Label.objects.filter(pk__in=add_label_ids + remove_label_ids).values( 'id', 'color', 'name') def get_inventory_id(self, q, inventory_type): return getattr(q, "{}view_id".format(inventory_type)) def exclude(self, qs, inventory_type, label_ids): exclude = {label: [] for label in label_ids} for q in qs: if q.statuslabel_id in label_ids: inventory_id = self.get_inventory_id(q, inventory_type) exclude[q.statuslabel_id].append(inventory_id) return exclude def filter_by_inventory(self, qs, inventory_type, inventory_ids): if inventory_ids: filterdict = { "{}view__pk__in".format(inventory_type): inventory_ids } qs = qs.filter(**filterdict) return qs def label_factory(self, inventory_type, label_id, inventory_id): Model = self.models[inventory_type] # Ensure the the label org and inventory org are the same inventory_parent_org_id = getattr(Model, "{}view".format(inventory_type)).get_queryset().get(pk=inventory_id)\ .cycle.organization.get_parent().id label_super_org_id = Model.statuslabel.get_queryset().get( pk=label_id).super_organization_id if inventory_parent_org_id == label_super_org_id: create_dict = { 'statuslabel_id': label_id, "{}view_id".format(inventory_type): inventory_id } return Model(**create_dict) else: raise IntegrityError( 'Label with super_organization_id={} cannot be applied to a record with parent ' 'organization_id={}.'.format(label_super_org_id, inventory_parent_org_id)) def add_labels(self, qs, inventory_type, inventory_ids, add_label_ids): added = [] if add_label_ids: model = self.models[inventory_type] exclude = self.exclude(qs, inventory_type, add_label_ids) new_inventory_labels = [ self.label_factory(inventory_type, label_id, pk) for label_id in add_label_ids for pk in inventory_ids if pk not in exclude[label_id] ] model.objects.bulk_create(new_inventory_labels) added = [ self.get_inventory_id(m, inventory_type) for m in new_inventory_labels ] return added def remove_labels(self, qs, inventory_type, remove_label_ids): removed = [] if remove_label_ids: rqs = qs.filter(statuslabel_id__in=remove_label_ids) removed = [self.get_inventory_id(q, inventory_type) for q in rqs] rqs.delete() return removed @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'add_label_ids': ['integer'], 'remove_label_ids': ['integer'], 'inventory_ids': ['integer'], }, required=['inventory_ids'], description='Properties:\n' '- add_label_ids: label ids to add to the inventories\n' '- remove_label_ids: label ids to remove from the inventories\n' '- inventory_ids: List of inventory IDs')) @has_perm_class('can_modify_data') def put(self, request, inventory_type): """ Updates label assignments to inventory items. Returns:: { 'status': {string} 'success' or 'error' 'message': {string} Error message if error 'num_updated': {integer} Number of properties/taxlots updated 'labels': [ List of labels affected. { 'color': {string} 'id': {int} 'label': {'string'} 'name': {string} }... ] } """ add_label_ids = request.data.get('add_label_ids', []) remove_label_ids = request.data.get('remove_label_ids', []) inventory_ids = request.data.get('inventory_ids', []) organization_id = request.query_params['organization_id'] error = None # ensure add_label_ids and remove_label_ids are different if not set(add_label_ids).isdisjoint(remove_label_ids): error = self.errors['disjoint'] elif not organization_id: error = self.errors['missing_org'] elif len(inventory_ids) == 0: error = self.errors['missing_inventory_ids'] if error: result = {'status': 'error', 'message': str(error)} status_code = error.status_code else: qs = self.get_queryset(inventory_type, organization_id) qs = self.filter_by_inventory(qs, inventory_type, inventory_ids) removed = self.remove_labels(qs, inventory_type, remove_label_ids) added = self.add_labels(qs, inventory_type, inventory_ids, add_label_ids) num_updated = len(set(added).union(removed)) labels = self.get_label_desc(add_label_ids, remove_label_ids) result = { 'status': 'success', 'num_updated': num_updated, 'labels': labels } status_code = status.HTTP_200_OK return response.Response(result, status=status_code)
class DataQualityCheckViewSet(viewsets.ViewSet): """ Handles Data Quality API operations within Inventory backend. (1) Post, wait, get… (2) Respond with what changed """ # Remove lookup_field once data_quality_check_id is used and "pk" can be used lookup_field = 'organization_id' @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.base_field( "organization_id", "IN_PATH", "Organization ID - identifier used to specify a DataQualityCheck", True, "TYPE_INTEGER") ], request_body=AutoSchemaHelper.schema_factory( { 'property_view_ids': ['integer'], 'taxlot_view_ids': ['integer'], }, description='An object containing IDs of the records to perform' ' data quality checks on. Should contain two keys- ' 'property_view_ids and taxlot_view_ids, each of ' 'which is an array of appropriate IDs.', ), responses={ 200: AutoSchemaHelper.schema_factory({ 'num_properties': 'integer', 'num_taxlots': 'integer', 'progress_key': 'string', 'progress': {}, }) }) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['POST']) def start(self, request, organization_id): """ This API endpoint will create a new data_quality check process in the background, on potentially a subset of properties/taxlots, and return back a query key """ body = request.data property_view_ids = body['property_view_ids'] taxlot_view_ids = body['taxlot_view_ids'] property_state_ids = PropertyView.objects.filter( id__in=property_view_ids).values_list('state_id', flat=True) taxlot_state_ids = TaxLotView.objects.filter( id__in=taxlot_view_ids).values_list('state_id', flat=True) # For now, organization_id is the only key currently used to identify DataQualityChecks return_value = do_checks(organization_id, property_state_ids, taxlot_state_ids) return JsonResponse({ 'num_properties': len(property_state_ids), 'num_taxlots': len(taxlot_state_ids), # TODO #239: Deprecate progress_key from here and just use the 'progess.progress_key' 'progress_key': return_value['progress_key'], 'progress': return_value, }) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field("run_id", True, "Import file ID or cache key"), ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=False, methods=['GET']) def results_csv(self, request): """ Download a CSV of the results from a data quality run based on either the ID that was given during the creation of the data quality task or the ID of the import file which had it's records checked. Note that it is not related to objects in the database, since the results are stored in redis! """ run_id = request.query_params.get('run_id') if run_id is None: return JsonResponse( { 'status': 'error', 'message': 'must include Import file ID or cache key as run_id' }, status=status.HTTP_400_BAD_REQUEST) data_quality_results = get_cache_raw( DataQualityCheck.cache_key(run_id)) response = HttpResponse(content_type='text/csv') response[ 'Content-Disposition'] = 'attachment; filename="Data Quality Check Results.csv"' writer = csv.writer(response) if data_quality_results is None: writer.writerow(['Error']) writer.writerow(['data quality results not found']) return response writer.writerow([ 'Table', 'Address Line 1', 'PM Property ID', 'Tax Lot ID', 'Custom ID', 'Field', 'Applied Label', 'Condition', 'Error Message', 'Severity' ]) for row in data_quality_results: for result in row['data_quality_results']: writer.writerow([ row['data_quality_results'][0]['table_name'], row['address_line_1'], row['pm_property_id'] if 'pm_property_id' in row else None, row['jurisdiction_tax_lot_id'] if 'jurisdiction_tax_lot_id' in row else None, row['custom_id_1'], result['formatted_field'], result.get('label', None), result['condition'], # the detailed_message field can have units which has superscripts/subscripts, so unidecode it! unidecode(result['detailed_message']), result['severity'] ]) return response @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field("run_id", True, "Import file ID or cache key"), ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=False, methods=['GET']) def results(self, request): """ Return the results of a data quality run based on either the ID that was given during the creation of the data quality task or the ID of the import file which had it's records checked. Note that it is not related to objects in the database, since the results are stored in redis! """ data_quality_id = request.query_params['run_id'] data_quality_results = get_cache_raw( DataQualityCheck.cache_key(data_quality_id)) return JsonResponse({'data': data_quality_results})
class GeocodeViewSet(viewsets.ViewSet, OrgMixin): @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'property_view_ids': ['integer'], 'taxlot_view_ids': ['integer'], }, description='IDs by inventory type for records to be geocoded.' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def geocode_by_ids(self, request): """ Submit a request to geocode property and tax lot records. """ body = dict(request.data) org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') if property_view_ids: property_views = PropertyView.objects.filter( id__in=property_view_ids, cycle__organization_id=org_id ) properties = PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')) ) geocode_buildings(properties) if taxlot_view_ids: taxlot_views = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=org_id ) taxlots = TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')) ) geocode_buildings(taxlots) return JsonResponse({'status': 'success'}) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'property_view_ids': ['integer'], 'taxlot_view_ids': ['integer'], }, description='IDs by inventory type for records to be used in building a geocoding summary.' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('can_view_data') @action(detail=False, methods=['POST']) def confidence_summary(self, request): """ Generate a summary of geocoding confidence values for property and tax lot records. """ body = dict(request.data) org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') result = {} if property_view_ids: property_views = PropertyView.objects.filter( id__in=property_view_ids, cycle__organization_id=org_id ) result["properties"] = { 'not_geocoded': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__isnull=True ).count(), 'high_confidence': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='High' ).count(), 'low_confidence': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='Low' ).count(), 'manual': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' ).count(), 'missing_address_components': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' ).count(), } if taxlot_view_ids: taxlot_views = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=org_id ) result["tax_lots"] = { 'not_geocoded': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__isnull=True ).count(), 'high_confidence': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='High' ).count(), 'low_confidence': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='Low' ).count(), 'manual': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' ).count(), 'missing_address_components': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' ).count(), } return result
class TaxLotPropertyViewSet(GenericViewSet): """ The TaxLotProperty field is used to return the properties and tax lots from the join table. This method presently only works with the CSV, but should eventually be extended to be the viewset for any tax lot / property join API call. """ renderer_classes = (JSONRenderer, ) serializer_class = TaxLotPropertySerializer @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field("cycle_id", True, "Cycle ID"), AutoSchemaHelper.query_string_field( "inventory_type", False, "Either 'taxlots' or 'properties' and defaults to 'properties'." ), ], request_body=AutoSchemaHelper.schema_factory( { 'ids': ['integer'], 'filename': 'string', 'export_type': 'string', 'profile_id': 'integer' }, description='- ids: (View) IDs for records to be exported\n' '- filename: desired filename including extension (defaulting to \'ExportedData.{export_type}\')\n' '- export_types: \'csv\', \'geojson\', \'xlsx\' (defaulting to \'csv\')\n' '- profile_id: Column List Profile ID to use for customizing fields included in export' ), ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=False, methods=['POST']) def export(self, request): """ Download a collection of the TaxLot and Properties in multiple formats. """ cycle_pk = request.query_params.get('cycle_id', None) if not cycle_pk: return JsonResponse({ 'status': 'error', 'message': 'Must pass in cycle_id as query parameter' }) org_id = request.query_params['organization_id'] if 'profile_id' not in request.data: profile_id = None else: if request.data['profile_id'] == 'None' or request.data[ 'profile_id'] == '': profile_id = None else: profile_id = request.data['profile_id'] # get the class to operate on and the relationships view_klass_str = request.query_params.get('inventory_type', 'properties') view_klass = INVENTORY_MODELS[view_klass_str] # Set the first column to be the ID column_name_mappings = OrderedDict([('id', 'ID')]) column_ids, add_column_name_mappings, columns_from_database = ColumnListProfile.return_columns( org_id, profile_id, view_klass_str) column_name_mappings.update(add_column_name_mappings) select_related = ['state', 'cycle'] ids = request.data.get('ids', []) filter_str = {'cycle': cycle_pk} if hasattr(view_klass, 'property'): select_related.append('property') prefetch_related = ['labels'] filter_str = {'property__organization_id': org_id} if ids: filter_str['id__in'] = ids # always export the labels and notes column_name_mappings['property_notes'] = 'Property Notes' column_name_mappings['property_labels'] = 'Property Labels' elif hasattr(view_klass, 'taxlot'): select_related.append('taxlot') prefetch_related = ['labels'] filter_str = {'taxlot__organization_id': org_id} if ids: filter_str['id__in'] = ids # always export the labels and notes column_name_mappings['taxlot_notes'] = 'Tax Lot Notes' column_name_mappings['taxlot_labels'] = 'Tax Lot Labels' model_views = view_klass.objects.select_related( *select_related).prefetch_related(*prefetch_related).filter( **filter_str).order_by('id') # get the data in a dict which includes the related data data = TaxLotProperty.get_related(model_views, column_ids, columns_from_database) # add labels and notes for i, record in enumerate(model_views): label_string = [] note_string = [] for label in list(record.labels.all().order_by('name')): label_string.append(label.name) for note in list(record.notes.all().order_by('created')): note_string.append(note.created.astimezone().strftime( "%Y-%m-%d %I:%M:%S %p") + "\n" + note.text) if hasattr(record, 'property'): data[i]['property_labels'] = ','.join(label_string) data[i]['property_notes'] = '\n----------\n'.join(note_string) elif hasattr(record, 'taxlot'): data[i]['taxlot_labels'] = ','.join(label_string) data[i]['taxlot_notes'] = '\n----------\n'.join(note_string) # force the data into the same order as the IDs if ids: order_dict = {obj_id: index for index, obj_id in enumerate(ids)} if view_klass_str == 'properties': view_id_str = 'property_view_id' else: view_id_str = 'taxlot_view_id' data.sort(key=lambda inventory_obj: order_dict[inventory_obj[ view_id_str]]) export_type = request.data.get('export_type', 'csv') filename = request.data.get('filename', f"ExportedData.{export_type}") if export_type == "csv": return self._csv_response(filename, data, column_name_mappings) elif export_type == "geojson": return self._json_response(filename, data, column_name_mappings) elif export_type == "xlsx": return self._spreadsheet_response(filename, data, column_name_mappings) def _csv_response(self, filename, data, column_name_mappings): response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="{}"'.format( filename) writer = csv.writer(response) # check the first item in the header and make sure that it isn't ID (it can be id, or iD). # excel doesn't like the first item to be ID in a CSV header = list(column_name_mappings.values()) if header[0] == 'ID': header[0] = 'id' writer.writerow(header) # iterate over the results to preserve column order and write row. for datum in data: row = [] for column in column_name_mappings: row_result = datum.get(column, None) # Try grabbing the value out of the related field if not found yet. if row_result is None and datum.get('related'): row_result = datum['related'][0].get(column, None) # Convert quantities (this is typically handled in the JSON Encoder, but that isn't here). if isinstance(row_result, ureg.Quantity): row_result = row_result.magnitude elif isinstance(row_result, datetime.datetime): row_result = row_result.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(row_result, datetime.date): row_result = row_result.strftime("%Y-%m-%d") row.append(row_result) writer.writerow(row) return response def _spreadsheet_response(self, filename, data, column_name_mappings): response = HttpResponse( content_type= 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) response['Content-Disposition'] = 'attachment; filename="{}"'.format( filename) scenario_keys = ('id', 'name', 'description', 'annual_site_energy_savings', 'annual_source_energy_savings', 'annual_cost_savings', 'analysis_state', 'analysis_state_message', 'annual_electricity_savings', 'annual_natural_gas_savings', 'annual_site_energy', 'annual_source_energy', 'annual_natural_gas_energy', 'annual_electricity_energy', 'annual_peak_demand', 'annual_site_energy_use_intensity', 'annual_source_energy_use_intensity') scenario_key_mappings = { 'annual_site_energy_savings': 'annual_site_energy_savings_mmbtu', 'annual_source_energy_savings': 'annual_source_energy_savings_mmbtu', 'annual_cost_savings': 'annual_cost_savings_dollars', 'annual_site_energy': 'annual_site_energy_kbtu', 'annual_site_energy_use_intensity': 'annual_site_energy_use_intensity_kbtu_ft2', 'annual_source_energy': 'annual_source_energy_kbtu', 'annual_source_energy_use_intensity': 'annual_source_energy_use_intensity_kbtu_ft2', 'annual_natural_gas_energy': 'annual_natural_gas_energy_mmbtu', 'annual_electricity_energy': 'annual_electricity_energy_mmbtu', 'annual_peak_demand': 'annual_peak_demand_kw', 'annual_electricity_savings': 'annual_electricity_savings_kbtu', 'annual_natural_gas_savings': 'annual_natural_gas_savings_kbtu' } property_measure_keys = ('id', 'property_measure_name', 'measure_id', 'cost_mv', 'cost_total_first', 'cost_installation', 'cost_material', 'cost_capital_replacement', 'cost_residual_value') measure_keys = ('name', 'display_name', 'category', 'category_display_name') # find measures and scenarios for i, record in enumerate(data): measures = PropertyMeasure.objects.filter( property_state_id=record['property_state_id']) record['measures'] = measures scenarios = Scenario.objects.filter( property_state_id=record['property_state_id']) record['scenarios'] = scenarios output = io.BytesIO() wb = xlsxwriter.Workbook(output, {'remove_timezone': True}) # add tabs ws1 = wb.add_worksheet('Properties') ws2 = wb.add_worksheet('Measures') ws3 = wb.add_worksheet('Scenarios') ws4 = wb.add_worksheet('Scenario Measure Join Table') ws5 = wb.add_worksheet('Meter Readings') bold = wb.add_format({'bold': True}) row = 0 row2 = 0 col2 = 0 row3 = 0 col3 = 0 row4 = 0 row5 = 0 for index, val in enumerate(list(column_name_mappings.values())): # Do not write the first element as ID, this causes weird issues with Excel. if index == 0 and val == 'ID': ws1.write(row, index, 'id', bold) else: ws1.write(row, index, val, bold) # iterate over the results to preserve column order and write row. add_m_headers = True add_s_headers = True for datum in data: row += 1 id = None for index, column in enumerate(column_name_mappings): if column == 'id': id = datum.get(column, None) row_result = datum.get(column, None) # Try grabbing the value out of the related field if not found yet. if row_result is None and datum.get('related'): row_result = datum['related'][0].get(column, None) # Convert quantities (this is typically handled in the JSON Encoder, but that isn't here). if isinstance(row_result, ureg.Quantity): row_result = row_result.magnitude elif isinstance(row_result, datetime.datetime): row_result = row_result.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(row_result, datetime.date): row_result = row_result.strftime("%Y-%m-%d") ws1.write(row, index, row_result) # measures for index, m in enumerate(datum['measures']): if add_m_headers: # grab headers for key in property_measure_keys: ws2.write(row2, col2, key, bold) col2 += 1 for key in measure_keys: ws2.write(row2, col2, 'measure ' + key, bold) col2 += 1 add_m_headers = False row2 += 1 col2 = 0 for key in property_measure_keys: ws2.write(row2, col2, getattr(m, key)) col2 += 1 for key in measure_keys: ws2.write(row2, col2, getattr(m.measure, key)) col2 += 1 # scenarios (and join table) # join table ws4.write('A1', 'property_id', bold) ws4.write('B1', 'scenario_id', bold) ws4.write('C1', 'measure_id', bold) for index, s in enumerate(datum['scenarios']): scenario_id = s.id if add_s_headers: # grab headers for key in scenario_keys: # double check scenario_key_mappings in case a different header is desired if key in scenario_key_mappings.keys(): key = scenario_key_mappings[key] ws3.write(row3, col3, key, bold) col3 += 1 add_s_headers = False row3 += 1 col3 = 0 for key in scenario_keys: ws3.write(row3, col3, getattr(s, key)) col3 += 1 for sm in s.measures.all(): row4 += 1 ws4.write(row4, 0, id) ws4.write(row4, 1, scenario_id) ws4.write(row4, 2, sm.id) # scenario meter readings ws5.write('A1', 'scenario_id', bold) ws5.write('B1', 'meter_id', bold) ws5.write('C1', 'type', bold) ws5.write('D1', 'start_time', bold) ws5.write('E1', 'end_time', bold) ws5.write('F1', 'reading', bold) ws5.write('G1', 'units', bold) ws5.write('H1', 'is_virtual', bold) # datetime formatting date_format = wb.add_format({'num_format': 'yyyy-mm-dd hh:mm:ss'}) for index, s in enumerate(datum['scenarios']): scenario_id = s.id # retrieve meters meters = Meter.objects.filter(scenario_id=scenario_id) for m in meters: # retrieve readings readings = MeterReading.objects.filter( meter_id=m.id).order_by('start_time') for r in readings: row5 += 1 ws5.write(row5, 0, scenario_id) ws5.write(row5, 1, m.id) the_type = next((item for item in Meter.ENERGY_TYPES if item[0] == m.type), None) the_type = the_type[1] if the_type is not None else None ws5.write( row5, 2, the_type ) # use energy type enum to determine reading type ws5.write_datetime(row5, 3, r.start_time, date_format) ws5.write_datetime(row5, 4, r.end_time, date_format) ws5.write(row5, 5, r.reading) # this is now a float field ws5.write(row5, 6, r.source_unit) ws5.write(row5, 7, m.is_virtual) wb.close() # xlsx_data contains the Excel file xlsx_data = output.getvalue() response.write(xlsx_data) return response def _json_response(self, filename, data, column_name_mappings): polygon_fields = [ "bounding_box", "centroid", "property_footprint", "taxlot_footprint", "long_lat" ] features = [] # extract related records related_records = self._extract_related(data) # append related_records to data complete_data = data + related_records for datum in complete_data: feature = {"type": "Feature", "properties": {}} for key, value in datum.items(): if value is None: continue if isinstance(value, ureg.Quantity): value = value.magnitude elif isinstance(value, datetime.datetime): value = value.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(value, datetime.date): value = value.strftime("%Y-%m-%d") if value and any(k in key for k in polygon_fields): """ If object is a polygon and is populated, add the 'geometry' key-value-pair in the appropriate GeoJSON format. When the first geometry is added, the correct format is established. When/If a second geometry is added, this is appended alongside the previous geometry. """ individual_geometry = {} # long_lat if key == 'long_lat': coordinates = self._serialized_point(value) # point individual_geometry = { "coordinates": coordinates, "type": "Point" } else: # polygons coordinates = self._serialized_coordinates(value) individual_geometry = { "coordinates": [coordinates], "type": "Polygon" } if feature.get("geometry", None) is None: feature["geometry"] = { "type": "GeometryCollection", "geometries": [individual_geometry] } else: feature["geometry"]["geometries"].append( individual_geometry) else: """ Non-polygon data """ display_key = column_name_mappings.get(key, key) feature["properties"][display_key] = value # # store point geometry in case you need it # if display_key == "Longitude": # point_geometry[0] = value # if display_key == "Latitude": # point_geometry[1] = value """ Before appending feature, ensure that if there is no geometry recorded. Note that the GeoJson will not render if no lat/lng """ # add style information if feature["properties"].get("property_state_id") is not None: feature["properties"]["stroke"] = "#185189" # buildings color elif feature["properties"].get("taxlot_state_id") is not None: feature["properties"]["stroke"] = "#10A0A0" # buildings color feature["properties"]["marker-color"] = "#E74C3C" # feature["properties"]["stroke-width"] = 3 feature["properties"]["fill-opacity"] = 0 # append feature features.append(feature) response_dict = { "type": "FeatureCollection", "crs": { "type": "EPSG", "properties": { "code": 4326 } }, "features": features } response = JsonResponse(response_dict) response['Content-Disposition'] = 'attachment; filename="{}"'.format( filename) return response def _serialized_coordinates(self, polygon_wkt): string_coord_pairs = polygon_wkt.lstrip('POLYGON (').rstrip(')').split( ', ') coordinates = [] for coord_pair in string_coord_pairs: float_coords = [float(coord) for coord in coord_pair.split(' ')] coordinates.append(float_coords) return coordinates def _serialized_point(self, point_wkt): string_coords = point_wkt.lstrip('POINT (').rstrip(')').split(', ') coordinates = [] for coord in string_coords[0].split(' '): coordinates.append(float(coord)) return coordinates def _extract_related(self, data): # extract all related records into a separate array related = [] # figure out if we are dealing with properties or taxlots if data[0].get("property_state_id", None) is not None: is_property = True elif data[0].get("taxlot_state_id", None) is not None: is_property = False for datum in data: if datum.get("related", None) is not None: for record in datum["related"]: related.append(record) # make array unique if is_property: unique = [dict(p) for p in set(tuple(i.items()) for i in related)] else: unique = [dict(p) for p in set(tuple(i.items()) for i in related)] return unique
class UploadViewSet(viewsets.ViewSet): """ Endpoint to upload data files to, if uploading to local file storage. Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES`` Returns:: { 'import_record': True, 'import_file_id': The ID of the newly-uploaded ImportFile } """ parser_classes = (FormParser, MultiPartParser) @swagger_auto_schema( request_body=no_body, manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.upload_file_field(name='file', required=True, description='File to Upload'), AutoSchemaHelper.form_integer_field( name='import_record', required=True, description= 'the dataset ID you want to associate this file with.'), AutoSchemaHelper.form_string_field( name='source_type', required=True, description= 'the type of file (e.g. "Portfolio Raw" or "Assessed Raw")'), AutoSchemaHelper.form_string_field( name='source_program_version', required=False, description= 'the version of the file as related to the source_type'), ]) @has_perm_class('can_modify_data') @api_endpoint_class @ajax_request_class def create(self, request): """ Upload a new file to an import_record. This is a multipart/form upload. """ if len(request.FILES) == 0: return JsonResponse({ 'success': False, 'message': "Must pass file in as a Multipart/Form post" }) # Fineuploader requires the field to be qqfile it appears... so why not support both? ugh. if 'qqfile' in request.data: the_file = request.data['qqfile'] else: the_file = request.data['file'] filename = the_file.name path = os.path.join(settings.MEDIA_ROOT, "uploads", filename) # Get a unique filename using the get_available_name method in FileSystemStorage s = FileSystemStorage() path = s.get_available_name(path) # verify the directory exists if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) # save the file with open(path, 'wb+') as temp_file: for chunk in the_file.chunks(): temp_file.write(chunk) org_id = request.query_params.get('organization_id', None) import_record_pk = request.POST.get('import_record', request.GET.get('import_record')) try: record = ImportRecord.objects.get(pk=import_record_pk, super_organization_id=org_id) except ImportRecord.DoesNotExist: # clean up the uploaded file os.unlink(path) return JsonResponse({ 'success': False, 'message': "Import Record %s not found" % import_record_pk }) source_type = request.POST.get('source_type', request.GET.get('source_type')) # Add Program & Version fields (empty string if not given) kw_fields = { field: request.POST.get(field, request.GET.get(field, '')) for field in ['source_program', 'source_program_version'] } f = ImportFile.objects.create(import_record=record, uploaded_filename=filename, file=path, source_type=source_type, **kw_fields) return JsonResponse({'success': True, "import_file_id": f.pk}) @staticmethod def _get_pint_var_from_pm_value_object(pm_value): units = pint.UnitRegistry() if '@uom' in pm_value and '#text' in pm_value: # this is the correct expected path for unit-based attributes string_value = pm_value['#text'] try: float_value = float(string_value) except ValueError: return { 'success': False, 'message': 'Could not cast value to float: \"%s\"' % string_value } original_unit_string = pm_value['@uom'] if original_unit_string == 'kBtu': pint_val = float_value * units.kBTU elif original_unit_string == 'kBtu/ft²': pint_val = float_value * units.kBTU / units.sq_ft elif original_unit_string == 'Metric Tons CO2e': pint_val = float_value * units.metric_ton elif original_unit_string == 'kgCO2e/ft²': pint_val = float_value * units.kilogram / units.sq_ft else: return { 'success': False, 'message': 'Unsupported units string: \"%s\"' % original_unit_string } return {'success': True, 'pint_value': pint_val} @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'import_record_id': 'integer', 'properties': [{ 'address_1': 'string', 'city': 'string', 'state_province': 'string', 'postal_code': 'string', 'county': 'string', 'country': 'string', 'property_name': 'string', 'property_id': 'integer', 'year_built': 'integer', }], }, required=['import_record_id', 'properties'], description='An object containing meta data for a property')) @has_perm_class('can_modify_data') @api_endpoint_class @ajax_request_class @action(detail=False, methods=['POST'], parser_classes=(JSONParser, )) def create_from_pm_import(self, request): """ Create an import_record from a PM import request. TODO: The properties key here is going to be an enormous amount of XML data at times, need to change this This allows the PM import workflow to be treated essentially the same as a standard file upload The process comprises the following steps: * Get a unique file name for this portfolio manager import """ doing_pint = False if 'properties' not in request.data: return JsonResponse( { 'success': False, 'message': "Must pass properties in the request body." }, status=status.HTTP_400_BAD_REQUEST) # base file name (will be appended with a random string to ensure uniqueness if multiple on the same day) today_date = datetime.datetime.today().strftime('%Y-%m-%d') file_name = "pm_import_%s.csv" % today_date # create a folder to keep pm_import files path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports", file_name) # Get a unique filename using the get_available_name method in FileSystemStorage s = FileSystemStorage() path = s.get_available_name(path) # verify the directory exists if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) # This list should cover the core keys coming from PM, ensuring that they map easily # We will also look for keys not in this list and just map them to themselves # pm_key_to_column_heading_map = { # 'address_1': 'Address', # 'city': 'City', # 'state_province': 'State', # 'postal_code': 'Zip', # 'county': 'County', # 'country': 'Country', # 'property_name': 'Property Name', # 'property_id': 'Property ID', # 'year_built': 'Year Built', # } # so now it looks like we *don't* need to override these, but instead we should leave all the headers as-is # I'm going to leave this in here for right now, but if it turns out that we don't need it after testing, # then I'll remove it entirely pm_key_to_column_heading_map = {} # We will also create a list of values that are used in PM export to indicate a value wasn't available # When we import them into SEED here we will be sure to not write those values pm_flagged_bad_string_values = [ 'Not Available', 'Unable to Check (not enough data)', 'No Current Year Ending Date', ] # We will make a pass through the first property to get the list of unexpected keys for pm_property in request.data['properties']: for pm_key_name, _ in pm_property.items(): if pm_key_name not in pm_key_to_column_heading_map: pm_key_to_column_heading_map[pm_key_name] = pm_key_name break # Create the header row of the csv file first rows = [] header_row = [] for _, csv_header in pm_key_to_column_heading_map.items(): header_row.append(csv_header) rows.append(header_row) num_properties = len(request.data['properties']) property_num = 0 last_time = datetime.datetime.now() _log.debug("About to try to import %s properties from ESPM" % num_properties) _log.debug("Starting at %s" % last_time) # Create a single row for each building for pm_property in request.data['properties']: # report some helpful info every 20 properties property_num += 1 if property_num % 20 == 0: new_time = datetime.datetime.now() _log.debug("On property number %s; current time: %s" % (property_num, new_time)) this_row = [] # Loop through all known PM variables for pm_variable, _ in pm_key_to_column_heading_map.items(): # Initialize this to False for each pm_variable we will search through added = False # Check if this PM export has this variable in it if pm_variable in pm_property: # If so, create a convenience variable to store it this_pm_variable = pm_property[pm_variable] # Next we need to check type. If it is a string, we will add it here to avoid parsing numerics # However, we need to be sure to not add the flagged bad strings. # However, a flagged value *could* be a value property name, and we would want to allow that if isinstance(this_pm_variable, basestring): if pm_variable == 'property_name': this_row.append(this_pm_variable) added = True elif pm_variable == 'property_notes': sanitized_string = this_pm_variable.replace( '\n', ' ') this_row.append(sanitized_string) added = True elif this_pm_variable not in pm_flagged_bad_string_values: this_row.append(this_pm_variable) added = True # If it isn't a string, it should be a dictionary, storing numeric data and units, etc. else: # As long as it is a valid dictionary, try to get a meaningful value out of it if this_pm_variable and '#text' in this_pm_variable and this_pm_variable[ '#text'] != 'Not Available': # Coerce the value into a proper set of Pint units for us if doing_pint: new_var = UploadViewSet._get_pint_var_from_pm_value_object( this_pm_variable) if new_var['success']: pint_value = new_var['pint_value'] this_row.append(pint_value.magnitude) added = True # TODO: What to do with the pint_value.units here? else: this_row.append( float(this_pm_variable['#text'])) added = True # And finally, if we haven't set the added flag, give the csv column a blank value if not added: this_row.append('') # Then add this property row of data rows.append(this_row) # Then write the actual data out as csv with open(path, 'w', encoding='utf-8') as csv_file: pm_csv_writer = csv.writer(csv_file) for row_num, row in enumerate(rows): pm_csv_writer.writerow(row) # Look up the import record (data set) org_id = request.data['organization_id'] import_record_pk = request.data['import_record_id'] try: record = ImportRecord.objects.get(pk=import_record_pk, super_organization_id=org_id) except ImportRecord.DoesNotExist: # clean up the uploaded file os.unlink(path) return JsonResponse({ 'success': False, 'message': "Import Record %s not found" % import_record_pk }) # Create a new import file object in the database f = ImportFile.objects.create( import_record=record, uploaded_filename=file_name, file=path, source_type=SEED_DATA_SOURCES[PORTFOLIO_RAW], **{ 'source_program': 'PortfolioManager', 'source_program_version': '1.0' }) # Return the newly created import file ID return JsonResponse({'success': True, 'import_file_id': f.pk})
class AnalysisViewSet(viewsets.ViewSet): serializer_class = AnalysisSerializer model = Analysis @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=CreateAnalysisSerializer, ) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def create(self, request): serializer = CreateAnalysisSerializer(data=request.data) if not serializer.is_valid(): return JsonResponse({ 'status': 'error', 'message': 'Bad request', 'errors': serializer.errors }) analysis = serializer.save( user_id=request.user.id, organization_id=request.query_params['organization_id']) pipeline = AnalysisPipeline.factory(analysis) try: progress_data = pipeline.prepare_analysis( serializer.validated_data['property_view_ids']) return JsonResponse({ 'status': 'success', 'progress_key': progress_data['progress_key'], 'progress': progress_data, }) except AnalysisPipelineException as e: return JsonResponse({ 'status': 'error', 'message': str(e) }, status=HTTP_409_CONFLICT) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(True), AutoSchemaHelper.query_integer_field('property_id', False, 'Property ID') ]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def list(self, request): organization_id = request.query_params.get('organization_id', None) property_id = request.query_params.get('property_id', None) analyses = [] if property_id is not None: analyses_queryset = (Analysis.objects.filter( organization=organization_id, analysispropertyview__property=property_id).distinct(). order_by('id')) else: analyses_queryset = (Analysis.objects.filter( organization=organization_id).order_by('id')) for analysis in analyses_queryset: property_view_info = analysis.get_property_view_info(property_id) serialized_analysis = AnalysisSerializer(analysis).data serialized_analysis.update(property_view_info) analyses.append(serialized_analysis) return JsonResponse({'status': 'success', 'analyses': analyses}) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field(True)]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def retrieve(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': "Requested analysis doesn't exist in this organization." }, status=HTTP_409_CONFLICT) serialized_analysis = AnalysisSerializer(analysis).data property_view_info = analysis.get_property_view_info() serialized_analysis.update(property_view_info) return JsonResponse({ 'status': 'success', 'analysis': serialized_analysis }) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['post']) def start(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) pipeline = AnalysisPipeline.factory(analysis) progress_data = pipeline.start_analysis() return JsonResponse({ 'status': 'success', 'progress_key': progress_data['progress_key'], 'progress': progress_data, }) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Requested analysis doesn\'t exist in this organization.' }, status=HTTP_409_CONFLICT) except AnalysisPipelineException as e: return JsonResponse({ 'status': 'error', 'message': str(e) }, status=HTTP_409_CONFLICT) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['post']) def stop(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) pipeline = AnalysisPipeline.factory(analysis) pipeline.stop() return JsonResponse({ 'status': 'success', }) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Requested analysis doesn\'t exist in this organization.' }, status=HTTP_409_CONFLICT) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def destroy(self, request, pk): organization_id = int(request.query_params.get('organization_id', 0)) try: analysis = Analysis.objects.get(id=pk, organization_id=organization_id) pipeline = AnalysisPipeline.factory(analysis) pipeline.delete() return JsonResponse({ 'status': 'success', }) except Analysis.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Requested analysis doesn\'t exist in this organization.' }, status=HTTP_409_CONFLICT)
class UbidViewSet(viewsets.ViewSet, OrgMixin): @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'property_view_ids': ['integer'], 'taxlot_view_ids': ['integer'], }, description= 'IDs by inventory type for records to have their UBID decoded.')) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def decode_by_ids(self, request): """ Submit a request to decode UBIDs for property and tax lot records. """ body = dict(request.data) org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') if property_view_ids: property_views = PropertyView.objects.filter( id__in=property_view_ids, cycle__organization_id=org_id) properties = PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id'))) decode_unique_ids(properties) if taxlot_view_ids: taxlot_views = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=org_id) taxlots = TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id'))) decode_unique_ids(taxlots) return JsonResponse({'status': 'success'}) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'property_view_ids': ['integer'], 'taxlot_view_ids': ['integer'], }, description= 'IDs by inventory type for records to be used in building a UBID decoding summary.' )) @ajax_request_class @has_perm_class('can_view_data') @action(detail=False, methods=['POST']) def decode_results(self, request): """ Generate a summary of populated, unpopulated, and decoded UBIDs for property and tax lot records. """ body = dict(request.data) org_id = self.get_organization(request) ubid_unpopulated = 0 ubid_successfully_decoded = 0 ubid_not_decoded = 0 ulid_unpopulated = 0 ulid_successfully_decoded = 0 ulid_not_decoded = 0 property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') if property_view_ids: property_views = PropertyView.objects.filter( id__in=property_view_ids, cycle__organization_id=org_id) property_states = PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id'))) ubid_unpopulated = property_states.filter( ubid__isnull=True).count() ubid_successfully_decoded = property_states.filter( ubid__isnull=False, bounding_box__isnull=False, centroid__isnull=False).count() # for ubid_not_decoded, bounding_box could be populated from a GeoJSON import ubid_not_decoded = property_states.filter( ubid__isnull=False, centroid__isnull=True).count() if taxlot_view_ids: taxlot_views = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=org_id) taxlot_states = TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id'))) ulid_unpopulated = taxlot_states.filter(ulid__isnull=True).count() ulid_successfully_decoded = taxlot_states.filter( ulid__isnull=False, bounding_box__isnull=False, centroid__isnull=False).count() # for ulid_not_decoded, bounding_box could be populated from a GeoJSON import ulid_not_decoded = taxlot_states.filter( ulid__isnull=False, centroid__isnull=True).count() result = { "ubid_unpopulated": ubid_unpopulated, "ubid_successfully_decoded": ubid_successfully_decoded, "ubid_not_decoded": ubid_not_decoded, "ulid_unpopulated": ulid_unpopulated, "ulid_successfully_decoded": ulid_successfully_decoded, "ulid_not_decoded": ulid_not_decoded, } return result
class ColumnViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet, OrgCreateUpdateMixin): """ create: Create a new Column within a specified org or user's currently activated org. update: Update a column and modify which dataset it belongs to. delete: Deletes a single column. """ raise_exception = True serializer_class = ColumnSerializer renderer_classes = (JSONRenderer, ) model = Column pagination_class = None parser_classes = (JSONParser, FormParser) def get_queryset(self): # check if the request is properties or taxlots org_id = self.get_organization(self.request) return Column.objects.filter(organization_id=org_id) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(required=False), AutoSchemaHelper.query_string_field( name='inventory_type', required=False, description= 'Which inventory type is being matched (for related fields and naming)' '\nDefault: "property"'), AutoSchemaHelper.query_boolean_field( name='only_used', required=False, description= 'Determine whether or not to show only the used fields ' '(i.e. only columns that have been mapped)' '\nDefault: "false"'), AutoSchemaHelper.query_boolean_field( name='display_units', required=False, description='If true, any columns that have units will have them' ' added as a suffix to the display_name' '\nDefault: "false"'), ], ) @api_endpoint_class @ajax_request_class def list(self, request): """ Retrieves all columns for the user's organization including the raw database columns. Will return all the columns across both the Property and Tax Lot tables. The related field will be true if the column came from the other table that is not the 'inventory_type' (which defaults to Property) """ organization_id = self.get_organization(self.request) inventory_type = request.query_params.get('inventory_type', 'property') only_used = json.loads(request.query_params.get('only_used', 'false')) columns = Column.retrieve_all(organization_id, inventory_type, only_used) organization = Organization.objects.get(pk=organization_id) if json.loads(request.query_params.get('display_units', 'false')): columns = [add_pint_unit_suffix(organization, x) for x in columns] return JsonResponse({ 'status': 'success', 'columns': columns, }) @swagger_auto_schema_org_query_param @ajax_request_class def retrieve(self, request, pk=None): """ This API endpoint retrieves a Column """ organization_id = self.get_organization(self.request) # check if column exists for the organization try: c = Column.objects.get(pk=pk) except Column.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'column with id {} does not exist'.format(pk) }, status=status.HTTP_404_NOT_FOUND) if c.organization.id != organization_id: return JsonResponse( { 'status': 'error', 'message': 'Organization ID mismatch between column and organization' }, status=status.HTTP_400_BAD_REQUEST) return JsonResponse({ 'status': 'success', 'column': ColumnSerializer(c).data }) @ajax_request_class @has_perm_class('can_modify_data') def update(self, request, pk=None): organization_id = request.query_params.get('organization_id', None) request.data['shared_field_type'] = request.data['sharedFieldType'] del request.data['sharedFieldType'] # Ensure ComStock uniqueness across properties and taxlots together if request.data['comstock_mapping'] is not None: Column.objects.filter(organization_id=organization_id, comstock_mapping=request.data['comstock_mapping']) \ .update(comstock_mapping=None) return super(ColumnViewSet, self).update(request, pk) @ajax_request_class @has_perm_class('can_modify_data') def destroy(self, request, pk=None): org_id = self.get_organization(request) try: column = Column.objects.get(id=pk, organization_id=org_id) except Column.DoesNotExist: return JsonResponse( { 'success': False, 'message': 'Cannot find column in org=%s with pk=%s' % (org_id, pk) }, status=status.HTTP_404_NOT_FOUND) if not column.is_extra_data: return JsonResponse( { 'success': False, 'message': 'Only extra_data columns can be deleted' }, status=status.HTTP_400_BAD_REQUEST) # Delete key from jsonb data if column.table_name == 'PropertyState': states = PropertyState.objects.filter( organization_id=org_id, data_state=DATA_STATE_MATCHING, extra_data__has_key=column.column_name) state_count = states.count() # far faster than iterating states, popping the key, and saving, but doesn't update the hash # with connection.cursor() as cursor: # cursor.execute("UPDATE seed_propertystate " # "SET extra_data = extra_data - %s " # "WHERE organization_id = %s AND data_state = %s", # [column.column_name, org_id, DATA_STATE_MATCHING]) elif column.table_name == 'TaxLotState': states = TaxLotState.objects.filter( organization_id=org_id, data_state=DATA_STATE_MATCHING, extra_data__has_key=column.column_name) state_count = states.count() # far faster than iterating states, popping the key, and saving, but doesn't update the hash # with connection.cursor() as cursor: # cursor.execute("UPDATE seed_taxlotstate " # "SET extra_data = extra_data - %s " # "WHERE organization_id = %s AND data_state = %s", # [column.column_name, org_id, DATA_STATE_MATCHING]) else: return JsonResponse( { 'success': False, 'message': 'Unexpected table_name \'%s\' for column with pk=%s' % (column.table_name, pk) }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) # Pop the key and update the hash for state in states: state.extra_data.pop(column.column_name) state.save() # Delete all mappings from raw column names to the mapped column, then delete the mapped column ColumnMapping.objects.filter(column_mapped=column).delete() column.delete() table_display_name = column.table_name if state_count == 1 else column.table_name + 's' return JsonResponse( { 'success': True, 'message': 'Removed \'%s\' from %s %s' % (column.column_name, state_count, table_display_name) }, status=status.HTTP_200_OK) @swagger_auto_schema(request_body=AutoSchemaHelper.schema_factory({ 'new_column_name': 'string', 'overwrite': 'boolean' })) @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def rename(self, request, pk=None): """ This API endpoint renames a Column """ org_id = self.get_organization(request) try: column = Column.objects.get(id=pk, organization_id=org_id) except Column.DoesNotExist: return JsonResponse( { 'success': False, 'message': 'Cannot find column in org=%s with pk=%s' % (org_id, pk) }, status=status.HTTP_404_NOT_FOUND) new_column_name = request.data.get('new_column_name', None) overwrite = request.data.get('overwrite', False) if not new_column_name: return JsonResponse( { 'success': False, 'message': 'You must specify the name of the new column as "new_column_name"' }, status=status.HTTP_400_BAD_REQUEST) result = column.rename_column(new_column_name, overwrite) if not result[0]: return JsonResponse( { 'success': False, 'message': 'Unable to rename column with message: "%s"' % result[1] }, status=status.HTTP_400_BAD_REQUEST) else: return JsonResponse({'success': True, 'message': result[1]}) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_string_field( 'inventory_type', required=True, description='Inventory Type, either "property" or "taxlot"') ]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @action(detail=False, methods=['GET']) def mappable(self, request): """ List only inventory columns that are mappable """ organization_id = int(request.query_params.get('organization_id')) inventory_type = request.query_params.get('inventory_type') if inventory_type not in ['property', 'taxlot']: return JsonResponse({ 'status': 'error', 'message': 'Query param `inventory_type` must be "property" or "taxlot"' }) columns = Column.retrieve_mapping_columns(organization_id, inventory_type) return JsonResponse({'status': 'success', 'columns': columns})
class UserViewSet(viewsets.ViewSet): raise_exception = True def validate_request_user(self, pk, request): try: user = User.objects.get(pk=pk) except ObjectDoesNotExist: return False, JsonResponse( {'status': 'error', 'message': "Could not find user with pk = " + str(pk)}, status=status.HTTP_404_NOT_FOUND) if not user == request.user: return False, JsonResponse( {'status': 'error', 'message': "Cannot access user with pk = " + str(pk)}, status=status.HTTP_403_FORBIDDEN) return True, user @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'org_name': 'string', 'first_name': 'string', 'last_name': 'string', 'role': 'string', 'email': 'string', }, required=['first_name', 'last_name', 'role', 'email'], description='An object containing meta data for a new user:\n' '-org_name: New organization name if creating a new organization for this user\n' '-first_name: First name of new user\n' '-last_name: Last name of new user\n' '-role: one of owner, member, or viewer\n' '-email: Email address of the new user' ), responses={ 200: AutoSchemaHelper.schema_factory( { 'status': 'string', 'message': 'string', 'org': 'string', 'org_created': 'boolean', 'username': '******', 'user_id': 'string', }, description='Properties:\n' '-org: name of new org (or existing org)\n' '-org_created: true if new org created\n' '-username: username of new user\n' '-user_id: user id (pk) of new user' ) } ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_owner') def create(self, request): """ Creates a new SEED user. One of 'organization_id' or 'org_name' is needed. Sends invitation email to the new user. """ body = request.data org_name = body.get('org_name') org_id = request.query_params.get('organization_id', None) if (org_name and org_id) or (not org_name and not org_id): return JsonResponse({ 'status': 'error', 'message': 'Choose either an existing org or provide a new one' }, status=status.HTTP_409_CONFLICT) first_name = body['first_name'] last_name = body['last_name'] email = body['email'] username = body['email'] user, created = User.objects.get_or_create(username=username.lower()) if org_id: org = Organization.objects.get(pk=org_id) org_created = False else: org, _, _ = create_organization(user, org_name) org_created = True # Add the user to the org. If this is the org's first user, # the user becomes the owner/admin automatically. # see Organization.add_member() if not org.is_member(user): org.add_member(user) if body.get('role'): # check if this is a dict, if so, grab the value out of 'value' role = body['role'] try: _get_role_from_js(role) except Exception: return JsonResponse({'status': 'error', 'message': 'valid arguments for role are [viewer, member, ' 'owner]'}, status=status.HTTP_400_BAD_REQUEST) OrganizationUser.objects.filter( organization_id=org.pk, user_id=user.pk ).update(role_level=_get_role_from_js(role)) if created: user.set_unusable_password() user.email = email user.first_name = first_name user.last_name = last_name user.save() try: domain = request.get_host() except Exception: domain = 'seed-platform.org' invite_to_seed( domain, user.email, default_token_generator.make_token(user), user.pk, first_name ) return JsonResponse({ 'status': 'success', 'message': user.email, 'org': org.name, 'org_created': org_created, 'username': user.username, 'user_id': user.id }) @swagger_auto_schema( responses={ 200: ListUsersResponseSerializer, } ) @ajax_request_class @has_perm_class('requires_superuser') def list(self, request): """ Retrieves all users' email addresses and IDs. Only usable by superusers. """ users = [] for user in User.objects.only('id', 'email'): users.append({'email': user.email, 'user_id': user.id}) return JsonResponse({'users': users}) @ajax_request_class @api_endpoint_class @action(detail=False, methods=['GET']) def current(self, request): """ Returns the id (primary key) for the current user to allow it to be passed to other user related endpoints --- type: pk: description: Primary key for the current user required: true type: string """ return JsonResponse({'pk': request.user.id}) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'role': 'string', }, description='new role for user', ), ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_owner') @action(detail=True, methods=['PUT']) def role(self, request, pk=None): """ Updates a user's role within an organization. """ body = request.data role = _get_role_from_js(body['role']) user_id = pk organization_id = request.query_params.get('organization_id', None) is_last_member = not OrganizationUser.objects.filter( organization_id=organization_id, ).exclude(user_id=user_id).exists() if is_last_member: return JsonResponse({ 'status': 'error', 'message': 'an organization must have at least one member' }, status=status.HTTP_409_CONFLICT) is_last_owner = not OrganizationUser.objects.filter( organization_id=organization_id, role_level=ROLE_OWNER, ).exclude(user_id=user_id).exists() if is_last_owner: return JsonResponse({ 'status': 'error', 'message': 'an organization must have at least one owner level member' }, status=status.HTTP_409_CONFLICT) OrganizationUser.objects.filter( user_id=user_id, organization_id=organization_id ).update(role_level=role) return JsonResponse({'status': 'success'}) @swagger_auto_schema( responses={ 200: user_response_schema, } ) @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): """ Retrieves the a user's first_name, last_name, email and api key if exists by user ID (pk). """ ok, content = self.validate_request_user(pk, request) if ok: user = content else: return content return JsonResponse({ 'status': 'success', 'first_name': user.first_name, 'last_name': user.last_name, 'email': user.email, 'api_key': user.api_key, }) @ajax_request_class @action(detail=True, methods=['POST']) def generate_api_key(self, request, pk=None): """ Generates a new API key --- parameter_strategy: replace parameters: - name: pk description: User ID / primary key type: integer required: true paramType: path type: status: description: success or error type: string required: true api_key: description: the new API key for this user type: string required: true """ ok, content = self.validate_request_user(pk, request) if ok: user = content else: return content user.generate_key() return { 'status': 'success', 'api_key': User.objects.get(pk=pk).api_key } @swagger_auto_schema( request_body=AutoSchemaHelper.schema_factory({ 'first_name': 'string', 'last_name': 'string', 'email': 'string' }), description='An object containing meta data for a updated user: \n' '- Required - first_name, last_name, email', responses={ 200: user_response_schema, } ) @api_endpoint_class @ajax_request_class def update(self, request, pk=None): """ Updates the user's first name, last name, and email """ body = request.data ok, content = self.validate_request_user(pk, request) if ok: user = content else: return content json_user = body user.first_name = json_user.get('first_name') user.last_name = json_user.get('last_name') user.email = json_user.get('email') user.username = json_user.get('email') user.save() return JsonResponse({ 'status': 'success', 'first_name': user.first_name, 'last_name': user.last_name, 'email': user.email, 'api_key': user.api_key, }) @swagger_auto_schema( request_body=AutoSchemaHelper.schema_factory( { 'current_password': '******', 'password_1': 'string', 'password_2': 'string', }, description='Fill in the current and new matching passwords'), ) @ajax_request_class @action(detail=True, methods=['PUT']) def set_password(self, request, pk=None): """ sets/updates a user's password, follows the min requirement of django password validation settings in config/settings/common.py """ body = request.data ok, content = self.validate_request_user(pk, request) if ok: user = content else: return content current_password = body.get('current_password') p1 = body.get('password_1') p2 = body.get('password_2') if not user.check_password(current_password): return JsonResponse({'status': 'error', 'message': 'current password is not valid'}, status=status.HTTP_400_BAD_REQUEST) if p1 is None or p1 != p2: return JsonResponse({'status': 'error', 'message': 'entered password do not match'}, status=status.HTTP_400_BAD_REQUEST) try: validate_password(p2) except ValidationError as e: return JsonResponse({'status': 'error', 'message': e.messages[0]}, status=status.HTTP_400_BAD_REQUEST) user.set_password(p1) user.save() return JsonResponse({'status': 'success'}) @ajax_request_class def get_actions(self, request): """returns all actions""" return { 'status': 'success', 'actions': list(PERMS.keys()), } @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'actions': ['string'], }, description='A list of actions to check: examples include (requires_parent_org_owner, ' 'requires_owner, requires_member, requires_viewer, ' 'requires_superuser, can_create_sub_org, can_remove_org)' ), responses={ 200: AutoSchemaHelper.schema_factory({ 'auth': { 'action_name': 'boolean' } }) } ) @ajax_request_class @action(detail=True, methods=['POST']) def is_authorized(self, request, pk=None): """ Checks the auth for a given action, if user is the owner of the parent org then True is returned for each action """ ok, content = self.validate_request_user(pk, request) if ok: user = content else: return content actions, org, error, message = self._parse_is_authenticated_params(request) if error: return JsonResponse({ 'status': 'error', 'message': message }, status=status.HTTP_400_BAD_REQUEST) # If the only action requested is 'requires_superuser' no need to check an org affiliation if len(actions) == 1 and actions[0] == 'requires_superuser': return JsonResponse( {'status': 'success', 'auth': {'requires_superuser': user.is_superuser}}) auth = self._try_parent_org_auth(user, org, actions) if auth: return JsonResponse({'status': 'success', 'auth': auth}) try: ou = OrganizationUser.objects.get(user=user, organization=org) except OrganizationUser.DoesNotExist: return JsonResponse({'status': 'error', 'message': 'user does not exist'}) auth = {action: PERMS[action](ou) for action in actions} return JsonResponse({'status': 'success', 'auth': auth}) def _parse_is_authenticated_params(self, request): """checks if the org exists and if the actions are present :param request: the request :returns: tuple (actions, org, error, message) """ error = False message = "" body = request.data if not body.get('actions'): message = 'no actions to check' error = True org_id = request.query_params.get('organization_id', None) if org_id == '': message = 'organization id is undefined' error = True org = None else: try: org = Organization.objects.get(pk=org_id) except Organization.DoesNotExist: message = 'organization does not exist' error = True org = None return body.get('actions'), org, error, message def _try_parent_org_auth(self, user, organization, actions): """checks the parent org for permissions, if the user is not the owner of the parent org, then None is returned. :param user: the request user :param organization: org to check its parent :param actions: list of str actions to check :returns: a dict of action permission resolutions or None """ try: ou = OrganizationUser.objects.get( user=user, organization=organization.parent_org, role_level=ROLE_OWNER ) except OrganizationUser.DoesNotExist: return None return { action: PERMS['requires_owner'](ou) for action in actions } @swagger_auto_schema( responses={ 200: AutoSchemaHelper.schema_factory({ 'show_shared_buildings': 'boolean' }) } ) @ajax_request_class @action(detail=True, methods=['GET']) def shared_buildings(self, request, pk=None): """ Get the request user's ``show_shared_buildings`` attr """ ok, content = self.validate_request_user(pk, request) if ok: user = content else: return content return JsonResponse({ 'status': 'success', 'show_shared_buildings': user.show_shared_buildings, }) @swagger_auto_schema_org_query_param @ajax_request_class @action(detail=True, methods=['PUT']) def default_organization(self, request, pk=None): """ Sets the user's default organization """ ok, content = self.validate_request_user(pk, request) if ok: user = content else: return content user.default_organization_id = request.query_params.get('organization_id', None) user.save() return {'status': 'success'} @has_perm_class('requires_superuser') @ajax_request_class @action(detail=True, methods=['PUT']) def deactivate(self, request, pk=None): """ Deactivates a user """ try: user_id = pk user = SEEDUser.objects.get( id=user_id ) user.deactivate_user() return JsonResponse({ 'status': 'successfully deactivated', 'data': user.email }) except Exception as e: return JsonResponse({ 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST)
class ImportFileViewSet(viewsets.ViewSet): raise_exception = True queryset = ImportFile.objects.all() @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): """ Retrieves details about an ImportFile. """ import_file_id = pk orgs = request.user.orgs.all() try: import_file = ImportFile.objects.get(pk=import_file_id) d = ImportRecord.objects.filter(super_organization__in=orgs, pk=import_file.import_record_id) except ObjectDoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not access an import file with this ID' }, status=status.HTTP_403_FORBIDDEN) # check if user has access to the import file if not d.exists(): return JsonResponse( { 'status': 'error', 'message': 'Could not locate import file with this ID', 'import_file': {}, }, status=status.HTTP_400_BAD_REQUEST) f = obj_to_dict(import_file) f['name'] = import_file.filename_only if not import_file.uploaded_filename: f['uploaded_filename'] = import_file.filename f['dataset'] = obj_to_dict(import_file.import_record) return JsonResponse({ 'status': 'success', 'import_file': f, }) @swagger_auto_schema_org_query_param @has_perm_class('requires_viewer') @api_endpoint_class @ajax_request_class @action(detail=True, methods=['GET']) def first_five_rows(self, request, pk=None): """ Retrieves the first five rows of an ImportFile. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) if import_file.cached_second_to_fifth_row is None: return JsonResponse( { 'status': 'error', 'message': 'Internal problem occurred, import file first five rows not cached' }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) ''' import_file.cached_second_to_fifth_row is a field that contains the first 5 lines of data from the file, split on newlines, delimited by ROW_DELIMITER. This becomes an issue when fields have newlines in them, so the following is to handle newlines in the fields. In the case of only one data column there will be no ROW_DELIMITER. ''' header = import_file.cached_first_row.split(ROW_DELIMITER) data = import_file.cached_second_to_fifth_row return JsonResponse({ 'status': 'success', 'first_five_rows': convert_first_five_rows_to_list(header, data) }) @swagger_auto_schema_org_query_param @has_perm_class('requires_viewer') @api_endpoint_class @ajax_request_class @action(detail=True, methods=['GET']) def raw_column_names(self, request, pk=None): """ Retrieves a list of all column names from an ImportFile. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return JsonResponse({ 'status': 'success', 'raw_columns': import_file.first_row_columns }) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], responses={200: MappingResultsResponseSerializer}) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST'], url_path='mapping_results') def mapping_results(self, request, pk=None): """ Retrieves a paginated list of Properties and Tax Lots for an import file after mapping. """ import_file_id = pk org_id = request.query_params.get('organization_id', None) org = Organization.objects.get(pk=org_id) try: # get the field names that were in the mapping import_file = ImportFile.objects.get( pk=import_file_id, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # List of the only fields to show field_names = import_file.get_cached_mapped_columns # set of fields fields = { 'PropertyState': ['id', 'extra_data', 'lot_number'], 'TaxLotState': ['id', 'extra_data'] } columns_from_db = Column.retrieve_all(org_id) property_column_name_mapping = {} taxlot_column_name_mapping = {} for field_name in field_names: # find the field from the columns in the database for column in columns_from_db: if column['table_name'] == 'PropertyState' and \ field_name[0] == 'PropertyState' and \ field_name[1] == column['column_name']: property_column_name_mapping[ column['column_name']] = column['name'] if not column['is_extra_data']: fields['PropertyState'].append( field_name[1]) # save to the raw db fields continue elif column['table_name'] == 'TaxLotState' and \ field_name[0] == 'TaxLotState' and \ field_name[1] == column['column_name']: taxlot_column_name_mapping[ column['column_name']] = column['name'] if not column['is_extra_data']: fields['TaxLotState'].append( field_name[1]) # save to the raw db fields continue inventory_type = request.data.get('inventory_type', 'all') result = {'status': 'success'} if inventory_type == 'properties' or inventory_type == 'all': properties = PropertyState.objects.filter( import_file_id=import_file_id, data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], merge_state__in=[ MERGE_STATE_UNKNOWN, MERGE_STATE_NEW ]).only(*fields['PropertyState']).order_by('id') property_results = [] for prop in properties: prop_dict = TaxLotProperty.model_to_dict_with_mapping( prop, property_column_name_mapping, fields=fields['PropertyState'], exclude=['extra_data']) prop_dict.update( TaxLotProperty.extra_data_to_dict_with_mapping( prop.extra_data, property_column_name_mapping, fields=prop.extra_data.keys(), ).items()) prop_dict = apply_display_unit_preferences(org, prop_dict) property_results.append(prop_dict) result['properties'] = property_results if inventory_type == 'taxlots' or inventory_type == 'all': tax_lots = TaxLotState.objects.filter( import_file_id=import_file_id, data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], merge_state__in=[MERGE_STATE_UNKNOWN, MERGE_STATE_NEW ]).only(*fields['TaxLotState']).order_by('id') tax_lot_results = [] for tax_lot in tax_lots: tax_lot_dict = TaxLotProperty.model_to_dict_with_mapping( tax_lot, taxlot_column_name_mapping, fields=fields['TaxLotState'], exclude=['extra_data']) tax_lot_dict.update( TaxLotProperty.extra_data_to_dict_with_mapping( tax_lot.extra_data, taxlot_column_name_mapping, fields=tax_lot.extra_data.keys(), ).items()) tax_lot_dict = apply_display_unit_preferences( org, tax_lot_dict) tax_lot_results.append(tax_lot_dict) result['tax_lots'] = tax_lot_results return result @staticmethod def has_coparent(state_id, inventory_type, fields=None): """ Return the coparent of the current state id based on the inventory type. If fields are given (as a list), then it will only return the fields specified of the state model object as a dictionary. :param state_id: int, ID of PropertyState or TaxLotState :param inventory_type: string, either properties | taxlots :param fields: list, either None or list of fields to return :return: dict or state object, If fields is not None then will return state_object """ state_model = PropertyState if inventory_type == 'properties' else TaxLotState # TODO: convert coparent to instance method, not class method audit_entry, audit_count = state_model.coparent(state_id) if audit_count == 0: return False if audit_count > 1: return JsonResponse( { 'status': 'error', 'message': 'Internal problem occurred, more than one merge record found' }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return audit_entry[0] @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=AutoSchemaHelper.schema_factory({ 'remap': 'boolean', 'mark_as_done': 'boolean', })) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def map(self, request, pk=None): """ Starts a background task to convert imported raw data into PropertyState and TaxLotState, using user's column mappings. """ body = request.data remap = body.get('remap', False) mark_as_done = body.get('mark_as_done', True) org_id = request.query_params.get('organization_id', None) import_file = ImportFile.objects.filter( pk=pk, import_record__super_organization_id=org_id) if not import_file.exists(): return { 'status': 'error', 'message': 'ImportFile {} does not exist'.format(pk) } # return remap_data(import_file_id) return JsonResponse(map_data(pk, remap, mark_as_done)) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def start_system_matching_and_geocoding(self, request, pk=None): """ Starts a background task to attempt automatic matching between buildings in an ImportFile with other existing buildings within the same org. """ org_id = request.query_params.get('organization_id', None) try: ImportFile.objects.get(pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return geocode_and_match_buildings_task(pk) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def start_data_quality_checks(self, request, pk=None): """ Starts a background task to attempt automatic matching between buildings in an ImportFile with other existing buildings within the same org. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return_value = do_checks(org_id, None, None, import_file.pk) # step 5: create a new model instance return JsonResponse({ 'progress_key': return_value['progress_key'], 'progress': return_value, }) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def validate_use_cases(self, request, pk=None): """ Starts a background task to call BuildingSync's use case validation tool. """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) return task_validate_use_cases(import_file.pk) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=AutoSchemaHelper.schema_factory( {'cycle_id': 'string'})) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def start_save_data(self, request, pk=None): """ Starts a background task to import raw data from an ImportFile into PropertyState objects as extra_data. If the cycle_id is set to year_ending then the cycle ID will be set to the year_ending column for each record in the uploaded file. Note that the year_ending flag is not yet enabled. """ body = request.data import_file_id = pk if not import_file_id: return JsonResponse( { 'status': 'error', 'message': 'must pass file_id of file to save' }, status=status.HTTP_400_BAD_REQUEST) org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=import_file_id, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) cycle_id = body.get('cycle_id') if not cycle_id: return JsonResponse( { 'status': 'error', 'message': 'must pass cycle_id of the cycle to save the data' }, status=status.HTTP_400_BAD_REQUEST) elif cycle_id == 'year_ending': _log.error("NOT CONFIGURED FOR YEAR ENDING OPTION AT THE MOMENT") return JsonResponse( { 'status': 'error', 'message': 'SEED is unable to parse year_ending at the moment' }, status=status.HTTP_400_BAD_REQUEST) else: # find the cycle cycle = Cycle.objects.get(id=cycle_id) if cycle: # assign the cycle id to the import file object import_file.cycle = cycle import_file.save() else: return JsonResponse( { 'status': 'error', 'message': 'cycle_id was invalid' }, status=status.HTTP_400_BAD_REQUEST) return JsonResponse(task_save_raw(import_file.id)) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def mapping_done(self, request, pk=None): """ Tell the backend that the mapping is complete. """ import_file_id = pk if not import_file_id: return JsonResponse( { 'status': 'error', 'message': 'must pass import_file_id' }, status=status.HTTP_400_BAD_REQUEST) org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=import_file_id, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'no import file with given id' }, status=status.HTTP_404_NOT_FOUND) import_file.mapping_done = True import_file.save() return JsonResponse({'status': 'success', 'message': ''}) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def matching_and_geocoding_results(self, request, pk=None): """ Retrieves the number of matched and unmatched properties & tax lots for a given ImportFile record. Specifically for new imports """ org_id = request.query_params.get('organization_id', None) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # property views associated with this imported file (including merges) properties_new = [] properties_matched = list( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED, ).values_list('id', flat=True)) # Check audit log in case PropertyStates are listed as "new" but were merged into a different property properties = list( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_NEW, )) for state in properties: audit_creation_id = PropertyAuditLog.objects.only('id').exclude( import_filename=None).get(state_id=state.id, name='Import Creation') if PropertyAuditLog.objects.exclude( record_type=AUDIT_USER_EDIT).filter( parent1_id=audit_creation_id).exists(): properties_matched.append(state.id) else: properties_new.append(state.id) tax_lots_new = [] tax_lots_matched = list( TaxLotState.objects.only('id').filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED, ).values_list('id', flat=True)) # Check audit log in case TaxLotStates are listed as "new" but were merged into a different tax lot taxlots = list( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_NEW, )) for state in taxlots: audit_creation_id = TaxLotAuditLog.objects.only('id').exclude( import_filename=None).get(state_id=state.id, name='Import Creation') if TaxLotAuditLog.objects.exclude( record_type=AUDIT_USER_EDIT).filter( parent1_id=audit_creation_id).exists(): tax_lots_matched.append(state.id) else: tax_lots_new.append(state.id) # Construct Geocode Results property_geocode_results = { 'high_confidence': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='High')), 'low_confidence': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='Low')), 'manual': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Manually geocoded (N/A)')), 'missing_address_components': len( PropertyState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Missing address components (N/A)')), } tax_lot_geocode_results = { 'high_confidence': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='High')), 'low_confidence': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence__startswith='Low')), 'manual': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Manually geocoded (N/A)')), 'missing_address_components': len( TaxLotState.objects.filter( import_file__pk=import_file.pk, data_state=DATA_STATE_MATCHING, geocoding_confidence='Missing address components (N/A)')), } # merge in any of the matching results from the JSON field return { 'status': 'success', 'import_file_records': import_file.matching_results_data.get('import_file_records', None), 'properties': { 'initial_incoming': import_file.matching_results_data.get( 'property_initial_incoming', None), 'duplicates_against_existing': import_file.matching_results_data.get( 'property_duplicates_against_existing', None), 'duplicates_within_file': import_file.matching_results_data.get( 'property_duplicates_within_file', None), 'merges_against_existing': import_file.matching_results_data.get( 'property_merges_against_existing', None), 'merges_between_existing': import_file.matching_results_data.get( 'property_merges_between_existing', None), 'merges_within_file': import_file.matching_results_data.get( 'property_merges_within_file', None), 'new': import_file.matching_results_data.get('property_new', None), 'geocoded_high_confidence': property_geocode_results.get('high_confidence'), 'geocoded_low_confidence': property_geocode_results.get('low_confidence'), 'geocoded_manually': property_geocode_results.get('manual'), 'geocode_not_possible': property_geocode_results.get('missing_address_components'), }, 'tax_lots': { 'initial_incoming': import_file.matching_results_data.get( 'tax_lot_initial_incoming', None), 'duplicates_against_existing': import_file.matching_results_data.get( 'tax_lot_duplicates_against_existing', None), 'duplicates_within_file': import_file.matching_results_data.get( 'tax_lot_duplicates_within_file', None), 'merges_against_existing': import_file.matching_results_data.get( 'tax_lot_merges_against_existing', None), 'merges_between_existing': import_file.matching_results_data.get( 'tax_lot_merges_between_existing', None), 'merges_within_file': import_file.matching_results_data.get( 'tax_lot_merges_within_file', None), 'new': import_file.matching_results_data.get('tax_lot_new', None), 'geocoded_high_confidence': tax_lot_geocode_results.get('high_confidence'), 'geocoded_low_confidence': tax_lot_geocode_results.get('low_confidence'), 'geocoded_manually': tax_lot_geocode_results.get('manual'), 'geocode_not_possible': tax_lot_geocode_results.get('missing_address_components'), } } @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def mapping_suggestions(self, request, pk): """ Returns suggested mappings from an uploaded file's headers to known data fields. """ organization_id = request.query_params.get('organization_id', None) result = {'status': 'success'} membership = OrganizationUser.objects.select_related('organization') \ .get(organization_id=organization_id, user=request.user) organization = membership.organization # For now, each organization holds their own mappings. This is non-ideal, but it is the # way it is for now. In order to move to parent_org holding, then we need to be able to # dynamically match columns based on the names and not the db id (or support many-to-many). # parent_org = organization.get_parent() try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=organization.pk) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # Get a list of the database fields in a list, these are the db columns and the extra_data columns property_columns = Column.retrieve_mapping_columns( organization.pk, 'property') taxlot_columns = Column.retrieve_mapping_columns( organization.pk, 'taxlot') # If this is a portfolio manager file, then load in the PM mappings and if the column_mappings # are not in the original mappings, default to PM if import_file.from_portfolio_manager: pm_mappings = simple_mapper.get_pm_mapping( import_file.first_row_columns, resolve_duplicates=True) suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization_id), previous_mapping=get_column_mapping, map_args=[organization], default_mappings=pm_mappings, thresh=80) elif import_file.from_buildingsync: bsync_mappings = xml_mapper.build_column_mapping() suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization_id), previous_mapping=get_column_mapping, map_args=[organization], default_mappings=bsync_mappings, thresh=80) else: # All other input types suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization.pk), previous_mapping=get_column_mapping, map_args=[organization], thresh= 80 # percentage match that we require. 80% is random value for now. ) # replace None with empty string for column names and PropertyState for tables # TODO #239: Move this fix to build_column_mapping for m in suggested_mappings: table, destination_field, _confidence = suggested_mappings[m] if destination_field is None: suggested_mappings[m][1] = '' # Fix the table name, eventually move this to the build_column_mapping for m in suggested_mappings: table, _destination_field, _confidence = suggested_mappings[m] # Do not return the campus, created, updated fields... that is force them to be in the property state if not table or table == 'Property': suggested_mappings[m][0] = 'PropertyState' elif table == 'TaxLot': suggested_mappings[m][0] = 'TaxLotState' result['suggested_column_mappings'] = suggested_mappings result['property_columns'] = property_columns result['taxlot_columns'] = taxlot_columns return JsonResponse(result) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def destroy(self, request, pk): """ Deletes an import file """ organization_id = int(request.query_params.get('organization_id', None)) try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=organization_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # check if the import record exists for the file and organization d = ImportRecord.objects.filter(super_organization_id=organization_id, pk=import_file.import_record.pk) if not d.exists(): return JsonResponse( { 'status': 'error', 'message': 'user does not have permission to delete file', }, status=status.HTTP_403_FORBIDDEN) # This does not actually delete the object because it is a NonDeletableModel import_file.delete() return JsonResponse({'status': 'success'}) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'view_id', required=True, description='ID for property view') ]) @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def greenbutton_meters_preview(self, request, pk): """ Returns validated type units and proposed imports """ org_id = request.query_params.get('organization_id') view_id = request.query_params.get('view_id') try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) parser = reader.GreenButtonParser(import_file.local_file) raw_meter_data = list(parser.data) try: property_id = PropertyView.objects.get( pk=view_id, cycle__organization_id=org_id).property_id except PropertyView.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find property with pk=' + str(view_id) }, status=status.HTTP_400_BAD_REQUEST) meters_parser = MetersParser(org_id, raw_meter_data, source_type=Meter.GREENBUTTON, property_id=property_id) result = {} result["validated_type_units"] = meters_parser.validated_type_units() result["proposed_imports"] = meters_parser.proposed_imports import_file.matching_results_data['property_id'] = property_id import_file.save() return result @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @ajax_request_class @has_perm_class('requires_member') @action(detail=True, methods=['GET']) def pm_meters_preview(self, request, pk): """ Returns validated type units, proposed imports and unlinkable PM ids """ org_id = request.query_params.get('organization_id') try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=org_id) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) parser = reader.MCMParser(import_file.local_file, sheet_name='Meter Entries') raw_meter_data = list(parser.data) meters_parser = MetersParser(org_id, raw_meter_data) result = {} result["validated_type_units"] = meters_parser.validated_type_units() result["proposed_imports"] = meters_parser.proposed_imports result["unlinkable_pm_ids"] = meters_parser.unlinkable_pm_ids return result
StatusLabel as Label, ) from seed.serializers.labels import ( LabelSerializer, ) from seed.utils.api import drf_api_endpoint from django.utils.decorators import method_decorator from seed.utils.api_schema import AutoSchemaHelper from seed.utils.labels import filter_labels_for_inv_type from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet @method_decorator( name='retrieve', decorator=swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ) ]), ) @method_decorator( name='list', decorator=swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ) ]), ) @method_decorator(
class DatasetViewSet(viewsets.ViewSet): raise_exception = True @swagger_auto_schema_org_query_param @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('can_view_data') def list(self, request): """ Retrieves all datasets for the user's organization. """ org_id = request.query_params.get('organization_id', None) org = Organization.objects.get(pk=org_id) datasets = [] for d in ImportRecord.objects.filter(super_organization=org): importfiles = [obj_to_dict(f) for f in d.files] dataset = obj_to_dict(d) dataset['importfiles'] = importfiles if d.last_modified_by: dataset['last_modified_by'] = d.last_modified_by.email dataset['updated_at'] = convert_to_js_timestamp(d.updated_at) datasets.append(dataset) return JsonResponse({ 'status': 'success', 'datasets': datasets, }) @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field() ], request_body=AutoSchemaHelper.schema_factory( {'dataset': 'string'}, description='The new name for this dataset')) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def update(self, request, pk=None): """ Updates the name of a dataset (ImportRecord). """ organization_id = request.query_params.get('organization_id', None) if organization_id is None: return JsonResponse({ 'status': 'error', 'message': 'Missing organization_id query parameter' }) try: organization_id = int(organization_id) except ValueError: return JsonResponse({ 'status': 'error', 'message': 'Bad (non-numeric) organization_id' }) dataset_id = pk name = request.data['dataset'] # check if user has access to the dataset d = ImportRecord.objects.filter(super_organization_id=organization_id, pk=dataset_id) if not d.exists(): return JsonResponse( { 'status': 'error', 'message': 'user does not have permission to update dataset', }, status=status.HTTP_400_BAD_REQUEST) d = d[0] d.name = name d.save() return JsonResponse({ 'status': 'success', }) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_view_data') def retrieve(self, request, pk=None): """ Retrieves a dataset (ImportRecord). """ organization_id = request.query_params.get('organization_id', None) if organization_id is None: return JsonResponse( { 'status': 'error', 'message': 'Missing organization_id query parameter' }, status=status.HTTP_400_BAD_REQUEST) try: organization_id = int(organization_id) except ValueError: return JsonResponse( { 'status': 'error', 'message': 'Bad (non-numeric) organization_id' }, status=status.HTTP_400_BAD_REQUEST) valid_orgs = OrganizationUser.objects.filter( user_id=request.user.id).values_list( 'organization_id', flat=True).order_by('organization_id') if organization_id not in valid_orgs: return JsonResponse( { 'status': 'error', 'message': 'Cannot access datasets for this organization id', }, status=status.HTTP_403_FORBIDDEN) # check if dataset exists try: d = ImportRecord.objects.get(pk=pk) except ImportRecord.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'dataset with id {} does not exist'.format(pk) }, status=status.HTTP_404_NOT_FOUND) if d.super_organization_id != organization_id: return JsonResponse( { 'status': 'error', 'message': 'Organization ID mismatch between dataset and organization' }, status=status.HTTP_400_BAD_REQUEST) dataset = obj_to_dict(d) importfiles = [] for f in d.files: importfile = obj_to_dict(f) if not f.uploaded_filename: importfile['name'] = f.filename_only else: importfile['name'] = f.uploaded_filename importfiles.append(importfile) dataset['importfiles'] = importfiles if d.last_modified_by: dataset['last_modified_by'] = d.last_modified_by.email dataset['updated_at'] = convert_to_js_timestamp(d.updated_at) return JsonResponse({ 'status': 'success', 'dataset': dataset, }) @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def destroy(self, request, pk=None): """ Deletes a dataset (ImportRecord). """ organization_id = int(request.query_params.get('organization_id', None)) dataset_id = pk # check if user has access to the dataset d = ImportRecord.objects.filter(super_organization_id=organization_id, pk=dataset_id) if not d.exists(): return JsonResponse( { 'status': 'error', 'message': 'user does not have permission to deactivate dataset', }, status=status.HTTP_403_FORBIDDEN) d = d[0] d.delete() return JsonResponse({'status': 'success'}) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( {'name': 'string'}, description='Name of the dataset to be created'), responses={ 200: AutoSchemaHelper.schema_factory({ 'id': 'integer', 'name': 'string', }) }) @require_organization_id_class @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def create(self, request): """ Creates a new empty dataset (ImportRecord). """ body = request.data org_id = int(request.query_params.get('organization_id', None)) try: org = Organization.objects.get(pk=org_id) except Organization.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'organization_id not provided' }, status=status.HTTP_400_BAD_REQUEST) record = ImportRecord.objects.create(name=body['name'], app='seed', start_time=timezone.now(), created_at=timezone.now(), last_modified_by=request.user, super_organization=org, owner=request.user) return JsonResponse({ 'status': 'success', 'id': record.pk, 'name': record.name }) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], responses={ 200: AutoSchemaHelper.schema_factory({ 'datasets_count': 'integer', }) }) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @require_organization_id_class @action(detail=False, methods=['GET']) def count(self, request): """ Retrieves the number of datasets for an org. """ org_id = int(request.query_params.get('organization_id', None)) datasets_count = ImportRecord.objects.filter( super_organization_id=org_id).count() return JsonResponse({ 'status': 'success', 'datasets_count': datasets_count })
class ColumnListProfileViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet): """ API endpoint for returning Column List Profiles create: Create a new list profile. The list of columns is an array of column primary keys. If using Swagger, then this will be enters as a list with returns between each primary key. JSON POST Example: { "name": "some new name 3", "profile_location": "List View Profile", "inventory_type": "Tax Lot", "columns": [ {"id": 1, "pinned": false, "order": 10}, {"id": 5, "pinned": true, "order": 14}, {"id": 7, "pinned": true, "order": 14} ] } """ serializer_class = ColumnListProfileSerializer model = ColumnListProfile filter_backends = (ColumnListProfileFilterBackend, ) pagination_class = None # force_parent = True # Ideally the column list profiles would inherit from the parent, # but not yet. # Overridden to augment with protected ComStock list profile if enabled def retrieve(self, request, *args, **kwargs): org_id = self.get_organization(self.request) try: org = Organization.objects.get(pk=org_id) except Organization.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'organization with id %s does not exist' % org_id }, status=status.HTTP_404_NOT_FOUND) if not org.comstock_enabled or kwargs['pk'] != 'null': return super(ColumnListProfileViewSet, self).retrieve(request, args, kwargs) result = { 'status': 'success', 'data': { 'id': None, 'name': 'ComStock', 'profile_location': VIEW_LOCATION_TYPES[VIEW_LIST][1], 'inventory_type': VIEW_LIST_INVENTORY_TYPE[VIEW_LIST_PROPERTY][1], 'columns': self.list_comstock_columns(org_id) } } return JsonResponse(result, status=status.HTTP_200_OK) # Overridden to augment with protected ComStock list profile if enabled @swagger_auto_schema(manual_parameters=[ AutoSchemaHelper.query_org_id_field( required=False, description= "Optional org id which overrides the users (default) current org id" ), AutoSchemaHelper.query_string_field( name='inventory_type', required=True, description="'Property' or 'Tax Lot' for filtering."), AutoSchemaHelper.query_string_field( name='profile_location', required=True, description= "'List View Profile' or 'Detail View Profile' for filtering."), ]) def list(self, request, *args, **kwargs): org_id = self.get_organization(self.request) try: org = Organization.objects.get(pk=org_id) except Organization.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'organization with id %s does not exist' % org_id }, status=status.HTTP_404_NOT_FOUND) inventory_type = request.query_params.get('inventory_type') profile_location = request.query_params.get('profile_location') if not org.comstock_enabled or inventory_type == 'Tax Lot' or profile_location == 'Detail View Profile': return super(ColumnListProfileViewSet, self).list(request, args, kwargs) queryset = self.filter_queryset(self.get_queryset()) page = self.paginate_queryset(queryset) if page is not None: serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) results = list(queryset) base_profiles = self.get_serializer(results, many=True).data # Add ComStock columns base_profiles.append({ "id": None, "name": "ComStock", "profile_location": profile_location, "inventory_type": inventory_type, "columns": self.list_comstock_columns(org_id) }) return Response(base_profiles) @staticmethod def list_comstock_columns(org_id): comstock_columns = Column.objects.filter(organization_id=org_id, comstock_mapping__isnull=False) \ .order_by('comstock_mapping') results = [] for index, column in enumerate(comstock_columns): results.append({ "id": column.id, "pinned": False, "order": index + 1, "column_name": column.column_name, "table_name": column.table_name, "comstock_mapping": column.comstock_mapping }) return results
class TaxlotViewSet(viewsets.ViewSet, OrgMixin, ProfileIdMixin): renderer_classes = (JSONRenderer,) serializer_class = TaxLotSerializer parser_classes = (JSONParser,) _organization = None @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field(required=True)], request_body=AutoSchemaHelper.schema_factory( { 'selected': ['integer'], }, description='IDs for taxlots to be checked for which labels are applied.' ) ) @action(detail=False, methods=['POST']) def labels(self, request): """ Returns a list of all labels where the is_applied field in the response pertains to the labels applied to taxlot_view """ labels = StatusLabel.objects.filter( super_organization=self.get_parent_org(self.request) ).order_by("name").distinct() super_organization = self.get_organization(request) # TODO: refactor to avoid passing request here return get_labels(request, labels, super_organization, 'taxlot_view') def _get_filtered_results(self, request, profile_id): page = request.query_params.get('page', 1) per_page = request.query_params.get('per_page', 1) org_id = request.query_params.get('organization_id', None) cycle_id = request.query_params.get('cycle') # check if there is a query paramater for the profile_id. If so, then use that one profile_id = request.query_params.get('profile_id', profile_id) if not org_id: return JsonResponse( {'status': 'error', 'message': 'Need to pass organization_id as query parameter'}, status=status.HTTP_400_BAD_REQUEST) if cycle_id: cycle = Cycle.objects.get(organization_id=org_id, pk=cycle_id) else: cycle = Cycle.objects.filter(organization_id=org_id).order_by('name') if cycle: cycle = cycle.first() else: return JsonResponse({ 'status': 'error', 'message': 'Could not locate cycle', 'pagination': { 'total': 0 }, 'cycle_id': None, 'results': [] }) # Return taxlot views limited to the 'taxlot_view_ids' list. Otherwise, if selected is empty, return all if 'taxlot_view_ids' in request.data and request.data['taxlot_view_ids']: taxlot_views_list = TaxLotView.objects.select_related('taxlot', 'state', 'cycle') \ .filter(id__in=request.data['taxlot_view_ids'], taxlot__organization_id=org_id, cycle=cycle) \ .order_by('id') else: taxlot_views_list = TaxLotView.objects.select_related('taxlot', 'state', 'cycle') \ .filter(taxlot__organization_id=org_id, cycle=cycle) \ .order_by('id') paginator = Paginator(taxlot_views_list, per_page) try: taxlot_views = paginator.page(page) page = int(page) except PageNotAnInteger: taxlot_views = paginator.page(1) page = 1 except EmptyPage: taxlot_views = paginator.page(paginator.num_pages) page = paginator.num_pages org = Organization.objects.get(pk=org_id) # Retrieve all the columns that are in the db for this organization columns_from_database = Column.retrieve_all(org_id, 'taxlot', False) # This uses an old method of returning the show_columns. There is a new method that # is preferred in v2.1 API with the ProfileIdMixin. if profile_id is None: show_columns = None elif profile_id == -1: show_columns = list(Column.objects.filter( organization_id=org_id ).values_list('id', flat=True)) else: try: profile = ColumnListProfile.objects.get( organization=org, id=profile_id, profile_location=VIEW_LIST, inventory_type=VIEW_LIST_TAXLOT ) show_columns = list(ColumnListProfileColumn.objects.filter( column_list_profile_id=profile.id ).values_list('column_id', flat=True)) except ColumnListProfile.DoesNotExist: show_columns = None related_results = TaxLotProperty.get_related(taxlot_views, show_columns, columns_from_database) # collapse units here so we're only doing the last page; we're already a # realized list by now and not a lazy queryset unit_collapsed_results = [apply_display_unit_preferences(org, x) for x in related_results] response = { 'pagination': { 'page': page, 'start': paginator.page(page).start_index(), 'end': paginator.page(page).end_index(), 'num_pages': paginator.num_pages, 'has_next': paginator.page(page).has_next(), 'has_previous': paginator.page(page).has_previous(), 'total': paginator.count }, 'cycle_id': cycle.id, 'results': unit_collapsed_results } return JsonResponse(response) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'cycle', required=True, description='The ID of the cycle to get taxlots' ), AutoSchemaHelper.query_integer_field( 'page', required=False, description='The current page of taxlots to return' ), AutoSchemaHelper.query_integer_field( 'per_page', required=False, description='The number of items per page to return' ), AutoSchemaHelper.query_integer_field( 'profile_id', required=False, description='The ID of the column profile to use' ) ] ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') def list(self, request): """ List all the properties """ return self._get_filtered_results(request, profile_id=-1) @swagger_auto_schema( request_body=AutoSchemaHelper.schema_factory( { 'organization_id': 'integer', 'profile_id': 'integer', 'cycle_ids': ['integer'], }, required=['organization_id', 'cycle_ids'], description='Properties:\n' '- organization_id: ID of organization\n' '- profile_id: Either an id of a list settings profile, ' 'or undefined\n' '- cycle_ids: The IDs of the cycle to get taxlots' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @action(detail=False, methods=['POST']) def filter_by_cycle(self, request): """ List all the taxlots with all columns """ # NOTE: we are using a POST http method b/c swagger and django handle # arrays differently in query parameters. ie this is just simpler org_id = request.data.get('organization_id', None) profile_id = request.data.get('profile_id', -1) cycle_ids = request.data.get('cycle_ids', []) if not org_id: return JsonResponse( {'status': 'error', 'message': 'Need to pass organization_id as query parameter'}, status=status.HTTP_400_BAD_REQUEST) response = taxlots_across_cycles(org_id, profile_id, cycle_ids) return JsonResponse(response) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'cycle', required=True, description='The ID of the cycle to get tax lots' ), AutoSchemaHelper.query_integer_field( 'page', required=False, description='The current page of taxlots to return' ), AutoSchemaHelper.query_integer_field( 'per_page', required=False, description='The number of items per page to return' ), ], request_body=AutoSchemaHelper.schema_factory( { 'profile_id': 'integer', 'taxlot_view_ids': ['integer'], }, required=['profile_id'], description='Properties:\n' '- profile_id: Either an id of a list settings profile, or undefined\n' '- taxlot_view_ids: List of taxlot view ids' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @action(detail=False, methods=['POST']) def filter(self, request): """ List all the tax lots """ if 'profile_id' not in request.data: profile_id = None else: if request.data['profile_id'] == 'None': profile_id = None else: profile_id = request.data['profile_id'] return self._get_filtered_results(request, profile_id=profile_id) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), ], request_body=AutoSchemaHelper.schema_factory( { 'taxlot_view_ids': ['integer'] }, required=['taxlot_view_ids'], description='Properties:\n' '- taxlot_view_ids: Array containing tax lot state ids to merge' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def merge(self, request): """ Merge multiple tax lot records into a single new record, and run this new record through a match and merge round within it's current Cycle. """ body = request.data organization_id = int(request.query_params.get('organization_id', None)) taxlot_view_ids = body.get('taxlot_view_ids', []) taxlot_states = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=organization_id ).values('id', 'state_id') # get the state ids in order according to the given view ids taxlot_states_dict = {t['id']: t['state_id'] for t in taxlot_states} taxlot_state_ids = [ taxlot_states_dict[view_id] for view_id in taxlot_view_ids if view_id in taxlot_states_dict ] if len(taxlot_state_ids) != len(taxlot_view_ids): return { 'status': 'error', 'message': 'All records not found.' } # Check the number of taxlot_state_ids to merge if len(taxlot_state_ids) < 2: return JsonResponse({ 'status': 'error', 'message': 'At least two ids are necessary to merge' }, status=status.HTTP_400_BAD_REQUEST) merged_state = merge_taxlots(taxlot_state_ids, organization_id, 'Manual Match') merge_count, link_count, view_id = match_merge_link(merged_state.taxlotview_set.first().id, 'TaxLotState') result = { 'status': 'success' } result.update({ 'match_merged_count': merge_count, 'match_link_count': link_count, }) return result @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def unmerge(self, request, pk=None): """ Unmerge a taxlot view into two taxlot views """ try: old_view = TaxLotView.objects.select_related( 'taxlot', 'cycle', 'state' ).get( id=pk, taxlot__organization_id=self.request.GET['organization_id'] ) except TaxLotView.DoesNotExist: return { 'status': 'error', 'message': 'taxlot view with id {} does not exist'.format(pk) } # Duplicate pairing paired_view_ids = list(TaxLotProperty.objects.filter(taxlot_view_id=old_view.id) .order_by('property_view_id').values_list('property_view_id', flat=True)) # Capture previous associated labels label_ids = list(old_view.labels.all().values_list('id', flat=True)) notes = old_view.notes.all() for note in notes: note.taxlot_view = None merged_state = old_view.state if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED: return { 'status': 'error', 'message': 'taxlot view with id {} is not a merged taxlot view'.format(pk) } log = TaxLotAuditLog.objects.select_related('parent_state1', 'parent_state2').filter( state=merged_state ).order_by('-id').first() if log.parent_state1 is None or log.parent_state2 is None: return { 'status': 'error', 'message': 'taxlot view with id {} must have two parent states'.format(pk) } state1 = log.parent_state1 state2 = log.parent_state2 cycle_id = old_view.cycle_id # Clone the taxlot record twice old_taxlot = old_view.taxlot new_taxlot = old_taxlot new_taxlot.id = None new_taxlot.save() new_taxlot_2 = TaxLot.objects.get(pk=new_taxlot.pk) new_taxlot_2.id = None new_taxlot_2.save() # If the canonical TaxLot is NOT associated to another -View if not TaxLotView.objects.filter(taxlot_id=old_view.taxlot_id).exclude(pk=old_view.id).exists(): TaxLot.objects.get(pk=old_view.taxlot_id).delete() # Create the views new_view1 = TaxLotView( cycle_id=cycle_id, taxlot_id=new_taxlot.id, state=state1 ) new_view2 = TaxLotView( cycle_id=cycle_id, taxlot_id=new_taxlot_2.id, state=state2 ) # Mark the merged state as deleted merged_state.merge_state = MERGE_STATE_DELETE merged_state.save() # Change the merge_state of the individual states if log.parent1.name in ['Import Creation', 'Manual Edit'] and log.parent1.import_filename is not None: # State belongs to a new record state1.merge_state = MERGE_STATE_NEW else: state1.merge_state = MERGE_STATE_MERGED if log.parent2.name in ['Import Creation', 'Manual Edit'] and log.parent2.import_filename is not None: # State belongs to a new record state2.merge_state = MERGE_STATE_NEW else: state2.merge_state = MERGE_STATE_MERGED # In most cases data_state will already be 3 (DATA_STATE_MATCHING), but if one of the parents was a # de-duplicated record then data_state will be 0. This step ensures that the new states will be 3. state1.data_state = DATA_STATE_MATCHING state2.data_state = DATA_STATE_MATCHING state1.save() state2.save() # Delete the audit log entry for the merge log.delete() old_view.delete() new_view1.save() new_view2.save() # Asssociate labels label_objs = StatusLabel.objects.filter(pk__in=label_ids) new_view1.labels.set(label_objs) new_view2.labels.set(label_objs) # Duplicate notes to the new views for note in notes: created = note.created updated = note.updated note.id = None note.taxlot_view = new_view1 note.save() ids = [note.id] note.id = None note.taxlot_view = new_view2 note.save() ids.append(note.id) # Correct the created and updated times to match the original note Note.objects.filter(id__in=ids).update(created=created, updated=updated) for paired_view_id in paired_view_ids: TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view1.id, property_view_id=paired_view_id).save() TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view2.id, property_view_id=paired_view_id).save() return { 'status': 'success', 'view_id': new_view1.id } @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['GET']) def links(self, request, pk=None): """ Get taxlot details for each linked taxlot across org cycles """ organization_id = request.query_params.get('organization_id', None) base_view = TaxLotView.objects.select_related('cycle').filter( pk=pk, cycle__organization_id=organization_id ) if base_view.exists(): result = {'data': []} linked_views = TaxLotView.objects.select_related('cycle').filter( taxlot_id=base_view.get().taxlot_id, cycle__organization_id=organization_id ).order_by('-cycle__start') for linked_view in linked_views: state_data = TaxLotStateSerializer(linked_view.state).data state_data['cycle_id'] = linked_view.cycle.id state_data['view_id'] = linked_view.id result['data'].append(state_data) return JsonResponse(result, status=status.HTTP_200_OK) else: result = { 'status': 'error', 'message': 'property view with id {} does not exist in given organization'.format(pk) } return JsonResponse(result) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def match_merge_link(self, request, pk=None): """ Runs match merge link for an individual taxlot. Note that this method can return a view_id of None if the given -View was not involved in a merge. """ org_id = request.query_params.get('organization_id', None) taxlot_view = TaxLotView.objects.get( pk=pk, cycle__organization_id=org_id ) merge_count, link_count, view_id = match_merge_link(taxlot_view.pk, 'TaxLotState') result = { 'view_id': view_id, 'match_merged_count': merge_count, 'match_link_count': link_count, } return JsonResponse(result) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'property_id', required=True, description='The property id to pair up with this taxlot' ) ] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['PUT']) def pair(self, request, pk=None): """ Pair a property to this taxlot """ organization_id = int(request.query_params.get('organization_id')) property_id = int(request.query_params.get('property_id')) taxlot_id = int(pk) return pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, True) @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'property_id', required=True, description='The property id to unpair from this taxlot' ) ] ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['PUT']) def unpair(self, request, pk=None): """ Unpair a property from this taxlot """ organization_id = int(request.query_params.get('organization_id')) property_id = int(request.query_params.get('property_id')) taxlot_id = int(pk) return pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, False) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'taxlot_view_ids': ['integer'] }, required=['taxlot_view_ids'], description='A list of taxlot view ids to delete' ) ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['DELETE']) def batch_delete(self, request): """ Batch delete several tax lots """ org_id = request.query_params.get('organization_id', None) taxlot_view_ids = request.data.get('taxlot_view_ids', []) taxlot_state_ids = TaxLotView.objects.filter( id__in=taxlot_view_ids, cycle__organization_id=org_id ).values_list('state_id', flat=True) resp = TaxLotState.objects.filter(pk__in=Subquery(taxlot_state_ids)).delete() if resp[0] == 0: return JsonResponse({'status': 'warning', 'message': 'No action was taken'}) return JsonResponse({'status': 'success', 'taxlots': resp[1]['seed.TaxLotState']}) def _get_taxlot_view(self, taxlot_pk): try: taxlot_view = TaxLotView.objects.select_related( 'taxlot', 'cycle', 'state' ).get( id=taxlot_pk, taxlot__organization_id=self.request.GET['organization_id'] ) result = { 'status': 'success', 'taxlot_view': taxlot_view } except TaxLotView.DoesNotExist: result = { 'status': 'error', 'message': 'taxlot view with id {} does not exist'.format(taxlot_pk) } return result def get_history(self, taxlot_view): """Return history in reverse order""" # access the history from the property state history, master = taxlot_view.state.history() # convert the history and master states to StateSerializers master['state'] = TaxLotStateSerializer(master['state_data']).data del master['state_data'] del master['state_id'] for h in history: h['state'] = TaxLotStateSerializer(h['state_data']).data del h['state_data'] del h['state_id'] return history, master def _get_properties(self, taxlot_view_pk): property_view_pks = TaxLotProperty.objects.filter( taxlot_view_id=taxlot_view_pk ).values_list('property_view_id', flat=True) property_views = PropertyView.objects.filter( pk__in=property_view_pks ).select_related('cycle', 'state') properties = [] for property_view in property_views: properties.append(PropertyViewSerializer(property_view).data) return properties @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_view_data') def retrieve(self, request, pk): """ Get taxlot details """ result = self._get_taxlot_view(pk) if result.get('status', None) != 'error': taxlot_view = result.pop('taxlot_view') result.update(TaxLotViewSerializer(taxlot_view).data) # remove TaxLotView id from result result.pop('id') result['state'] = TaxLotStateSerializer(taxlot_view.state).data result['properties'] = self._get_properties(taxlot_view.pk) result['history'], master = self.get_history(taxlot_view) result = update_result_with_master(result, master) return JsonResponse(result, status=status.HTTP_200_OK) else: return JsonResponse(result, status=status.HTTP_404_NOT_FOUND) @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=UpdateTaxLotPayloadSerializer, ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def update(self, request, pk): """ Update a taxlot and run the updated record through a match and merge round within it's current Cycle. """ data = request.data result = self._get_taxlot_view(pk) if result.get('status', 'error') != 'error': taxlot_view = result.pop('taxlot_view') taxlot_state_data = TaxLotStateSerializer(taxlot_view.state).data # get the taxlot state information from the request new_taxlot_state_data = data['state'] # set empty strings to None for key, val in new_taxlot_state_data.items(): if val == '': new_taxlot_state_data[key] = None changed_fields, previous_data = get_changed_fields(taxlot_state_data, new_taxlot_state_data) if not changed_fields: result.update( {'status': 'success', 'message': 'Records are identical'} ) return JsonResponse(result, status=status.HTTP_204_NO_CONTENT) else: # Not sure why we are going through the pain of logging this all right now... need to # reevaluate this. log = TaxLotAuditLog.objects.select_related().filter( state=taxlot_view.state ).order_by('-id').first() # if checks above pass, create an exact copy of the current state for historical purposes if log.name == 'Import Creation': # Add new state by removing the existing ID. taxlot_state_data.pop('id') # Remove the import_file_id for the first edit of a new record # If the import file has been deleted and this value remains the serializer won't be valid taxlot_state_data.pop('import_file') new_taxlot_state_serializer = TaxLotStateSerializer( data=taxlot_state_data ) if new_taxlot_state_serializer.is_valid(): # create the new property state, and perform an initial save / moving relationships new_state = new_taxlot_state_serializer.save() # then assign this state to the property view and save the whole view taxlot_view.state = new_state taxlot_view.save() TaxLotAuditLog.objects.create(organization=log.organization, parent1=log, parent2=None, parent_state1=log.state, parent_state2=None, state=new_state, name='Manual Edit', description=None, import_filename=log.import_filename, record_type=AUDIT_USER_EDIT) result.update( {'state': new_taxlot_state_serializer.data} ) # save the property view so that the datetime gets updated on the property. taxlot_view.save() else: result.update({ 'status': 'error', 'message': 'Invalid update data with errors: {}'.format( new_taxlot_state_serializer.errors)} ) return JsonResponse(result, status=status.HTTP_422_UNPROCESSABLE_ENTITY) # redo assignment of this variable in case this was an initial edit taxlot_state_data = TaxLotStateSerializer(taxlot_view.state).data if 'extra_data' in new_taxlot_state_data: taxlot_state_data['extra_data'].update( new_taxlot_state_data['extra_data'] ) taxlot_state_data.update( {k: v for k, v in new_taxlot_state_data.items() if k != 'extra_data'} ) log = TaxLotAuditLog.objects.select_related().filter( state=taxlot_view.state ).order_by('-id').first() if log.name in ['Manual Edit', 'Manual Match', 'System Match', 'Merge current state in migration']: # Convert this to using the serializer to save the data. This will override the # previous values in the state object. # Note: We should be able to use partial update here and pass in the changed # fields instead of the entire state_data. updated_taxlot_state_serializer = TaxLotStateSerializer( taxlot_view.state, data=taxlot_state_data ) if updated_taxlot_state_serializer.is_valid(): # create the new property state, and perform an initial save / moving # relationships updated_taxlot_state_serializer.save() result.update( {'state': updated_taxlot_state_serializer.data} ) # save the property view so that the datetime gets updated on the property. taxlot_view.save() Note.create_from_edit(request.user.id, taxlot_view, new_taxlot_state_data, previous_data) merge_count, link_count, view_id = match_merge_link(taxlot_view.id, 'TaxLotState') result.update({ 'view_id': view_id, 'match_merged_count': merge_count, 'match_link_count': link_count, }) return JsonResponse(result, status=status.HTTP_200_OK) else: result.update({ 'status': 'error', 'message': 'Invalid update data with errors: {}'.format( updated_taxlot_state_serializer.errors)} ) return JsonResponse(result, status=status.HTTP_422_UNPROCESSABLE_ENTITY) else: result = { 'status': 'error', 'message': 'Unrecognized audit log name: ' + log.name } return JsonResponse(result, status=status.HTTP_204_NO_CONTENT) else: return JsonResponse(result, status=status.HTTP_404_NOT_FOUND)