Example #1
0
    def update_tcms_from_save(self, json_data, save_counter):
        # Check save_counter vs queued_save_counters.
        queued_save_counter = get_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY, None)
        if not queued_save_counter or save_counter > queued_save_counter:
            if not get_cache_state(self.UPDATING_TCMS_KEY, None):
                set_cache_state(self.UPDATING_TCMS_KEY, True)
                for d in json.loads(json_data):

                    tcm = TableColumnMapping.objects.get(pk=d["pk"])
                    for field_name in TableColumnMapping.fields_to_save:
                        if not field_name == "pk":
                            setattr(tcm, field_name, d[field_name])
                    tcm.was_a_human_decision = True
                    tcm.save()

                if get_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY, False) is not False:
                    queued_data = get_cache_raw(self.QUEUED_TCM_DATA_KEY)
                    queued_time = get_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY)
                    delete_cache(self.QUEUED_TCM_DATA_KEY)
                    delete_cache(self.QUEUED_TCM_SAVE_COUNTER_KEY)
                    delete_cache(self.UPDATING_TCMS_KEY)
                    self.update_tcms_from_save(queued_data, queued_time)

                delete_cache(self.UPDATING_TCMS_KEY)
                delete_cache(self.QUEUED_TCM_DATA_KEY)
                delete_cache(self.QUEUED_TCM_SAVE_COUNTER_KEY)
                return True

            else:
                set_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY, save_counter)
                set_cache_raw(self.QUEUED_TCM_DATA_KEY, json_data)
        return False
Example #2
0
File: views.py Project: Maalka/seed
def get_csv(request):
    """
    Download a csv of the results.
    """

    import_file_id = request.GET.get('import_file_id')
    cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="Data Cleansing Results.csv"'

    writer = csv.writer(response)
    writer.writerow(['Address Line 1', 'PM Property ID', 'Tax Lot ID', 'Custom ID', 'Field',
                     'Error Message', 'Severity'])
    for row in cleansing_results:
        for result in row['cleansing_results']:
            field = result['field']
            if field in Cleansing.ASSESSOR_FIELDS_BY_COLUMN:
                field = Cleansing.ASSESSOR_FIELDS_BY_COLUMN[field]['title']
            writer.writerow([
                row['address_line_1'],
                row['pm_property_id'],
                row['tax_lot_id'],
                row['custom_id_1'],
                field,
                result['message'],
                result['severity']
            ])

    return response
Example #3
0
 def get_cleansing_results(self, request, pk=None):
     """
     Retrieve the details of the cleansing script.
     ---
     type:
         status:
             required: true
             type: string
             description: either success or error
         message:
             type: string
             description: additional information, if any
         progress:
             type: integer
             description: integer percent of completion
         data:
             type: JSON
             description: object describing the results of the cleansing
     parameter_strategy: replace
     parameters:
         - name: pk
           description: Import file ID
           required: true
           paramType: path
     """
     import_file_id = pk
     cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
     return JsonResponse({
         'status': 'success',
         'message': 'Cleansing complete',
         'progress': 100,
         'data': cleansing_results
     })
Example #4
0
def get_csv(request):
    """
    Download a csv of the results.
    """

    import_file_id = request.GET.get("import_file_id")
    cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
    response = HttpResponse(content_type="text/csv")
    response["Content-Disposition"] = 'attachment; filename="Data Cleansing Results.csv"'

    writer = csv.writer(response)
    writer.writerow(
        ["Address Line 1", "PM Property ID", "Tax Lot ID", "Custom ID", "Field", "Error Message", "Severity"]
    )
    for row in cleansing_results:
        for result in row["cleansing_results"]:
            writer.writerow(
                [
                    row["address_line_1"],
                    row["pm_property_id"],
                    row["tax_lot_id"],
                    row["custom_id_1"],
                    result["formatted_field"],
                    result["detailed_message"],
                    result["severity"],
                ]
            )

    return response
Example #5
0
def get_cleansing_results(request):
    """
    Retrieve the details of the cleansing script.
    """

    import_file_id = request.GET.get("import_file_id")
    cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))

    return {"status": "success", "message": "Cleansing complete", "progress": 100, "data": cleansing_results}
Example #6
0
File: views.py Project: Maalka/seed
def get_cleansing_results(request):
    """
    Retrieve the details of the cleansing script.
    """

    import_file_id = request.GET.get('import_file_id')
    cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))

    # add in additional fields for view
    for i, row in enumerate(cleansing_results):
        for j, result in enumerate(row['cleansing_results']):
            if result['field'] in Cleansing.ASSESSOR_FIELDS_BY_COLUMN:
                result['field'] = Cleansing.ASSESSOR_FIELDS_BY_COLUMN[result['field']]['title']

    return {
        'status': 'success',
        'message': 'Cleansing complete',
        'progress': 100,
        'data': cleansing_results
    }
Example #7
0
    def get_csv(self, request, pk=None):
        """
        Download a csv of the results.
        ---
        type:
            status:
                required: true
                type: string
                description: either success or error
            progress_key:
                type: integer
                description: ID of background job, for retrieving job progress
        parameter_strategy: replace
        parameters:
            - name: pk
              description: Import file ID
              required: true
              paramType: path
        """

        import_file_id = pk
        cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="Data Cleansing Results.csv"'

        writer = csv.writer(response)
        writer.writerow(['Address Line 1', 'PM Property ID', 'Tax Lot ID', 'Custom ID', 'Field',
                         'Error Message', 'Severity'])
        for row in cleansing_results:
            for result in row['cleansing_results']:
                writer.writerow([
                    row['address_line_1'],
                    row['pm_property_id'],
                    row['tax_lot_id'],
                    row['custom_id_1'],
                    result['formatted_field'],
                    result['detailed_message'],
                    result['severity']
                ])

        return response
Example #8
0
    def save_to_cache(self, file_pk):
        """
        Save the results to the cache database. The data in the cache are
        stored as a list of dictionaries. The data in this class are stored as
        a dict of dict. This is important to remember because the data from the
        cache cannot be simply loaded into the above structure.

        :param file_pk: Import file primary key
        :return: None
        """

        # change the format of the data in the cache. Make this a list of
        # objects instead of object of objects.
        existing_results = get_cache_raw(Cleansing.cache_key(file_pk))

        l = []
        for key, value in self.results.iteritems():
            l.append(value)

        existing_results = existing_results + l

        z = sorted(existing_results, key=lambda k: k["id"])
        set_cache_raw(Cleansing.cache_key(file_pk), z, 3600)  # save the results for 1 hour
Example #9
0
 def premerge_estimated_seconds_remaining(self):
     return get_cache_raw(self.premerge_seconds_remaining_key)
Example #10
0
 def estimated_seconds_remaining(self):
     return get_cache_raw(self.merge_seconds_remaining_key)
Example #11
0
    def test_whole_org_match_merge_link_preview_endpoint_taxlots(self):
        # Cycle 1 / ImportFile 1 - Create 1 taxlot
        base_taxlot_details = {
            'jurisdiction_tax_lot_id': '1st Non-Match Set',
            'city': 'City 1',
            'district': 'Match Set',
            'import_file_id': self.import_file_1.id,
            'data_state': DATA_STATE_MAPPING,
            'no_default_data': False,
        }

        tls_1 = self.taxlot_state_factory.get_taxlot_state(
            **base_taxlot_details)

        self.import_file_1.mapping_done = True
        self.import_file_1.save()
        geocode_and_match_buildings_task(self.import_file_1.id)

        # Cycle 2 / ImportFile 2 - Create 1 unlinked taxlot
        base_taxlot_details['jurisdiction_tax_lot_id'] = '2nd Non-Match Set'
        base_taxlot_details['district'] = 'Match Set'
        base_taxlot_details['import_file_id'] = self.import_file_2.id
        tls_2 = self.taxlot_state_factory.get_taxlot_state(
            **base_taxlot_details)

        self.import_file_2.mapping_done = True
        self.import_file_2.save()
        geocode_and_match_buildings_task(self.import_file_2.id)

        # Check there doesn't exist links
        self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id,
                            tls_2.taxlotview_set.first().taxlot_id)

        url = reverse('api:v3:organizations-match-merge-link-preview',
                      args=[self.org.id])
        post_params = json.dumps({
            "inventory_type": "taxlots",
            "add": ['district'],
            "remove": ['jurisdiction_tax_lot_id']
        })
        raw_result = self.client.post(url,
                                      post_params,
                                      content_type='application/json')

        # Check there *still* doesn't exist links
        self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id,
                            tls_2.taxlotview_set.first().taxlot_id)

        self.assertEqual(200, raw_result.status_code)

        raw_content = json.loads(raw_result.content)

        identifier = ProgressData.from_key(
            raw_content['progress_key']).data['unique_id']
        result_key = "org_match_merge_link_result__%s" % identifier
        raw_summary = get_cache_raw(result_key)

        summary = {str(k): v
                   for k, v in raw_summary.items() if v}  # ignore empty cycles

        # Check format of summary
        self.assertCountEqual(
            [str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys())

        # Check that preview shows links would be created
        self.assertEqual(summary[str(self.cycle_1.id)][0]['id'],
                         summary[str(self.cycle_2.id)][0]['id'])

        # try to get result using results endpoint
        get_result_url = reverse(
            'api:v3:organizations-match-merge-link-result',
            args=[self.org.id]) + '?match_merge_link_id=' + str(identifier)

        get_result_raw_response = self.client.get(get_result_url)
        raw_summary = json.loads(get_result_raw_response.content)

        summary = {str(k): v
                   for k, v in raw_summary.items() if v}  # ignore empty cycles

        # Check format of summary
        self.assertCountEqual(
            [str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys())

        # Check that preview shows links would be created
        self.assertEqual(summary[str(self.cycle_1.id)][0]['id'],
                         summary[str(self.cycle_2.id)][0]['id'])