示例#1
0
文件: jobs.py 项目: zspencer/calc
def _create_contract_batches(upload_source, batch_size=5000, rows=None):
    if rows is None:
        r10_file = ContentFile(upload_source.original_file)
        converter = Region10SpreadsheetConverter(r10_file)
        rows = converter.convert_next()

    contracts = []
    bad_rows = []
    i = 0

    for row in rows:
        try:
            c = Region10Loader.make_contract(row, upload_source=upload_source)
            contracts.append(c)
        except (ValueError, ValidationError) as e:
            bad_rows.append(row)
        i += 1
        if i == batch_size:
            yield contracts, bad_rows
            i = 0
            contracts = []
            bad_rows = []

    if i > 0:
        yield contracts, bad_rows
示例#2
0
文件: jobs.py 项目: 18F/calc
def _create_contract_batches(upload_source, batch_size=5000, rows=None):
    if rows is None:
        r10_file = ContentFile(upload_source.original_file)
        converter = Region10SpreadsheetConverter(r10_file)
        rows = converter.convert_next()

    contracts = []
    bad_rows = []
    i = 0

    for row in rows:
        try:
            c = Region10Loader.make_contract(row, upload_source=upload_source)
            contracts.append(c)
        except (ValueError, ValidationError) as e:
            bad_rows.append(row)
        i += 1
        if i == batch_size:
            yield contracts, bad_rows
            i = 0
            contracts = []
            bad_rows = []

    if i > 0:
        yield contracts, bad_rows
示例#3
0
    def handle(self, *args, **options):
        log = logging.getLogger('contracts')

        log.info("Begin load_data task")

        log.info("Deleting existing contract records")
        Contract.objects.all().delete()

        filename = options['filename']
        if not filename or not os.path.exists(filename):
            raise ValueError('invalid filename')

        log.info("Processing new datafile")

        # create BulkUploadContractSource to associate with the new Contracts
        f = open(filename, 'rb')
        upload_source = BulkUploadContractSource.objects.create(
            has_been_loaded=True,
            original_file=f.read(),
            file_mime_type="text/csv",
            procurement_center=BulkUploadContractSource.REGION_10
        )
        f.close()

        contracts = Region10Loader().load_file(
            filename, upload_source=upload_source
        )

        log.info("Inserting records")
        Contract.objects.bulk_create(contracts)

        log.info("Updating search index")
        call_command('update_search_field')

        log.info("End load_data task")
示例#4
0
def _process_bulk_upload(upload_source):
    file = ContentFile(upload_source.original_file)
    converter = Region10SpreadsheetConverter(file)

    contracts_logger.info("Deleting contract objects related to region 10.")

    # Delete existing contracts identified by the same
    # procurement_center
    Contract.objects.filter(
        upload_source__procurement_center=BulkUploadContractSource.REGION_10
    ).delete()

    contracts = []
    bad_rows = []

    contracts_logger.info("Generating new contract objects.")

    for row in converter.convert_next():
        try:
            c = Region10Loader.make_contract(row, upload_source=upload_source)
            contracts.append(c)
        except (ValueError, ValidationError) as e:
            bad_rows.append(row)

    contracts_logger.info("Saving new contract objects.")

    # Save new contracts
    Contract.objects.bulk_create(contracts)

    contracts_logger.info("Updating full-text search indexes.")

    # Update search field on Contract models
    Contract._fts_manager.update_search_field()

    # Update the upload_source
    upload_source.has_been_loaded = True
    upload_source.save()

    return len(contracts), len(bad_rows)