def _create_contract_batches(upload_source, batch_size=5000, rows=None): if rows is None: r10_file = ContentFile(upload_source.original_file) converter = Region10SpreadsheetConverter(r10_file) rows = converter.convert_next() contracts = [] bad_rows = [] i = 0 for row in rows: try: c = Region10Loader.make_contract(row, upload_source=upload_source) contracts.append(c) except (ValueError, ValidationError) as e: bad_rows.append(row) i += 1 if i == batch_size: yield contracts, bad_rows i = 0 contracts = [] bad_rows = [] if i > 0: yield contracts, bad_rows
def handle(self, *args, **options): log = logging.getLogger('contracts') log.info("Begin load_data task") log.info("Deleting existing contract records") Contract.objects.all().delete() filename = options['filename'] if not filename or not os.path.exists(filename): raise ValueError('invalid filename') log.info("Processing new datafile") # create BulkUploadContractSource to associate with the new Contracts f = open(filename, 'rb') upload_source = BulkUploadContractSource.objects.create( has_been_loaded=True, original_file=f.read(), file_mime_type="text/csv", procurement_center=BulkUploadContractSource.REGION_10 ) f.close() contracts = Region10Loader().load_file( filename, upload_source=upload_source ) log.info("Inserting records") Contract.objects.bulk_create(contracts) log.info("Updating search index") call_command('update_search_field') log.info("End load_data task")
def _process_bulk_upload(upload_source): file = ContentFile(upload_source.original_file) converter = Region10SpreadsheetConverter(file) contracts_logger.info("Deleting contract objects related to region 10.") # Delete existing contracts identified by the same # procurement_center Contract.objects.filter( upload_source__procurement_center=BulkUploadContractSource.REGION_10 ).delete() contracts = [] bad_rows = [] contracts_logger.info("Generating new contract objects.") for row in converter.convert_next(): try: c = Region10Loader.make_contract(row, upload_source=upload_source) contracts.append(c) except (ValueError, ValidationError) as e: bad_rows.append(row) contracts_logger.info("Saving new contract objects.") # Save new contracts Contract.objects.bulk_create(contracts) contracts_logger.info("Updating full-text search indexes.") # Update search field on Contract models Contract._fts_manager.update_search_field() # Update the upload_source upload_source.has_been_loaded = True upload_source.save() return len(contracts), len(bad_rows)