def _import_results(self, mapping, result_file, conn): # Map SF field names to local db column names sf_header = result_file.readline().strip().decode("utf-8").split(",") columns = [] for sf in sf_header: if sf == "Records not found for this query": return if sf: column = mapping["fields"].get(sf) if not column: column = mapping.get("lookups", {}).get(sf, {}).get("key_field") if column: columns.append(column) if not columns: return record_type = mapping.get("record_type") if record_type: columns.append("record_type") processor = log_progress( process_incoming_rows(result_file, record_type), self.logger) data_file = IteratorBytesIO(processor) self._sql_bulk_insert_from_csv(conn, mapping["table"], columns, data_file) self.session.commit()
def _import_results(self, mapping, result_file, conn): # Map SF field names to local db column names sf_header = [ name.strip('"') for name in result_file.readline().strip().decode("utf-8").split(",") ] columns = [] lookup_keys = [] for sf in sf_header: if sf == "Records not found for this query": return if sf: column = mapping.get("fields", {}).get(sf) if not column: lookup = mapping.get("lookups", {}).get(sf, {}) if lookup: lookup_keys.append(sf) column = get_lookup_key_field(lookup, sf) if column: columns.append(column) if not columns: return record_type = mapping.get("record_type") if record_type: columns.append("record_type") processor = log_progress( process_incoming_rows(result_file, record_type), self.logger ) data_file = IteratorBytesIO(processor) if mapping["oid_as_pk"]: self._sql_bulk_insert_from_csv(conn, mapping["table"], columns, data_file) else: # If using the autogenerated id field, split out the CSV file from the Bulk API # into two separate files and load into the main table and the sf_id_table with tempfile.TemporaryFile("w+b") as f_values: with tempfile.TemporaryFile("w+b") as f_ids: data_file_values, data_file_ids = self._split_batch_csv( data_file, f_values, f_ids ) self._sql_bulk_insert_from_csv( conn, mapping["table"], columns, data_file_values ) self._sql_bulk_insert_from_csv( conn, mapping["sf_id_table"], ["sf_id"], data_file_ids ) if "RecordTypeId" in mapping["fields"]: self._extract_record_types( mapping["sf_object"], mapping["record_type_table"], conn ) self.session.commit() if lookup_keys and not mapping["oid_as_pk"]: self._convert_lookups_to_id(mapping, lookup_keys)
def _import_results(self, mapping, result_file, conn): # Map SF field names to local db column names sf_header = [ name.strip('"') for name in result_file.readline().strip().decode("utf-8").split(",") ] columns = [] lookup_keys = [] for sf in sf_header: if sf == "Records not found for this query": return if sf: column = mapping.get("fields", {}).get(sf) if not column: lookup = mapping.get("lookups", {}).get(sf, {}) if lookup: lookup_keys.append(sf) column = get_lookup_key_field(lookup, sf) if column: columns.append(column) if not columns: return record_type = mapping.get("record_type") if record_type: columns.append("record_type") processor = log_progress( process_incoming_rows(result_file, record_type), self.logger ) data_file = IteratorBytesIO(processor) if mapping["oid_as_pk"]: self._sql_bulk_insert_from_csv(conn, mapping["table"], columns, data_file) else: # If using the autogenerated id field, split out the CSV file from the Bulk API # into two separate files and load into the main table and the sf_id_table with tempfile.TemporaryFile("w+b") as f_values: with tempfile.TemporaryFile("w+b") as f_ids: data_file_values, data_file_ids = self._split_batch_csv( data_file, f_values, f_ids ) self._sql_bulk_insert_from_csv( conn, mapping["table"], columns, data_file_values ) self._sql_bulk_insert_from_csv( conn, mapping["sf_id_table"], ["sf_id"], data_file_ids ) self.session.commit() if lookup_keys and not mapping["oid_as_pk"]: self._convert_lookups_to_id(mapping, lookup_keys)
def _import_results(self, mapping, step): """Ingest results from the Bulk API query.""" conn = self.session.connection() # Map SF field names to local db column names fields = self._fields_for_mapping(mapping) columns = [] lookup_keys = [] for field_name in fields: column = mapping.get("fields", {}).get(field_name) if not column: lookup = mapping.get("lookups", {}).get(field_name, {}) if lookup: lookup_keys.append(field_name) column = lookup.get_lookup_key_field() if column: columns.append(column) if not columns: return record_type = mapping.get("record_type") if record_type: columns.append("record_type") # FIXME: log_progress needs to know our batch size, when made configurable. record_iterator = log_progress(step.get_results(), self.logger) if record_type: record_iterator = (record + [record_type] for record in record_iterator) if mapping["oid_as_pk"]: self._sql_bulk_insert_from_records( connection=conn, table=mapping["table"], columns=columns, record_iterable=record_iterator, ) else: # If using the autogenerated id field, split out the returned records # into two separate files and load into the main table and the sf_id_table with tempfile.TemporaryFile("w+", newline="") as f_values: with tempfile.TemporaryFile("w+", newline="") as f_ids: data_file_values, data_file_ids = self._split_batch_csv( record_iterator, f_values, f_ids) self._sql_bulk_insert_from_records( connection=conn, table=mapping["table"], columns=columns, record_iterable=csv.reader(data_file_values), ) self._sql_bulk_insert_from_records( connection=conn, table=mapping["sf_id_table"], columns=["sf_id"], record_iterable=csv.reader(data_file_ids), ) if "RecordTypeId" in mapping["fields"]: self._extract_record_types(mapping["sf_object"], mapping["record_type_table"], conn) self.session.commit() if lookup_keys and not mapping["oid_as_pk"]: self._convert_lookups_to_id(mapping, lookup_keys)
def test_log_progress(self): logger = mock.Mock() for x in utils.log_progress(range(3), logger, batch_size=1): pass self.assertEqual(4, logger.info.call_count)
def test_log_progress(self): logger = mock.Mock() for x in utils.log_progress(range(3), logger, batch_size=1): pass assert 4 == logger.info.call_count
def _import_results(self, mapping, step): """Ingest results from the Bulk API query.""" conn = self.session.connection() # Map SF field names to local db column names field_map = mapping.get_complete_field_map(include_id=True) columns = [field_map[f] for f in field_map] # Get values in insertion order. record_type = mapping.record_type if record_type: columns.append("record_type") # TODO: log_progress needs to know our batch size, when made configurable. record_iterator = log_progress(step.get_results(), self.logger) if record_type: record_iterator = (record + [record_type] for record in record_iterator) # Convert relative dates to stable dates. if mapping.anchor_date: date_context = mapping.get_relative_date_context(self.org_config) if date_context[0] or date_context[1]: record_iterator = (adjust_relative_dates( mapping, date_context, record, DataOperationType.QUERY) for record in record_iterator) # Set Name field as blank for Person Account "Account" records. if (mapping.sf_object == "Account" and "Name" in field_map and self.org_config.is_person_accounts_enabled): # Bump indices by one since record's ID is the first column. Name_index = columns.index(mapping.fields["Name"]) IsPersonAccount_index = columns.index( mapping.fields["IsPersonAccount"]) def strip_name_field(record): nonlocal Name_index, IsPersonAccount_index if record[IsPersonAccount_index].lower() == "true": record[Name_index] = "" return record record_iterator = (strip_name_field(record) for record in record_iterator) if mapping.get_oid_as_pk(): self._sql_bulk_insert_from_records( connection=conn, table=mapping.table, columns=columns, record_iterable=record_iterator, ) else: # If using the autogenerated id field, split out the returned records # into two separate streams and load into the main table and the sf_id_table values, ids = itertools.tee(record_iterator) f_values = (row[1:] for row in values) f_ids = (row[:1] for row in ids) values_chunks = self._sql_bulk_insert_from_records_incremental( connection=conn, table=mapping.table, columns=columns[1:], # Strip off the Id column record_iterable=f_values, ) ids_chunks = self._sql_bulk_insert_from_records_incremental( connection=conn, table=mapping.get_sf_id_table(), columns=["sf_id"], record_iterable=f_ids, ) # do the inserts one chunk at a time based on all of the # generators nested previously. consume(zip(values_chunks, ids_chunks)) if "RecordTypeId" in mapping.fields: self._extract_record_types(mapping.sf_object, mapping.get_source_record_type_table(), conn) self.session.commit()
def _import_results(self, mapping, step): """Ingest results from the Bulk API query.""" conn = self.session.connection() # Map SF field names to local db column names fields = self._fields_for_mapping(mapping) columns = [] lookup_keys = [] for field_name in fields: column = mapping.get("fields", {}).get(field_name) if not column: lookup = mapping.get("lookups", {}).get(field_name, {}) if lookup: lookup_keys.append(field_name) column = lookup.get_lookup_key_field() if column: columns.append(column) if not columns: return record_type = mapping.get("record_type") if record_type: columns.append("record_type") # FIXME: log_progress needs to know our batch size, when made configurable. record_iterator = log_progress(step.get_results(), self.logger) if record_type: record_iterator = (record + [record_type] for record in record_iterator) # Set Name field as blank for Person Account "Account" records. if (mapping["sf_object"] == "Account" and "Name" in mapping.get("fields", {}) and self.org_config.is_person_accounts_enabled): # Bump indices by one since record's ID is the first column. Name_index = columns.index(mapping["fields"]["Name"]) + 1 IsPersonAccount_index = ( columns.index(mapping["fields"]["IsPersonAccount"]) + 1) def strip_name_field(record): nonlocal Name_index, IsPersonAccount_index if record[IsPersonAccount_index] == "true": record[Name_index] = "" return record record_iterator = (strip_name_field(record) for record in record_iterator) if mapping["oid_as_pk"]: self._sql_bulk_insert_from_records( connection=conn, table=mapping["table"], columns=columns, record_iterable=record_iterator, ) else: # If using the autogenerated id field, split out the returned records # into two separate files and load into the main table and the sf_id_table with tempfile.TemporaryFile("w+", newline="", encoding="utf-8") as f_values: with tempfile.TemporaryFile("w+", newline="", encoding="utf-8") as f_ids: data_file_values, data_file_ids = self._split_batch_csv( record_iterator, f_values, f_ids) self._sql_bulk_insert_from_records( connection=conn, table=mapping["table"], columns=columns, record_iterable=csv.reader(data_file_values), ) self._sql_bulk_insert_from_records( connection=conn, table=mapping["sf_id_table"], columns=["sf_id"], record_iterable=csv.reader(data_file_ids), ) if "RecordTypeId" in mapping["fields"]: self._extract_record_types(mapping["sf_object"], mapping["record_type_table"], conn) self.session.commit() if lookup_keys and not mapping["oid_as_pk"]: self._convert_lookups_to_id(mapping, lookup_keys)