def _import_results(self, mapping, result_file, conn):
# Map SF field names to local db column names
sf_header = result_file.readline().strip().decode("utf-8").split(",")
columns = []
for sf in sf_header:
if sf == "Records not found for this query":
return
if sf:
column = mapping["fields"].get(sf)
if not column:
column = mapping.get("lookups",
{}).get(sf, {}).get("key_field")
if column:
columns.append(column)
if not columns:
return
record_type = mapping.get("record_type")
if record_type:
columns.append("record_type")
processor = log_progress(
process_incoming_rows(result_file, record_type), self.logger)
data_file = IteratorBytesIO(processor)
self._sql_bulk_insert_from_csv(conn, mapping["table"], columns,
data_file)
self.session.commit()
def _import_results(self, mapping, result_file, conn):
# Map SF field names to local db column names
sf_header = [
name.strip('"')
for name in result_file.readline().strip().decode("utf-8").split(",")
]
columns = []
lookup_keys = []
for sf in sf_header:
if sf == "Records not found for this query":
return
if sf:
column = mapping.get("fields", {}).get(sf)
if not column:
lookup = mapping.get("lookups", {}).get(sf, {})
if lookup:
lookup_keys.append(sf)
column = get_lookup_key_field(lookup, sf)
if column:
columns.append(column)
if not columns:
return
record_type = mapping.get("record_type")
if record_type:
columns.append("record_type")
processor = log_progress(
process_incoming_rows(result_file, record_type), self.logger
)
data_file = IteratorBytesIO(processor)
if mapping["oid_as_pk"]:
self._sql_bulk_insert_from_csv(conn, mapping["table"], columns, data_file)
else:
# If using the autogenerated id field, split out the CSV file from the Bulk API
# into two separate files and load into the main table and the sf_id_table
with tempfile.TemporaryFile("w+b") as f_values:
with tempfile.TemporaryFile("w+b") as f_ids:
data_file_values, data_file_ids = self._split_batch_csv(
data_file, f_values, f_ids
)
self._sql_bulk_insert_from_csv(
conn, mapping["table"], columns, data_file_values
)
self._sql_bulk_insert_from_csv(
conn, mapping["sf_id_table"], ["sf_id"], data_file_ids
)
if "RecordTypeId" in mapping["fields"]:
self._extract_record_types(
mapping["sf_object"], mapping["record_type_table"], conn
)
self.session.commit()
if lookup_keys and not mapping["oid_as_pk"]:
self._convert_lookups_to_id(mapping, lookup_keys)
def _import_results(self, mapping, result_file, conn):
# Map SF field names to local db column names
sf_header = [
name.strip('"')
for name in result_file.readline().strip().decode("utf-8").split(",")
]
columns = []
lookup_keys = []
for sf in sf_header:
if sf == "Records not found for this query":
return
if sf:
column = mapping.get("fields", {}).get(sf)
if not column:
lookup = mapping.get("lookups", {}).get(sf, {})
if lookup:
lookup_keys.append(sf)
column = get_lookup_key_field(lookup, sf)
if column:
columns.append(column)
if not columns:
return
record_type = mapping.get("record_type")
if record_type:
columns.append("record_type")
processor = log_progress(
process_incoming_rows(result_file, record_type), self.logger
)
data_file = IteratorBytesIO(processor)
if mapping["oid_as_pk"]:
self._sql_bulk_insert_from_csv(conn, mapping["table"], columns, data_file)
else:
# If using the autogenerated id field, split out the CSV file from the Bulk API
# into two separate files and load into the main table and the sf_id_table
with tempfile.TemporaryFile("w+b") as f_values:
with tempfile.TemporaryFile("w+b") as f_ids:
data_file_values, data_file_ids = self._split_batch_csv(
data_file, f_values, f_ids
)
self._sql_bulk_insert_from_csv(
conn, mapping["table"], columns, data_file_values
)
self._sql_bulk_insert_from_csv(
conn, mapping["sf_id_table"], ["sf_id"], data_file_ids
)
self.session.commit()
if lookup_keys and not mapping["oid_as_pk"]:
self._convert_lookups_to_id(mapping, lookup_keys)
def _import_results(self, mapping, step):
"""Ingest results from the Bulk API query."""
conn = self.session.connection()
# Map SF field names to local db column names
fields = self._fields_for_mapping(mapping)
columns = []
lookup_keys = []
for field_name in fields:
column = mapping.get("fields", {}).get(field_name)
if not column:
lookup = mapping.get("lookups", {}).get(field_name, {})
if lookup:
lookup_keys.append(field_name)
column = lookup.get_lookup_key_field()
if column:
columns.append(column)
if not columns:
return
record_type = mapping.get("record_type")
if record_type:
columns.append("record_type")
# FIXME: log_progress needs to know our batch size, when made configurable.
record_iterator = log_progress(step.get_results(), self.logger)
if record_type:
record_iterator = (record + [record_type]
for record in record_iterator)
if mapping["oid_as_pk"]:
self._sql_bulk_insert_from_records(
connection=conn,
table=mapping["table"],
columns=columns,
record_iterable=record_iterator,
)
else:
# If using the autogenerated id field, split out the returned records
# into two separate files and load into the main table and the sf_id_table
with tempfile.TemporaryFile("w+", newline="") as f_values:
with tempfile.TemporaryFile("w+", newline="") as f_ids:
data_file_values, data_file_ids = self._split_batch_csv(
record_iterator, f_values, f_ids)
self._sql_bulk_insert_from_records(
connection=conn,
table=mapping["table"],
columns=columns,
record_iterable=csv.reader(data_file_values),
)
self._sql_bulk_insert_from_records(
connection=conn,
table=mapping["sf_id_table"],
columns=["sf_id"],
record_iterable=csv.reader(data_file_ids),
)
if "RecordTypeId" in mapping["fields"]:
self._extract_record_types(mapping["sf_object"],
mapping["record_type_table"], conn)
self.session.commit()
if lookup_keys and not mapping["oid_as_pk"]:
self._convert_lookups_to_id(mapping, lookup_keys)
def test_log_progress(self):
logger = mock.Mock()
for x in utils.log_progress(range(3), logger, batch_size=1):
pass
self.assertEqual(4, logger.info.call_count)
def test_log_progress(self):
logger = mock.Mock()
for x in utils.log_progress(range(3), logger, batch_size=1):
pass
assert 4 == logger.info.call_count
def _import_results(self, mapping, step):
"""Ingest results from the Bulk API query."""
conn = self.session.connection()
# Map SF field names to local db column names
field_map = mapping.get_complete_field_map(include_id=True)
columns = [field_map[f]
for f in field_map] # Get values in insertion order.
record_type = mapping.record_type
if record_type:
columns.append("record_type")
# TODO: log_progress needs to know our batch size, when made configurable.
record_iterator = log_progress(step.get_results(), self.logger)
if record_type:
record_iterator = (record + [record_type]
for record in record_iterator)
# Convert relative dates to stable dates.
if mapping.anchor_date:
date_context = mapping.get_relative_date_context(self.org_config)
if date_context[0] or date_context[1]:
record_iterator = (adjust_relative_dates(
mapping, date_context, record, DataOperationType.QUERY)
for record in record_iterator)
# Set Name field as blank for Person Account "Account" records.
if (mapping.sf_object == "Account" and "Name" in field_map
and self.org_config.is_person_accounts_enabled):
# Bump indices by one since record's ID is the first column.
Name_index = columns.index(mapping.fields["Name"])
IsPersonAccount_index = columns.index(
mapping.fields["IsPersonAccount"])
def strip_name_field(record):
nonlocal Name_index, IsPersonAccount_index
if record[IsPersonAccount_index].lower() == "true":
record[Name_index] = ""
return record
record_iterator = (strip_name_field(record)
for record in record_iterator)
if mapping.get_oid_as_pk():
self._sql_bulk_insert_from_records(
connection=conn,
table=mapping.table,
columns=columns,
record_iterable=record_iterator,
)
else:
# If using the autogenerated id field, split out the returned records
# into two separate streams and load into the main table and the sf_id_table
values, ids = itertools.tee(record_iterator)
f_values = (row[1:] for row in values)
f_ids = (row[:1] for row in ids)
values_chunks = self._sql_bulk_insert_from_records_incremental(
connection=conn,
table=mapping.table,
columns=columns[1:], # Strip off the Id column
record_iterable=f_values,
)
ids_chunks = self._sql_bulk_insert_from_records_incremental(
connection=conn,
table=mapping.get_sf_id_table(),
columns=["sf_id"],
record_iterable=f_ids,
)
# do the inserts one chunk at a time based on all of the
# generators nested previously.
consume(zip(values_chunks, ids_chunks))
if "RecordTypeId" in mapping.fields:
self._extract_record_types(mapping.sf_object,
mapping.get_source_record_type_table(),
conn)
self.session.commit()
def _import_results(self, mapping, step):
"""Ingest results from the Bulk API query."""
conn = self.session.connection()
# Map SF field names to local db column names
fields = self._fields_for_mapping(mapping)
columns = []
lookup_keys = []
for field_name in fields:
column = mapping.get("fields", {}).get(field_name)
if not column:
lookup = mapping.get("lookups", {}).get(field_name, {})
if lookup:
lookup_keys.append(field_name)
column = lookup.get_lookup_key_field()
if column:
columns.append(column)
if not columns:
return
record_type = mapping.get("record_type")
if record_type:
columns.append("record_type")
# FIXME: log_progress needs to know our batch size, when made configurable.
record_iterator = log_progress(step.get_results(), self.logger)
if record_type:
record_iterator = (record + [record_type]
for record in record_iterator)
# Set Name field as blank for Person Account "Account" records.
if (mapping["sf_object"] == "Account"
and "Name" in mapping.get("fields", {})
and self.org_config.is_person_accounts_enabled):
# Bump indices by one since record's ID is the first column.
Name_index = columns.index(mapping["fields"]["Name"]) + 1
IsPersonAccount_index = (
columns.index(mapping["fields"]["IsPersonAccount"]) + 1)
def strip_name_field(record):
nonlocal Name_index, IsPersonAccount_index
if record[IsPersonAccount_index] == "true":
record[Name_index] = ""
return record
record_iterator = (strip_name_field(record)
for record in record_iterator)
if mapping["oid_as_pk"]:
self._sql_bulk_insert_from_records(
connection=conn,
table=mapping["table"],
columns=columns,
record_iterable=record_iterator,
)
else:
# If using the autogenerated id field, split out the returned records
# into two separate files and load into the main table and the sf_id_table
with tempfile.TemporaryFile("w+", newline="",
encoding="utf-8") as f_values:
with tempfile.TemporaryFile("w+", newline="",
encoding="utf-8") as f_ids:
data_file_values, data_file_ids = self._split_batch_csv(
record_iterator, f_values, f_ids)
self._sql_bulk_insert_from_records(
connection=conn,
table=mapping["table"],
columns=columns,
record_iterable=csv.reader(data_file_values),
)
self._sql_bulk_insert_from_records(
connection=conn,
table=mapping["sf_id_table"],
columns=["sf_id"],
record_iterable=csv.reader(data_file_ids),
)
if "RecordTypeId" in mapping["fields"]:
self._extract_record_types(mapping["sf_object"],
mapping["record_type_table"], conn)
self.session.commit()
if lookup_keys and not mapping["oid_as_pk"]:
self._convert_lookups_to_id(mapping, lookup_keys)
