def _get_make_records(self, fitment_data):
make_retriever = DataRetriever(
VehicleMake,
VehicleMake.objects.filter(
name__in=fitment_data.make_storage['makes']), ('name', ))
return make_retriever.bulk_get_or_create(
fitment_data.make_storage['storage_objects'])
def _bulk_create_digital_assets(self, product_lookup):
product_assets_to_create = list()
product_asset_objects = dict()
digital_asset_objects = dict()
urls = set()
def _append_digital_asset(_part_number, _asset,
_digital_asset_type_record):
urls.add(_asset['url'])
digital_asset_key = _asset['url'] + _digital_asset_type_record.name
digital_asset_objects[digital_asset_key] = {
'url': _asset['url'],
'file_size_bytes': _asset['file_size_bytes'],
'type': _digital_asset_type_record
}
if _part_number not in product_asset_objects:
product_asset_objects[_part_number] = list()
product_asset_objects[_part_number].append({
'url':
_asset['url'],
'asset_type':
_digital_asset_type_record.name,
'display_sequence':
_asset['display_sequence']
})
for product_data in product_lookup.values():
digital_assets = product_data['digital_assets']
if digital_assets:
for digital_asset in digital_assets:
digital_asset_type_record = self._get_digital_asset_type_record(
digital_asset['asset_type'])
_append_digital_asset(product_data['part_number'],
digital_asset,
digital_asset_type_record)
if urls:
digital_assets_retriever = DataRetriever(
DigitalAsset, DigitalAsset.objects.filter(url__in=urls), (
'url',
'type__name',
))
digital_asset_records = digital_assets_retriever.bulk_get_or_create(
digital_asset_objects)
for part_number, product_assets in product_asset_objects.items():
product_record = product_lookup[part_number]['product_record']
for product_asset in product_assets:
digital_asset_id = digital_asset_records[
product_asset['url'] + product_asset['asset_type']]
product_assets_to_create.append(
ProductDigitalAsset(
digital_asset_id=digital_asset_id,
display_sequence=product_asset['display_sequence'],
product=product_record))
if product_assets_to_create:
ProductDigitalAsset.objects.bulk_create(product_assets_to_create)
def _get_model_records(self, fitment_data, make_records):
model_storage = fitment_data.model_storage['storage_objects']
for model_key, model_object in model_storage.items():
model_object["make_id"] = make_records[model_object.pop("make")]
model_retriever = DataRetriever(
VehicleModel,
VehicleModel.objects.filter(
name__in=fitment_data.model_storage['models'],
make_id__in=make_records.values()).select_related("make"), (
"make__name",
"name",
))
return model_retriever.bulk_get_or_create(model_storage)
def _get_sub_model_records(self, fitment_data, model_records):
sub_model_storage = fitment_data.sub_model_storage['storage_objects']
if sub_model_storage:
for sub_model_key, sub_model_object in sub_model_storage.items():
sub_model_object['model_id'] = model_records[
sub_model_object.pop('model')]
sub_model_retriever = DataRetriever(
VehicleSubModel,
VehicleSubModel.objects.filter(
name__in=fitment_data.sub_model_storage['sub_models'],
model_id__in=model_records.values()).select_related(
"model__make"), (
"model__make__name",
"model__name",
"name",
))
return sub_model_retriever.bulk_get_or_create(sub_model_storage)
return None
def _store_fitment(self, fitment_data, vehicle_records):
fitment_storage_objects = fitment_data.part_fitment_storage[
'storage_objects']
product_retriever = DataRetriever(
Product,
Product.objects.filter(
brand=fitment_data.brand_record,
part_number__in=fitment_storage_objects.keys()),
("part_number", ))
product_fitment_objects = list()
for part_number, storage_objects in fitment_storage_objects.items():
for storage_object in storage_objects.values():
storage_object['product_id'] = product_retriever.get_instance(
storage_object.pop('product'))
storage_object['vehicle_id'] = vehicle_records[
storage_object.pop('vehicle')]
product_fitment_objects.append(
ProductFitment(**storage_object))
if product_fitment_objects:
ProductFitment.objects.bulk_create(product_fitment_objects)
def _get_vehicle_records(self, fitment_data, make_records, model_records,
sub_model_records, engine_records):
vehicle_storage = fitment_data.vehicle_storage['storage_objects']
vehicle_year_storage = dict()
for vehicle_key, vehicle_object in vehicle_storage.items():
vehicle_object['make_id'] = make_records[vehicle_object.pop(
'make')]
vehicle_object['model_id'] = model_records[vehicle_object.pop(
'model')]
if vehicle_object['sub_model'] and sub_model_records:
vehicle_object['sub_model_id'] = sub_model_records[
vehicle_object.pop('sub_model')]
if vehicle_object['engine'] and engine_records:
vehicle_object['engine_id'] = engine_records[
vehicle_object.pop('engine')]
vehicle_years = vehicle_object.pop("years")
for vehicle_year in vehicle_years:
vehicle_year_key = str(vehicle_year) + vehicle_key
vehicle_year_storage[vehicle_year_key] = {
'vehicle': vehicle_key,
'year': vehicle_year
}
vehicle_retriever = DataRetriever(
Vehicle,
Vehicle.objects.filter(
model_id__in=model_records.values()).select_related(
"make", "model", "sub_model", "engine",
"engine__fuel_delivery", "engine__fuel_type",
"engine__aspiration"),
("make__name", "model__name", "sub_model__name",
"engine__configuration", "engine__liters",
"engine__fuel_type__name", "engine__fuel_delivery__name",
"engine__engine_code", "engine__aspiration__name"))
vehicle_records = vehicle_retriever.bulk_get_or_create(vehicle_storage)
for vehicle_year_object in vehicle_year_storage.values():
vehicle_year_object['vehicle_id'] = vehicle_records[
vehicle_year_object.pop('vehicle')]
vehicle_year_retriever = DataRetriever(
VehicleYear,
VehicleYear.objects.filter(
vehicle_id__in=vehicle_records.values()).select_related(
"vehicle", "vehicle__make", "vehicle__model",
"vehicle__sub_model", "vehicle__engine",
"vehicle__engine__fuel_delivery",
"vehicle__engine__fuel_type",
"vehicle__engine__aspiration"),
("year", "vehicle__make__name", "vehicle__model__name",
"vehicle__sub_model__name", "vehicle__engine__configuration",
"vehicle__engine__liters", "vehicle__engine__fuel_type__name",
"vehicle__engine__fuel_delivery__name",
"vehicle__engine__engine_code",
"vehicle__engine__aspiration__name"))
vehicle_year_retriever.bulk_get_or_create(vehicle_year_storage)
return vehicle_records
def _get_engine_records(self, fitment_data):
engine_storage = fitment_data.engine_storage['storage_objects']
if engine_storage:
for engine_key, engine_object in engine_storage.items():
engine_object['fuel_type'] = self._get_fuel_type(
engine_object['fuel_type'])
engine_object['fuel_delivery'] = self._get_fuel_delivery(
engine_object['fuel_delivery'])
engine_object['aspiration'] = self._get_aspiration(
engine_object['aspiration'])
engine_retriever = DataRetriever(
VehicleEngine,
VehicleEngine.objects.filter(
configuration__in=fitment_data.
engine_storage['configurations']).filter(
Q(engine_code__isnull=True)
| Q(engine_code__in=fitment_data.
engine_storage['engine_codes'])).select_related(),
("configuration", "liters", "fuel_type__name",
"fuel_delivery__name", "engine_code", "aspiration__name"))
return engine_retriever.bulk_get_or_create(engine_storage)
return None
def _store_chunks(self, categories, parts_categories, part_numbers):
category_records = dict()
existing_categories = Category.objects.filter(name__in=categories)
for existing_category in existing_categories:
category_records[existing_category.name] = existing_category
for category in categories:
if category not in category_records:
category_records[category] = Category.objects.create(
name=category)
for part_category_data in parts_categories.values():
part_category_data['category'] = category_records[
part_category_data.pop("category")]
part_category_retriever = DataRetriever(
ProductCategoryLookup,
ProductCategoryLookup.objects.filter(
part_number__in=part_numbers).select_related("category"), (
"brand_short_name",
"category__name",
"part_number",
))
part_category_retriever.bulk_get_or_create(parts_categories)
def _clean_fitment_data(self, fitment_data):
"""
This method cleans the fitment data.
1. If the fitment_data input is the same as database, do not store
2. If the fitment_data and database differ, delete existing records from the database and re-insert new records
3. If the fitment_data is storing a part that does not exist, remove it
"""
existing_fitment_lookup = dict()
product_fitment_to_delete = list()
part_fitment_storage = fitment_data.part_fitment_storage
existing_product_lookup = Product.objects.filter(
brand=fitment_data.brand_record,
part_number__in=part_fitment_storage['storage_objects'].keys(
)).values_list("part_number", flat=True)
part_fitment_storage['storage_objects'] = {
key: value
for key, value in part_fitment_storage['storage_objects'].items()
if key in existing_product_lookup
}
existing_fitment_records = ProductFitment.objects.filter(
product__brand=fitment_data.brand_record,
product__part_number__in=part_fitment_storage['storage_objects'].
keys())
existing_fitment_records = existing_fitment_records.select_related(
"product", "vehicle", "vehicle__make", "vehicle__model",
"vehicle__sub_model", "vehicle__engine",
"vehicle__engine__fuel_delivery", "vehicle__engine__fuel_type",
"vehicle__engine__aspiration")
vehicle_key_parts = [
"vehicle__make__name", "vehicle__model__name",
"vehicle__sub_model__name", "vehicle__engine__configuration",
"vehicle__engine__liters", "vehicle__engine__fuel_type__name",
"vehicle__engine__fuel_delivery__name",
"vehicle__engine__engine_code", "vehicle__engine__aspiration__name"
]
vehicle_fitment_key_parts = [
"start_year", "end_year"
] + vehicle_key_parts + ["fitment_info_1", "fitment_info_2"]
existing_fitment_records = existing_fitment_records.values(
*(["id", "product__part_number"] + vehicle_fitment_key_parts))
for existing_fitment_record in existing_fitment_records:
vehicle_fitment_key = DataRetriever.get_record_key(
existing_fitment_record, vehicle_fitment_key_parts)
vehicle_key = DataRetriever.get_record_key(existing_fitment_record,
vehicle_key_parts)
part_number = existing_fitment_record['product__part_number']
if part_number not in existing_fitment_lookup:
existing_fitment_lookup[part_number] = dict()
existing_fitment_lookup[part_number][vehicle_fitment_key] = {
'product': part_number,
'vehicle': vehicle_key,
'start_year': existing_fitment_record['start_year'],
'end_year': existing_fitment_record['end_year'],
'fitment_info_1': existing_fitment_record['fitment_info_1'],
'fitment_info_2': existing_fitment_record['fitment_info_2'],
'fitment_id': existing_fitment_record['id']
}
if existing_fitment_lookup:
for part_number in list(
part_fitment_storage['storage_objects'].keys()):
if part_number in existing_fitment_lookup:
new_part_fitment_storage = part_fitment_storage[
'storage_objects'][part_number]
existing_part_fitment_storage = existing_fitment_lookup[
part_number]
for existing_fitment_key, existing_fitment_data in existing_part_fitment_storage.items(
):
fitment_id = existing_fitment_data.pop('fitment_id')
if existing_fitment_key not in new_part_fitment_storage:
product_fitment_to_delete.append(fitment_id)
# TODO, clean up unused vehicle data
else:
new_fitment_data = new_part_fitment_storage[
existing_fitment_key]
if existing_fitment_data != new_fitment_data:
product_fitment_to_delete.append(fitment_id)
else:
del part_fitment_storage['storage_objects'][
part_number][existing_fitment_key]
if len(part_fitment_storage['storage_objects']
[part_number]) == 0:
del part_fitment_storage[
'storage_objects'][part_number]
if product_fitment_to_delete:
ProductFitment.objects.filter(
id__in=product_fitment_to_delete).delete()
return part_fitment_storage
def _bulk_create_attributes(self, product_lookup):
category_records = set()
attribute_names = set()
attribute_values = set()
attribute_objects = dict()
attribute_value_objects = dict()
for product_data in product_lookup.values():
attributes = product_data['attributes']
if attributes:
category_record = product_data['product_record'].category
category_records.add(category_record)
for attribute in attributes:
attribute_names.add(attribute['type'])
attribute_values.add(attribute['value'])
attribute_key = category_record.name + attribute['type']
attribute_objects[attribute_key] = {
'name': attribute['type'],
'category': category_record
}
attribute_value_key = attribute_key + attribute['value']
attribute_value_objects[attribute_value_key] = {
'value': attribute['value'],
'attribute': attribute_key
}
if attribute_objects:
attribute_retriever = DataRetriever(
Attribute,
Attribute.objects.filter(
name__in=attribute_names,
category__in=category_records).select_related('category'),
(
'category__name',
'name',
))
attribute_records = attribute_retriever.bulk_get_or_create(
attribute_objects)
for attribute_value_config in attribute_value_objects.values():
attribute_value_config['attribute_id'] = attribute_records[
attribute_value_config.pop('attribute')]
attribute_value_retriever = DataRetriever(
AttributeValue,
AttributeValue.objects.filter(
value__in=attribute_values,
attribute_id__in=attribute_records.values()).
select_related('attribute').select_related(
'attribute__category'), (
'attribute__category__name',
'attribute__name',
'value',
))
attribute_value_records = attribute_value_retriever.bulk_get_or_create(
attribute_value_objects)
product_attributes_to_create = list()
for product_data in product_lookup.values():
attributes = product_data['attributes']
if attributes:
product_record = product_data['product_record']
category_record = product_record.category
category_name = category_record.name
for attribute in attributes:
attribute_id = attribute_records.get(category_name +
attribute['type'])
attribute_value_id = attribute_value_records.get(
category_name + attribute['type'] +
attribute['value'])
product_attributes_to_create.append(
ProductAttribute(attribute_id=attribute_id,
value_id=attribute_value_id,
product=product_record))
if product_attributes_to_create:
ProductAttribute.objects.bulk_create(
product_attributes_to_create)