def __init__(self, *args, **kwargs):
super(BestBuyTyresSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
self.brand_fixes['Bridgestone'] = ["b'stone", 'b/stone', 'bridestone', 'bridgestohne', 'brridgestone']
self.brand_fixes['Continental'] = ['conti', 'contiental', 'continenal', 'continntal', 'contintenal']
self.brand_fixes['Dunlop'] = ['dlp']
self.brand_fixes['Goodyear'] = ['g’year', 'g’yr', 'g/year', 'goodyea', 'gy', 'gyr']
self.brand_fixes['Michelin'] = ['mich']
self.brand_fixes['Pirelli'] = ['pir', 'pire', 'pireelli']
#self.brand_fixes['Uniroyal'] = ['uni']
self.custom_man_marks = {
'(LEXUS FITMENT)': '',
'()': '',
'(BMW FITMENT)': '*',
'(RAV 4)': '',
'(BMW)': '*'
}
self.errors = []
def __init__(self, *args, **kwargs):
super(EtyresSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
self.custom_man_marks[' JAGUAR FITMENT'] = 'J'
self.custom_man_marks[' RAV4 FITMENT'] = ''
self.custom_man_marks[' NISSAN JUKE FITMENT'] = ''
self.custom_man_marks[' (PORSCHE FITMENT)'] = 'N0'
self.custom_man_marks[' LEXUS FITMENT'] = ''
self.custom_man_marks[' PRIUS FITMENT'] = ''
self.custom_man_marks[' TOYOTA AURIS FITMENT'] = ''
self.custom_man_marks[' - TOYOTA RAV4 FITMENT'] = ''
self.custom_man_marks[' BMW MINI FITMENT'] = '*'
self.custom_man_marks[' AUDI FITMENT'] = 'AO'
self.custom_man_marks[' JAG FITMENT'] = 'J'
self.custom_man_marks[' FERRARI MASERATI FITMENT'] = ''
self.custom_man_marks[' MASERATI FITMENT'] = ''
self.custom_man_marks[' - BMW FITMENT'] = '*'
self.custom_man_marks[' ASTON MARTIN FITMENT'] = ''
self.custom_man_marks[' MERCEDES & RENAULT FITMENT'] = 'MO'
def __init__(self, *args, **kwargs):
super(KwikFitSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
def __init__(self, *args, **kwargs):
super(MyTyresSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
self.errors = []
self.search_history = set()
def __init__(self, *args, **kwargs):
super(EtyresSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
def __init__(self, *args, **kwargs):
super(EtyresSpider, self).__init__(*args, **kwargs)
dispatcher.connect(self.spider_closed, signals.spider_closed)
self.matcher = Matcher(self.log)
self.all_man_marks = {}
self.custom_man_marks = {}
self.tyre_sizes = []
self.tyre_widths = {}
self.tyre_profiles = {}
self.tyre_rims = {}
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
self.ip_codes = {}
self.ip_codes_filename = os.path.join(HERE, 'etyres_ip_codes.csv')
if os.path.exists(self.ip_codes_filename):
with open(self.ip_codes_filename) as f:
reader = csv.DictReader(f)
for row in reader:
self.ip_codes[row['identifier']] = row['ip_code']
self.custom_man_marks[' JAGUAR FITMENT'] = 'J'
self.custom_man_marks[' RAV4 FITMENT'] = ''
self.custom_man_marks[' NISSAN JUKE FITMENT'] = ''
self.custom_man_marks[' (PORSCHE FITMENT)'] = 'N0'
self.custom_man_marks[' LEXUS FITMENT'] = ''
self.custom_man_marks[' PRIUS FITMENT'] = ''
self.custom_man_marks[' TOYOTA AURIS FITMENT'] = ''
self.custom_man_marks[' - TOYOTA RAV4 FITMENT'] = ''
self.custom_man_marks[' BMW MINI FITMENT'] = '*'
self.custom_man_marks[' AUDI FITMENT'] = 'AO'
self.custom_man_marks[' JAG FITMENT'] = 'J'
self.custom_man_marks[' FERRARI MASERATI FITMENT'] = ''
self.custom_man_marks[' MASERATI FITMENT'] = ''
self.custom_man_marks[' - BMW FITMENT'] = '*'
self.custom_man_marks[' ASTON MARTIN FITMENT'] = ''
self.custom_man_marks[' MERCEDES & RENAULT FITMENT'] = 'MO'
def __init__(self, *args, **kwargs):
super(EcraterSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'lego.csv')) as f:
reader = csv.reader(f)
self.products = {
prod[2]: prod[3].decode('utf8')
for prod in reader
}
dispatcher.connect(self.spider_closed, signals.spider_closed)
if os.path.exists(os.path.join(HERE, 'ecrater_products.csv')):
shutil.copy(os.path.join(HERE, 'ecrater_products.csv'),
os.path.join(HERE, 'ecrater_products.csv.bak'))
# Errors
self.errors = []
def __init__(self, *args, **kwargs):
super(HusqvarnaDEEbaySpider, self).__init__()
self._ebay_url = 'http://www.ebay.de'
self._search_fields = ['brand', 'sku']
self._all_vendors = True
self._meta_fields = [('name', 'name'), ('price', 'price'),
('brand', 'brand'), ('category', 'category')]
self._match_fields = ('sku', )
self._check_valid_item = self.__valid_item_
self._converted_price = False
self._check_diff_ratio = True
self._re_sku = re.compile(r'(\d{3,})')
self._look_related = False
self.__collected_items = set()
self._check_diff_ratio = True
self.matcher = Matcher(self.log)
def __init__(self, *args, **kwargs):
super(LegoUsaEbaySpider, self).__init__()
self._csv_file = os.path.join(self.HERE, 'lego.csv')
self._converted_price = True
self._ebay_url = 'http://www.ebay.com'
self._search_fields = [3, 2]
self._all_vendors = True
self._look_related = False
self._meta_fields = [('sku', 2), ('name', 3), ('price', 4),
('category', 1)]
self._match_fields = ('sku', 'identifier')
self._check_valid_item = self._valid_item_
self._re_sku = re.compile(r'(\d{3,})')
self._check_diff_ratio = True
# self._ratio_accuracy = 60
self.matcher = Matcher(self.log)
def __init__(self, *args, **kwargs):
super(TyrebookersSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
self.tyre_sizes = self.tyre_sizes[::-1]
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
self.already_processed = []
self.custom_man_marks[' Merc'] = 'MO'
self.custom_man_marks[' BMW'] = '*'
self.custom_man_marks[' Audi'] = 'AO'
def __init__(self, *args, **kwargs):
super(EvenTyresSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
self.tyre_sizes = []
self.all_man_marks = {}
self.manually_matched = []
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
new_row = row.copy()
self.tyre_sizes.append(new_row)
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
self.errors = []
def __init__(self, *args, **kwargs):
super(LoveTyresSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
self.images = {}
self.all_man_marks = {}
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
if os.path.exists(self.images_filename):
with open(self.images_filename) as f:
reader = csv.DictReader(f)
for row in reader:
self.images[row['product_url']] = row['image_url']
self.errors = []
dispatcher.connect(self.spider_closed, signals.spider_closed)
def __init__(self, *args, **kwargs):
super(TyreGiantSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
with open(os.path.join(HERE, 'manmarks.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.all_man_marks[row['code']] = row['manufacturer_mark']
self.brands = [row['Brand'] for row in self.tyre_sizes]
self.search_history = set()
self.finished = False
dispatcher.connect(self.spider_idle, signals.spider_idle)
def __init__(self, *args, **kwargs):
super(MyTyresSpider, self).__init__(*args, **kwargs)
dispatcher.connect(self.spider_closed, signals.spider_closed)
self.matcher = Matcher(self.log)
with open(os.path.join(HERE, 'mtsstockcodes.csv')) as f:
reader = csv.DictReader(f)
for row in reader:
self.tyre_sizes.append(row)
self.ip_codes = {}
self.ip_codes_filename = os.path.join(HERE, 'mytyres_ip_codes.csv')
if os.path.exists(self.ip_codes_filename):
with open(self.ip_codes_filename) as f:
reader = csv.DictReader(f)
for row in reader:
self.ip_codes[row['identifier']] = row['ip_code']
self.errors = []
self.search_history = set()
def __init__(self, *args, **kwargs):
super(TrovaprezziSpider, self).__init__(*args, **kwargs)
self.matcher = Matcher(self.log)
dispatcher.connect(self.spider_idle, signals.spider_idle)
"""
>>> find_man_mark('bmw')
'*'
>>> find_man_mark('Mercedes')
'MO'
>>> find_man_mark('por')
'N0'
"""
man_marks = load_manufacturers_marks()
for code, manufacturer_mark in man_marks.items():
if mark.lower() in code.lower():
return manufacturer_mark
return ''
matcher = Matcher(logging.error)
def match_name(search_name,
new_item,
match_threshold=90,
important_words=None):
r = matcher.match_ratio(search_name, new_item, important_words)
return r >= match_threshold
def match_pattern(pattern, name, match_threshold=70):
"""
>>> match_pattern('B 250 ECOPIA', 'B250ECO')
True
>>> match_pattern('DSPORT', 'Dueler Sport')
