def __init__(self, config, logger):
self.htmlparser = etree.HTMLParser(recover=True)
self.xmlparser = etree.XMLParser(recover=True, ns_clean=True)
self.config = config.get('parser_rules')
self.site_name = config.get('name')
self.encoding = config.get('encoding') or 'utf-8'
self.logger = logger
self.stores = {}
sess = db.makesession()()
for store in sess.query(db.Store).all():
self.stores.update({store.store_name.lower(): store.id})
sess.close()
def __init__(self, config, logger):
self.htmlparser = etree.HTMLParser(recover=True)
self.xmlparser = etree.XMLParser(recover=True, ns_clean=True)
self.config = config.get("parser_rules")
self.site_name = config.get("name")
self.encoding = config.get("encoding") or "utf-8"
self.logger = logger
self.stores = {}
sess = db.makesession()()
for store in sess.query(db.Store).all():
self.stores.update({store.store_name.lower(): store.id})
sess.close()
def save_data(self, data):
# name, category_id, store_id, search_title, url, description=None,
# price_ytl = None, price_dollar=None, price_euro=None, vat_rate=None
store_id = self.stores.get(self.site_name.lower())
# var mı diye yoklamalı
session = db.makesession()()
try:
former = (
session.query(db.StoreProductTemp)
.filter(
and_(
db.StoreProductTemp.product_name == data.get("name").encode("utf-8"),
db.StoreProductTemp.url == data.get("url").encode("utf-8"),
)
)
.first()
)
former.store_id = store_id
former.thumb_url = data.get("thumb_url").encode("utf-8")
former.price_ytl = data.get("price_ytl")
former.price_dollar = data.get("price_dollar")
former.price_euro = data.get("price_euro")
former.vat_rate = 18 # TODO: hardcode
former.last_update = datetime.now()
former.description = data.get("description")
session.commit()
self.logger.debug("updated %s" % former.product_name)
except:
try:
sp = db.StoreProductTemp(
data.get("name"),
1, # category_id için ne yapmalı
store_id,
"sorctaytil",
data.get("url"),
description=data.get("description"),
thumb_url=data.get("thumb_url"),
price_ytl=data.get("price_ytl"),
price_dollar=data.get("price_dollar"),
price_euro=data.get("price_euro"),
vat_rate=18,
)
sp.save()
except:
self.logger.error("Parser_save_data: %s" % traceback.format_exc())
session.close()
def save_data(self, data):
# name, category_id, store_id, search_title, url, description=None,
# price_ytl = None, price_dollar=None, price_euro=None, vat_rate=None
store_id = self.stores.get(self.site_name.lower())
# var mı diye yoklamalı
session = db.makesession()()
try:
former = session.query(db.StoreProductTemp).filter(
and_(db.StoreProductTemp.product_name==data.get('name').encode('utf-8'),
db.StoreProductTemp.url==data.get('url').encode('utf-8'))).first()
former.store_id = store_id
former.thumb_url = data.get('thumb_url').encode('utf-8')
former.price_ytl = data.get('price_ytl')
former.price_dollar = data.get('price_dollar')
former.price_euro = data.get('price_euro')
former.vat_rate = 18 # TODO: hardcode
former.last_update = datetime.now()
former.description = data.get('description')
session.commit()
self.logger.debug('updated %s' % former.product_name)
except:
try:
sp = db.StoreProductTemp(data.get('name'),
1, # category_id için ne yapmalı
store_id,
'sorctaytil',
data.get('url'),
description = data.get('description'),
thumb_url = data.get('thumb_url'),
price_ytl = data.get('price_ytl'),
price_dollar = data.get('price_dollar'),
price_euro = data.get('price_euro'),
vat_rate = 18)
sp.save()
except:
self.logger.error('Parser_save_data: %s' % traceback.format_exc())
session.close()
