def spider_closed(self, spider):
"""Handles spider_closed signal from end of scraping.
Handles usual end operations for scraper like writing xml, exporting
to database and sending appropriate mail message."""
msg = ""
if self.counter < self.total:
msg += "\nScraper didn't go through all products, please report"
msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
# filename for writing xml
self.xml.write_xml(self.name, self.d['file'])
msg += self.exc.create_message(self.counter)
msg += "\n{0}".format(self.temp_msg)
exp = CommonExport()
# part for exporting to database here
if self.d['upload']:
try:
exp.xml_to_db(self.name, self.d['file'], "e2b3b658-16d5-4059-a9df-3c212c817d2c")
msg += "\n\nExport to database successful"
except StandardError:
msg += "\n\nExport to database failed"
else:
msg += "\n\nUpload to database not selected"
msg += self.log.get_message()
from modules.mail import Mail
mail = Mail()
mail.send_mail(msg, "Express scraper report")
def spider_closed(self, spider):
"""Handles spider_closed signal from end of scraping.
Handles usual end operations for scraper like writing xml, exporting
to database and sending appropriate mail message."""
msg = "Ran: {0}".format(datetime.now())
if self.counter < self.total:
msg += "\nScraper didn't go through all products, please report"
msg += "\n\nScraped %d product out of %d\n\n" % (self.counter,
self.total)
# filename for writing xml
if self.d['database']:
try:
self.database.connect()
filename = self.database.get_name(self.d['catalog_id'])
self.database.update_db(self.products)
self.database.disconnect()
msg += "\nRan from interface.\n"
except:
msg += "\nUpdating database failed, please report."
else:
msg += "\nRan from console.\n"
filename = self.d['file']
self.xml.write_xml(self.name, filename)
msg += self.exc.create_message(self.counter)
if self.d['upload']:
exp = CommonExport()
try:
exp.xml_to_db(self.name, filename,
"4ea95a81-90fb-49e2-837e-acf5ab58f574")
msg += "\n\nExport to database successful"
except StandardError:
msg += "\n\nExport to database failed"
else:
msg += "\n\nUpload to database not selected"
# part for exporting to database here
from modules.mail import Mail
mail = Mail()
try:
mail.send_mail(msg, "Burton: {0}".format(filename))
if self.d['email']:
mail.send_mail(msg, "Burton: {0}".format(filename),
self.d['email'])
except:
msg += "\nSending mail failed."
if self.d['database']:
path = "logs/{0}".format(self.name)
if not os.path.exists(path):
os.makedirs(path)
with open("{0}/{1}".format(path, filename), 'w') as f:
f.write(msg)
def spider_closed(self, spider):
"""Handles spider_closed signal from end of scraping.
Handles usual end operations for scraper like writing xml, exporting
to database and sending appropriate mail message."""
msg = "Ran: {0}".format(datetime.now())
if self.counter < self.total:
msg += "\nScraper didn't go through all products, please report"
msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
# filename for writing xml
if self.d["database"]:
try:
self.database.connect()
filename = self.database.get_name(self.d["catalog_id"])
self.database.update_db(self.products)
self.database.disconnect()
msg += "\nRan from interface.\n"
except:
msg += "\nUpdating database failed, please report."
else:
msg += "\nRan from console.\n"
filename = self.d["file"]
self.xml.write_xml(self.name, filename)
msg += self.exc.create_message(self.counter)
if self.d["upload"]:
exp = CommonExport()
try:
exp.xml_to_db(self.name, filename, "4a9f5955-9b8e-4e13-84ef-95f937dbc00d")
msg += "\n\nExport to database successful"
except StandardError:
msg += "\n\nExport to database failed"
else:
msg += "\n\nUpload to database not selected"
## part for exporting to database here
from modules.mail import Mail
mail = Mail()
try:
mail.send_mail(msg, "GuitarCenter: {0}".format(filename))
if self.d["email"]:
mail.send_mail(msg, "GuitarCenter: {0}".format(filename), self.d["email"])
except:
msg += "\nSending mail failed."
if self.d["database"]:
path = "logs/{0}".format(self.name)
if not os.path.exists(path):
os.makedirs(path)
with open("{0}/{1}".format(path, filename), "w") as f:
f.write(msg)
def spider_closed(self, spider):
"""Handles spider_closed signal from end of scraping.
Handles usual end operations for scraper like writing xml, exporting
to database and sending appropriate mail message."""
msg = ""
if self.counter < self.total:
msg += "\nScraper didn't go through all products, please report"
msg += "\n\nScraped {0} product out of {1}\n\n".format(self.counter, self.total)
# filename for writing xml
if self.d['database']:
try:
self.database.connect()
filename = self.database.get_name(self.d['catalog_id'])
self.database.update_db(self.products)
self.database.disconnect()
msg += "\nRan from interface.\n"
except:
msg += "\nUpdating database failed, please report."
else:
msg += "\nRan from console.\n"
filename = self.d['file']
self.xml.write_xml(self.name, filename)
msg += self.exc.create_message(self.counter)
if self.d['upload']:
exp = CommonExport()
#try:
exp.xml_to_db(self.name, filename, "29eac9ea-8c57-4d22-baf4-3f1471dc3ab6")
msg += "\n\nExport to database successful"
#except StandardError:
#msg += "\n\nExport to database failed"
else:
msg += "\n\nUpload to database not selected"
from modules.mail import Mail
mail = Mail()
try:
mail.send_mail(msg, "KennethCole: {0}".format(filename))
if self.d['email']:
mail.send_mail(msg, "KennethCole: {0}".format(filename), self.d['email'])
except:
msg += "\nSending mail failed."
if self.d['database']:
path = 'logs/{0}'.format(self.name)
if not os.path.exists(path):
os.makedirs(path)
with open("{0}/{1}".format(path, filename), 'w') as f:
f.write(msg)
def spider_closed(self, spider):
"""Handles spider_closed signal from end of scraping.
Handles usual end operations for scraper like writing xml, exporting
to database and sending appropriate mail message."""
msg = "Ran: {0}".format(datetime.now())
if self.counter < self.total:
msg += "\nScraper didn't go through all products, please report"
msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
# filename for writing xml"""
if self.d['database']:
try:
self.database.connect()
filename = self.database.get_name(self.d['catalog_id'])
self.database.update_db(self.products)
self.database.disconnect()
msg += "\nRan from interface.\n"
except:
msg += "\nUpdating database failed, please report."
else:
msg += "\nRan from console.\n"
filename = self.d['file']
self.xml.write_xml(self.name, filename)
msg += self.exc.create_message(self.counter)
if self.d['upload']:
exp = CommonExport()
try:
exp.xml_to_db(self.name, filename, "5097450b-2c49-49d4-b47a-55b1bc652c78")
msg += "\n\nExport to database successful"
except StandardError:
msg += "\n\nExport to database failed"
else:
msg += "\n\nUpload to database not selected"
## part for exporting to database here
from modules.mail import Mail
mail = Mail()
"""try:
def spider_closed(self, spider):
"""Handles spider_closed signal from end of scraping.
Handles usual end operations for scraper like writing xml, exporting
to database and sending appropriate mail message."""
msg = party.get_settings_message(self.d)
if self.counter < self.total:
msg += "\nScraper didn't go through all products, please report"
msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
# filename for writing xml
if self.d['database']:
try:
self.database.connect()
filename = self.database.get_name(self.d['catalog_id'])
self.database.update_db(self.products)
self.database.disconnect()
msg += "\nRan from interface.\n"
except:
msg += "\nUpdating database failed, please report."
else:
msg += "\nRan from console.\n"
filename = self.d['file']
logname = filename
filename = "{0}_{1}".format(filename, self.d['lang'])
self.xml.write_xml(self.name, filename)
msg += self.exc.create_message(self.counter)
from modules.mail import Mail
from modules.export_to_db import CommonExport
exp = CommonExport()
if self.upload:
try:
if self.d['lang'] == 'us':
exp.xml_to_db(self.name, filename, "55892247-1b92-4ff9-a8a3-33cc976f9341")
else:
exp.xml_to_db(self.name, filename, "9cb6c676-c14f-403b-b94f-b981184e1de0")
msg += "\n\nExport to database successful"
except StandardError:
msg += "\n\nExport to database failed"
else:
msg += "\n\nUpload to database not selected"
mail = Mail()
try:
mail.send_mail(msg, "Partylite: {0}".format(filename))
if self.d['email']:
mail.send_mail(msg, "Partylite: {0}".format(filename), self.d['email'])
except:
msg += "\nSending mail failed."
if self.d['database']:
path = 'logs/{0}'.format(self.name)
if not os.path.exists(path):
os.makedirs(path)
with open("{0}/{1}".format(path, logname), 'w') as f:
f.write(msg)
