def spider_closed(self, spider): """Handles spider_closed signal from end of scraping. Handles usual end operations for scraper like writing xml, exporting to database and sending appropriate mail message.""" msg = "" if self.counter < self.total: msg += "\nScraper didn't go through all products, please report" msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total) # filename for writing xml self.xml.write_xml(self.name, self.d['file']) msg += self.exc.create_message(self.counter) msg += "\n{0}".format(self.temp_msg) exp = CommonExport() # part for exporting to database here if self.d['upload']: try: exp.xml_to_db(self.name, self.d['file'], "e2b3b658-16d5-4059-a9df-3c212c817d2c") msg += "\n\nExport to database successful" except StandardError: msg += "\n\nExport to database failed" else: msg += "\n\nUpload to database not selected" msg += self.log.get_message() from modules.mail import Mail mail = Mail() mail.send_mail(msg, "Express scraper report")
def spider_closed(self, spider): """Handles spider_closed signal from end of scraping. Handles usual end operations for scraper like writing xml, exporting to database and sending appropriate mail message.""" msg = "Ran: {0}".format(datetime.now()) if self.counter < self.total: msg += "\nScraper didn't go through all products, please report" msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total) # filename for writing xml if self.d['database']: try: self.database.connect() filename = self.database.get_name(self.d['catalog_id']) self.database.update_db(self.products) self.database.disconnect() msg += "\nRan from interface.\n" except: msg += "\nUpdating database failed, please report." else: msg += "\nRan from console.\n" filename = self.d['file'] self.xml.write_xml(self.name, filename) msg += self.exc.create_message(self.counter) if self.d['upload']: exp = CommonExport() try: exp.xml_to_db(self.name, filename, "4ea95a81-90fb-49e2-837e-acf5ab58f574") msg += "\n\nExport to database successful" except StandardError: msg += "\n\nExport to database failed" else: msg += "\n\nUpload to database not selected" # part for exporting to database here from modules.mail import Mail mail = Mail() try: mail.send_mail(msg, "Burton: {0}".format(filename)) if self.d['email']: mail.send_mail(msg, "Burton: {0}".format(filename), self.d['email']) except: msg += "\nSending mail failed." if self.d['database']: path = "logs/{0}".format(self.name) if not os.path.exists(path): os.makedirs(path) with open("{0}/{1}".format(path, filename), 'w') as f: f.write(msg)
def spider_closed(self, spider): """Handles spider_closed signal from end of scraping. Handles usual end operations for scraper like writing xml, exporting to database and sending appropriate mail message.""" msg = "Ran: {0}".format(datetime.now()) if self.counter < self.total: msg += "\nScraper didn't go through all products, please report" msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total) # filename for writing xml if self.d["database"]: try: self.database.connect() filename = self.database.get_name(self.d["catalog_id"]) self.database.update_db(self.products) self.database.disconnect() msg += "\nRan from interface.\n" except: msg += "\nUpdating database failed, please report." else: msg += "\nRan from console.\n" filename = self.d["file"] self.xml.write_xml(self.name, filename) msg += self.exc.create_message(self.counter) if self.d["upload"]: exp = CommonExport() try: exp.xml_to_db(self.name, filename, "4a9f5955-9b8e-4e13-84ef-95f937dbc00d") msg += "\n\nExport to database successful" except StandardError: msg += "\n\nExport to database failed" else: msg += "\n\nUpload to database not selected" ## part for exporting to database here from modules.mail import Mail mail = Mail() try: mail.send_mail(msg, "GuitarCenter: {0}".format(filename)) if self.d["email"]: mail.send_mail(msg, "GuitarCenter: {0}".format(filename), self.d["email"]) except: msg += "\nSending mail failed." if self.d["database"]: path = "logs/{0}".format(self.name) if not os.path.exists(path): os.makedirs(path) with open("{0}/{1}".format(path, filename), "w") as f: f.write(msg)
def spider_closed(self, spider): """Handles spider_closed signal from end of scraping. Handles usual end operations for scraper like writing xml, exporting to database and sending appropriate mail message.""" msg = "" if self.counter < self.total: msg += "\nScraper didn't go through all products, please report" msg += "\n\nScraped {0} product out of {1}\n\n".format(self.counter, self.total) # filename for writing xml if self.d['database']: try: self.database.connect() filename = self.database.get_name(self.d['catalog_id']) self.database.update_db(self.products) self.database.disconnect() msg += "\nRan from interface.\n" except: msg += "\nUpdating database failed, please report." else: msg += "\nRan from console.\n" filename = self.d['file'] self.xml.write_xml(self.name, filename) msg += self.exc.create_message(self.counter) if self.d['upload']: exp = CommonExport() #try: exp.xml_to_db(self.name, filename, "29eac9ea-8c57-4d22-baf4-3f1471dc3ab6") msg += "\n\nExport to database successful" #except StandardError: #msg += "\n\nExport to database failed" else: msg += "\n\nUpload to database not selected" from modules.mail import Mail mail = Mail() try: mail.send_mail(msg, "KennethCole: {0}".format(filename)) if self.d['email']: mail.send_mail(msg, "KennethCole: {0}".format(filename), self.d['email']) except: msg += "\nSending mail failed." if self.d['database']: path = 'logs/{0}'.format(self.name) if not os.path.exists(path): os.makedirs(path) with open("{0}/{1}".format(path, filename), 'w') as f: f.write(msg)
def spider_closed(self, spider): """Handles spider_closed signal from end of scraping. Handles usual end operations for scraper like writing xml, exporting to database and sending appropriate mail message.""" msg = "Ran: {0}".format(datetime.now()) if self.counter < self.total: msg += "\nScraper didn't go through all products, please report" msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total) # filename for writing xml""" if self.d['database']: try: self.database.connect() filename = self.database.get_name(self.d['catalog_id']) self.database.update_db(self.products) self.database.disconnect() msg += "\nRan from interface.\n" except: msg += "\nUpdating database failed, please report." else: msg += "\nRan from console.\n" filename = self.d['file'] self.xml.write_xml(self.name, filename) msg += self.exc.create_message(self.counter) if self.d['upload']: exp = CommonExport() try: exp.xml_to_db(self.name, filename, "5097450b-2c49-49d4-b47a-55b1bc652c78") msg += "\n\nExport to database successful" except StandardError: msg += "\n\nExport to database failed" else: msg += "\n\nUpload to database not selected" ## part for exporting to database here from modules.mail import Mail mail = Mail() """try:
def spider_closed(self, spider): """Handles spider_closed signal from end of scraping. Handles usual end operations for scraper like writing xml, exporting to database and sending appropriate mail message.""" msg = party.get_settings_message(self.d) if self.counter < self.total: msg += "\nScraper didn't go through all products, please report" msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total) # filename for writing xml if self.d['database']: try: self.database.connect() filename = self.database.get_name(self.d['catalog_id']) self.database.update_db(self.products) self.database.disconnect() msg += "\nRan from interface.\n" except: msg += "\nUpdating database failed, please report." else: msg += "\nRan from console.\n" filename = self.d['file'] logname = filename filename = "{0}_{1}".format(filename, self.d['lang']) self.xml.write_xml(self.name, filename) msg += self.exc.create_message(self.counter) from modules.mail import Mail from modules.export_to_db import CommonExport exp = CommonExport() if self.upload: try: if self.d['lang'] == 'us': exp.xml_to_db(self.name, filename, "55892247-1b92-4ff9-a8a3-33cc976f9341") else: exp.xml_to_db(self.name, filename, "9cb6c676-c14f-403b-b94f-b981184e1de0") msg += "\n\nExport to database successful" except StandardError: msg += "\n\nExport to database failed" else: msg += "\n\nUpload to database not selected" mail = Mail() try: mail.send_mail(msg, "Partylite: {0}".format(filename)) if self.d['email']: mail.send_mail(msg, "Partylite: {0}".format(filename), self.d['email']) except: msg += "\nSending mail failed." if self.d['database']: path = 'logs/{0}'.format(self.name) if not os.path.exists(path): os.makedirs(path) with open("{0}/{1}".format(path, logname), 'w') as f: f.write(msg)