예제 #1
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = ""
     if self.counter < self.total:
         msg += "\nScraper didn't go through all products, please report"
     msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
     # filename for writing xml
     self.xml.write_xml(self.name, self.d['file'])
     msg += self.exc.create_message(self.counter)
     msg += "\n{0}".format(self.temp_msg)
     exp = CommonExport()
     # part for exporting to database here
     if self.d['upload']:
         try:
             exp.xml_to_db(self.name, self.d['file'], "e2b3b658-16d5-4059-a9df-3c212c817d2c")
             msg += "\n\nExport to database successful"
         except StandardError:
             msg += "\n\nExport to database failed"
     else:
         msg += "\n\nUpload to database not selected"
     msg += self.log.get_message()
     from modules.mail import Mail
     mail = Mail()
     mail.send_mail(msg, "Express scraper report")
예제 #2
0
 def end_with_message(self, msg):
     if not msg:
         msg = "Script has been terminated with no message provided"
     print msg
     from modules.mail import Mail
     mail = Mail()
     mail.send_mail(msg, self.name)
     import os
     os._exit(1)
예제 #3
0
    def end_with_message(self, msg):
        if not msg:
            msg = "Script has been terminated with no message provided"
        print msg
        from modules.mail import Mail

        mail = Mail()
        mail.send_mail(msg, self.name)
        import os

        os._exit(1)
예제 #4
0
 def end_script(self):
     print "Script terminated after {0} errors".format(self.maximum)
     nl = "\n"
     msg = "Scraping terminated after\
            error occurred on {0} urls.".format(self.maximum)
     l = "\n".join(self.list_100)
     msg += nl + nl + "those errors occurred on:" + nl
     msg += l
     from modules.mail import Mail
     mail = Mail()
     mail.send_mail(msg, self.name)
     import os
     os._exit(1)
예제 #5
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = "Ran: {0}\n".format(datetime.now())
     if self.total - self.number:
         msg += "{0} id(s) from id list weren't found in feed".format(self.total - self.number)
         basic.warning(msg)
     else:
         msg += "All ids found in feed."
         basic.green(msg)
     # filename for writing xml
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.no_urls)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     #if self.d['upload']:
         #exp = CommonExport()
         #try:
             #exp.xml_to_db(self.name, self.d['file'], "40b029c9-dff7-4bc1-b8bc-ef062960b24d")
             #msg += "\n\nExport to database successful"
         #except StandardError:
             #msg += "\n\nExport to database failed"
     #else:
         #msg += "\n\nUpload to database not selected"
     from modules.mail import Mail
     mail = Mail()
     try:
         mail.send_mail(msg, "CelebratingHome: {0}".format(filename))
         if self.d['email']:
             mail.send_mail(msg, "CelebratingHome: {0}".format(filename), self.d['email'])
     except:
         msg += "\nSending mail failed."
     if self.d['database']:
         path = "logs/{0}".format(self.name)
         if not os.path.exists(path):
             os.makedirs(path)
         with open("{0}/{1}".format(path, filename), 'w') as f:
             f.write(msg)
예제 #6
0
    def spider_closed(self, spider):
        """Handles spider_closed signal from end of scraping.
        Handles usual end operations for scraper like writing xml, exporting
        to database and sending appropriate mail message."""
        msg = "Ran: {0}".format(datetime.now())
        if self.counter < self.total:
            msg += "\nScraper didn't go through all products, please report"
        msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
        # filename for writing xml
        if self.d["database"]:
            try:
                self.database.connect()
                filename = self.database.get_name(self.d["catalog_id"])
                self.database.update_db(self.products)
                self.database.disconnect()
                msg += "\nRan from interface.\n"
            except:
                msg += "\nUpdating database failed, please report."
        else:
            msg += "\nRan from console.\n"
            filename = self.d["file"]
        self.xml.write_xml(self.name, filename)
        msg += self.exc.create_message(self.counter)
        if self.d["upload"]:
            exp = CommonExport()
            try:
                exp.xml_to_db(self.name, filename, "4a9f5955-9b8e-4e13-84ef-95f937dbc00d")
                msg += "\n\nExport to database successful"
            except StandardError:
                msg += "\n\nExport to database failed"
        else:
            msg += "\n\nUpload to database not selected"
        ## part for exporting to database here
        from modules.mail import Mail

        mail = Mail()
        try:
            mail.send_mail(msg, "GuitarCenter: {0}".format(filename))
            if self.d["email"]:
                mail.send_mail(msg, "GuitarCenter: {0}".format(filename), self.d["email"])
        except:
            msg += "\nSending mail failed."
        if self.d["database"]:
            path = "logs/{0}".format(self.name)
            if not os.path.exists(path):
                os.makedirs(path)
            with open("{0}/{1}".format(path, filename), "w") as f:
                f.write(msg)
예제 #7
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = ""
     if self.counter < self.total:
         msg += "\nScraper didn't go through all products, please report"
     msg += "\n\nScraped %d product out of %d\n\n" % (self.counter,
                                                      self.total)
     # filename for writing xml
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.products)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     #if self.d['upload']:
     #exp = CommonExport()
     #try:
     #exp.xml_to_db(self.name, filename, "4b0d6b52-7b05-4e54-9d87-dfe77ac270c9")
     #msg += "\n\nExport to database successful"
     #except StandardError:
     #msg += "\n\nExport to database failed"
     #else:
     #msg += "\n\nUpload to database not selected"
     ## part for exporting to database here
     from modules.mail import Mail
     mail = Mail()
     try:
         mail.send_mail(msg, "Lydias: {0}".format(filename))
     except:
         msg += "\nSending mail failed."
     if self.d['database']:
         path = "logs/{0}".format(self.name)
         if not os.path.exists(path):
             os.makedirs(path)
         with open("{0}/{1}".format(path, filename), 'w') as f:
             f.write(msg)
예제 #8
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = ""
     if self.counter < self.total:
         msg += "\nScraper didn't go through all products, please report"
     msg += "\n\nScraped {0} product out of {1}\n\n".format(self.counter, self.total)
     # filename for writing xml
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.products)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     if self.d['upload']:
         exp = CommonExport()
         #try:
         exp.xml_to_db(self.name, filename, "29eac9ea-8c57-4d22-baf4-3f1471dc3ab6")
         msg += "\n\nExport to database successful"
         #except StandardError:
             #msg += "\n\nExport to database failed"
     else:
         msg += "\n\nUpload to database not selected"
     from modules.mail import Mail
     mail = Mail()
     try:
         mail.send_mail(msg, "KennethCole: {0}".format(filename))
         if self.d['email']:
             mail.send_mail(msg, "KennethCole: {0}".format(filename), self.d['email'])
     except:
         msg += "\nSending mail failed."
     if self.d['database']:
         path = 'logs/{0}'.format(self.name)
         if not os.path.exists(path):
             os.makedirs(path)
         with open("{0}/{1}".format(path, filename), 'w') as f:
             f.write(msg)
예제 #9
0
    def end_script(self):
        print "Script terminated after {0} errors".format(self.maximum)
        nl = "\n"
        msg = "Scraping terminated after\
               error occurred on {0} urls.".format(
            self.maximum
        )
        l = "\n".join(self.list_100)
        msg += nl + nl + "those errors occurred on:" + nl
        msg += l
        from modules.mail import Mail

        mail = Mail()
        mail.send_mail(msg, self.name)
        import os

        os._exit(1)
예제 #10
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = ""
     if self.counter < self.total:
         msg += "\nScraper didn't go through all products, please report"
     msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
     # filename for writing xml
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.products)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     #if self.d['upload']:
         #exp = CommonExport()
         #try:
             #exp.xml_to_db(self.name, filename, "4b0d6b52-7b05-4e54-9d87-dfe77ac270c9")
             #msg += "\n\nExport to database successful"
         #except StandardError:
             #msg += "\n\nExport to database failed"
     #else:
         #msg += "\n\nUpload to database not selected"
     ## part for exporting to database here
     from modules.mail import Mail
     mail = Mail()
     try:
         mail.send_mail(msg, "Lydias: {0}".format(filename))
     except:
         msg += "\nSending mail failed."
     if self.d['database']:
         path = "logs/{0}".format(self.name)
         if not os.path.exists(path):
             os.makedirs(path)
         with open("{0}/{1}".format(path, filename), 'w') as f:
             f.write(msg)
예제 #11
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = party.get_settings_message(self.d)
     if self.counter < self.total:
         msg += "\nScraper didn't go through all products, please report"
     msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
     # filename for writing xml
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.products)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     logname = filename
     filename = "{0}_{1}".format(filename, self.d['lang'])
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     from modules.mail import Mail
     from modules.export_to_db import CommonExport
     exp = CommonExport()
     if self.upload:
         try:
             if self.d['lang'] == 'us':
                 exp.xml_to_db(self.name, filename, "55892247-1b92-4ff9-a8a3-33cc976f9341")
             else:
                 exp.xml_to_db(self.name, filename, "9cb6c676-c14f-403b-b94f-b981184e1de0")
             msg += "\n\nExport to database successful"
         except StandardError:
             msg += "\n\nExport to database failed"
     else:
         msg += "\n\nUpload to database not selected"
     mail = Mail()
     try:
         mail.send_mail(msg, "Partylite: {0}".format(filename))
         if self.d['email']:
             mail.send_mail(msg, "Partylite: {0}".format(filename), self.d['email'])
     except:
         msg += "\nSending mail failed."
     if self.d['database']:
         path = 'logs/{0}'.format(self.name)
         if not os.path.exists(path):
             os.makedirs(path)
         with open("{0}/{1}".format(path, logname), 'w') as f:
             f.write(msg)
예제 #12
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = "Ran: {0}\n".format(datetime.now())
     if self.total - self.number:
         msg += "{0} id(s) from id list weren't found in feed".format(
             self.total - self.number)
         basic.warning(msg)
     else:
         msg += "All ids found in feed."
         basic.green(msg)
     # filename for writing xml
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.no_urls)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     #if self.d['upload']:
     #exp = CommonExport()
     #try:
     #exp.xml_to_db(self.name, self.d['file'], "40b029c9-dff7-4bc1-b8bc-ef062960b24d")
     #msg += "\n\nExport to database successful"
     #except StandardError:
     #msg += "\n\nExport to database failed"
     #else:
     #msg += "\n\nUpload to database not selected"
     from modules.mail import Mail
     mail = Mail()
     try:
         mail.send_mail(msg, "CelebratingHome: {0}".format(filename))
         if self.d['email']:
             mail.send_mail(msg, "CelebratingHome: {0}".format(filename),
                            self.d['email'])
     except:
         msg += "\nSending mail failed."
     if self.d['database']:
         path = "logs/{0}".format(self.name)
         if not os.path.exists(path):
             os.makedirs(path)
         with open("{0}/{1}".format(path, filename), 'w') as f:
             f.write(msg)
예제 #13
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = "Ran: {0}".format(datetime.now())
     if self.counter < self.total:
         msg += "\nScraper didn't go through all products, please report"
     msg += "\n\nScraped %d product out of %d\n\n" % (self.counter,
                                                      self.total)
     # filename for writing xml
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.products)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     if self.d['upload']:
         exp = CommonExport()
         try:
             exp.xml_to_db(self.name, filename,
                           "1ccd39a5-af4e-47cc-aebe-e0dede5b14d8")
             msg += "\n\nExport to database successful"
         except StandardError:
             msg += "\n\nExport to database failed"
     else:
         msg += "\n\nUpload to database not selected"
     from modules.mail import Mail
     mail = Mail()
     try:
         mail.send_mail(msg, "Sportmann: {0}".format(filename))
         if self.d['email']:
             mail.send_mail(msg, "Sportmann: {0}".format(filename),
                            self.d['email'])
     except:
         msg += "\nSending mail failed."
     if self.d['database']:
         path = "logs/{0}".format(self.name)
         if not os.path.exists(path):
             os.makedirs(path)
         with open("{0}/{1}".format(path, filename), 'w') as f:
             f.write(msg)
예제 #14
0
 def spider_closed(self, spider):
     """Handles spider_closed signal from end of scraping.
     Handles usual end operations for scraper like writing xml, exporting
     to database and sending appropriate mail message."""
     msg = "Ran: {0}".format(datetime.now())
     if self.counter < self.total:
         msg += "\nScraper didn't go through all products, please report"
     msg += "\n\nScraped %d product out of %d\n\n" % (self.counter, self.total)
     # filename for writing xml"""
     if self.d['database']:
         try:
             self.database.connect()
             filename = self.database.get_name(self.d['catalog_id'])
             self.database.update_db(self.products)
             self.database.disconnect()
             msg += "\nRan from interface.\n"
         except:
             msg += "\nUpdating database failed, please report."
     else:
         msg += "\nRan from console.\n"
         filename = self.d['file']
     self.xml.write_xml(self.name, filename)
     msg += self.exc.create_message(self.counter)
     if self.d['upload']:
         exp = CommonExport()
         try:
             exp.xml_to_db(self.name, filename, "5097450b-2c49-49d4-b47a-55b1bc652c78")
             msg += "\n\nExport to database successful"
         except StandardError:
             msg += "\n\nExport to database failed"
     else:
         msg += "\n\nUpload to database not selected"
     ## part for exporting to database here
     from modules.mail import Mail
     mail = Mail()
     """try:
예제 #15
0
# Logging
app.debug_log_format = '%(asctime)s\t[%(levelname)s]\t%(message)s'
log_file = os.path.join(app.root_path, 'logs/app.log')
file_handler = RotatingFileHandler(log_file, backupCount=3, maxBytes=100000)
file_handler.setFormatter(
    Formatter('%(asctime)s\t[%(levelname)s]\t%(message)s'))
file_handler.setLevel(logging.INFO)
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)

login_manager = LoginManager()
login_manager.init_app(app)

db = SQLAlchemy(app)

gravatar = Gravatar(app,
                    size=100,
                    rating='g',
                    default='retro',
                    force_default=False,
                    use_ssl=False,
                    base_url=None)

# Mail
server_root = app.config.get('SERVER_ROOT')
from_addr = app.config.get('MAIL_FROM_ADDRESS')
api_key = app.config.get('POSTMARK_API_KEY')
mail = Mail(server_root=server_root, from_addr=from_addr, api_key=api_key)

import modules.views
예제 #16
0
def get_callbacks(args):
    callbacks = []
    if args.mail:
        callbacks.append(Mail(sender=args.mail[0], receivers=args.mail))
    return callbacks