示例#1
0
    def save_Preset(self):
        if len(self.Dialog_Window.window.Name_Input.text()) == 0:
            QMessageBox.about(self, "eMag Price Tracker",
                              "You must choose a name for the preset!")
        list = self.get_Presets()
        if str(self.Dialog_Window.window.Name_Input.text()) in list:
            QMessageBox.about(self, "eMag Price Tracker",
                              "The preset already exists!")
        else:
            db_manager = DatabaseManager('Database.db')
            if db_manager.check_database():
                cursor = db_manager.conn.cursor()
            cursor.execute("""create table if not exists Products(
                link text,
                preset text,
                previous_price integer
            )""")

            for i in range(self.window.Tracker_List.count()):
                cursor.execute(
                    """INSERT INTO Products VALUES(?,?,?)""",
                    (str(self.window.Tracker_List.item(i).text()),
                     str(self.Dialog_Window.window.Name_Input.text()), 0))
            cursor.execute("SELECT * FROM Products")

            print(cursor.fetchall())
            db_manager.conn.commit()
            db_manager.close_connection()
            self.window.Tracker_List.clear()
            self.Dialog_Window.hide()
示例#2
0
 def get_Presets(self):
     db_manager = DatabaseManager('Database.db')
     db_manager.check_database()
     cursor = db_manager.conn.cursor()
     cursor.execute("SELECT DISTINCT(preset) FROM Products")
     list = cursor.fetchall()
     list = [x[0] for x in list]
     return list
示例#3
0
    def check_price(self, products, Notif_List, email):

        header = {
            "User-Agent":
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
        }
        # initialize the prices
        previous_prices = [x[1] for x in products]
        links = [x[0] for x in products]
        if 0 in previous_prices:
            self.initialize_prices(products)
        # checking the prices every 900sec
        db_manager = DatabaseManager('Database.db')
        db_manager.check_database()
        cursor = db_manager.conn.cursor()
        while True:
            for i in range(0, len(links)):
                page = requests.get(links[i], headers=header)
                main = BeautifulSoup(page.content, 'html.parser')
                price = main.find(class_="product-new-price").get_text()
                name = main.find(class_="page-title").get_text()
                name = name.strip()
                price = price.strip()
                price = self.number(price[0:len(price) - 6])
                if price < previous_prices[i]:
                    aux = previous_prices[i] - price
                    print_message = "Price has been dropped by {0} lei for product {1}".format(
                        aux, name)
                    previous_prices[i] = price
                    subject = "Price was decreased by {0} lei for product {1}".format(
                        aux, name)
                    body = "Check the link {}".format(links[i])
                    message = f"Subject: {subject}\n\n{body}"
                    self.send_mail(message, email)
                    ########################################
                    cursor.execute(
                        "UPDATE Products SET previous_price =? WHERE link=?",
                        (previous_prices[i], links[i]))
                    Notif_List.addItem(QListWidgetItem(print_message))
                    db_manager.conn.commit()
                else:
                    if price < previous_prices[i]:
                        print_message = "Price was increased by {0} lei for product {1}".format(
                            price - previous_prices[i], name)
                        Notif_List.addItem(QListWidgetItem(print_message))
                    else:
                        print_message = "Price it's the same for product {0}".format(
                            name)
                        Notif_List.addItem(QListWidgetItem(print_message))
                print(print_message)
            time.sleep(300)
示例#4
0
 def load_Preset(self):
     self.window.Notif_List.clear()
     preset_name = self.Select_Window.window.List.currentItem().text()
     db_manager = DatabaseManager('Database.db')
     db_manager.check_database()
     cursor = db_manager.conn.cursor()
     cursor.execute(
         "SELECT link,previous_price FROM Products WHERE preset=?",
         (preset_name, ))
     self.Select_Window.hide()
     if str(preset_name) is not None:
         self.window.Status_Label.setText(str(preset_name) + " loaded!")
     #########################################
     self.start_thread(cursor.fetchall(), self.window.Notif_List,
                       self.email)
     self.preset_is_loaded = True
示例#5
0
 def __init__(self):
     self.db = DatabaseManager.DatabaseManager()
     self.threadPool = ThreadPoolExecutor(10)
     asyncio.get_event_loop().set_default_executor(self.threadPool)
     self.users = Users.User()
     self.chats = Chats.Chats()
     self.private_key: RsaKey = None
     self.public_key: RsaKey = None
     self.load_key()
示例#6
0
 def delete_Preset(self):
     preset_name = self.Select_Window2.window.List.currentItem().text()
     db_manager = DatabaseManager('Database.db')
     db_manager.check_database()
     cursor = db_manager.conn.cursor()
     cursor.execute("DELETE FROM Products WHERE preset=?", (preset_name, ))
     cursor.execute("SELECT * FROM Products")
     print(cursor.fetchall())
     db_manager.conn.commit()
     db_manager.close_connection()
     self.Select_Window2.hide()
示例#7
0
 def initialize_prices(self, products):
     # initialize the prices
     previous_prices = [x[1] for x in products]
     links = [x[0] for x in products]
     header = {
         "User-Agent":
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
     }
     for i in range(0, len(links)):
         page = requests.get(str(links[i]), headers=header)
         main = BeautifulSoup(page.content, 'html.parser')
         price = main.find(class_="product-new-price").get_text()
         price = price.strip()
         price = self.number(price[0:len(price) - 6])
         previous_prices[i] = price
     db_manager = DatabaseManager('Database.db')
     db_manager.check_database()
     cursor = db_manager.conn.cursor()
     for i in range(0, len(previous_prices)):
         cursor.execute(
             "UPDATE Products SET previous_price =? WHERE link=?",
             (previous_prices[i], links[i]))
     db_manager.conn.commit()
     db_manager.close_connection()
示例#8
0
 def __init__(self):
     QObject.__init__(self)
     AbstractCrawler.__init__(self)
     self.settings = QSettings("RFCode", "TwitterCrawler")
     self.db = DatabaseManager()
     self.max_id = 0
示例#9
0
class Crawler(QObject, AbstractCrawler):

    db = None
    """
    @type: DatabaseManager
    """

    rest = None
    """
    @type: RestCrawler
    """

    streaming = None
    """
    @type: StreamingCrawler
    """

    auth = None
    """
    @type: OAuthHandler
    """

    threadPool = []
    """
    @type: Array
    """

    def __init__(self):
        QObject.__init__(self)
        AbstractCrawler.__init__(self)
        self.settings = QSettings("RFCode", "TwitterCrawler")
        self.db = DatabaseManager()
        self.max_id = 0

    def authInit(self):
        self.rest = RestCrawler(self.auth)
        self.streaming = StreamingCrawler(self.auth, headers={"User-Agent": "TwitterCrawler/1.0"})
        self.rest.restDataReady.connect(self.updateSearchStep)
        self.streaming.listener.streamingDataReady.connect(self.updateSearchStep)
        self.streaming.listener.streamingError.connect(self.errorHandler)

    def getAuthUrl(self):
        self.auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
        return self.auth.get_authorization_url()

    def setAuthAccess(self, verifier):
        self.auth.get_access_token(verifier)
        self.settings.setValue("auth_key", self.auth.access_token.key)
        self.settings.setValue("auth_secret", self.auth.access_token.secret)

    def login(self):
        key = self.settings.value("auth_key")
        secret = self.settings.value("auth_secret")
        if key == None or secret == None:
            return False
        if self.auth == None:
            self.auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
        self.auth.set_access_token(key, secret)
        self.authInit()
        return True

    @Slot(int)
    def errorHandler(self, code):
        print "error", code
        if code >= 600:
            return
        self.stop()

    #        if code == 420: #rate limit error
    #            pass #TODO
    #        elif code == 600: #streaming parsing error
    #            pass
    #        elif code == 601: #database insertion error
    #            pass

    @Slot("QVariant")
    def updateSearchStep(self, step):
        for i in range(len(step) - 1, -1, -1):
            dbId = self.db.addTweet(
                step[i]["userName"],
                step[i]["tweet"],
                step[i]["year"],
                step[i]["month"],
                step[i]["day"],
                step[i]["hour"],
                step[i]["minute"],
                step[i]["second"],
            )
            step[i]["dbId"] = dbId
            if dbId == -1:
                print "duplicated"
                step.pop(i)
            else:
                for h in step[i]["hashtags"]:
                    self.db.addHashtag(dbId, h)
                for l in step[i]["links"]:
                    self.db.addLink(dbId, l)
                if step[i]["location"] != False:
                    self.db.addLocation(dbId, step[i]["location"]["lat"], step[i]["location"]["lon"])
            if step[i]["id"] != None and self.max_id < step[i]["id"]:
                self.max_id = step[i]["id"]
        self.db.commit()

    @AbstractCrawler.crawlingAction
    @AbstractCrawler.traceHistory
    def getTweetsInsideArea(self, lat1, lon1, lat2, lon2, crawler=REST_CRAWLER | STREAMING_CRAWLER, **parameters):
        """Get tweets inside the given bounding box"""
        if (crawler & REST_CRAWLER) == REST_CRAWLER:
            self.threadPool.append(MyThread(self.rest.getTweetsInsideArea, lat1, lon1, lat2, lon2, **parameters))
        if (crawler & STREAMING_CRAWLER) == STREAMING_CRAWLER:
            self.threadPool.append(MyThread(self.streaming.getTweetsInsideArea, lat1, lon1, lat2, lon2, **parameters))

    @AbstractCrawler.crawlingAction
    @AbstractCrawler.traceHistory
    def getTweetsByContent(self, content, crawler=REST_CRAWLER | STREAMING_CRAWLER, **parameters):
        if (crawler & REST_CRAWLER) == REST_CRAWLER:
            self.threadPool.append(MyThread(self.rest.getTweetsByContent, content, **parameters))
        if (crawler & STREAMING_CRAWLER) == STREAMING_CRAWLER:
            self.threadPool.append(MyThread(self.streaming.getTweetsByContent, content))

    @AbstractCrawler.crawlingAction
    @AbstractCrawler.traceHistory
    def getTweetsByUser(self, username, crawler=REST_CRAWLER | STREAMING_CRAWLER, **parameters):
        if (crawler & REST_CRAWLER) == REST_CRAWLER:
            self.threadPool.append(MyThread(self.rest.getTweetsByUser, username, **parameters))
        if (crawler & STREAMING_CRAWLER) == STREAMING_CRAWLER:
            self.threadPool.append(MyThread(self.streaming.getTweetsByUser, username, **parameters))

    def export(self, output):
        self.db.dumpDb(output)

    def stop(self):
        self.max_id = 0
        removable = []
        self.streaming.stop()
        for i in range(len(self.threadPool)):
            if not self.threadPool[i].isRunning():
                removable.append(i)
        for i in removable:
            self.threadPool.pop(i)
        self.db.commit()
示例#10
0
 def __init__(self):
     QObject.__init__(self)
     AbstractCrawler.__init__(self)
     self.settings = QSettings("RFCode", "TwitterCrawler")
     self.db = DatabaseManager()
     self.max_id = 0
示例#11
0
class Crawler(QObject, AbstractCrawler):

    db = None
    """
    @type: DatabaseManager
    """

    rest = None
    """
    @type: RestCrawler
    """

    streaming = None
    """
    @type: StreamingCrawler
    """

    auth = None
    """
    @type: OAuthHandler
    """

    threadPool = []
    """
    @type: Array
    """
    def __init__(self):
        QObject.__init__(self)
        AbstractCrawler.__init__(self)
        self.settings = QSettings("RFCode", "TwitterCrawler")
        self.db = DatabaseManager()
        self.max_id = 0

    def authInit(self):
        self.rest = RestCrawler(self.auth)
        self.streaming = StreamingCrawler(
            self.auth, headers={"User-Agent": "TwitterCrawler/1.0"})
        self.rest.restDataReady.connect(self.updateSearchStep)
        self.streaming.listener.streamingDataReady.connect(
            self.updateSearchStep)
        self.streaming.listener.streamingError.connect(self.errorHandler)

    def getAuthUrl(self):
        self.auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
        return self.auth.get_authorization_url()

    def setAuthAccess(self, verifier):
        self.auth.get_access_token(verifier)
        self.settings.setValue("auth_key", self.auth.access_token.key)
        self.settings.setValue("auth_secret", self.auth.access_token.secret)

    def login(self):
        key = self.settings.value("auth_key")
        secret = self.settings.value("auth_secret")
        if key == None or secret == None:
            return False
        if self.auth == None:
            self.auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
        self.auth.set_access_token(key, secret)
        self.authInit()
        return True

    @Slot(int)
    def errorHandler(self, code):
        print "error", code
        if code >= 600:
            return
        self.stop()
#        if code == 420: #rate limit error
#            pass #TODO
#        elif code == 600: #streaming parsing error
#            pass
#        elif code == 601: #database insertion error
#            pass

    @Slot("QVariant")
    def updateSearchStep(self, step):
        for i in range(len(step) - 1, -1, -1):
            dbId = self.db.addTweet(step[i]["userName"], step[i]["tweet"],
                                    step[i]["year"], step[i]["month"],
                                    step[i]["day"], step[i]["hour"],
                                    step[i]["minute"], step[i]["second"])
            step[i]["dbId"] = dbId
            if dbId == -1:
                print "duplicated"
                step.pop(i)
            else:
                for h in step[i]["hashtags"]:
                    self.db.addHashtag(dbId, h)
                for l in step[i]["links"]:
                    self.db.addLink(dbId, l)
                if step[i]["location"] != False:
                    self.db.addLocation(dbId, step[i]["location"]["lat"],
                                        step[i]["location"]["lon"])
            if step[i]["id"] != None and self.max_id < step[i]["id"]:
                self.max_id = step[i]["id"]
        self.db.commit()

    @AbstractCrawler.crawlingAction
    @AbstractCrawler.traceHistory
    def getTweetsInsideArea(self,
                            lat1,
                            lon1,
                            lat2,
                            lon2,
                            crawler=REST_CRAWLER | STREAMING_CRAWLER,
                            **parameters):
        '''Get tweets inside the given bounding box'''
        if (crawler & REST_CRAWLER) == REST_CRAWLER:
            self.threadPool.append(
                MyThread(self.rest.getTweetsInsideArea, lat1, lon1, lat2, lon2,
                         **parameters))
        if (crawler & STREAMING_CRAWLER) == STREAMING_CRAWLER:
            self.threadPool.append(
                MyThread(self.streaming.getTweetsInsideArea, lat1, lon1, lat2,
                         lon2, **parameters))

    @AbstractCrawler.crawlingAction
    @AbstractCrawler.traceHistory
    def getTweetsByContent(self,
                           content,
                           crawler=REST_CRAWLER | STREAMING_CRAWLER,
                           **parameters):
        if (crawler & REST_CRAWLER) == REST_CRAWLER:
            self.threadPool.append(
                MyThread(self.rest.getTweetsByContent, content, **parameters))
        if (crawler & STREAMING_CRAWLER) == STREAMING_CRAWLER:
            self.threadPool.append(
                MyThread(self.streaming.getTweetsByContent, content))

    @AbstractCrawler.crawlingAction
    @AbstractCrawler.traceHistory
    def getTweetsByUser(self,
                        username,
                        crawler=REST_CRAWLER | STREAMING_CRAWLER,
                        **parameters):
        if (crawler & REST_CRAWLER) == REST_CRAWLER:
            self.threadPool.append(
                MyThread(self.rest.getTweetsByUser, username, **parameters))
        if (crawler & STREAMING_CRAWLER) == STREAMING_CRAWLER:
            self.threadPool.append(
                MyThread(self.streaming.getTweetsByUser, username,
                         **parameters))

    def export(self, output):
        self.db.dumpDb(output)

    def stop(self):
        self.max_id = 0
        removable = []
        self.streaming.stop()
        for i in range(len(self.threadPool)):
            if not self.threadPool[i].isRunning():
                removable.append(i)
        for i in removable:
            self.threadPool.pop(i)
        self.db.commit()
示例#12
0
 def setUp(self) -> None:
     sys.argv = [sys.argv[0]]
     self.db = DatabaseManager.DatabaseManager(memory_db=True)
     self.ws = WebSocketServer.WebSocketServer()
示例#13
0
 def main(cls):
     db_manage = DatabaseManager.DatabaseManager()
     db_manage.db_init()
     WebSocketServer.WebSocketServer.start_server()