Пример #1
0
class shortcodeMine:
    #shortcode var
    filename = "shortcodes.txt"

    #API var
    apiLimit = 5000
    apiCurrent = 0
    clientPosition = 0
    clientLength = 0

    def __init__(self):
        self.ig_config = ig_config
        self.clientLength = len(ig_config)
        print self.clientLength

    def initDB(self):
        self.db = Db()
        self.db.connect()

    def initIg(self):
        pos = self.clientPosition
        self.api = client.InstagramAPI(
            client_id=self.ig_config[pos]['client_id'],
            client_secret=self.ig_config[pos]['client_secret']
        )
Пример #2
0
def adminviewrating():
    ob = Db()
    res = ob.select(
        "select avg(rating.rate)as rate1,doctor.* from rating, doctor where rating.did=doctor.did group by rating.did"
    )

    return render_template("adminviewrating.html", a=res)
Пример #3
0
    def registerButton(self):
        """
        sends the info to the database and checks if the fields are acceptable

        """
        name = self.txtName.text()
        email = self.txtEmail.text()
        username = self.txtUsername.text()
        password = self.txtPassword.text()
        password2 = self.txtPassword2.text()
        if self.checkFields(username, name, email, password):
            self.showMessage("Error", "All fields must be filled")
        else:
            if (self.checkPassword(password, password2)):
                if (Db().registerCheck(username, email)):
                    Db().insertTable(name, username, email, password)
                    self.showMessage("Success", "Registration successful")
                    self.clearField()
                    self.trainModel()
                else:
                    self.showMessage("Error",
                                     "Username or Email already existing")
                    self.clearField()
            else:
                self.showMessage("Error", "Passwords doesn't match")
Пример #4
0
class SuggestionApp:
    def __init__(self):
        self.db = Db()

    # Add the word to database, skip if it already exsists
    def addToDb(self, word, lang="Eng", skip_if_exsists=True):
        words = self.tokenizeString(word)
        self.db.addList(words, lang=lang)
        return

    # tokenize the string
    # split on whitespaces.
    # Also return the complete sentence except the last word as it might be incomplete
    # filter empty strings
    def tokenizeString(self, string):
        splitString = string.split()

        if len(string) == len(string.strip()):
            if (len(splitString) < 2): return []
            return filter(None, splitString + [" ".join(splitString[:-1])])
        else:
            return filter(None, splitString + [" ".join(splitString[:])])

    # search for the word in db and return Suggestions
    # Add new words to DB
    def getSuggestions(self, query, lang="Eng", limit=100):
        # Add new words to DB
        self.addToDb(query)
        suggestions = self.db.getWordsByQuery(query, lang=lang, limit=limit)
        # get words from DB
        return suggestions
Пример #5
0
class igMine:

    # API var
    apiLimit = 5000
    apiCurrent = 0
    clientPosition = 0
    clientLength = 0

    # mine var
    usernames = ["sex_on_water", "lavimeer", "vita_century"]

    def __init__(self):
        self.ig_config = ig_config
        self.clientLength = len(ig_config)
        print self.clientLength

    def initDB(self):
        self.db = Db()
        self.db.connect()

    def initIg(self):
        pos = self.clientPosition
        self.api = client.InstagramAPI(
            client_id=self.ig_config[pos]["client_id"], client_secret=self.ig_config[pos]["client_secret"]
        )

    def search(self, user_id, max_id):
        user = self.api.user(user_id)
        media_search = self.api.user_recent_media(user_id=user_id, count=35, max_id=max_id)
        return media_search

    def getUserId(self, username):
        user_search = self.api.user_search(username)
        return int(user_search[0].id)
Пример #6
0
def mongo_db_test():
    from database_mongo import Db
    import time
    db = Db()
    db.init()
    current_timestamp = time.time() * 1000
    db.add_pending_messages("asdasdasd", 1, "{}", current_timestamp)
    db.add_pending_messages("asdasdasd", 2, "{}")
    time.time() * 1000

    print db.fetch_inbox_messages("asdasdasd", -1, -1, current_timestamp - 100)
Пример #7
0
 def loginCheck(self):
     username = self.txtUsername.text()
     password = self.txtPassword.text()
     getDb = Db()
     result = getDb.loginCheck(username, password)
     if (result):
         self.welcomePage()
         self.clearField()
         print(result)
     else:
         print("password wrong")
         self.showMessage("Warning", "Invalid Username and Password")
Пример #8
0
 def saveContact(self):
     id = self.id
     first_name = self.first_name
     last_name = self.last_name
     telephone = self.telephone
     email = self.email
     url = self.url
     notes = self.notes
     db = Db()
     if (first_name != None and first_name != "" and last_name != None
             and last_name != ""):
         db.saveContact(id, first_name, last_name, telephone, email, url,
                        notes)
Пример #9
0
def mongo_db_test():
    from database_mongo import Db
    import time
    db = Db()
    db.init()
    current_timestamp = time.time()*1000
    db.add_pending_messages("asdasdasd", 1, "{}", current_timestamp)
    db.add_pending_messages("asdasdasd", 2, "{}")
    time.time()*1000
    
    print db.fetch_inbox_messages("asdasdasd", -1, -1, current_timestamp-100)
Пример #10
0
def upload():	
	form = UploadForm()
	if form.validate_on_submit():
		flash('TXT registered')
		db = Db()
		filename = secure_filename(form.upload.data.filename)
		form.upload.data.save(os.path.join(app.config['UPLOADED_TEXT_DEST'], filename))
		SQL =  'INSERT INTO text (author_name, file_location, description, title, text_type, pub_date) VALUES (%s, %s, %s, %s, %s, %s);'
		data = (form.author.data, "uploads/" + form.upload.data.filename, form.description.data, form.title.data, form.textType.data, form.pub_date.data)
		db.execute(SQL, data)
		return redirect(url_for('pick'))

	else:
		filename = None
	return render_template('upload.html', form=form, filename=filename)
Пример #11
0
 def loginCheck(self):
     username = self.txtUsername.text()
     print(username)
     fh = open('database/username.txt', 'w+')
     fh.write(str(username))
     fh.close()
     password = self.txtPassword.text()
     getDb = Db()
     result = getDb.loginCheck(username, password)
     if (result):
         self.welcomePage()
         self.clearField()
         print(result)
     else:
         print("密码错误哦,再输一次吧(≖ω≖✿)")
         self.showMessage("Warning", "诶,这是不合理的用户名和密码哦~(¯•ω•¯)")
 def registerButton(self):
     name = self.txtName.text()
     email = self.txtEmail.text()
     username = self.txtUsername.text()
     password = self.txtPassword.text()
     password2 = self.txtPassword2.text()
     if self.checkFields(username,name,email,password):
         self.showMessage("Error", "All fields must be filled")
     else:
         if(self.checkPassword(password,password2)):
             insertDb = Db()
             Db().insertTable(name,username,email,password)
             self.showMessage("Success","Registration successul")
             self.clearField()
         else:
             self.showMessage("Error","Passwords doesn't match")
Пример #13
0
 def __init__(self, *args, **kwargs):
     discord.Client.__init__(self, *args, **kwargs)
     self.redis_url = kwargs.get('redis_url')
     self.db = Db(self.redis_url)
     self.plugin_manager = PluginManager(self)
     self.plugin_manager.load_all()
     self.last_messages = []
Пример #14
0
def main():
    db = Db.instance()
    try:
        app.setup()
        app.run()
    finally:
        # close the app
        app.close()
def main():
    db = Db.instance()
    try:
        app.setup()
        app.run()
    finally:
        # close the app
        app.close()
Пример #16
0
        def save_name(message):
            connection = Db().connect()
            connection.cursor().execute(
                'UPDATE users SET name=? WHERE chat_id=?',
                (message.text, message.chat.id))
            connection.commit()
            connection.close()

            get_company = bot.send_message(chat_id=message.chat.id,
                                           text=msg['company'])
            bot.register_next_step_handler(get_company, save_company)
Пример #17
0
 def registerButton(self):
     username = self.txtUsername.text()
     password = self.txtPassword.text()
     password2 = self.txtPassword2.text()
     if self.checkFields(username, password):
         self.showMessage("Error", "小主,所有信息都要填写哦|д・)っ")
     else:
         if (self.checkPassword(password, password2)):
             insertDb = Db()
             Db().insertTable(username, password)
             self.showMessage("Success", "注册成功啦!(⁄⁄•⁄ω⁄•⁄⁄)")
             self.clearField()
             #                self.Dialog.setVisible(False)
             self.homWindow = QtWidgets.QDialog()
             self.ui = Ui_Dialog5()
             self.ui.setupUi(self.homWindow)
             self.homWindow.show()
         else:
             self.showMessage("Error", "前后密码不匹配哦⊙̆̈_⊙̆̈")
Пример #18
0
 def __init__(self, *args, **kwargs):
     super().__init__(*args, **kwargs)
     self.redis_url = kwargs.get('redis_url')
     self.mongo_url = kwargs.get('mongo_url')
     self.dd_agent_url = kwargs.get('dd_agent_url')
     self.db = Db(self.redis_url, self.mongo_url, self.loop)
     self.plugin_manager = PluginManager(self)
     self.plugin_manager.load_all()
     self.last_messages = []
     self.stats = DDAgent(self.dd_agent_url)
Пример #19
0
class igMine:

    #API var
    apiLimit = 5000
    apiCurrent = 0
    clientPosition = 0
    clientLength = 0

    #mine var
    lat = location_config.lat
    lng = location_config.lng
    city = location_config.city
    distance = 5000

    def __init__(self):
        self.ig_config = ig_config
        self.clientLength = len(ig_config)
        print self.clientLength

    def initDB(self):
        self.db = Db()
        self.db.connect()

    def initIg(self):
        pos = self.clientPosition
        self.api = client.InstagramAPI(
            client_id=self.ig_config[pos]['client_id'],
            client_secret=self.ig_config[pos]['client_secret']
        )

    def search(self, max_timestamp, min_timestamp):
        media_search = self.api.media_search(
            lat=self.lat,
            lng=self.lng,
            distance=self.distance,
            max_timestamp = max_timestamp,
            min_timestamp = min_timestamp
        )

        return media_search
Пример #20
0
def mongo_db_test_1():
    # to test fetch node
    from database_mongo import Db
    import time
    db = Db()
    db.init(user_name="abhinav", password="******", host="104.199.129.250", namespace="samosa_messaging_v2")
    
#     print db.fetch_inbox_messages("df44c88cdaa36e4de1c5bf9c36968078", -1, -1, 1468524512476)
#
    db.update_node_info("g3cK4ljIyh1468240301", num_connections=1650, num_max_connections=1500)
    print db.get_a_connection_node()
Пример #21
0
def hours():
    form = HoursForm()

    if form.validate_on_submit():
        email = form.email.data

        DB = Db()
        tutor = DB.execute(
            "SELECT tutorID, hours FROM tutors WHERE email = %s AND status = 1",
            (email, ))

        # Check if email is registered as a tutor
        if not tutor:
            flash(
                Markup(
                    f"This email doesn't seem to be registered as a tutor. Please check to make sure you typed it correctly, \
                or <a href={ url_for('contact') }>contact us</a> if you think this is an error."
                ), "error")
            return render_template("hours.html", form=form)

        # Convert hours to integer if hours is a whole number
        hours = tutor[0]['hours']
        hours = hours if hours % 1 != 0 else int(hours)

        if hours == 1:
            flash(
                Markup(
                    f"You currently have 1 volunteer hour. Please <a href={ url_for('contact') }>contact us</a> if you think this is an error."
                ))
        else:
            flash(
                Markup(
                    f"You currently have { hours } volunteer hours. Please <a href={ url_for('contact') }>contact us</a> if you think this is an error."
                ))

        return redirect(url_for("hours"))

    return render_template("hours.html", form=form)
Пример #22
0
def mongo_db_test_1():
    # to test fetch node
    from database_mongo import Db
    import time
    db = Db()
    db.init(user_name="abhinav",
            password="******",
            host="104.199.129.250",
            namespace="samosa_messaging_v2")

    #     print db.fetch_inbox_messages("df44c88cdaa36e4de1c5bf9c36968078", -1, -1, 1468524512476)
    #
    db.update_node_info("g3cK4ljIyh1468240301",
                        num_connections=1650,
                        num_max_connections=1500)
    print db.get_a_connection_node()
Пример #23
0
    def check_password(self):
        """
        checks the login info
        """
        msg = QMessageBox()
        username = self.lineEdit_username.text()
        password = self.lineEdit_password.text()
        getDb = Db()
        result = getDb.loginCheck(username, password)
        if result:
            msg = QMessageBox()
            msg.setWindowIcon(QtGui.QIcon('eye-512.png'))
            msg.setWindowTitle("Login")
            msg.setText("Success")
            x = msg.exec_()  # this will show our message box
            self.login()

        else:
            msg = QMessageBox()
            msg.setWindowIcon(QtGui.QIcon('eye-512.png'))
            msg.setWindowTitle("Login")
            msg.setText("Incorrect Password")
            x = msg.exec_()  # this will show our message box
Пример #24
0
def init(db_path):
    if (not utils.is_database_exists(db_path)):
        print('Database was not found. Do you want to create a new one?')
        a = input('Yes/no: ')
        if (a == 'no'):
            sys.exit(1)
        else:
            db = Db(db_path)
            db.init()
            db.close()
Пример #25
0
def process_pre_checkout_query(
        pre_checkout_query: telebot.types.PreCheckoutQuery):
    connection = Db().connect()
    connection.cursor().execute(
        'INSERT INTO payments(chat_id, status, date) VALUES (?,?,?)',
        (pre_checkout_query.from_user.id, 'Not paid',
         datetime.now().strftime('%d.%m.%y')))
    connection.commit()

    bot.answer_pre_checkout_query(pre_checkout_query.id, ok=True)
Пример #26
0
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.redis_url = kwargs.get('redis_url')
        self.mongo_url = kwargs.get('mongo_url')
        self.dd_agent_url = kwargs.get('dd_agent_url')
        self.sentry_dsn = kwargs.get('sentry_dsn')
        self.db = Db(self.redis_url, self.mongo_url, self.loop)
        self.plugin_manager = PluginManager(self)
        self.plugin_manager.load_all()
        self.last_messages = []
        self.stats = DDAgent(self.dd_agent_url)
        self.voice_sessions_ids = dict()

        if self.shard_id is not None:
            self.shard = [self.shard_id, self.shard_count]
        else:
            self.shard = [0, 1]
Пример #27
0
 def __init__(self, myconfig):
     # 线程池, 
     self.thread_pool = ThreadPool(myconfig.threadnum)
     # 已访问的url集合
     self.visited_urls = set()
     # set 不是线程安全,所以这里加一把锁
     self.visited_urls_lock = threading.Lock()
     # 未访问的url集合
     self.will_visited_urls = deque()
     self.will_visited_urls.append(myconfig.url)
     self.temp_q = deque()
     self.cur_depth = 0
     self.status = ""
     self.myconfig = myconfig
     MyLogger(myconfig.logfile, myconfig.loglevel)
     #MyLogger(myconfig.logfile, loglevel = 5)  # debug
     self.db = Db()
Пример #28
0
def show_stocks(searchterm = '', page = None):
	search = SearchForm()
	db = Db()
	SQL = "SELECT ticker, company_name FROM stocks WHERE ticker LIKE %s OR company_name LIKE %s ORDER BY ticker;"
	data =  ('%', '%')	
	if search.validate_on_submit():
		search_data = search.search.data
		data = (search_data + '%', search_data + '%')
		db.execute(SQL, data)
		entries = [dict(ticker = row[0], company = row[1]) for row in db.fetchall()]
		return render_template('index.html', search = SearchForm(),  entries = entries)

	else:
		db.execute(SQL, data)
		entries = [dict(ticker = row[0], company = row[1]) for row in db.fetchall()]
		return render_template('index.html',search = search, entries=entries)
Пример #29
0
def test2():
    from websocket._core import create_connection
    from bson import json_util
    import cookies
    from database import Db
    import config
    import time

    db = Db()
    db.init()
    # this is where user connects to nodes
    db.remove_client_nodes("abhinav")
    client_node_id1 = db.create_node("abhinav", None, None, None)  # client
    client_node_id2 = db.create_node("abhinav", None, None, None)  # client

    print "creating nodes", client_node_id1, client_node_id2

    auth_key_client1 = cookies.create_signed_value(
        config.SERVER_SECRET, "auth",
        json_util.dumps({'node_id': client_node_id1}))

    auth_key_client2 = cookies.create_signed_value(
        config.SERVER_SECRET, "auth",
        json_util.dumps({'node_id': client_node_id2}))

    ws1 = create_connection("ws://192.168.1.146:8081/connect?auth_key=" +
                            auth_key_client1)
    print "created_connection 1"

    ws2 = create_connection("ws://192.168.1.146:8083/connect?auth_key=" +
                            auth_key_client2)
    print "created_connection 2"

    time.sleep(1)

    ws1.send(
        json_util.dumps({
            'dest_id': client_node_id2,
            "payload": "Hello world"
        }))
    time.sleep(0.1)
    print ws2.recv()
    time.sleep(0.1)
    print ws1.recv()

    ws1.send_close()
    ws2.send_close()
Пример #30
0
        def save_company(message):
            connection = Db().connect()
            connection.cursor().execute(
                'UPDATE users SET company=? WHERE chat_id=?',
                (message.text, message.chat.id))
            connection.commit()
            connection.close()

            markup1 = telebot.types.InlineKeyboardMarkup(row_width=1)
            btn1 = telebot.types.InlineKeyboardButton(text='Получить доступ',
                                                      callback_data='invoice')
            btn2 = telebot.types.InlineKeyboardButton(text='Договор оферты',
                                                      callback_data='dogovor',
                                                      url='https://google.com')
            btn3 = telebot.types.InlineKeyboardButton(text='О компании',
                                                      callback_data='about',
                                                      url='https://google.com')
            markup1.add(btn1, btn2, btn3)
            bot.send_message(chat_id=message.chat.id,
                             text=msg['start'],
                             reply_markup=markup1)
Пример #31
0
 def contact_handler(message):
     try:
         connection = Db().connect()
         connection.cursor().execute(
             'INSERT INTO users(chat_id, username, phone, date) VALUES (?,?,?,?)',
             (message.chat.id,
              message.from_user.username, message.contact.phone_number,
              datetime.now().strftime('%d.%m.%y')))
         connection.commit()
         connection.close()
         markup = telebot.types.ReplyKeyboardRemove()
         get_name = bot.send_message(chat_id=message.chat.id,
                                     text=msg['name'],
                                     reply_markup=markup)
         bot.register_next_step_handler(get_name, save_name)
     except:
         bot.send_message(
             chat_id=message.chat.id,
             text=
             'Вы не можете зарегистрироваться без username. Для того, чтобы его указать, перейдите в настройки вашего Telegram и укажите username'
         )
         pass
Пример #32
0
def test2():
    from websocket._core import create_connection
    from bson import json_util
    import cookies
    from database import Db
    import config
    import time
    
    db = Db()    
    db.init()
    # this is where user connects to nodes    
    db.remove_client_nodes("abhinav")    
    client_node_id1 = db.create_node("abhinav", None, None, None)# client
    client_node_id2 = db.create_node("abhinav", None, None , None)# client
    
    print "creating nodes" , client_node_id1, client_node_id2
    
    auth_key_client1 = cookies.create_signed_value(config.SERVER_SECRET , "auth", json_util.dumps({'node_id':client_node_id1}))
    
    auth_key_client2 = cookies.create_signed_value(config.SERVER_SECRET , "auth",json_util.dumps({'node_id':client_node_id2}))
        
    ws1 = create_connection("ws://192.168.1.146:8081/connect?auth_key="+auth_key_client1)
    print "created_connection 1"
    
    
    ws2 = create_connection("ws://192.168.1.146:8083/connect?auth_key="+auth_key_client2)
    print "created_connection 2"

    time.sleep(1)
    
    
    ws1.send(json_util.dumps({'dest_id':client_node_id2, "payload":"Hello world" }))
    time.sleep(0.1)
    print ws2.recv()
    time.sleep(0.1)
    print ws1.recv()
    
    ws1.send_close()
    ws2.send_close()
Пример #33
0
    :param proc_db:
    :return diff:
    """
    diff = []
    i = 0
    while True:
        if proc_file[i] != proc_db[i]:
            diff.append(proc_file[i])
            diff.append(proc_db[i])
            break
        else:
            i += 1
    return diff


if __name__ == '__main__':
    db = Db()
    pf = Procfile('testdata/stored_procs.sql')

    procname = 'dbo.uspGetBillOfMaterials'
    proc_file = pf.get_procedure(procname)
    proc_db = db.get_procedure(procname)
    actual = compare_two_list(proc_file, proc_db)
    if actual:
        print('Különbség van az adatbázisban és a fájlban a ' + procname +
              ' eljárásban:')
        print('Fájl')
        print(actual[0])
        print('DB')
        print(actual[1])
Пример #34
0
 def getContactFromId(self):
     db = Db()
     contact = db.getContactfromId(self.id)
     return contact
Пример #35
0
 def getContactFromText(self, text):
     db = Db()
     contacts = db.getContactFromText(text)
     return contacts
Пример #36
0
 def updateContact(self):
     db = Db()
     db.updateContact(self.first_name, self.last_name, self.telephone,
                      self.email, self.url, self.notes, self.id)
Пример #37
0
 def showContacts(self):
     db = Db()
     contacts = db.getContacts()
     return contacts
Пример #38
0
 def getContactFromTagAndText(self, text):
     db = Db()
     result = db.getContactFromTagAndText(text, self.tag)
     return result
Пример #39
0
 def deleteContactTagsFromId(self):
     db = Db()
     id = self.id
     db.deleteContactTagsFromId(id)
Пример #40
0
 def saveContactTag(self):
     db = Db()
     tag_name = self.tag
     id = self.id
     if (tag_name != "" and tag_name != None and id != "" and id != None):
         db.insertTagContact(id, tag_name)
Пример #41
0
class Crawler():
    def __init__(self, myconfig):
        # 线程池, 
        self.thread_pool = ThreadPool(myconfig.threadnum)
        # 已访问的url集合
        self.visited_urls = set()
        # set 不是线程安全,所以这里加一把锁
        self.visited_urls_lock = threading.Lock()
        # 未访问的url集合
        self.will_visited_urls = deque()
        self.will_visited_urls.append(myconfig.url)
        self.temp_q = deque()
        self.cur_depth = 0
        self.status = ""
        self.myconfig = myconfig
        MyLogger(myconfig.logfile, myconfig.loglevel)
        #MyLogger(myconfig.logfile, loglevel = 5)  # debug
        self.db = Db()
        
    
    def start(self):
        self.status = "start"
        while self.cur_depth < self.myconfig.depth:
            if self.status == "stop":
                break
            try:
                while self.will_visited_urls:
                    url = self.will_visited_urls.popleft()
                    # 添加工作,这里基本上没有阻塞,因为是在主线程里,只是负责
                    # 添加工作,真正执行工作是在线程里做的
                 
                    self.thread_pool.add_job(self.handler, url)
                #
                # TODO:
                # 通知线程有活干了,这里可以看出是在将will_visited_urls的url
                # 都添加后才通知线程去干活的,这样设计,粒度似乎有点粗?
                # 如果还想节省时间的话,可以在url的数目 >= 线程初始数目的时候,就通知
                # 线程池里的线程开始干活,如果url的数目 < 线程初始数目的时候,等都
                # 添加完之后,再通知
                
                #print ">>>>>>>>  give event to threads in thread pool"
                # 通知线程池里的线程开始新一轮的抓取
                self.thread_pool.event_do_job()
                # 主动退出调度,让子线程有时间可以执行
                time.sleep(3)
            except Empty:
                # 需要访问的url没有了
                logging.info("no url right now")
            finally:
                
                # 必须等线程池里的线程工作做完之后,才算本次深度的访问结束
                # 这里做的处理是如果线程池里面有线程,则睡3s,再读,
                # 直到线程池里的工作线程为0才停下来
                # 这样才算本次深度的抓取完毕
                while True:
                    #print "thread waiting num is %d, config thread num is %d" % (self.thread_pool.get_thread_waiting_num(), self.myconfig.thread)
                    if self.thread_pool.get_thread_waiting_num() == self.myconfig.threadnum:
                        # 如果等待的线程数目等于线程初始数目,则说明,所有线程都执行完毕
                        # 所以break
                        break
                    else:
                        # 有线程仍然在执行,则说明, 本次深度的访问还没有结束
                        # 睡眠等待
                        time.sleep(10)
                #此次深度的访问结束,深度加一
                self.cur_depth += 1
                logging.info("crawler depth now is %s" % str(self.cur_depth))
                if self.cur_depth > self.myconfig.depth:
                    break
                # 从url中抓到的网页都放到了temp_q中,
                # 将temp_q中的网页从新给 will_visited_urls,继续
                self.will_visited_urls = self.temp_q
                self.temp_q = deque()
                
                
        # 所有深度的url都抓取完毕 or 爬虫退出
        self.thread_pool.stop_threads()
        logging.info("crawler exit")
        return
        
            
    def handler(self, url):
        content= self.get_html_content(url)
        if content == "" or content == None:
            # 无法获取content,直接返回
            return
        # 添加此url为已访问过
        self.add_url_to_visited(url)
        if content.find(self.myconfig.key) != -1:
            self.db.save_data(url, self.myconfig.key, content)
        try:
            hrefs = self.get_hrefs(content, url)
        except StandardError, se:
            logging.error("error: %s" % (se))
            print se
            # log
            # 无法获取 hrefs
            return
        # 如果获得了hrefs
        if hrefs:
            # 将hrefs添加到 temp_q中,等本级深度访问完毕之后再访问
            for link in hrefs:
                # 最后的考验
                if not self.is_url_visited(link) \
                            and link not in self.will_visited_urls \
                            and link not in self.temp_q:
                    #print "put %s into temp_q" % link 
                    self.temp_q.append(link)
Пример #42
0
 def initDB(self):
     self.db = Db()
     self.db.connect()
Пример #43
0
def show_stock(ticker = None):
	form = StockForm()
	db = Db()
	SQL = "SELECT company_name FROM stocks where ticker = %s;"
	data = (ticker,)
	db.execute(SQL, data)
	company = db.fetchall()[0][0]
	price_SQL = "SELECT open_price, close_price, high, low, pdate FROM stock_prices WHERE ticker = %s ORDER BY pdate;"
	volume_SQL = "SELECT volume, vdate FROM stock_volumes where ticker = %s ORDER BY vdate;"
	dividend_SQL = "SELECT price, ddate from stock_dividends where ticker = %s ORDER BY ddate;"
	price_dates = []
	open_prices = []
	high_prices = []
	low_prices  = []
	close_prices = []
	volumes = []
	dividends = []	
	db.execute(price_SQL, data)	
	for row in db.fetchall():
		date_js = int(time.mktime(row[4].timetuple())) * 1000
		open_prices.append([date_js, float(row[0])])
		high_prices.append([date_js, float(row[2])])
		low_prices.append([date_js,  float(row[3])])
		close_prices.append([date_js, float(row[1])])
	db.execute(volume_SQL, data)
	for row in db.fetchall():
		date_js = int(time.mktime(row[1].timetuple())) * 1000
		volumes.append([date_js, float(row[0])])
	db.execute(dividend_SQL, data)
	for row in db.fetchall():
		date_js = int(time.mktime(row[1].timetuple())) * 1000
		dividends.append([date_js, float(row[0])])
	if form.validate_on_submit():
		date = str(form.date_field.data).split()[0]
		nyt = NYT_Parser()
		nyt.find_articles(company, date)
		articles = nyt.get_news()
		return render_template('stock.html', form = form, open_prices = open_prices,close_prices = close_prices, volumes = volumes, dividends = dividends, company = company, ticker = ticker, validated = True, articles = articles) 
	else:
		return render_template('stock.html', form = form, open_prices = open_prices,close_prices = close_prices, volumes = volumes, dividends = dividends, company = company, ticker = ticker)