def payLoan(): venmo_id = request.form['venmo_name'] payment_amount = float(request.form['amount']) account_info = db.get_collection(account_collection_str) account_info_record = account_info.find_one({"user_id" : venmo_id}) current_chequing = account_info_record['chequing_balance'] current_debt = account_info_record['debt_amount'] # Need to be careful to maintain good state if (payment_amount < 0 or payment_amount > current_chequing or payment_amount > current_debt): return jsonify(**{ "success": False }) current_chequing -= payment_amount; current_debt -= payment_amount; account_info.update({"user_id" : venmo_id}, {"$set": { "chequing_balance" : current_chequing, "debt_amount" : current_debt }}) return jsonify(**{ "success" : True })
def page(page): ln = 10 conf = config.get_config() cdn_domain = conf.get('cdn', 'domain') try: page = int(page) except ValueError: page = 1 timeline = db.get_collection('timeline') db_items = timeline.find( skip=(page - 1) * ln, limit=ln, sort=[('date', pymongo.DESCENDING)], ) items = [] for item in db_items: try: item['date'] = str(item['date'].strftime('%Y-%m-%d')) except: pass item['_id'] = str(item['_id']) item['cdn_path'] = os.path.join(cdn_domain, item['cdn_path']) items.append(item) # pagination total = int(math.ceil(float(timeline.find().count()) / ln)) pagination = get_pagination(page, total) return render_template( 'timeline.html', items=items, pagination=pagination, )
def withdrawAmount(): venmo_id = request.form['venmo_name'] withdraw_amount = float(request.form['amount']) account_info = db.get_collection(account_collection_str) account_info_record = account_info.find_one({"user_id" : venmo_id}) current_amount = account_info_record['chequing_balance'] if (withdraw_amount > current_amount): return jsonify(**{ "success": False }) else: print venmo.post_payment( account_info_record['access_token'], venmo_id, 'Cash out made', withdraw_amount, ) current_amount = current_amount - withdraw_amount account_info.update({"user_id" : venmo_id}, {"$set": {"chequing_balance" : current_amount}}) return jsonify(**{ "success": True })
def transferToSavings(): venmo_id = request.form['venmo_name'] transfer_amount = float(request.form['amount']) account_info = db.get_collection(account_collection_str) account_info_record = account_info.find_one({"user_id" : venmo_id}) current_chequing = account_info_record['chequing_balance'] current_savings = account_info_record['savings_balance'] if (transfer_amount > current_chequing): return jsonify(**{ "success": False }) else: current_savings = current_savings + transfer_amount; current_chequing = current_chequing - transfer_amount; account_info.update({"user_id" : venmo_id}, {"$set": { "chequing_balance" : current_chequing, "savings_balance" : current_savings }}) return jsonify(**{ "success": True })
def add_user(phoneNumber): user_collection = get_collection("quotelee", "users") user_collection.insert({ "phoneNo": phoneNumber, "status": "active" }) send_welcome_message(phoneNumber)
def message_detail(id=None): msgs = db.get_collection("msg") msg = msgs.get(id) if msg: return template("message_detail", msg=msg) else: abort(404, "Message not found")
def parse(self, response): selector = Selector(response=response) articles = selector.xpath('/html/body/section/div/div/article') timeline = db.get_collection('timeline') for item in articles: try: title = item.xpath('header/h2/a/text()').extract()[0] # link URL url = item.xpath('header/h2/a/@href').extract()[0] description = item.xpath('p[3]/text()').extract()[0] description = self._join_text(description) # image URL img = item.xpath('p[2]/a/span/span/img/@data-original').extract()[0] # [tag1, tag2, ...] tags = item.xpath('p[4]/span[2]/a/text()').extract() # YYYY-MM-DD date = item.xpath('p[1]/text()').extract()[0].strip() date = date.split(" ", 1)[1] date = self._parse_date(date) # label of category category = item.xpath('header/a/text()').extract()[0] self.save(title=title, url=url, description=description, img=img, tags=tags, date=date, category=category) except IndexError: continue next_page = selector.xpath(u'/html/body/section/div/div/div/ul/li/a[text()="下一页"]/@href').extract()[0] yield Request(response.urljoin(next_page), self.parse)
def getFeeds(): feeds = get_collection(host_mongodb, db_name="feeds") feed_urls = [{ "url": feed['url'], "item_id": str(feed['_id']) } for feed in feeds.find()] return {"feeds": feed_urls}
def register(): venmo_id = request.form['venmo_name'] name = request.form['user_name'] p_venmo_id = request.form['parent_venmo_name'] p_name = request.form['parent_name'] account_collection = db.get_collection(account_collection_str) account_collection.insert({ "user_id" : venmo_id, "user_name" : name, "chequing_balance" : 0, "savings_balance" : 0, "parent_id" : p_venmo_id, "parent_name" : p_name, "allowance_amount" : 0, "savings_amount" : 0, "savings_interest_rate" : 0, "debt_amount" : 0, "loan_interest_rate" : 0, "user_token": "u_token", "access_token": "a_token", "refresh_token": "r_token" }) # TODO: Venmo fields are totally sucky return jsonify(**{ "success": True })
def page(page): ln = 10 try: page = int(page) except ValueError: page = 1 timeline = db.get_collection('timeline') db_items = timeline.find( skip=(page - 1) * ln, limit=ln, sort=[('date', pymongo.DESCENDING)], ) #{'date': {'$exist': True}}, items = [] for item in db_items: try: item['date'] = str(item['date'].strftime('%Y-%m-%d')) except: pass item['_id'] = str(item['_id']) items.append(item) # pagination total = int(math.ceil(float(timeline.find().count()) / ln)) pagination = get_pagination(page, total) return render_template( 'timeline.html', items=items, pagination=pagination, )
def takeLoan(): venmo_id = request.form['venmo_name'] loan_amount = float(request.form['amount']) account_info = db.get_collection(account_collection_str) account_info_record = account_info.find_one({"user_id" : venmo_id}) current_chequing = account_info_record['chequing_balance'] current_debt = account_info_record['debt_amount'] if (loan_amount < 0): return jsonify(**{ "success": False }) current_chequing += loan_amount; current_debt += loan_amount; account_info.update({"user_id" : venmo_id}, {"$set": { "chequing_balance" : current_chequing, "debt_amount" : current_debt }}) return jsonify(**{ "success" : True })
def stats(): msgs = db.get_collection("msg") stats = { "count": len(msgs), "num_authors": len(set(msg.get("author") for msg in msgs.values())), "longest": max(len(msg.get("text", "")) for msg in msgs.values()), } return template("stats", **stats)
def get_random_quote(): quote_collection = get_collection("quotelee", "quotes") random_quote = quote_collection.aggregate( [{'$sample':{'size':1}}] ) for quote in random_quote: return quote
def unsubscribe_user(phoneNumber): user_collection = get_collection("quotelee", "users") user_collection.update_one( {"phoneNo": phoneNumber}, { "$set": { "status": "inactive" } })
def report(bot, channel, user, args): p = get_collection('cases') query_str = { '$or': [ {'has_owner': False}, {'$and': [ {'nno': True}, {'mute': False} ]}, {'$and': [ {'status': 'WoRH'}, {'mute': False}, {'has_owner': False}, ]}, ] } cases = p.find(query_str) for case in cases: if case['region'] == 'HK': msg_prefix = 'HK_Case' elif case['region'] == 'TW': msg_prefix = 'TW_Case' elif case['region'] == 'KR': msg_prefix = 'KR_Case' elif case['region'] == 'CN': msg_prefix = 'CN_Case' else: msg_prefix = 'Global_Case' if case['severity'] == '1': sev_str = BOLD + RED + 'S1' + CLEAR elif case['severity'] == '2': sev_str = BOLD + ORANGE + 'S2' + CLEAR else: sev_str = case['severity'] if case['sbt'] != '-': if 0 <= int(case['sbt'].replace(',', '')) < 60: sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR elif int(case['sbt'].replace(',', '')) < 0: sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR else: sbt_str = GREEN + case['sbt'] + CLEAR else: sbt_str = case['sbt'] msg_str = "%s: [%s%s%s] [%s] %s%s%s%s [%s %s] %s NNO:%s %s" % ( msg_prefix, BOLD, case['_id'], CLEAR, case['account'], BOLD, BLUE, case['subject'], CLEAR, sbt_str, sev_str, case['status'], case['nno'], case['url'], ) bot.msg(channel, user + ' ' + msg_str)
def resubscribe_user(phoneNumber): user_collection = get_collection("quotelee", "users") user_collection.update_one({ "phoneNo": phoneNumber}, { "$set": { "status": "active" } }) send_resubscribe_message(phoneNumber)
def fetch_quote(input_query): tags = process_input(input_query) result_id = find_nearest_neighbor(tags) if result_id == False: return error_message() else: quote_collection = get_collection("quotelee", "quotes") quote_object = quote_collection.find_one({'_id': result_id}) return quote_object
def report(bot, channel, user, args): p = get_collection('cases') query_str = { '$or': [ { 'has_owner': False }, { '$and': [{ 'nno': True }, { 'mute': False }] }, { '$and': [ { 'status': 'WoRH' }, { 'mute': False }, { 'has_owner': False }, ] }, ] } cases = p.find(query_str) for case in cases: if case['region'] == 'HK': msg_prefix = 'HK_Case' elif case['region'] == 'TW': msg_prefix = 'TW_Case' elif case['region'] == 'KR': msg_prefix = 'KR_Case' elif case['region'] == 'CN': msg_prefix = 'CN_Case' else: msg_prefix = 'Global_Case' if case['severity'] == '1': sev_str = BOLD + RED + 'S1' + CLEAR elif case['severity'] == '2': sev_str = BOLD + ORANGE + 'S2' + CLEAR else: sev_str = case['severity'] if case['sbt'] != '-': if 0 <= int(case['sbt'].replace(',', '')) < 60: sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR elif int(case['sbt'].replace(',', '')) < 0: sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR else: sbt_str = GREEN + case['sbt'] + CLEAR
def getNews(): news = get_collection(host_mongodb) result = [] for elem in news.find()[0:20]: new_elem = elem elem["elem_id"] = str(elem["_id"]) del elem["_id"] result.append(new_elem) return {"result": result}
def deleteNewInvestment(): investment_id = request.form['investment_id'] new_investments_collection = db.get_collection(new_investments_str) new_investments_collection.remove({'investment_id': investment_id}) return jsonify(**{ "success": True })
def addBoughtInvestment(user_id, investment_id, amount): new_investments_collection = db.get_collection(new_investments_str) new_investments_collection.update({'investment_id': investment_id}, {"$set": {"is_purchased" : True}}) investment = new_investments_collection.find_one({'investment_id': investment_id}) term_length = int(investment['term_length']) rate = float(investment['rate']) bought_investments_collection = db.get_collection(bought_investments_str) bought_investments_collection.insert({ "name" : investment['name'], "term_length": term_length, "rate": investment['rate'], "end_date": util.add_months_to_today(term_length), "amount": amount, "final_amount": util.calculate_investment(rate, term_length, amount), "is_redeemed": False, "investment_id": investment_id, "user_id": user_id, })
def addNewInvestment(venmo_id, name, term, rate): new_investments_collection = db.get_collection(new_investments_str) investment_id = str(uuid.uuid4()) new_investments_collection.insert({ "user_id": venmo_id, "name": name, "term_length" : term, "rate" : rate, "investment_id": investment_id, "is_purchased": False, })
def addFeeds(feeds: Feeds): db = get_collection(host_mongodb, db_name="feeds") result_ids = [] for feed in feeds.feeds: if not db.find_one({"url": feed}): LOGGER.info(str(feed)) # TODO set up type new_id = db.insert_one({ "url": feed, "status": "PENDING" }).inserted_id result_ids.append(str(new_id)) return {"ids": result_ids}
def direct(token): coll = get_collection() doc = coll.find_one({"token": str(token)}) close_db() if doc is None: return redirect(url_for("index")) if 'ads' in doc: timeout = getenv("ADS_REDIRECT_TIMEOUT") if timeout is not None: return render_template("ads.html", timeout=int(timeout)) else: return render_template("ads.html", timeout=5) else: return redirect("http://" + str(doc["url"]))
def setCurrentInvestment(): venmo_id = request.form['venmo_name'] investment_id = request.form['investment_id'] amount = float(request.form['amount']) account_info = db.find_one(account_collection_str, {"user_id" : str(venmo_id)}) current_chequing = float(account_info['chequing_balance']) new_investments_collection = db.get_collection(new_investments_str) investment = new_investments_collection.find_one({'investment_id': investment_id}) # post as long as value is 0 if amount < 0 or current_chequing < amount or investment['is_purchased'] == True: return jsonify(**{ "success": False }) else: addBoughtInvestment(venmo_id, investment_id, amount, ) new_chequing_balance = current_chequing - amount account_info = db.get_collection(account_collection_str) account_info.update({"user_id" : venmo_id}, {"$set": {"chequing_balance" : new_chequing_balance}}) new_investment_list = db.find(new_investments_str, {"user_id" : venmo_id, "is_purchased": False }) current_investment_list = db.find(bought_investments_str, {"user_id" : venmo_id }) return jsonify(**{ "chequing_balance": new_chequing_balance, "new_investments" : new_investment_list, "my_investments" : current_investment_list, })
def unmute(bot, channel, user, cid): p = get_collection('cases') case = p.find_one({'_id': cid}) if case: case['mute'] = False p.update({'_id': cid}, case) if channel == bot.nick: bot.reply(user, 'mute case: %s' % cid) else: bot.msg(channel, '%s, mute case: %s' % (user, cid)) else: if channel == bot.nick: bot.reply(user, 'I can\'t find the case %s' % cid) else: bot.msg(channel, '%s, I can\'t find the case: %s' % (user, cid))
def escalation_parser(html_source): """ Give the HTML code to pyquery decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue. Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/ """ report = pq(html_source.decode("utf-8")) table = report("table.reportTable.tabularReportTable") trs = table("tr.even") current_case_list = [] prev_case_list = [] p = get_collection("escalation") for i in xrange(0, len(trs)): case = {} tds = trs.eq(i)("td") case['_id'] = tds.eq(3).text() res = p.find_one({'_id': case['_id']}) if res: continue else: case['owner'] = tds.eq(1).text() case['account'] = tds.eq(2).text() case['subject'] = tds.eq(5).text() case['severity'] = tds.eq(8).text() case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href") case['checked'] = False current_case_list.append(case['_id']) try: p.insert(case) except DuplicateKeyError: print 'Case exist:', sys.exc_info()[0] print case['_id'] except: print 'Unexpected error:', sys.exc_info()[0] query_str = {'checked': False} prev_case_list = map(lambda x: x.get('_id'), list(p.find(query_str, {'_id': 1}))) taken_list = list(set(prev_case_list) - set(current_case_list)) for _id in taken_list: p.update({'_id': _id}, {'$set': {'checked': True}})
def send_daily_quote(): quote = get_random_quote() full_quote = quote['quote'] + " - " + quote['author'] users = get_documents("quotelee", "users") for user in users: if(user["status"] == "active"): client.messages.create(to=user["phoneNo"], from_=twilio_phone_number, body=full_quote) currDateTime = datetime.now(timezone('US/Pacific')).strftime('%m-%d-%Y %H:%M') numberOfUsers = users.count() daily_quotes = get_collection("quotelee", "daily_quotes") daily_quotes.insert({ "quote": quote['quote'], "author": quote['author'], "datetime": currDateTime, "numberOfUsers": numberOfUsers })
def take(bot, channel, user, cid): p = get_collection('cases') cid = cid.split()[0] try: int(cid) except: bot.msg(channel, '%s: case id incorrect' % user) return False case = p.find_one({'_id': cid}) if case is None: bot.msg(channel, '%s: case id incorrect' % user) return False case['owner'] = user case['mute'] = False case['has_owner'] = True if not case['nno']: case['nno'] = False p.update({'_id': cid}, case) bot.msg(channel, 'gss_china: %s take case %s %s' % (user, case['_id'], case['subject']))
def setAllowance(): venmo_id = request.form['venmo_name'] rate = float(request.form['rate']) # post as long as value is 0 if rate < 0: return jsonify(**{ "success": False }) else: account_info = db.get_collection(account_collection_str) account_info.update({"user_id" : venmo_id}, {"$set": {"allowance_amount": rate}}) return jsonify(**{ "success": True })
def setSavingsAmount(): venmo_id = request.form['venmo_name'] savings_amount = float(request.form['amount']) # post as long as value is 0 if savings_amount < 0: return jsonify(**{ "success": False }) else: account_info = db.get_collection(account_collection_str) account_info.update({"user_id" : venmo_id}, {"$set": {"savings_amount": savings_amount}}) return jsonify(**{ "success": True })
def escalation_parser(html_source): """ Give the HTML code to pyquery decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue. Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/ """ report = pq(html_source.decode("utf-8")) table = report("table.reportTable.tabularReportTable") trs = table("tr.even") current_case_list = [] prev_case_list = [] p = get_collection("escalation") for i in xrange(0, len(trs)): case = {} tds = trs.eq(i)("td") case['_id'] = tds.eq(3).text() res = p.find_one({'_id': case['_id']}) if res: continue else: case['owner'] = tds.eq(1).text() case['account'] = tds.eq(2).text() case['subject'] = tds.eq(5).text() case['severity'] = tds.eq(8).text() case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr( "href") case['checked'] = False current_case_list.append(case['_id']) try: p.insert(case) except DuplicateKeyError: print 'Case exist:', sys.exc_info()[0] print case['_id'] except: print 'Unexpected error:', sys.exc_info()[0]
def index(): if request.method == "POST": url = sub("^http.*://", "", str(request.form["url"])) coll = get_collection() if coll.find_one({"url": url}) is not None: token = coll.find_one({"url": url})["token"] else: nbytes = getenv("TOKEN_NBYTES") if nbytes is not None: token = token_urlsafe(int(getenv("TOKEN_NBYTES"))) else: token = token_urlsafe(4) if getenv("ADS") is not None: coll.insert_one({"url": url, "token": token, "ads": True}) else: coll.insert_one({"url": url, "token": token}) close_db() return render_template("index.html", token=token) else: return render_template("index.html")
def ncq_parser(html_source): """ Give the HTML code to pyquery decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue. Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/ """ global SBR_MSG sbr_dic={} report = pq(html_source.decode("utf-8")) table = report("table.reportTable.tabularReportTable") trs = table("tr") lens = len(trs) p = get_collection("ncq") for i in xrange(1, lens-2): key = trs.eq(i)("td").eq(6).text() if key == '-': key = "No_SBR" if not sbr_dic.has_key(key): sbr_dic[key] = 1 elif sbr_dic.has_key(key): sbr_dic[key] += 1 #print sbr_dic sbr_msg='' for key in sbr_dic.keys(): sbr_msg += key sbr_msg += ' ' sbr_msg += str(sbr_dic[key]) sbr_msg += ' ' sbr_msg = "Global NCQ SBR : " + sbr_msg SBR_MSG = sbr_msg p.update({'_id': 200}, {'$set': {'content': SBR_MSG}})
def take(bot, channel, user, cid): p = get_collection('cases') cid = cid.split()[0] try: int(cid) except: bot.msg(channel, '%s: case id incorrect' % user) return False case = p.find_one({'_id': cid}) if case is None: bot.msg(channel, '%s: case id incorrect' % user) return False case['owner'] = user case['mute'] = False case['has_owner'] = True if not case['nno']: case['nno'] = False p.update({'_id': cid}, case) bot.msg( channel, 'gss_china: %s take case %s %s' % (user, case['_id'], case['subject']))
def parse(self, response): selector = Selector(response=response) articles = selector.xpath('/html/body/section/div/div/article') timeline = db.get_collection('timeline') for item in articles: try: title = item.xpath('header/h2/a/text()').extract()[0] # link URL url = item.xpath('header/h2/a/@href').extract()[0] description = item.xpath('p[3]/text()').extract()[0] description = self._join_text(description) # image URL img = item.xpath( 'p[2]/a/span/span/img/@data-original').extract()[0] # [tag1, tag2, ...] tags = item.xpath('p[4]/span[2]/a/text()').extract() # YYYY-MM-DD date = item.xpath('p[1]/text()').extract()[0].strip() date = date.split(" ", 1)[1] date = self._parse_date(date) # label of category category = item.xpath('header/a/text()').extract()[0] if category not in self.__class__.white_list: continue self.save(title=title, url=url, description=description, img=img, tags=tags, date=date, category=category) except IndexError: continue next_page = selector.xpath( u'/html/body/section/div/div/div/ul/li/a[text()="下一页"]/@href' ).extract()[0] yield Request(response.urljoin(next_page), self.parse)
def ncq_parser(html_source): """ Give the HTML code to pyquery decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue. Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/ """ global SBR_MSG sbr_dic = {} report = pq(html_source.decode("utf-8")) table = report("table.reportTable.tabularReportTable") trs = table("tr") lens = len(trs) p = get_collection("ncq") for i in xrange(1, lens - 2): key = trs.eq(i)("td").eq(6).text() if key == '-': key = "No_SBR" if not sbr_dic.has_key(key): sbr_dic[key] = 1 elif sbr_dic.has_key(key): sbr_dic[key] += 1 #print sbr_dic sbr_msg = '' for key in sbr_dic.keys(): sbr_msg += key sbr_msg += ' ' sbr_msg += str(sbr_dic[key]) sbr_msg += ' ' sbr_msg = "Global NCQ SBR : " + sbr_msg SBR_MSG = sbr_msg p.update({'_id': 200}, {'$set': {'content': SBR_MSG}})
def getSaleandRentCsvFor(state): collection = get_collection() rent = list( collection.find({ "State": state, "Status": { "$in": ["Townhouse for rent", "Condo for rent", "House for rent"] } })) sale = list( collection.find({ "State": state, "Status": { "$nin": ["Townhouse for rent", "Condo for rent", "House for rent"] } })) write_multi_data_to_csv(state + "_rent.csv", rent) write_multi_data_to_csv(state + "_sale.csv", sale) genrate_historical_data_for(state) remove_rent_entries_from(state + "_history.csv", state + "_history_without_rent.csv")
def save(self, **kwargs): if self.__class__.exist_count > 10: raise RuntimeError('Exceed maximum retry times for database') try: # add source and chinese source kwargs['src'] = self.__class__.name kwargs['ch_src'] = self.__class__.ch_name # make cdn path img_url = kwargs['img'].encode("utf-8") cdn_path = md5.new(img_url).hexdigest() kwargs['cdn_path'] = cdn_path.decode("utf-8") # insert into database timeline = db.get_collection('timeline') timeline.insert(kwargs) # actually upload image here if not self.__class__.img_store.exists(cdn_path): self.__class__.img_store.upload_remote_image(cdn_path, img_url) except DuplicateKeyError: self.__class__.exist_count += 1
def parse(self, response): selector = Selector(response=response) articles = selector.xpath('//*[@id="main"]/*/div[@class="post-box"]') timeline = db.get_collection('timeline') for item in articles: try: title = item.xpath( 'div[@class="post-header"]/p/a/text()').extract()[0] # link URL url = item.xpath( 'div[@class="post-header"]/p/a/@href').extract()[0] description = item.xpath( '*/div[@class="post-expert"]/text()').extract()[0] description = self._join_text(description) # image URL img = item.xpath( '*/div[@class="post-info"]/a/img/@data-original').extract( )[0] # YYYY-MM-DD #date = item.xpath('*/div[@class="post-date"]/text()').extract()[0].strip() date = item.xpath( 'div[@class="post-content"]/div[@class="post-footer"]/div[@class="post-date"]/text()' ).extract()[0] date = datetime.strptime(date, '%Y-%m-%d') self.save(title=title, url=url, description=description, img=img, date=date) except IndexError: continue next_page = selector.xpath( u'//*/div[@class="page-navigator"]/li/a[text()="下一页 »"]/@href' ).extract()[0] yield Request(response.urljoin(next_page), self.parse)
def report_parser(html_source): """ Give the HTML code to pyquery decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue. Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/ """ report = pq(html_source.decode("utf-8")) table = report("table.reportTable.tabularReportTable") trs = table("tr") lens = len(trs) current_case_list = [] prev_case_list = [] p = get_collection('cases') for i in xrange(0, lens): if trs.eq(i).hasClass("breakRowClass1.breakRowClass1Top"): res = re.search(contry_pattern, trs.eq(i).text()) if res: region = res.group(1) record = res.group(2) if region.startswith("Hong"): region = 'HK' elif region.startswith("Tai"): region = 'TW' elif region.startswith("Kor"): region = 'KR' elif region.startswith("Chin"): region = 'CN' else: region = 'Global' for k in xrange(i + 1, i + int(record) + 2): if trs.eq(k).hasClass("even"): case = {} tds = trs.eq(k)("td") # for testing purpose and get the fields and filed seq number #for i in xrange(len(tds)): #print "%s: %s" % (i, tds.eq(i).text()) case['_id'] = tds.eq(2).text() case['owner'] = tds.eq(1).text() case['subject'] = tds.eq(3).text() case['sbt'] = tds.eq(4).text() case['severity'] = tds.eq(5).text()[0] if tds.eq(14).text() == "-": case['sbr'] = "No SBR" else: case['sbr'] = tds.eq(14).text() status = tds.eq(6).text().strip() if status == 'Waiting on Red Hat': case['status'] = 'WoRH' elif status == 'Waiting on Customer': case['status'] = 'WoCU' else: case['status'] = status case['mute'] = False case['inter_stat'] = tds.eq(7).text() case['nno'] = True if tds.eq(8).text() == 'Any' else False case['account'] = tds.eq(11).text() case['region'] = region case['has_owner'] = True if case['nno'] else False case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href") current_case_list.append(case['_id']) res = p.find_one({'_id': case['_id']}) if res: #continue case['mute'] = res['mute'] try: p.update({'_id': case['_id']}, case) except: print 'Unexpected error:', sys.exc_info()[0] else: #case['owner'] = tds.eq(1).text() #case['subject'] = tds.eq(3).text() #case['sbt'] = tds.eq(4).text() #case['severity'] = tds.eq(5).text()[0] #status = tds.eq(6).text().strip() #if status == 'Waiting on Red Hat': #case['status'] = 'WoRH' #elif status == 'Waiting on Customer': #case['status'] = 'WoCU' #case['mute'] = False #case['inter_stat'] = tds.eq(7).text() #case['nno'] = True if tds.eq(8).text() == 'Any' else False #case['account'] = tds.eq(11).text() #case['region'] = region #case['has_owner'] = True if case['nno'] else False #case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href") try: p.insert(case) except DuplicateKeyError: print "Case exist:", sys.exc_info()[0] except: print "Unexpected error:", sys.exc_info()[0] #Get cases in prev_case_list but not in current_case_list, mark them to has_owner, remove nno tag and unmute query_str = {'$or': [ {'has_owner': False}, {'nno': True}, ] } #print current_case_list prev_case_list = map(lambda x: x.get('_id'), list(p.find(query_str, {'_id': 1}))) #print prev_case_list taken_list = list(set(prev_case_list) - set(current_case_list)) for _id in taken_list: p.update({'_id': _id}, {'$set': {'has_owner': True, 'nno': False, 'mute': False}})
def runAnalysis(link: str, host_mongodb: str): news = get_collection(host_mongodb) if not news.find_one({'link': link}): news.insert_one({'link': link, "status": "PENDING"}) # TODO Apply ray LOGGER.debug("Working with this link: " + link)
def report_parser(html_source): """ Give the HTML code to pyquery decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue. Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/ """ report = pq(html_source.decode("utf-8")) table = report("table.reportTable.tabularReportTable") trs = table("tr") lens = len(trs) current_case_list = [] prev_case_list = [] p = get_collection('cases') for i in xrange(0, lens): if trs.eq(i).hasClass("breakRowClass1.breakRowClass1Top"): res = re.search(contry_pattern, trs.eq(i).text()) if res: region = res.group(1) record = res.group(2) if region.startswith("Hong"): region = 'HK' elif region.startswith("Tai"): region = 'TW' elif region.startswith("Kor"): region = 'KR' elif region.startswith("Chin"): region = 'CN' else: region = 'Global' for k in xrange(i + 1, i + int(record) + 2): if trs.eq(k).hasClass("even"): case = {} tds = trs.eq(k)("td") # for testing purpose and get the fields and filed seq number #for i in xrange(len(tds)): #print "%s: %s" % (i, tds.eq(i).text()) case['_id'] = tds.eq(2).text() case['owner'] = tds.eq(1).text() case['subject'] = tds.eq(3).text() case['sbt'] = tds.eq(4).text() case['severity'] = tds.eq(5).text()[0] if tds.eq(14).text() == "-": case['sbr'] = "No SBR" else: case['sbr'] = tds.eq(14).text() status = tds.eq(6).text().strip() if status == 'Waiting on Red Hat': case['status'] = 'WoRH' elif status == 'Waiting on Customer': case['status'] = 'WoCU' else: case['status'] = status case['mute'] = False case['inter_stat'] = tds.eq(7).text() case['nno'] = True if tds.eq(8).text() == 'Any' else False case['account'] = tds.eq(11).text() case['region'] = region case['has_owner'] = True if case['nno'] else False case['url'] = 'https://na7.salesforce.com' + tds.eq(3)( "a").attr("href") current_case_list.append(case['_id']) res = p.find_one({'_id': case['_id']}) if res: #continue case['mute'] = res['mute'] try: p.update({'_id': case['_id']}, case) except: print 'Unexpected error:', sys.exc_info()[0] else: #case['owner'] = tds.eq(1).text() #case['subject'] = tds.eq(3).text() #case['sbt'] = tds.eq(4).text() #case['severity'] = tds.eq(5).text()[0] #status = tds.eq(6).text().strip() #if status == 'Waiting on Red Hat': #case['status'] = 'WoRH' #elif status == 'Waiting on Customer': #case['status'] = 'WoCU' #case['mute'] = False #case['inter_stat'] = tds.eq(7).text() #case['nno'] = True if tds.eq(8).text() == 'Any' else False #case['account'] = tds.eq(11).text() #case['region'] = region #case['has_owner'] = True if case['nno'] else False #case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href") try: p.insert(case) except DuplicateKeyError: print "Case exist:", sys.exc_info()[0] except: print "Unexpected error:", sys.exc_info()[0]
def init_push(database_name, collection_Push): col_push = db.get_collection(database_name, collection_Push) return col_push
def check_subscription(phoneNumber): user_collection = get_collection("quotelee", "users") user = user_collection.find({"phoneNo": phoneNumber}) if user.count() == 0: add_user(phoneNumber)
from flask import Flask app = Flask(__name__) MONGODB_SERVER = "localhost" MONGODB_PORT = 27017 MONGODB_DB = "game_reviews" settings = { 'MONGODB_SERVER': MONGODB_SERVER, 'MONGODB_PORT': MONGODB_PORT, 'MONGODB_DB': MONGODB_DB } import db coll = db.get_collection(db.get_db(settings, False), 'reviews') from app import routes
import os import asyncio from discord.ext import commands # Custom modules import db import tfidf_summarizer client = commands.Bot(command_prefix='!') # This takes a while and should be run only once on startup mongo_client = db.connect_to_mongo(db.user, db.pswd) # Get the keyword list for test server (each guild will have their own list) # NOTE: This code must be updated to use with different guilds users_collection = db.get_collection(db.get_db(mongo_client, db.db_name), db.collections[1]) message_collection = db.get_collection(db.get_db(mongo_client, db.db_name), db.collections[2]) all_messages_collection = db.get_collection( db.get_db(mongo_client, db.db_name), db.collections[3]) keywords_collection = db.get_collection(db.get_db(mongo_client, db.db_name), db.collections[0]) @client.event async def on_ready(): print('Bot is ready.') @client.command(name='about') async def about(ctx):
def init(database_name, collection_Items, collection_Errors): log.open_log() col_items = db.get_collection(database_name, collection_Items) col_errors = db.get_collection(database_name, collection_Errors) return col_items, col_errors
def save(self, **kwargs): kwargs['src'] = self.__class__.name kwargs['ch_src'] = self.__class__.ch_name timeline = db.get_collection('timeline') timeline.insert(kwargs)
def message_list(): """ Shows a list of messages. """ msgs = db.get_collection("msg").values() return template("message_list", msgs=msgs)
def process_video( model, thre, input_path, output_path, require_fps, hat_color, person_color, fourcc='avc1', step=2, ): """处理视频并输出到指定目录 Arguments: model {torch.nn.Sequ} -- [使用的模型] input_path {[str]} -- [视频文件路径] require_fps {[int]} -- [输出的视频fps] fourcc {[str]} -- [opencv写文件编码格式] hat_color {[str]} -- [安全帽框颜色] person_color {[str]} -- [人头框颜色] """ video = mmcv.VideoReader(input_path,cache_capacity=40) # 图像分辨率 resolution = (video.width, video.height) video_fps = video.fps ds = DetectionSifter( int(video_fps), osp.basename(args.i).split('.')[0], resolution, get_collection(asyn=True) ) if require_fps is None: require_fps = video_fps if require_fps > video_fps: require_fps = video_fps vwriter = cv2.VideoWriter( output_path, VideoWriter_fourcc(*fourcc), require_fps, resolution ) t = threading.Thread(target=write_frame,args=(vwriter,)) t.start() # 跳步视频(每次读取step+1帧) #video = over_step_video(video,step) vlen = len(video) indexs = np.arange(vlen)[0:len(video):step] # 每次取step张图像,比如第一次取[0:3]第二次则取[2:5]确保探测一张跳过step-1张,相当于探测一张相当于探测step张 p = Process(model,step+1) for start_index in tqdm(indexs): end_index = start_index + step + 1 if end_index >= vlen: break origin_frames = video[start_index:end_index] origin_frames = np.array(origin_frames) frames_index = [start_index,start_index+step] origin_frames,psn_objects,hat_objects = p.get_img(origin_frames,frames_index) for psn_o in psn_objects: ds.add_object(*psn_o) Img_Q.put(origin_frames[:-1]) ds.clear() print('process finshed')