def payLoan():
venmo_id = request.form['venmo_name']
payment_amount = float(request.form['amount'])
account_info = db.get_collection(account_collection_str)
account_info_record = account_info.find_one({"user_id" : venmo_id})
current_chequing = account_info_record['chequing_balance']
current_debt = account_info_record['debt_amount']
# Need to be careful to maintain good state
if (payment_amount < 0 or payment_amount > current_chequing or payment_amount > current_debt):
return jsonify(**{
"success": False
})
current_chequing -= payment_amount;
current_debt -= payment_amount;
account_info.update({"user_id" : venmo_id},
{"$set": {
"chequing_balance" : current_chequing,
"debt_amount" : current_debt
}})
return jsonify(**{
"success" : True
})
def page(page):
ln = 10
conf = config.get_config()
cdn_domain = conf.get('cdn', 'domain')
try:
page = int(page)
except ValueError:
page = 1
timeline = db.get_collection('timeline')
db_items = timeline.find(
skip=(page - 1) * ln,
limit=ln,
sort=[('date', pymongo.DESCENDING)],
)
items = []
for item in db_items:
try:
item['date'] = str(item['date'].strftime('%Y-%m-%d'))
except:
pass
item['_id'] = str(item['_id'])
item['cdn_path'] = os.path.join(cdn_domain, item['cdn_path'])
items.append(item)
# pagination
total = int(math.ceil(float(timeline.find().count()) / ln))
pagination = get_pagination(page, total)
return render_template(
'timeline.html',
items=items,
pagination=pagination,
)
def withdrawAmount():
venmo_id = request.form['venmo_name']
withdraw_amount = float(request.form['amount'])
account_info = db.get_collection(account_collection_str)
account_info_record = account_info.find_one({"user_id" : venmo_id})
current_amount = account_info_record['chequing_balance']
if (withdraw_amount > current_amount):
return jsonify(**{
"success": False
})
else:
print venmo.post_payment(
account_info_record['access_token'],
venmo_id,
'Cash out made',
withdraw_amount,
)
current_amount = current_amount - withdraw_amount
account_info.update({"user_id" : venmo_id},
{"$set": {"chequing_balance" : current_amount}})
return jsonify(**{
"success": True
})
def transferToSavings():
venmo_id = request.form['venmo_name']
transfer_amount = float(request.form['amount'])
account_info = db.get_collection(account_collection_str)
account_info_record = account_info.find_one({"user_id" : venmo_id})
current_chequing = account_info_record['chequing_balance']
current_savings = account_info_record['savings_balance']
if (transfer_amount > current_chequing):
return jsonify(**{
"success": False
})
else:
current_savings = current_savings + transfer_amount;
current_chequing = current_chequing - transfer_amount;
account_info.update({"user_id" : venmo_id},
{"$set": {
"chequing_balance" : current_chequing,
"savings_balance" : current_savings
}})
return jsonify(**{
"success": True
})
def add_user(phoneNumber):
user_collection = get_collection("quotelee", "users")
user_collection.insert({
"phoneNo": phoneNumber,
"status": "active"
})
send_welcome_message(phoneNumber)
def message_detail(id=None):
msgs = db.get_collection("msg")
msg = msgs.get(id)
if msg:
return template("message_detail", msg=msg)
else:
abort(404, "Message not found")
def parse(self, response):
selector = Selector(response=response)
articles = selector.xpath('/html/body/section/div/div/article')
timeline = db.get_collection('timeline')
for item in articles:
try:
title = item.xpath('header/h2/a/text()').extract()[0]
# link URL
url = item.xpath('header/h2/a/@href').extract()[0]
description = item.xpath('p[3]/text()').extract()[0]
description = self._join_text(description)
# image URL
img = item.xpath('p[2]/a/span/span/img/@data-original').extract()[0]
# [tag1, tag2, ...]
tags = item.xpath('p[4]/span[2]/a/text()').extract()
# YYYY-MM-DD
date = item.xpath('p[1]/text()').extract()[0].strip()
date = date.split(" ", 1)[1]
date = self._parse_date(date)
# label of category
category = item.xpath('header/a/text()').extract()[0]
self.save(title=title, url=url, description=description,
img=img, tags=tags, date=date, category=category)
except IndexError:
continue
next_page = selector.xpath(u'/html/body/section/div/div/div/ul/li/a[text()="下一页"]/@href').extract()[0]
yield Request(response.urljoin(next_page), self.parse)
def getFeeds():
feeds = get_collection(host_mongodb, db_name="feeds")
feed_urls = [{
"url": feed['url'],
"item_id": str(feed['_id'])
} for feed in feeds.find()]
return {"feeds": feed_urls}
def register():
venmo_id = request.form['venmo_name']
name = request.form['user_name']
p_venmo_id = request.form['parent_venmo_name']
p_name = request.form['parent_name']
account_collection = db.get_collection(account_collection_str)
account_collection.insert({
"user_id" : venmo_id,
"user_name" : name,
"chequing_balance" : 0,
"savings_balance" : 0,
"parent_id" : p_venmo_id,
"parent_name" : p_name,
"allowance_amount" : 0,
"savings_amount" : 0,
"savings_interest_rate" : 0,
"debt_amount" : 0,
"loan_interest_rate" : 0,
"user_token": "u_token",
"access_token": "a_token",
"refresh_token": "r_token"
})
# TODO: Venmo fields are totally sucky
return jsonify(**{
"success": True
})
def page(page):
ln = 10
try:
page = int(page)
except ValueError:
page = 1
timeline = db.get_collection('timeline')
db_items = timeline.find(
skip=(page - 1) * ln,
limit=ln,
sort=[('date', pymongo.DESCENDING)],
)
#{'date': {'$exist': True}},
items = []
for item in db_items:
try:
item['date'] = str(item['date'].strftime('%Y-%m-%d'))
except:
pass
item['_id'] = str(item['_id'])
items.append(item)
# pagination
total = int(math.ceil(float(timeline.find().count()) / ln))
pagination = get_pagination(page, total)
return render_template(
'timeline.html',
items=items,
pagination=pagination,
)
def page(page):
ln = 10
conf = config.get_config()
cdn_domain = conf.get('cdn', 'domain')
try:
page = int(page)
except ValueError:
page = 1
timeline = db.get_collection('timeline')
db_items = timeline.find(
skip=(page - 1) * ln,
limit=ln,
sort=[('date', pymongo.DESCENDING)],
)
items = []
for item in db_items:
try:
item['date'] = str(item['date'].strftime('%Y-%m-%d'))
except:
pass
item['_id'] = str(item['_id'])
item['cdn_path'] = os.path.join(cdn_domain, item['cdn_path'])
items.append(item)
# pagination
total = int(math.ceil(float(timeline.find().count()) / ln))
pagination = get_pagination(page, total)
return render_template(
'timeline.html',
items=items,
pagination=pagination,
)
def takeLoan():
venmo_id = request.form['venmo_name']
loan_amount = float(request.form['amount'])
account_info = db.get_collection(account_collection_str)
account_info_record = account_info.find_one({"user_id" : venmo_id})
current_chequing = account_info_record['chequing_balance']
current_debt = account_info_record['debt_amount']
if (loan_amount < 0):
return jsonify(**{
"success": False
})
current_chequing += loan_amount;
current_debt += loan_amount;
account_info.update({"user_id" : venmo_id},
{"$set": {
"chequing_balance" : current_chequing,
"debt_amount" : current_debt
}})
return jsonify(**{
"success" : True
})
def stats():
msgs = db.get_collection("msg")
stats = {
"count": len(msgs),
"num_authors": len(set(msg.get("author") for msg in msgs.values())),
"longest": max(len(msg.get("text", "")) for msg in msgs.values()),
}
return template("stats", **stats)
def get_random_quote():
quote_collection = get_collection("quotelee", "quotes")
random_quote = quote_collection.aggregate(
[{'$sample':{'size':1}}]
)
for quote in random_quote:
return quote
def unsubscribe_user(phoneNumber):
user_collection = get_collection("quotelee", "users")
user_collection.update_one(
{"phoneNo": phoneNumber},
{ "$set": {
"status": "inactive"
}
})
def report(bot, channel, user, args):
p = get_collection('cases')
query_str = {
'$or': [
{'has_owner': False},
{'$and': [
{'nno': True},
{'mute': False}
]},
{'$and': [
{'status': 'WoRH'},
{'mute': False},
{'has_owner': False},
]},
]
}
cases = p.find(query_str)
for case in cases:
if case['region'] == 'HK':
msg_prefix = 'HK_Case'
elif case['region'] == 'TW':
msg_prefix = 'TW_Case'
elif case['region'] == 'KR':
msg_prefix = 'KR_Case'
elif case['region'] == 'CN':
msg_prefix = 'CN_Case'
else:
msg_prefix = 'Global_Case'
if case['severity'] == '1':
sev_str = BOLD + RED + 'S1' + CLEAR
elif case['severity'] == '2':
sev_str = BOLD + ORANGE + 'S2' + CLEAR
else:
sev_str = case['severity']
if case['sbt'] != '-':
if 0 <= int(case['sbt'].replace(',', '')) < 60:
sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR
elif int(case['sbt'].replace(',', '')) < 0:
sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR
else:
sbt_str = GREEN + case['sbt'] + CLEAR
else:
sbt_str = case['sbt']
msg_str = "%s: [%s%s%s] [%s] %s%s%s%s [%s %s] %s NNO:%s %s" % (
msg_prefix,
BOLD, case['_id'], CLEAR,
case['account'],
BOLD, BLUE, case['subject'], CLEAR,
sbt_str,
sev_str,
case['status'],
case['nno'],
case['url'],
)
bot.msg(channel, user + ' ' + msg_str)
def resubscribe_user(phoneNumber):
user_collection = get_collection("quotelee", "users")
user_collection.update_one({
"phoneNo": phoneNumber},
{ "$set": {
"status": "active"
}
})
send_resubscribe_message(phoneNumber)
def fetch_quote(input_query):
tags = process_input(input_query)
result_id = find_nearest_neighbor(tags)
if result_id == False:
return error_message()
else:
quote_collection = get_collection("quotelee", "quotes")
quote_object = quote_collection.find_one({'_id': result_id})
return quote_object
def report(bot, channel, user, args):
p = get_collection('cases')
query_str = {
'$or': [
{
'has_owner': False
},
{
'$and': [{
'nno': True
}, {
'mute': False
}]
},
{
'$and': [
{
'status': 'WoRH'
},
{
'mute': False
},
{
'has_owner': False
},
]
},
]
}
cases = p.find(query_str)
for case in cases:
if case['region'] == 'HK':
msg_prefix = 'HK_Case'
elif case['region'] == 'TW':
msg_prefix = 'TW_Case'
elif case['region'] == 'KR':
msg_prefix = 'KR_Case'
elif case['region'] == 'CN':
msg_prefix = 'CN_Case'
else:
msg_prefix = 'Global_Case'
if case['severity'] == '1':
sev_str = BOLD + RED + 'S1' + CLEAR
elif case['severity'] == '2':
sev_str = BOLD + ORANGE + 'S2' + CLEAR
else:
sev_str = case['severity']
if case['sbt'] != '-':
if 0 <= int(case['sbt'].replace(',', '')) < 60:
sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR
elif int(case['sbt'].replace(',', '')) < 0:
sbt_str = BOLD + ORANGE + case['sbt'] + CLEAR
else:
sbt_str = GREEN + case['sbt'] + CLEAR
def getNews():
news = get_collection(host_mongodb)
result = []
for elem in news.find()[0:20]:
new_elem = elem
elem["elem_id"] = str(elem["_id"])
del elem["_id"]
result.append(new_elem)
return {"result": result}
def deleteNewInvestment():
investment_id = request.form['investment_id']
new_investments_collection = db.get_collection(new_investments_str)
new_investments_collection.remove({'investment_id': investment_id})
return jsonify(**{
"success": True
})
def addBoughtInvestment(user_id, investment_id, amount):
new_investments_collection = db.get_collection(new_investments_str)
new_investments_collection.update({'investment_id': investment_id},
{"$set": {"is_purchased" : True}})
investment = new_investments_collection.find_one({'investment_id': investment_id})
term_length = int(investment['term_length'])
rate = float(investment['rate'])
bought_investments_collection = db.get_collection(bought_investments_str)
bought_investments_collection.insert({
"name" : investment['name'],
"term_length": term_length,
"rate": investment['rate'],
"end_date": util.add_months_to_today(term_length),
"amount": amount,
"final_amount": util.calculate_investment(rate, term_length, amount),
"is_redeemed": False,
"investment_id": investment_id,
"user_id": user_id,
})
def addNewInvestment(venmo_id, name, term, rate):
new_investments_collection = db.get_collection(new_investments_str)
investment_id = str(uuid.uuid4())
new_investments_collection.insert({
"user_id": venmo_id,
"name": name,
"term_length" : term,
"rate" : rate,
"investment_id": investment_id,
"is_purchased": False,
})
def addFeeds(feeds: Feeds):
db = get_collection(host_mongodb, db_name="feeds")
result_ids = []
for feed in feeds.feeds:
if not db.find_one({"url": feed}):
LOGGER.info(str(feed))
# TODO set up type
new_id = db.insert_one({
"url": feed,
"status": "PENDING"
}).inserted_id
result_ids.append(str(new_id))
return {"ids": result_ids}
def direct(token):
coll = get_collection()
doc = coll.find_one({"token": str(token)})
close_db()
if doc is None:
return redirect(url_for("index"))
if 'ads' in doc:
timeout = getenv("ADS_REDIRECT_TIMEOUT")
if timeout is not None:
return render_template("ads.html", timeout=int(timeout))
else:
return render_template("ads.html", timeout=5)
else:
return redirect("http://" + str(doc["url"]))
def setCurrentInvestment():
venmo_id = request.form['venmo_name']
investment_id = request.form['investment_id']
amount = float(request.form['amount'])
account_info = db.find_one(account_collection_str, {"user_id" : str(venmo_id)})
current_chequing = float(account_info['chequing_balance'])
new_investments_collection = db.get_collection(new_investments_str)
investment = new_investments_collection.find_one({'investment_id': investment_id})
# post as long as value is 0
if amount < 0 or current_chequing < amount or investment['is_purchased'] == True:
return jsonify(**{
"success": False
})
else:
addBoughtInvestment(venmo_id,
investment_id,
amount,
)
new_chequing_balance = current_chequing - amount
account_info = db.get_collection(account_collection_str)
account_info.update({"user_id" : venmo_id},
{"$set": {"chequing_balance" : new_chequing_balance}})
new_investment_list = db.find(new_investments_str, {"user_id" : venmo_id, "is_purchased": False })
current_investment_list = db.find(bought_investments_str, {"user_id" : venmo_id })
return jsonify(**{
"chequing_balance": new_chequing_balance,
"new_investments" : new_investment_list,
"my_investments" : current_investment_list,
})
def unmute(bot, channel, user, cid):
p = get_collection('cases')
case = p.find_one({'_id': cid})
if case:
case['mute'] = False
p.update({'_id': cid}, case)
if channel == bot.nick:
bot.reply(user, 'mute case: %s' % cid)
else:
bot.msg(channel, '%s, mute case: %s' % (user, cid))
else:
if channel == bot.nick:
bot.reply(user, 'I can\'t find the case %s' % cid)
else:
bot.msg(channel, '%s, I can\'t find the case: %s' % (user, cid))
def unmute(bot, channel, user, cid):
p = get_collection('cases')
case = p.find_one({'_id': cid})
if case:
case['mute'] = False
p.update({'_id': cid}, case)
if channel == bot.nick:
bot.reply(user, 'mute case: %s' % cid)
else:
bot.msg(channel, '%s, mute case: %s' % (user, cid))
else:
if channel == bot.nick:
bot.reply(user, 'I can\'t find the case %s' % cid)
else:
bot.msg(channel, '%s, I can\'t find the case: %s' % (user, cid))
def escalation_parser(html_source):
"""
Give the HTML code to pyquery
decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue.
Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/
"""
report = pq(html_source.decode("utf-8"))
table = report("table.reportTable.tabularReportTable")
trs = table("tr.even")
current_case_list = []
prev_case_list = []
p = get_collection("escalation")
for i in xrange(0, len(trs)):
case = {}
tds = trs.eq(i)("td")
case['_id'] = tds.eq(3).text()
res = p.find_one({'_id': case['_id']})
if res:
continue
else:
case['owner'] = tds.eq(1).text()
case['account'] = tds.eq(2).text()
case['subject'] = tds.eq(5).text()
case['severity'] = tds.eq(8).text()
case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href")
case['checked'] = False
current_case_list.append(case['_id'])
try:
p.insert(case)
except DuplicateKeyError:
print 'Case exist:', sys.exc_info()[0]
print case['_id']
except:
print 'Unexpected error:', sys.exc_info()[0]
query_str = {'checked': False}
prev_case_list = map(lambda x: x.get('_id'), list(p.find(query_str, {'_id': 1})))
taken_list = list(set(prev_case_list) - set(current_case_list))
for _id in taken_list:
p.update({'_id': _id}, {'$set': {'checked': True}})
def send_daily_quote():
quote = get_random_quote()
full_quote = quote['quote'] + " - " + quote['author']
users = get_documents("quotelee", "users")
for user in users:
if(user["status"] == "active"):
client.messages.create(to=user["phoneNo"], from_=twilio_phone_number, body=full_quote)
currDateTime = datetime.now(timezone('US/Pacific')).strftime('%m-%d-%Y %H:%M')
numberOfUsers = users.count()
daily_quotes = get_collection("quotelee", "daily_quotes")
daily_quotes.insert({
"quote": quote['quote'],
"author": quote['author'],
"datetime": currDateTime,
"numberOfUsers": numberOfUsers
})
def take(bot, channel, user, cid):
p = get_collection('cases')
cid = cid.split()[0]
try:
int(cid)
except:
bot.msg(channel, '%s: case id incorrect' % user)
return False
case = p.find_one({'_id': cid})
if case is None:
bot.msg(channel, '%s: case id incorrect' % user)
return False
case['owner'] = user
case['mute'] = False
case['has_owner'] = True
if not case['nno']:
case['nno'] = False
p.update({'_id': cid}, case)
bot.msg(channel, 'gss_china: %s take case %s %s' % (user, case['_id'], case['subject']))
def setAllowance():
venmo_id = request.form['venmo_name']
rate = float(request.form['rate'])
# post as long as value is 0
if rate < 0:
return jsonify(**{
"success": False
})
else:
account_info = db.get_collection(account_collection_str)
account_info.update({"user_id" : venmo_id},
{"$set": {"allowance_amount": rate}})
return jsonify(**{
"success": True
})
def setSavingsAmount():
venmo_id = request.form['venmo_name']
savings_amount = float(request.form['amount'])
# post as long as value is 0
if savings_amount < 0:
return jsonify(**{
"success": False
})
else:
account_info = db.get_collection(account_collection_str)
account_info.update({"user_id" : venmo_id},
{"$set": {"savings_amount": savings_amount}})
return jsonify(**{
"success": True
})
def escalation_parser(html_source):
"""
Give the HTML code to pyquery
decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue.
Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/
"""
report = pq(html_source.decode("utf-8"))
table = report("table.reportTable.tabularReportTable")
trs = table("tr.even")
current_case_list = []
prev_case_list = []
p = get_collection("escalation")
for i in xrange(0, len(trs)):
case = {}
tds = trs.eq(i)("td")
case['_id'] = tds.eq(3).text()
res = p.find_one({'_id': case['_id']})
if res:
continue
else:
case['owner'] = tds.eq(1).text()
case['account'] = tds.eq(2).text()
case['subject'] = tds.eq(5).text()
case['severity'] = tds.eq(8).text()
case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr(
"href")
case['checked'] = False
current_case_list.append(case['_id'])
try:
p.insert(case)
except DuplicateKeyError:
print 'Case exist:', sys.exc_info()[0]
print case['_id']
except:
print 'Unexpected error:', sys.exc_info()[0]
def index():
if request.method == "POST":
url = sub("^http.*://", "", str(request.form["url"]))
coll = get_collection()
if coll.find_one({"url": url}) is not None:
token = coll.find_one({"url": url})["token"]
else:
nbytes = getenv("TOKEN_NBYTES")
if nbytes is not None:
token = token_urlsafe(int(getenv("TOKEN_NBYTES")))
else:
token = token_urlsafe(4)
if getenv("ADS") is not None:
coll.insert_one({"url": url, "token": token, "ads": True})
else:
coll.insert_one({"url": url, "token": token})
close_db()
return render_template("index.html", token=token)
else:
return render_template("index.html")
def ncq_parser(html_source):
"""
Give the HTML code to pyquery
decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue.
Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/
"""
global SBR_MSG
sbr_dic={}
report = pq(html_source.decode("utf-8"))
table = report("table.reportTable.tabularReportTable")
trs = table("tr")
lens = len(trs)
p = get_collection("ncq")
for i in xrange(1, lens-2):
key = trs.eq(i)("td").eq(6).text()
if key == '-':
key = "No_SBR"
if not sbr_dic.has_key(key):
sbr_dic[key] = 1
elif sbr_dic.has_key(key):
sbr_dic[key] += 1
#print sbr_dic
sbr_msg=''
for key in sbr_dic.keys():
sbr_msg += key
sbr_msg += ' '
sbr_msg += str(sbr_dic[key])
sbr_msg += ' '
sbr_msg = "Global NCQ SBR : " + sbr_msg
SBR_MSG = sbr_msg
p.update({'_id': 200}, {'$set': {'content': SBR_MSG}})
def take(bot, channel, user, cid):
p = get_collection('cases')
cid = cid.split()[0]
try:
int(cid)
except:
bot.msg(channel, '%s: case id incorrect' % user)
return False
case = p.find_one({'_id': cid})
if case is None:
bot.msg(channel, '%s: case id incorrect' % user)
return False
case['owner'] = user
case['mute'] = False
case['has_owner'] = True
if not case['nno']:
case['nno'] = False
p.update({'_id': cid}, case)
bot.msg(
channel,
'gss_china: %s take case %s %s' % (user, case['_id'], case['subject']))
def parse(self, response):
selector = Selector(response=response)
articles = selector.xpath('/html/body/section/div/div/article')
timeline = db.get_collection('timeline')
for item in articles:
try:
title = item.xpath('header/h2/a/text()').extract()[0]
# link URL
url = item.xpath('header/h2/a/@href').extract()[0]
description = item.xpath('p[3]/text()').extract()[0]
description = self._join_text(description)
# image URL
img = item.xpath(
'p[2]/a/span/span/img/@data-original').extract()[0]
# [tag1, tag2, ...]
tags = item.xpath('p[4]/span[2]/a/text()').extract()
# YYYY-MM-DD
date = item.xpath('p[1]/text()').extract()[0].strip()
date = date.split(" ", 1)[1]
date = self._parse_date(date)
# label of category
category = item.xpath('header/a/text()').extract()[0]
if category not in self.__class__.white_list:
continue
self.save(title=title,
url=url,
description=description,
img=img,
tags=tags,
date=date,
category=category)
except IndexError:
continue
next_page = selector.xpath(
u'/html/body/section/div/div/div/ul/li/a[text()="下一页"]/@href'
).extract()[0]
yield Request(response.urljoin(next_page), self.parse)
def ncq_parser(html_source):
"""
Give the HTML code to pyquery
decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue.
Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/
"""
global SBR_MSG
sbr_dic = {}
report = pq(html_source.decode("utf-8"))
table = report("table.reportTable.tabularReportTable")
trs = table("tr")
lens = len(trs)
p = get_collection("ncq")
for i in xrange(1, lens - 2):
key = trs.eq(i)("td").eq(6).text()
if key == '-':
key = "No_SBR"
if not sbr_dic.has_key(key):
sbr_dic[key] = 1
elif sbr_dic.has_key(key):
sbr_dic[key] += 1
#print sbr_dic
sbr_msg = ''
for key in sbr_dic.keys():
sbr_msg += key
sbr_msg += ' '
sbr_msg += str(sbr_dic[key])
sbr_msg += ' '
sbr_msg = "Global NCQ SBR : " + sbr_msg
SBR_MSG = sbr_msg
p.update({'_id': 200}, {'$set': {'content': SBR_MSG}})
def getSaleandRentCsvFor(state):
collection = get_collection()
rent = list(
collection.find({
"State": state,
"Status": {
"$in":
["Townhouse for rent", "Condo for rent", "House for rent"]
}
}))
sale = list(
collection.find({
"State": state,
"Status": {
"$nin":
["Townhouse for rent", "Condo for rent", "House for rent"]
}
}))
write_multi_data_to_csv(state + "_rent.csv", rent)
write_multi_data_to_csv(state + "_sale.csv", sale)
genrate_historical_data_for(state)
remove_rent_entries_from(state + "_history.csv",
state + "_history_without_rent.csv")
def save(self, **kwargs):
if self.__class__.exist_count > 10:
raise RuntimeError('Exceed maximum retry times for database')
try:
# add source and chinese source
kwargs['src'] = self.__class__.name
kwargs['ch_src'] = self.__class__.ch_name
# make cdn path
img_url = kwargs['img'].encode("utf-8")
cdn_path = md5.new(img_url).hexdigest()
kwargs['cdn_path'] = cdn_path.decode("utf-8")
# insert into database
timeline = db.get_collection('timeline')
timeline.insert(kwargs)
# actually upload image here
if not self.__class__.img_store.exists(cdn_path):
self.__class__.img_store.upload_remote_image(cdn_path, img_url)
except DuplicateKeyError:
self.__class__.exist_count += 1
def parse(self, response):
selector = Selector(response=response)
articles = selector.xpath('//*[@id="main"]/*/div[@class="post-box"]')
timeline = db.get_collection('timeline')
for item in articles:
try:
title = item.xpath(
'div[@class="post-header"]/p/a/text()').extract()[0]
# link URL
url = item.xpath(
'div[@class="post-header"]/p/a/@href').extract()[0]
description = item.xpath(
'*/div[@class="post-expert"]/text()').extract()[0]
description = self._join_text(description)
# image URL
img = item.xpath(
'*/div[@class="post-info"]/a/img/@data-original').extract(
)[0]
# YYYY-MM-DD
#date = item.xpath('*/div[@class="post-date"]/text()').extract()[0].strip()
date = item.xpath(
'div[@class="post-content"]/div[@class="post-footer"]/div[@class="post-date"]/text()'
).extract()[0]
date = datetime.strptime(date, '%Y-%m-%d')
self.save(title=title,
url=url,
description=description,
img=img,
date=date)
except IndexError:
continue
next_page = selector.xpath(
u'//*/div[@class="page-navigator"]/li/a[text()="下一页 »"]/@href'
).extract()[0]
yield Request(response.urljoin(next_page), self.parse)
def save(self, **kwargs):
if self.__class__.exist_count > 10:
raise RuntimeError('Exceed maximum retry times for database')
try:
# add source and chinese source
kwargs['src'] = self.__class__.name
kwargs['ch_src'] = self.__class__.ch_name
# make cdn path
img_url = kwargs['img'].encode("utf-8")
cdn_path = md5.new(img_url).hexdigest()
kwargs['cdn_path'] = cdn_path.decode("utf-8")
# insert into database
timeline = db.get_collection('timeline')
timeline.insert(kwargs)
# actually upload image here
if not self.__class__.img_store.exists(cdn_path):
self.__class__.img_store.upload_remote_image(cdn_path, img_url)
except DuplicateKeyError:
self.__class__.exist_count += 1
def report_parser(html_source):
"""
Give the HTML code to pyquery
decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue.
Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/
"""
report = pq(html_source.decode("utf-8"))
table = report("table.reportTable.tabularReportTable")
trs = table("tr")
lens = len(trs)
current_case_list = []
prev_case_list = []
p = get_collection('cases')
for i in xrange(0, lens):
if trs.eq(i).hasClass("breakRowClass1.breakRowClass1Top"):
res = re.search(contry_pattern, trs.eq(i).text())
if res:
region = res.group(1)
record = res.group(2)
if region.startswith("Hong"):
region = 'HK'
elif region.startswith("Tai"):
region = 'TW'
elif region.startswith("Kor"):
region = 'KR'
elif region.startswith("Chin"):
region = 'CN'
else:
region = 'Global'
for k in xrange(i + 1, i + int(record) + 2):
if trs.eq(k).hasClass("even"):
case = {}
tds = trs.eq(k)("td")
# for testing purpose and get the fields and filed seq number
#for i in xrange(len(tds)):
#print "%s: %s" % (i, tds.eq(i).text())
case['_id'] = tds.eq(2).text()
case['owner'] = tds.eq(1).text()
case['subject'] = tds.eq(3).text()
case['sbt'] = tds.eq(4).text()
case['severity'] = tds.eq(5).text()[0]
if tds.eq(14).text() == "-":
case['sbr'] = "No SBR"
else:
case['sbr'] = tds.eq(14).text()
status = tds.eq(6).text().strip()
if status == 'Waiting on Red Hat':
case['status'] = 'WoRH'
elif status == 'Waiting on Customer':
case['status'] = 'WoCU'
else:
case['status'] = status
case['mute'] = False
case['inter_stat'] = tds.eq(7).text()
case['nno'] = True if tds.eq(8).text() == 'Any' else False
case['account'] = tds.eq(11).text()
case['region'] = region
case['has_owner'] = True if case['nno'] else False
case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href")
current_case_list.append(case['_id'])
res = p.find_one({'_id': case['_id']})
if res:
#continue
case['mute'] = res['mute']
try:
p.update({'_id': case['_id']}, case)
except:
print 'Unexpected error:', sys.exc_info()[0]
else:
#case['owner'] = tds.eq(1).text()
#case['subject'] = tds.eq(3).text()
#case['sbt'] = tds.eq(4).text()
#case['severity'] = tds.eq(5).text()[0]
#status = tds.eq(6).text().strip()
#if status == 'Waiting on Red Hat':
#case['status'] = 'WoRH'
#elif status == 'Waiting on Customer':
#case['status'] = 'WoCU'
#case['mute'] = False
#case['inter_stat'] = tds.eq(7).text()
#case['nno'] = True if tds.eq(8).text() == 'Any' else False
#case['account'] = tds.eq(11).text()
#case['region'] = region
#case['has_owner'] = True if case['nno'] else False
#case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href")
try:
p.insert(case)
except DuplicateKeyError:
print "Case exist:", sys.exc_info()[0]
except:
print "Unexpected error:", sys.exc_info()[0]
#Get cases in prev_case_list but not in current_case_list, mark them to has_owner, remove nno tag and unmute
query_str = {'$or': [
{'has_owner': False},
{'nno': True},
]
}
#print current_case_list
prev_case_list = map(lambda x: x.get('_id'), list(p.find(query_str, {'_id': 1})))
#print prev_case_list
taken_list = list(set(prev_case_list) - set(current_case_list))
for _id in taken_list:
p.update({'_id': _id}, {'$set': {'has_owner': True, 'nno': False, 'mute': False}})
def runAnalysis(link: str, host_mongodb: str):
news = get_collection(host_mongodb)
if not news.find_one({'link': link}):
news.insert_one({'link': link, "status": "PENDING"})
# TODO Apply ray
LOGGER.debug("Working with this link: " + link)
def report_parser(html_source):
"""
Give the HTML code to pyquery
decode to unicode before given to pyquery, it would solve underlaying CJK encoding issue.
Check pyquery doc for details to parse your report: http://packages.python.org/pyquery/
"""
report = pq(html_source.decode("utf-8"))
table = report("table.reportTable.tabularReportTable")
trs = table("tr")
lens = len(trs)
current_case_list = []
prev_case_list = []
p = get_collection('cases')
for i in xrange(0, lens):
if trs.eq(i).hasClass("breakRowClass1.breakRowClass1Top"):
res = re.search(contry_pattern, trs.eq(i).text())
if res:
region = res.group(1)
record = res.group(2)
if region.startswith("Hong"):
region = 'HK'
elif region.startswith("Tai"):
region = 'TW'
elif region.startswith("Kor"):
region = 'KR'
elif region.startswith("Chin"):
region = 'CN'
else:
region = 'Global'
for k in xrange(i + 1, i + int(record) + 2):
if trs.eq(k).hasClass("even"):
case = {}
tds = trs.eq(k)("td")
# for testing purpose and get the fields and filed seq number
#for i in xrange(len(tds)):
#print "%s: %s" % (i, tds.eq(i).text())
case['_id'] = tds.eq(2).text()
case['owner'] = tds.eq(1).text()
case['subject'] = tds.eq(3).text()
case['sbt'] = tds.eq(4).text()
case['severity'] = tds.eq(5).text()[0]
if tds.eq(14).text() == "-":
case['sbr'] = "No SBR"
else:
case['sbr'] = tds.eq(14).text()
status = tds.eq(6).text().strip()
if status == 'Waiting on Red Hat':
case['status'] = 'WoRH'
elif status == 'Waiting on Customer':
case['status'] = 'WoCU'
else:
case['status'] = status
case['mute'] = False
case['inter_stat'] = tds.eq(7).text()
case['nno'] = True if tds.eq(8).text() == 'Any' else False
case['account'] = tds.eq(11).text()
case['region'] = region
case['has_owner'] = True if case['nno'] else False
case['url'] = 'https://na7.salesforce.com' + tds.eq(3)(
"a").attr("href")
current_case_list.append(case['_id'])
res = p.find_one({'_id': case['_id']})
if res:
#continue
case['mute'] = res['mute']
try:
p.update({'_id': case['_id']}, case)
except:
print 'Unexpected error:', sys.exc_info()[0]
else:
#case['owner'] = tds.eq(1).text()
#case['subject'] = tds.eq(3).text()
#case['sbt'] = tds.eq(4).text()
#case['severity'] = tds.eq(5).text()[0]
#status = tds.eq(6).text().strip()
#if status == 'Waiting on Red Hat':
#case['status'] = 'WoRH'
#elif status == 'Waiting on Customer':
#case['status'] = 'WoCU'
#case['mute'] = False
#case['inter_stat'] = tds.eq(7).text()
#case['nno'] = True if tds.eq(8).text() == 'Any' else False
#case['account'] = tds.eq(11).text()
#case['region'] = region
#case['has_owner'] = True if case['nno'] else False
#case['url'] = 'https://na7.salesforce.com' + tds.eq(3)("a").attr("href")
try:
p.insert(case)
except DuplicateKeyError:
print "Case exist:", sys.exc_info()[0]
except:
print "Unexpected error:", sys.exc_info()[0]
def init_push(database_name, collection_Push):
col_push = db.get_collection(database_name, collection_Push)
return col_push
def check_subscription(phoneNumber):
user_collection = get_collection("quotelee", "users")
user = user_collection.find({"phoneNo": phoneNumber})
if user.count() == 0:
add_user(phoneNumber)
from flask import Flask
app = Flask(__name__)
MONGODB_SERVER = "localhost"
MONGODB_PORT = 27017
MONGODB_DB = "game_reviews"
settings = {
'MONGODB_SERVER': MONGODB_SERVER,
'MONGODB_PORT': MONGODB_PORT,
'MONGODB_DB': MONGODB_DB
}
import db
coll = db.get_collection(db.get_db(settings, False), 'reviews')
from app import routes
import os
import asyncio
from discord.ext import commands
# Custom modules
import db
import tfidf_summarizer
client = commands.Bot(command_prefix='!')
# This takes a while and should be run only once on startup
mongo_client = db.connect_to_mongo(db.user, db.pswd)
# Get the keyword list for test server (each guild will have their own list)
# NOTE: This code must be updated to use with different guilds
users_collection = db.get_collection(db.get_db(mongo_client, db.db_name),
db.collections[1])
message_collection = db.get_collection(db.get_db(mongo_client, db.db_name),
db.collections[2])
all_messages_collection = db.get_collection(
db.get_db(mongo_client, db.db_name), db.collections[3])
keywords_collection = db.get_collection(db.get_db(mongo_client, db.db_name),
db.collections[0])
@client.event
async def on_ready():
print('Bot is ready.')
@client.command(name='about')
async def about(ctx):
def init(database_name, collection_Items, collection_Errors):
log.open_log()
col_items = db.get_collection(database_name, collection_Items)
col_errors = db.get_collection(database_name, collection_Errors)
return col_items, col_errors
def save(self, **kwargs):
kwargs['src'] = self.__class__.name
kwargs['ch_src'] = self.__class__.ch_name
timeline = db.get_collection('timeline')
timeline.insert(kwargs)
def message_list():
"""
Shows a list of messages.
"""
msgs = db.get_collection("msg").values()
return template("message_list", msgs=msgs)
def process_video(
model,
thre,
input_path,
output_path,
require_fps,
hat_color,
person_color,
fourcc='avc1',
step=2,
):
"""处理视频并输出到指定目录
Arguments:
model {torch.nn.Sequ} -- [使用的模型]
input_path {[str]} -- [视频文件路径]
require_fps {[int]} -- [输出的视频fps]
fourcc {[str]} -- [opencv写文件编码格式]
hat_color {[str]} -- [安全帽框颜色]
person_color {[str]} -- [人头框颜色]
"""
video = mmcv.VideoReader(input_path,cache_capacity=40)
# 图像分辨率
resolution = (video.width, video.height)
video_fps = video.fps
ds = DetectionSifter(
int(video_fps),
osp.basename(args.i).split('.')[0],
resolution,
get_collection(asyn=True)
)
if require_fps is None:
require_fps = video_fps
if require_fps > video_fps:
require_fps = video_fps
vwriter = cv2.VideoWriter(
output_path,
VideoWriter_fourcc(*fourcc),
require_fps,
resolution
)
t = threading.Thread(target=write_frame,args=(vwriter,))
t.start()
# 跳步视频（每次读取step+1帧）
#video = over_step_video(video,step)
vlen = len(video)
indexs = np.arange(vlen)[0:len(video):step]
# 每次取step张图像，比如第一次取[0:3]第二次则取[2:5]确保探测一张跳过step-1张,相当于探测一张相当于探测step张
p = Process(model,step+1)
for start_index in tqdm(indexs):
end_index = start_index + step + 1
if end_index >= vlen:
break
origin_frames = video[start_index:end_index]
origin_frames = np.array(origin_frames)
frames_index = [start_index,start_index+step]
origin_frames,psn_objects,hat_objects = p.get_img(origin_frames,frames_index)
for psn_o in psn_objects:
ds.add_object(*psn_o)
Img_Q.put(origin_frames[:-1])
ds.clear()
print('process finshed')
