def test_documents(self):
aCollection = "col_test_documents"
anIndex = "idx_test_documents"
self._q(query.create_collection({"name": aCollection}))
self._q(
query.create_index({
"name": anIndex,
"source": query.collection(aCollection),
"active": True
}))
count = 56
data = [{} for x in range(count)]
self._q(
query.foreach(
query.lambda_(
"x",
query.create(query.collection(aCollection),
{"data": query.var("x")})), data))
self.assertEqual(
self._q(
query.select([0],
query.count(
query.paginate(
query.documents(
query.collection(aCollection)))))),
count)
self.assertEqual(
self._q(query.count(query.documents(
query.collection(aCollection)))), count)
def products():
if request.method == "GET":
category = request.args.get("category") or None
if category:
data = client.query(
q.map_(
lambda x: q.get(x),
q.paginate(
q.match(q.index("products_by_category"), category))))
response = [i["data"] for i in data["data"]]
for i in range(0, len(response)):
response[i]['id'] = str(data["data"][i]["ref"].id())
return jsonify(data=response)
data = client.query(
q.map_(lambda x: q.get(x),
q.paginate(q.documents(q.collection("products")))))
response = [i["data"] for i in data["data"]]
for i in range(0, len(response)):
response[i]['id'] = str(data["data"][i]["ref"].id())
return jsonify(data=response)
elif request.method == "POST":
request_data = request.get_json()
response = client.query(
q.create(q.collection("products"), {"data": request_data}))
return jsonify(id=response["ref"].id())
def reviews():
if request.method == "GET":
product = request.args.get('product')
if product:
data = client.query(
q.map_(
lambda x: q.get(x),
q.paginate(q.match(q.index("reviews_by_product"),
product))))
response = [i["data"] for i in data["data"]]
for i in range(0, len(response)):
response[i]['id'] = str(data["data"][i]["ref"].id())
return jsonify(data=response)
data = client.query(
q.map_(lambda x: q.get(x),
q.paginate(q.documents(q.collection("reviews")))))
response = [i["data"] for i in data["data"]]
for i in range(0, len(response)):
response[i]['id'] = str(data["data"][i]["ref"].id())
return jsonify(data=response)
elif request.method == "POST":
request_data = request.get_json()
response = client.query(
q.create(q.collection("reviews"), {"data": request_data}))
return jsonify(id=response["ref"].id())
def get_orders(secret, after=None, before=None, size=5):
client = FaunaClient(secret=secret)
return client.query(
q.map_(
lambda ref: q.get(ref),
q.paginate(q.documents(q.collection('orders')), size=size, after=after, before=before)
)
)
def get_categories(after=None, before=None, size=5):
return fauna.query(
q.map_(
lambda ref: q.get(ref),
q.paginate(q.documents(q.collection('categories')),
size=size,
after=after,
before=before)))
def _expr_documents(self, limit=10):
"""
Returns documents query
:param limit: int - limit for pagination
:return: query_reference: Expr - FaunaDB expression for documents
"""
return query.paginate(
set=query.documents(self._expr_collection(), ),
size=limit,
)
def check_for_update(context: CallbackContext):
logger.info("About to run subscription check")
# get all anime
all_animes = client.query(
q.paginate(q.documents(q.collection(animes)), size=100000))
for anime in all_animes['data']:
# get anime_info in the function send_update...
# if there are new episodes...
send_update_to_subscribed_users(anime.id())
logger.info("Subscription check finished")
def view_other_election():
elections = client.query(q.paginate(q.documents(
q.collection("Elections"))))
elections_ref = []
for i in elections["data"]:
elections_ref.append(q.get(q.ref(q.collection("Elections"), i.id())))
total_elections = client.query(elections_ref)
return render_template("view_other_elections.html",
total_elections=total_elections)
def things():
userSecret = request.headers.get('fauna-user-secret')
client = FaunaClient(secret=userSecret)
try:
result = client.query(
q.map_(q.lambda_("ref", q.get(q.var("ref"))),
q.paginate(q.documents(q.collection("Things")))))
things = map(
lambda doc: {
"id": doc["ref"].id(),
"name": doc["data"]["name"],
"color": doc["data"]["color"]
}, result["data"])
return {"things": list(things)}
except faunadb.errors.Unauthorized as exception:
error = exception.errors[0]
return {"code": error.code, "description": error.description}, 401
def get_products(name=None, category=None, sort_by=None, size=5, after=None, before=None):
sortDic = {
"price_asc": q.index('products_sort_by_price_asc'),
"price_desc": q.index('products_sort_by_price_desc'),
"name_asc": q.index('products_sort_by_name_asc'),
'created_at_asc': q.index('products_sort_by_created_asc')
}
matches = []
if(name):
matches.append(q.match(q.index('products_search_by_name'), name))
if(category):
matches.append(q.match(
q.index('products_search_by_category'),
q.ref(q.collection('categories'), category)
))
if(len(matches) == 0):
matches.append(q.documents(q.collection('products')))
return fauna.query(
q.map_(
lambda _, ref: q.get(ref),
q.paginate(
q.join(
q.intersection(matches),
sortDic[sort_by or 'name_asc']
),
size=size,
after=after,
before=before
)
)
)
def number_of_anime(update: Update, context: CallbackContext):
result = client.query(
q.count(q.paginate(q.documents(q.collection(animes)), size=100000)))
context.bot.send_message(chat_id=update.effective_chat.id,
text='Number of anime: ' + str(result['data'][0]))
def number_of_users(update: Update, context: CallbackContext):
user = User(update.effective_chat.id)
result = client.query(
q.count(q.paginate(q.documents(q.collection(users)), size=100000), ))
context.bot.send_message(chat_id=user.chat_id,
text='Number of users: ' + str(result['data'][0]))
def test_documents(self):
self.assertJson(query.documents(query.collection("users")),
'{"documents":{"collection":"users"}}')
def plain_message(update: Update, context: CallbackContext):
print(update.effective_message)
try:
bot_user = client.query(
q.get(q.ref(q.collection('users'), update.effective_chat.id)))
except errors.NotFound:
context.bot.send_message(
chat_id=update.effective_chat.id,
text=
'Sorry, I do not understand what you mean.\nPlease use the /help command to '
'discover what I can help you with.')
return
user = User(update.effective_chat.id)
last_command = bot_user['data']['last_command']
message = update.message.text
print(last_command)
if last_command == 'subscribe':
try:
search_results = scraper.get_anime(message, limit=15)
if len(search_results) == 0:
context.bot.send_message(
chat_id=user.chat_id,
text=
'Sorry but no search results were available for this anime'
)
else:
context.bot.send_message(
chat_id=user.chat_id,
text='Here are the search results for ' + message)
for result in search_results:
markup = [[
InlineKeyboardButton('Select',
callback_data='subscribe=' +
shorten(result['link']))
]]
context.bot.send_photo(
chat_id=user.chat_id,
photo=result['thumbnail'],
caption=result['title'],
timeout=5,
reply_markup=InlineKeyboardMarkup(markup))
# update last command
user.last_command = ''
except Exception as err:
log_error(err)
elif last_command == 'getlatest':
try:
search_results = scraper.get_anime(message, limit=15)
if len(search_results) == 0:
context.bot.send_message(
chat_id=user.chat_id,
text=
'Sorry but no search results were available for this anime'
)
else:
context.bot.send_message(
chat_id=user.chat_id,
text='Here are the search results for ' + message)
for result in search_results:
markup = [[
InlineKeyboardButton('Select',
callback_data='getlatest=' +
shorten(result['link']))
]]
context.bot.send_photo(
chat_id=user.chat_id,
photo=result['thumbnail'],
caption=result['title'],
timeout=5,
reply_markup=InlineKeyboardMarkup(markup))
# update last command
user.last_command = ''
except Exception as err:
log_error(err)
elif last_command == 'broadcast':
if user.is_admin():
context.bot.send_message(chat_id=user.chat_id,
text='Broadcasting message...')
try:
results = client.query(
q.paginate(q.documents(q.collection(users)), size=100000))
results = results['data']
# spin 5 processes
with Pool(5) as p:
res = p.map(send_broadcast, [[int(user_ref.id()), message]
for user_ref in results])
successful_broadcast = []
for i in res:
if i == 'success':
successful_broadcast.append(i)
logger.info('Message broadcast to ' +
str(len(successful_broadcast)) + ' users')
print(res)
# update user last command
user.last_command = ''
except Exception as err:
log_error(err)
else:
context.bot.send_message(chat_id=user.chat_id,
text="Only admins can use this command!")
else:
context.bot.send_message(
chat_id=user.chat_id,
text=
"Sorry, I do not understand what you mean.\nPlease use the /help command to "
"discover what I can help you with.")
def main():
FAUNA_SECRET = os.getenv("FAUNA_SECRET")
fClient = client.FaunaClient(FAUNA_SECRET, domain="db.us.fauna.com")
nlp = spacy.load("en_core_web_sm")
documents = fClient.query(
q.map_(
q.lambda_("x", q.get(q.var("x"))),
q.paginate(q.documents(q.collection("Article")), size=100000),
))
transcript_docs = []
for document in documents.get("data"):
data = document.get("data")
doc = nlp(document["data"]["title"])
extracted_date = None
extracted_title = document["data"]["title"]
multiword_list = [
"next week's", "simply wall", "three years", "zacks",
"virtual investor conference", "motley fool", "roadshow series",
"institutional investors conference", "speak at"
]
# check if extracted_title has any substrings in multiword list
pattern = re.compile(r'\b(?:' +
'|'.join(re.escape(s)
for s in multiword_list) + r')\b')
matches = pattern.findall(extracted_title.lower())
if len(matches) > 0:
print(matches)
print(extracted_title)
continue
for ent in doc.ents:
# ignore "dates" if they can be parsed as a number
if ent.label_ == "DATE":
try:
if ent.text.lower() in [
"today", "tomorrow", "yesterday", "decade",
"40-year", "friday", "thursday", "wednesday",
"tuesday", "monday", "sunday", "saturday",
"january", "February", "march", "april", "may",
"june", "july", "august", "september", "october",
"november", "december", "last year", "next week's",
"week", "-", "zacks", "roadshow", "participate",
"convention"
]:
continue
# check ent.text in multiword list contains
num = int(ent.text, 10)
except ValueError as e:
# print(e)
# dont want the number to be parsed as a date
extracted_date = dateparser.parse(ent.text)
if extracted_date != None:
if extracted_date.strftime('%Y-%m-%d') == datetime.now(
).strftime('%Y-%m-%d'):
extracted_date = None
continue
break
if extracted_date != None:
# re.IGNORECASE
# make sure its an earnings call
# must contain word quarter, results, annual report or months ended
fmt_date = extracted_date.strftime("%Y-%m-%d")
new_event = make_event_from_data(extracted_title, fmt_date)
if new_event:
send_message(extracted_title, [])
time.sleep(2)
extracted_date = None
# send to discord
continue