def get_cookies(self, code, url):
from mongo_handler import MongoHandler
col = MongoHandler().connect('col')
model_header = col.find_one({'code': code})['headers']['get']
page_content = self.session.post(url=url, headers=model_header)
cookies = requests.utils.dict_from_cookiejar(page_content.cookies)
return cookies
def register(*args, **kwargs): try: no_satisfy = kwargs["no_satisfy"] if len(no_satisfy) > 0: return jsonify(create_incomplete_form_error(no_satisfy)) else: mongo = MongoHandler() user = mongo.store_user(request.form) return jsonify(success_response) if user else jsonify(email_taken_error) except Exception as e: print(str(e)) return str(e)
def init():
mh = MongoHandler()
col = mh.connect('col')
code_filter = {'code': model_schema['code']}
existed = col.find_one(code_filter)
if not existed:
model_schema['createAt'] = datetime.datetime.now()
model_schema['headers']['updateAt'] = model_schema['createAt']
col.insert_one(model_schema)
else:
print('Document exists and it has been replaced.')
print("zjzdjy's info has been updated.")
print(col.find_one(code_filter))
def login(*args, **kwargs):
try:
no_satisfy = kwargs["no_satisfy"]
if len(no_satisfy) > 0:
return jsonify(create_incomplete_form_error(no_satisfy))
else:
mongo = MongoHandler()
authenticated, user = mongo.login(request.form)
if authenticated:
return jsonify(create_success_response("user", user))
return jsonify(login_error)
except Exception as e:
print(str(e))
return str(e)
def add_todo(todo: NewToDo):
logging.info("/add called. postdata: " + str(todo))
new_todo = {
"sender": todo.sender,
"receiver": todo.receiver,
"content": todo.content,
"start_time": todo.start_time,
"expected_finish_time": todo.expected_finish_time,
"actual_finish_time": "",
"status": 0,
"is_visible": True
}
mh = MongoHandler()
inserted = mh.collection.insert_one(new_todo)
if inserted:
logging.info("New todo added. id: {}".format(str(
inserted.inserted_id)))
executor.submit(
WeixinHandler().send_notification, todo.receiver, "新待办通知",
"<div>$userName={userid}$ 新增了待办：{content}</div>".format(
userid=todo.receiver, content=todo.content))
return JSONResponse(content={
"msg": "ok",
"id": str(inserted.inserted_id)
},
headers=cors_headers)
else:
logging.error("Adding todo failed.")
return JSONResponse(content={"msg": "error"}, headers=cors_headers)
async def finish_todo(todo: FinishToDo):
logging.info("/finish called. postdata: " + str(todo))
mh = MongoHandler()
mh.collection.update_one(
{
"_id": ObjectId(todo.todo_id),
"receiver": todo.receiver
}, {
"$set": {
"status": 1,
"actual_finish_time": todo.actual_finish_time.isoformat()
}
})
finished_todo = list(
mh.collection.find({
"_id": ObjectId(todo.todo_id),
"is_visible": True
}, {
"sender": 1,
"actual_finish_time": 1,
"content": 1
}))[0]
logging.info("finished_todo: " + str(finished_todo))
executor.submit(
WeixinHandler().send_notification, finished_todo["sender"], "待办完成通知",
"<div>$userName={userid}$ 已完成待办：{content}</div>".format(
userid=finished_todo["sender"], content=finished_todo["content"]))
return JSONResponse(content={
"msg": "ok",
"id": todo.todo_id
},
headers=cors_headers)
def stats():
"""
get videos ratings
"""
version = request.args.get('version')
stats = MongoHandler().get_statistics(version=version)
return json.dumps({"code": 200, "data": stats})
def get_unrated_videos(self):
unrated = []
videos = self.list_videos()
for vid in videos:
if not MongoHandler().get_statistics(
name=vid, version=os.path.getmtime(vid)):
unrated.append(vid)
return unrated
def update_members():
st = time.time()
mh = MongoHandler(collection="members")
wh = WeixinHandler(corpsecret=config.WX_MEMBERS_CORPSECRET)
all_members = wh.get_all_members()
if all_members:
mh.collection.delete_many({})
mh.collection.insert_many(all_members)
def list_members():
logging.info("/list_members called.")
mh = MongoHandler(collection="members")
members = list(mh.collection.find({}, {"_id": 0}))
return JSONResponse(content={
"msg": "ok",
"members": members
},
headers=cors_headers)
def list_sent_todo(todo: ListToDo):
logging.info("/list_sent called. postdata: " + str(todo))
mh = MongoHandler()
todos = list(
mh.collection.find({
"sender": todo.user_id,
"is_visible": True
}, {"is_visible": 0}))
for i in range(len(todos)):
todos[i]["_id"] = str(todos[i]["_id"])
todos[i]["expected_finish_time"] = str(
todos[i]["expected_finish_time"])
todos[i]["start_time"] = str(todos[i]["start_time"])
return JSONResponse(content={"data": todos}, headers=cors_headers)
def delete_todo(todo: DeleteToDo):
logging.info("/delete called. postdata: " + str(todo))
mh = MongoHandler()
mh.collection.update_one(
{
"_id": ObjectId(todo.todo_id),
"sender": todo.sender
}, {"$set": {
"is_visible": False
}})
return JSONResponse(content={
"msg": "ok",
"id": todo.todo_id
},
headers=cors_headers)
def rate():
"""
Upload a video rating
"""
data = json.loads(request.data)
version = data.get('version')
try:
name = data['name']
rating = data['rating']
except KeyError as e:
return json.dumps({"code": 404, "err": e})
feedback = data.get('feedback')
certified = data.get('certified', False)
MongoHandler().add_feedback(name,
version,
rating,
feedback=feedback,
certified=certified)
return json.dumps({"code": 200})
class AspxHandler(object):
def __init__(self):
self.mh = MongoHandler()
self.wh = WebHandler()
self.db = self.mh.connect('db')
self.col = self.mh.connect('col')
self.session = requests.session()
def send_request(self, web_code, req_method, *post_data):
web_info = self.col.find_one({
'code': web_code
})
try:
cookie_history = datetime.datetime.now() - web_info['cookies']['updateAt']
if cookie_history > datetime.timedelta(days=1):
self.mh.update_cookies(web_code)
except:
print('AspxHandler.send_request():')
print('There is no information about this web in the DB or it is not completed .')
print('Please create/fix the information first.')
return None
if req_method == 'get' or req_method == 'GET':
page_res = self.session.get(
url=web_info['url'],
headers=web_info['headers']['get'],
cookies=web_info['cookies']['content']
)
return page_res
elif req_method == 'post' or req_method == 'POST':
if post_data[0]:
page_res = self.session.post(
url=web_info['url'],
headers=web_info['headers']['post'],
cookies=web_info['cookies']['content'],
data=post_data[0]
)
return page_res
else:
print('AspxHandler.send_request():')
print('As you choose to use "POST", post_data is necessary.')
return None
else:
print('AspxHandler.send_request():')
print('Wrong input @ req_method.')
return None
def page_saver(self, html_response, file_path):
dir_path = os.path.dirname(file_path)
is_dir(dir_path)
with open(file_path, 'wb') as html_file:
html_file.write(html_response.content)
soup = BeautifulSoup(html_response.text, 'html5lib')
return soup
def get_viewstate(self, html_response):
soup = BeautifulSoup(html_response.text, 'html5lib')
view_state = soup.find('input', attrs={
'name': '__VIEWSTATE'
})['value']
return view_state
def __init__(self):
self.mh = MongoHandler()
self.wh = WebHandler()
self.db = self.mh.connect('db')
self.col = self.mh.connect('col')
self.session = requests.session()
from video_handler import VideoHandler
log = logging.getLogger("video_crowd_control")
parser = argparse.ArgumentParser(
description='Enable users to review some videos.')
parser.add_argument('--debug',
'-d',
dest='debug',
action='store_true',
help='Debug mode')
app = Flask(__name__)
app.secret_key = "test"
MongoHandler().init_db()
@app.before_first_request
def initialize():
logger = logging.getLogger("video_crowd_control")
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"""%(levelname)s in %(module)s [%(pathname)s:%(lineno)d]:\n%(message)s"""
)
ch.setFormatter(formatter)
logger.addHandler(ch)
import s3_handler
from pyspark import SparkContext
import json
from mongo_handler import MongoHandler
sc = SparkContext("local[*]", "Aggregator")
mongo = MongoHandler('localhost', 27017, 'camara')
def get_files():
deputados_file_list = s3_handler.list_files('00/dadosabertos-camara-deputados')
news_file_list = s3_handler.list_files('00/camara-news/json')
rdd_deputados = sc.parallelize(deputados_file_list['Contents'])
rdd_news = sc.parallelize(news_file_list['Contents'])
deputados = rdd_deputados.map(lambda file: s3_handler.get_file(file['Key']))\
.map(lambda x: json.loads(x))\
.collect()
news = rdd_news.map(lambda file: s3_handler.get_file(file['Key']))\
.map(lambda x: json.loads(x))\
.collect()
return deputados, news
deputados, news_list = get_files()
for deputado in deputados:
deputado['news'] = []
for news in news_list:
from mongo_handler import MongoHandler
from flask import Flask
from flask import jsonify
mongo = MongoHandler('localhost', 27017, 'camara')
app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False
@app.route('/news/ranking', methods = ['GET'])
def get_deputados_news_ranking():
deputados = mongo.find_all('deputados')
data = []
for deputado in deputados:
data.append({
'id': deputado['id'],
'name': deputado['nomeCivil'],
'partido': deputado['ultimoStatus']['siglaPartido'],
'news_count': len(deputado['news'])
})
return jsonify(sorted(data, key=lambda x: x['news_count'], reverse=True))
if __name__ == '__main__':
app.run()
import logging
from time import time, sleep
from config import mongo_credentials
from mongo_handler import MongoHandler
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(MongoHandler(**mongo_credentials))
logger.info('Starting', extra={"first_level": {"mongo_db_name": "LOGSSSS"}})
for e in range(0, 10):
try:
g = (10 / 0)
except:
start_time = time()
print("start send error")
logger.exception("no se puede dividir")
print(f"error save in {(time() - start_time)} sec")
continue
sleep(1)