def test_add_job(self):
job: IncartJob = IncartJob()
job.id = '3'
job.snippet = 'job_3'
repo = Repo(dal.session)
ok: bool = repo.add_incartjob(job)
self.assertTrue(ok)
def sync_clients_with_db() -> None:
dal.connect()
repo = Repo(dal.session)
doctors_id: List = repo.get_doctors_id() # id докторов из БД
threads = [] # потоки асинхронной обработки
offset = 0
result = get_api_all_clients(offset) # все клиенты из chat2desk (20 записей в странице
while is_continue(result):
meta = result['meta']
offset += meta['limit']
all_clients = result['data']
for client_data in all_clients:
client_id = client_data['id']
if client_id in doctors_id:
continue
# prepare threads
threads.append(threading.Thread(target=sync_client_with_db, args=(client_id,)))
# sync_client_with_db(client_id) # выполнить синхронизацию
# start
for thread in threads:
if thread.native_id is None:
thread.start()
result = get_api_all_clients(offset)
# finish
for thread in threads:
thread.join()
for data in whatsapp_data:
repo.add_doctor(data)
whatsapp_data.clear()
def test_job_relations(self):
repo = Repo(dal.session)
job1 = repo.get_incartjob(id='1')
self.assertIsNotNone(job1)
self.assertIsNotNone(job1.jobdoctors)
self.assertTrue(isinstance(job1.jobdoctors, List))
self.assertTrue(len(job1.jobdoctors), 2)
def run_job(self) -> None:
repo = Repo(self.dal.session)
job: IncartJob = repo.get_incartjob(self.job_id)
jobdoctor: JobDoctor = None
# искать исполнителя задания
if job.doctor_id is None:
jobdoctor = self.find_doctor(job)
# исполнитель не найден(не нашли кандидата)
if jobdoctor is None:
self.set_restart_datetime(job)
return
# исполнитель не найден (кандидат отказалася)
if job.doctor_id is None:
self.stop_job(jobdoctor)
return
# отправить задание на исполнение
self.send_job(jobdoctor)
if jobdoctor.job_finish_id is None:
self.stop_job(jobdoctor)
return
# задание не было выполнено в срок
if jobdoctor.job_finished is None:
self.stop_job(jobdoctor)
return
# доктор подтвердил выполнение задания
job.closed = datetime.now().astimezone(timezone.utc)
self.update_job(job)
if job.file_decrypt:
self.send_success(jobdoctor) # послать подтверждение выполнения
else:
self.send_error(jobdoctor) # послать сообщение
self.send_result(job)
def test_doctor_relations(self):
repo = Repo(dal.session)
doctor: Doctor = repo.get_doctor(id=1)
self.assertIsNotNone(doctor)
self.assertIsNotNone(doctor.jobdoctors)
self.assertTrue(isinstance(doctor.jobdoctors, List))
self.assertTrue(len(doctor.jobdoctors), 1)
def test_find_jobdoctor(self):
repo = Repo(dal.session)
job = repo.get_incartjob(id='1')
last_jobdoctor = job.jobdoctors[-1]
self.assertEqual(len(job.jobdoctors), 2)
self.assertIsNotNone(last_jobdoctor)
def test_update_jobdoctor(self):
repo = Repo(dal.session)
jobdoctor = repo.get_jobdoctor(doctor_id=1, job_id='1')
jobdoctor.request_id = 23
jobdoctor.request_started = datetime.datetime.now().astimezone(
datetime.timezone.utc)
ok: bool = repo.update_jobdoctor(jobdoctor)
self.assertTrue(ok)
def test_get_job_candidate_first(self):
repo = Repo(dal.session)
job = repo.get_incartjob('2')
candidate: Doctor = repo.get_job_candidate(job)
self.assertIsNotNone(candidate)
self.assertTrue(isinstance(candidate, Doctor))
self.assertEqual(candidate.id, 1)
def test_get_jobdoctor(self):
repo = Repo(dal.session)
jobdoctor = repo.get_jobdoctor(doctor_id=1, job_id='1')
self.assertIsNotNone(jobdoctor)
self.assertTrue(isinstance(jobdoctor, JobDoctor))
self.assertTrue(jobdoctor.doctor_id == 1)
self.assertTrue(jobdoctor.job_id == '1')
self.assertTrue(jobdoctor.doctor.id == 1)
self.assertTrue(jobdoctor.job.id == '1')
def test_update_job(self):
repo = Repo(dal.session)
job1 = repo.get_incartjob(id='1')
job1.doctor_id = 1
ok: bool = repo.update_incartjob(job1)
self.assertTrue(ok)
job2 = repo.get_incartjob(id='1')
self.assertEqual(job2.doctor_id, 1)
def load_queue_from_db() -> queue.Queue:
log_info("run: load_queue_from_db")
repo = Repo(dal.session)
jobs: List[IncartJob] = repo.get_unclosing_jobs()
q: queue.Queue = queue.Queue()
for job in jobs:
log_info(f"load_queue_from_db: job added to queue {job}")
q.put(job.id)
return q
def test_get_job_none_candidate(self):
repo = Repo(dal.session)
doc3: Doctor = repo.get_doctor(id=96881373) # Eugeny Bobylev
doc3.is_active = False # временно не доступен (temporarily unavailable)
job: IncartJob = repo.get_incartjob(
id="1") # у этого задания есть 2 обращения к док 1 и 2
doctor: Doctor = repo.get_job_candidate(job)
self.assertIsNone(doctor)
def test_create_jobdoctor(self):
repo = Repo(dal.session)
job = IncartJob()
job.id = '200'
job.snippet = 'test job'
doctor = repo.get_doctor(id=1)
jobdoctor = JobDoctor()
jobdoctor.job = job
jobdoctor.doctor = doctor
ok1: bool = repo.commit()
self.assertTrue(ok1)
jobdoctor2 = repo.get_jobdoctor(job_id=job.id, doctor_id=doctor.id)
self.assertIsNotNone(jobdoctor2)
self.assertEqual(jobdoctor.doctor_id, jobdoctor2.doctor_id)
self.assertTrue(jobdoctor.job_id, jobdoctor2.job_id)
self.assertNotEqual(jobdoctor, jobdoctor2)
def check_new_email():
log_info(f"run: check_new_email, new email every {check_new_email_interval} sec.")
srv = get_service()
new_messages = get_all_unread_emails(srv)
count = len(new_messages)
if count > 0:
log_info(f"run: check_new_email, has {count} new email(s)")
with dal.session_scope() as session:
repo = Repo(session)
for message in new_messages:
dct = parse_mail_message(message)
job = IncartJob.from_json(dct)
ok: bool = repo.add_incartjob(job)
if ok:
log_info(f"job added to db {job}")
jobid_queue.put(job.id)
# mark e-mail message as readed
labels = {"removeLabelIds": ['UNREAD'], "addLabelIds": []}
modify_message(srv, "me", message["id"], labels)
def create_app():
app = Flask(__name__)
Config.load()
for repo_name in Config.repos:
repo = Repo(repo_name)
app.register_blueprint(repo.blueprint)
repos.append(repo)
if not os.path.exists(Config.path_repos):
os.makedirs(Config.path_repos, exist_ok=True)
return app
def db_get_incartjob():
job_id = '1733014f3b7bea76'
with dal.session_scope() as session:
repo = Repo(session)
_task = repo.get_incartjob(job_id)
print(_task)
def clear_data_in_db() -> None:
# удалить данные о задании
repo = Repo(dal.session)
repo.clear_jobdoctors()
repo.clear_incartjobs()
from app.repo import Repo
from settings import CELERY_BROKER, CELERY_RESULTS, MONGODB_URI
from flask import Flask, jsonify, request
from flask_pymongo import PyMongo
from app.words_utils import proccess_articles
from app.tasks import fetch_for_day, executor
from app.news_fetcher import Fetcher
app = Flask(__name__)
app.config["CELERY_BROKER"] = CELERY_BROKER
app.config["CELERY_RESULTS"] = CELERY_RESULTS
app.config["MONGO_URI"] = MONGODB_URI
mongo = PyMongo(app, f"{MONGODB_URI}/news_analyzer")
repo = Repo(mongo)
@app.route("/", methods=["GET"])
def index_page():
return f"run client at client folder by npm start"
@app.route("/news", methods=["GET"])
def news_and_stats():
token = request.args.get("token")
amount = int(request.args.get("amount"))
days = int(request.args.get("days"))
articles, stats = repo.fetch_news_by_amount(amount)
if articles:
return jsonify({"articles": articles, "stats": stats})
def update_jobdoctor(self, jobdoctor: JobDoctor) -> bool:
self.log_info("run: update_jobdoctor")
repo = Repo(self.dal.session)
ok: bool = repo.update_jobdoctor(jobdoctor)
self.log_info(f"run: update_jobdoctor, result={ok}")
return ok
def update_job(self, job: IncartJob) -> bool:
self.log_info("run: update_job")
repo = Repo(self.dal.session)
ok: bool = repo.update_incartjob(job)
self.log_info(f"run: update_job, result={ok}")
return ok
def get_candidate(self, job: IncartJob) -> Doctor:
self.log_info("run: get_candidate")
repo = Repo(self.dal.session)
candidate: Doctor = repo.get_job_candidate(
job) # Бобылев Е.А. 96881373
return candidate
def test_repo_get_unclosing_jobs(self):
repo = Repo(dal.session)
jobs: List[IncartJob] = repo.get_unclosing_jobs()
all_jobs = dal.session.query(IncartJob).all()
self.assertTrue(len(jobs), len(all_jobs))
def test_get_doctor(self):
repo = Repo(dal.session)
doctor = repo.get_doctor(id=1)
self.assertIsNotNone(doctor)
self.assertTrue(doctor.id, 1)
def test_get_all_doctors(self):
repo = Repo(dal.session)
doctors = repo.get_all_doctors()
self.assertEqual(len(doctors), 3)
self.assertTrue(doctors[0].id, 1)
self.assertTrue(doctors[1], 2)
def test_get_nonexist_doctor(self):
repo = Repo(dal.session)
doctor = repo.get_doctor(id=-20000)
self.assertIsNone(doctor)
def test_get_job(self):
repo = Repo(dal.session)
job = repo.get_incartjob(id='1')
self.assertIsNotNone(job)
self.assertEqual('job_1', job.snippet)
