def run():
if (len(sys.argv) != 3):
print("Too few/not enough arguments.")
return
json = sys.argv[1]
lever_list = sys.argv[2]
applicant = Applicant()
applicant.read_from_json(json)
auto_lever = Auto_Lever()
lever_urls = open(lever_list, 'r').readlines()
lever_failed = open('failed' + os.sep + 'lever_failed.txt', 'w')
for url in lever_urls:
auto_lever.load_url(url)
auto_lever.load_applicant(applicant)
try:
response = auto_lever.run()
if (response != 'success'):
print(response, file=lever_failed)
print()
except:
print("Oops! Something went wrong. Skipping URL...")
print(url.strip(), file=lever_failed)
lever_failed.close()
def setup_app():
applist=[]
for qualty in [100,20,90,80,50,30,40,10,70,60]:
name = App()
name.quality = qualty
name.name = 'a'+str(qualty)
applist.append(name)
return applist
def test_sort_rank_interviewed_inst(self):
"""Test the sorting of institutions for the rank"""
app = App()
app.rank_inst = setup_inst() #List of institutions that will be sorted "Ranked"
app.sort_rank_interviewed_inst()
desired = ['i100', 'i90', 'i80', 'i70', 'i60', 'i50', 'i40', 'i30', 'i20', 'i10']
actual = [inst.name for inst in app.rank_inst]
self.assertEqual(actual, desired)
def setup_app():
applist = []
for qualty in [100, 20, 90, 80, 50, 30, 40, 10, 70, 60]:
name = App()
name.quality = qualty
name.name = 'a' + str(qualty)
applist.append(name)
return applist
def main():
print('\n_1_list_mentors: \n', Mentor._1_list_mentors())
print('\n_2_list_mentors_from_miskolc: \n', Mentor._2_list_mentors_from_miskolc())
print('\n_3_greatest_favourite_number: \n', Mentor._3_greatest_favourite_number())
print('\n_4_specific_applicant_by_first_name: \n', Applicant._4_specific_applicant_by_first_name())
print('\n_5_specific_applicant_by_email_domain: \n', Applicant._5_specific_applicant_by_email_domain())
print('\n_6_inserting_a_new_applicant: \n', Applicant._6_inserting_a_new_applicant())
print('\n_7_updating_data: \n', Applicant._7_updating_data())
print('\n_8_deleting_applicants: \n', Applicant._8_deleting_applicants())
def __init__(self, email: str) -> None:
self.failed_email_user_id = 0
self.bot_email = email
self.queue_from_bot = email
self.applicant = Applicant()
self.loop = asyncio.get_event_loop()
self.current_appeal: Optional[dict] = None
self.stop_timer = Timer(self.stop_appeal_sending, self.loop)
self.captcha_solver = CaptchaSolver()
self.user_captcha_text: Optional[str] = None
def test_sort_rank_interviewed_inst(self):
"""Test the sorting of institutions for the rank"""
app = App()
app.rank_inst = setup_inst(
) #List of institutions that will be sorted "Ranked"
app.sort_rank_interviewed_inst()
desired = [
'i100', 'i90', 'i80', 'i70', 'i60', 'i50', 'i40', 'i30', 'i20',
'i10'
]
actual = [inst.name for inst in app.rank_inst]
self.assertEqual(actual, desired)
def main():
print('\n_1_list_mentors: \n', Mentor._1_list_mentors())
print('\n_2_list_mentors_from_miskolc: \n',
Mentor._2_list_mentors_from_miskolc())
print('\n_3_greatest_favourite_number: \n',
Mentor._3_greatest_favourite_number())
print('\n_4_specific_applicant_by_first_name: \n',
Applicant._4_specific_applicant_by_first_name())
print('\n_5_specific_applicant_by_email_domain: \n',
Applicant._5_specific_applicant_by_email_domain())
print('\n_6_inserting_a_new_applicant: \n',
Applicant._6_inserting_a_new_applicant())
print('\n_7_updating_data: \n', Applicant._7_updating_data())
print('\n_8_deleting_applicants: \n', Applicant._8_deleting_applicants())
def create_applicants(cls):
for a in cls.applicants_list:
applicant = Applicant.create(name=a['name'],
application_code=a['application_code'],
home_cid=a['home_cid'],
school_cid=a['school_cid'],
email=a['email'])
def get_applicant(self, applicantID):
r = self.session.get("%sapi/Applicant/%s" % (self.url, applicantID),
headers=self.headers)
if r.status_code != 200:
raise JSONError("Status code %s returned. Json returned: \n\n%s" %
(r.status_code, r.text))
applicant_json = json.loads(json.dumps(r.text))
return Applicant(json_data=applicant_json)
def test_7_updating_data(self):
result = Applicant._7_updating_data()
self.assertEqual(type(result).__name__, "list")
self.assertEqual(len(result), 1)
for applicant in result:
self.assertEqual(type(applicant).__name__, "dict")
self.assertEqual(result[0], {
'phone_number': '003670/223-7459'
})
def test_rank_interviewed_inst(self):
"""Select institions that will be ranked"""
app=App()
app.quality = 50
app.num_applied_to = 5
app.observe = 1
app.applied_to_range = [.4, 1.2]
app.apply_list(setup_inst())
desired = {'i30','i40', 'i50', 'i60', 'i70'}
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(actual, desired)
def test_rank_interviewed_inst(self):
"""Select institions that will be ranked"""
app = App()
app.quality = 50
app.num_applied_to = 5
app.observe = 1
app.applied_to_range = [.4, 1.2]
app.apply_list(setup_inst())
desired = {'i30', 'i40', 'i50', 'i60', 'i70'}
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(actual, desired)
def test_6_inserting_a_new_applicant(self):
result = Applicant._6_inserting_a_new_applicant()
self.assertEqual(type(result).__name__, "list")
self.assertEqual(len(result), 1)
for applicant in result:
self.assertEqual(type(applicant).__name__, "dict")
self.assertEqual(result[0], {
'id': 11,
'first_name': 'Markus',
'last_name': 'Schaffarzyk',
'phone_number': '003620/725-2666',
'email': '*****@*****.**',
'application_code': 54823
})
def test_apply_list2(self):
'''Setup 2: number to apply to > number in desired range
range will be expanded
'''
app = App()
app.quality = 50
app.num_applied_to = 5
app.observe = 1
app.applied_to_range = [.8, 1.2]
app.apply_list(setup_inst())
desired = set(['i50', 'i60', 'i70', 'i80'])
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(desired, actual,
'desired '+str(desired)+' != '+ 'Actual '+str(actual))
def setup_app():
applist=[]
for qualty in [100,20,90,80,50,30,40,10,70,60, 25,95,85,55,35, 45, 15, 75, 65, 15, 10, 5, 1, 2, 3, 4, 5, 6, 7, 8, 9]:
name = App()
name.quality = qualty
name.observed_1 = .8
name.observed_2 = 1.25
name.applied_to = 3
name.name = 'a'+str(qualty)
applist.append(name)
return applist
def test_apply_list3(self):
"""Setup 3: number to apply to < number in desired range.
This may be ambiguous becuase the institutions ar randomly selected from
the list if larger than the number that will be applied to.
"""
app = App()
app.quality = 50
app.num_applied_to = 3
app.observe = 1
app.applied_to_range = [.5, 1.5]
app.apply_list(setup_inst())
desired = {'i40', 'i50', 'i60'}
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(len(desired), len(actual),
'desired '+str(len(desired))+' != '+ 'Actual '+str(len(actual)))
def test_apply_list2(self):
'''Setup 2: number to apply to > number in desired range
range will be expanded
'''
app = App()
app.quality = 50
app.num_applied_to = 5
app.observe = 1
app.applied_to_range = [.8, 1.2]
app.apply_list(setup_inst())
desired = set(['i50', 'i60', 'i70', 'i80'])
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(
desired, actual,
'desired ' + str(desired) + ' != ' + 'Actual ' + str(actual))
def test_apply_list1(self):
"""Setup 1: number to apply to = number in desired range
"""
app = App()
app.quality = 50
app.num_applied_to = 5
app.applied_to_range = [.4, 1.2]
app.apply_list(setup_inst())
desired = {'i30', 'i40', 'i50', 'i60', 'i70'}
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(desired, actual,
'desired '+str(desired)+' != '+ 'Actual '+str(actual))
def test_apply_list3(self):
"""Setup 3: number to apply to < number in desired range.
This may be ambiguous becuase the institutions ar randomly selected from
the list if larger than the number that will be applied to.
"""
app = App()
app.quality = 50
app.num_applied_to = 3
app.observe = 1
app.applied_to_range = [.5, 1.5]
app.apply_list(setup_inst())
desired = {'i40', 'i50', 'i60'}
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(
len(desired), len(actual), 'desired ' + str(len(desired)) +
' != ' + 'Actual ' + str(len(actual)))
def test_apply_list1(self):
"""Setup 1: number to apply to = number in desired range
"""
app = App()
app.quality = 50
app.num_applied_to = 5
app.applied_to_range = [.4, 1.2]
app.apply_list(setup_inst())
desired = {'i30', 'i40', 'i50', 'i60', 'i70'}
actual = set([inst.name for inst in app.applied_to])
self.assertEqual(
desired, actual,
'desired ' + str(desired) + ' != ' + 'Actual ' + str(actual))
def create_appl_interview_msg(cls):
try:
data_list = Applicant.to_newappl_msg()
msg_list = []
if len(data_list) >= 1:
for data in data_list:
msg = MIMEText('Hi ' + data['name'] + ","
+ "\n\nI am happy to inform you that we received your application to Codecool."
+ "\nThe closest Codecool School to you is in " + data['city'] + "."
+ "\nYour application code is " + str(data['ap_code']) + "."
+ "\n\nRegards,\nCodecool Team")
msg['Subject'] = 'Congratulation'
msg['From'] = '*****@*****.**'
msg['To'] = data['email']
msg_list.append(msg)
return msg_list
except StopIteration:
print("Can't create messages. Assign school and generate code fist.")
quit()
def schedule(cls):
"""finds free interviews and assigns applicant and mentor with matching city"""
a_notin_is = Applicant.select().join(cls, JOIN.LEFT_OUTER).where(cls.applicant_id==None)
for a in a_notin_is:
if a.school_cid is not None:
try:
mentor = Mentor.get(Mentor.school_id==a.school_cid)
interview = Interview.get(Interview.location==a.school_cid)
except Mentor.DoesNotExist:
print('No available mentor.')
break
except Interview.DoesNotExist:
print('No available interview time slot.')
break
cls.insert(applicant_id=a.aid, mentor_id=mentor.mid, interview_id=interview.iid).execute()
interview.update_to_reserved()
else:
print('Assign closest school to applicants first.')
break
def test_5_specific_applicant_by_email_domain_list_type_returned(self):
result = Applicant._5_specific_applicant_by_email_domain()
self.assertEqual(type(result).__name__, "list")
# Get user input to reach functions
user_input = input("Hello! Would you like to:\
\n1. Set up your database (WARNING: this will delete all your data!)\
\n2. Set up e-mail stuff\
\n3. Update new applicants with code and closest school\
\n4. Schedule interviews\
\n5. Send e-mails to new applicants\
\n")
if user_input == "1":
Build.create_tables()
Build.upload_data()
elif user_input == "2":
Connection.set_smtp()
elif user_input == "3":
Applicant.get_closest_school()
Applicant.update_appl_code()
elif user_input == "4":
InterviewSlot.schedule()
elif user_input == "5":
msg_list = OrmEmail.create_newappl_msg()
OrmEmail.send(msg_list)
else:
print("Bye")
def test_8_deleting_applicants(self):
result = Applicant._8_deleting_applicants()
self.assertEqual(type(result).__name__, "int")
self.assertEqual(result, 0)
def test_4_specific_applicant_by_first_name_list_type_returned(self):
result = Applicant._4_specific_applicant_by_first_name()
self.assertEqual(type(result).__name__, "list")
def test_4_specific_applicant_by_first_name_length(self):
result = Applicant._4_specific_applicant_by_first_name()
self.assertEqual(len(result), 1)
self.quality = max(min(r.gauss(50, 30), 100), 1) # on a scale 0-100
self.number_to_interview = 10 * tname.openings
self.observe_1 = r.gauss(1, .2)
self.observe_2 = tname.observe_2 = (r.gauss(1, .1)- tname.observe_1)
self.observed_1 = r.gauss(1, .2)
self.observed_2 = (r.gauss(1, .1)- tname.observed_1) # based on applicant interviewed, percentage
self.num_to_rank = 7 * tname.openings
self.accept_range = [.7, None] # as a % [.5, 1.5] if [.7, None] then there is no upper limit
#Make the list of Applicants
print('Make a list of Applicants')
all_applicants = []
for x in range(1, 5000):
tname = 'app'+str(x)
tname = Applicant()
tname.name = x
# Comment out the Following to use defualt
tname.quality = max(min(r.gauss(50, 20), 100), 1) # on a scale 0-100
tname.observe_1 = r.gauss(1, .2) # as a percentage, Applicant error in observing the institutions quality
tname.observe_2 = (r.gauss(1, .1)- tname.observe_1)
tname.observed_1 = r.gauss(1, .2) # as a percentage, Institutions error (as seen) in observing the applicants quality
tname.observed_2 = (r.gauss(1, .1)- tname.observed_1) # CORRECT change to observed_1 after interview default = 1
tname.applied_to_range = [.8, 1.2] # as a percentage, for example [0.8, 1.2]
tname.num_applied_to = 12 # the maximum number that the applicant applies to
all_applicants.append(tname) #Add Applicant to the list of Applicants
# Import institution mAtch data
# Make the list of Institutions
print('importing institution data')
def get_applicants(cls):
from applicant import Applicant
return [
Applicant(raw_applicant) for raw_applicant in cls.applicants_data
]
def test_5_specific_applicant_by_email_domain_length(self):
result = Applicant._5_specific_applicant_by_email_domain()
self.assertEqual(len(result), 1)
class Sender():
def __init__(self, email: str) -> None:
self.failed_email_user_id = 0
self.bot_email = email
self.queue_from_bot = email
self.applicant = Applicant()
self.loop = asyncio.get_event_loop()
self.current_appeal: Optional[dict] = None
self.stop_timer = Timer(self.stop_appeal_sending, self.loop)
self.captcha_solver = CaptchaSolver()
self.user_captcha_text: Optional[str] = None
def sending_in_progress(self):
return self.current_appeal is not None
def send_to_bot(self) -> HttpRabbit:
return HttpRabbit()
async def process_new_appeal(self,
message: aio_pika.IncomingMessage) -> None:
success = True
appeal = json.loads(message.body.decode("utf-8"))
self.convert_recipient(appeal['appeal'])
try:
success = await self.async_process_new_appeal(appeal)
except Exception:
logger.exception('Что-то пошло не так')
success = False
if success:
self.current_appeal = None
self.failed_email_user_id = 0
await message.ack()
logger.info(f'Обращение обработано ' +
f'user_id: {appeal["user_id"]} ' +
f'appeal_id: {appeal["appeal_id"]}')
else:
logger.info('Попробуем отправить еще раз')
await self.process_new_appeal(message)
def convert_recipient(self, appeal: dict) -> None:
department = appeal['police_department']
appeal['police_subdepartment'] = None
if department in config.DEPARTMENT_NAMES:
appeal['police_department'] = config.DEPARTMENT_NAMES[department]
elif department in config.MINSK_DEPARTMENT_NAMES:
appeal['police_department'] = config.DEPARTMENT_NAMES[config.MINSK]
appeal['police_subdepartment'] = \
config.MINSK_DEPARTMENT_NAMES[department]
async def solve_captcha(self, browser: webdriver.Remote) -> bool:
appeal: dict = self.current_appeal or dict()
email: str = self.get_value(appeal, 'sender_email', self.bot_email)
if self.failed_email_user_id == appeal['user_id']:
email = self.bot_email
self.current_appeal['sender_email'] = None
self.current_appeal['sender_email_password'] = ''
logger.info(f"Достали имейл: {email}")
proceed, success, captcha_text = await self.get_captcha_text(appeal,
email,
browser)
if not proceed:
return success
proceed = await self.process_captcha(captcha_text,
appeal['user_id'],
appeal['appeal_id'],
browser,
silent=True)
return proceed
async def async_process_new_appeal(self, appeal: dict) -> bool:
logger.info(f'Новое обращение: {pformat(appeal)}')
self.current_appeal = appeal
with brwsr.get_browser() as browser:
if not await self.solve_captcha(browser):
return False
proceed, url = await self.get_appeal_url()
if not proceed:
return False
return await self.send_appeal(url, browser)
async def get_captcha_text(
self,
appeal: dict,
email: str,
browser: webdriver.Remote) -> Tuple[bool, bool, str]:
try:
captcha_solution = await self.recognize_captcha(email, browser)
if captcha_solution is None:
logger.info("Капча не распозналась =(")
await self.send_captcha(appeal['appeal_id'],
appeal['user_id'],
email,
browser)
cancel, captcha_solution = \
await self.wait_for_input_or_cancel()
if cancel:
return False, True, ''
else:
logger.info("Капча распозналась.")
return True, True, captcha_solution
except BrowserError:
logger.info("Фейл браузинга")
return False, False, ''
except Exception:
logger.exception('ОЙ get_captcha_text')
return False, False, ''
async def wait_for_input_or_cancel(self) -> Tuple[bool, str]:
while True:
if not self.sending_in_progress():
return True, ''
if self.user_captcha_text:
text = self.user_captcha_text
self.user_captcha_text = None
return False, text
await asyncio.sleep(2)
async def send_captcha(self,
appeal_id: int,
user_id: int,
email: str,
browser: webdriver.Remote) -> None:
captcha = self.applicant.get_png_captcha(email, browser)
self.stop_timer.cock_it(config.CANCEL_TIMEOUT)
try:
await self.send_to_bot().send_captcha_url(captcha,
appeal_id,
user_id,
self.queue_from_bot)
except ErrorWhilePutInQueue as exc:
logger.error(exc.text)
if exc.data:
await self.send_to_bot().do_request(exc.data[0], exc.data[1])
async def recognize_captcha(self,
email: str,
browser: webdriver.Remote) -> Optional[str]:
svg_captcha = self.applicant.get_svg_captcha(email, browser)
return await self.captcha_solver.solve(svg_captcha)
async def process_bot_message(self,
message: aio_pika.IncomingMessage) -> None:
data = json.loads(message.body)
logger.info(f'Сообщение от бота: {data}')
email = self.get_value(data, 'sender_email', self.bot_email)
logger.info(f"Достали имейл: {email}")
user_id = data['user_id']
appeal_id = data['appeal_id']
# костыльчик пока бот не научится не хранить обращения, а парсить
# бот уже научился, но убирать страшно, такие дела
if not self.current_appeal \
or self.current_appeal['user_id'] != user_id \
or self.current_appeal['appeal_id'] != appeal_id:
return
if data['type'] == config.CAPTCHA_TEXT:
self.stop_timer.delete()
self.user_captcha_text = data['captcha_text']
elif data['type'] == config.CANCEL:
await self.stop_appeal_sending(local=True)
async def process_captcha(self,
captcha_text: str,
user_id: int,
appeal_id: int,
browser: webdriver.Remote,
silent=False) -> bool:
if self.applicant.enter_captcha_and_submit(
captcha_text, browser) != config.OK:
logger.info("Капча не подошла")
return False
if not silent:
await self.send_to_bot().send_status(user_id,
config.CAPTCHA_OK,
self.queue_from_bot,
appeal_id,
self.current_appeal['appeal'])
return True
async def send_appeal(self, url: str, browser: webdriver.Remote) -> bool:
try:
status_code, message = self.applicant.send_appeal(
self.current_appeal['appeal'], url, browser)
if status_code != config.OK:
logger.info(f"Ошибка при отправке - {message}")
return False
except BrowserError:
logger.info("Фейл браузинга")
return False
except ErrorWhilePutInQueue as exc:
logger.error(exc.text)
if exc.data:
await self.send_to_bot().do_request(exc.data[0], exc.data[1])
except RancidAppeal:
logger.info("Взяли протухшую форму обращения")
proceed, url = await self.get_appeal_url()
if not proceed:
return False
return await self.send_appeal(url, browser)
except Exception:
logger.exception('ОЙ send_appeal')
return False
await self.send_to_bot().send_status(self.current_appeal['user_id'],
config.OK,
self.queue_from_bot,
self.current_appeal['appeal_id'],
self.current_appeal['appeal'])
return True
async def get_appeal_url(self) -> Tuple[bool, str]:
email = self.get_value(self.current_appeal, 'sender_email', None)
password = self.get_value(self.current_appeal,
'sender_email_password',
config.EMAIL_PWD)
if not email:
email = self.bot_email
password = config.EMAIL_PWD
try:
url = self.applicant.get_appeal_url(email, password)
return True, url
except NoMessageFromPolice:
logger.info("Фейл почты. Не нашлось письмо.")
return False, ''
except AppealURLParsingFailed:
logger.info("Не удалось распарсить урл из письма.")
return False, ''
except EmailLoginError as exc:
logger.info(f'Не могу залогиниться {exc}')
await self.maybe_tell_user_about_broken_email(email)
return False, ''
async def maybe_tell_user_about_broken_email(self, email: str):
if email == self.bot_email:
return
self.failed_email_user_id = self.current_appeal['user_id']
await self.send_to_bot().send_status(self.current_appeal['user_id'],
config.BAD_EMAIL,
self.queue_from_bot,
self.current_appeal['appeal_id'],
self.current_appeal['appeal'])
async def start_sender(self, loop: AbstractEventLoop) -> None:
appeals = rabbit_amqp.Rabbit(config.RABBIT_EXCHANGE_MANAGING,
config.RABBIT_QUEUE_APPEAL,
config.RABBIT_AMQP_ADDRESS,
loop,
"appeals")
bot = rabbit_amqp.Rabbit(config.RABBIT_EXCHANGE_SENDING,
self.queue_from_bot,
config.RABBIT_AMQP_ADDRESS,
loop,
"bot_messages")
asyncio.gather(bot.start(self.process_bot_message, passive=False),
appeals.start(self.process_new_appeal, passive=True),
self.stop_timer.start())
logger.info(f"Воркер стартует.")
async def stop_appeal_sending(self, local=False):
logger.info(f"Останавливаем отправку обращения")
if not local:
await self.send_to_bot().send_sending_stopped(
self.current_appeal['appeal_id'],
self.current_appeal['user_id'],
self.queue_from_bot
)
self.current_appeal = None
self.user_captcha_text = None
self.stop_timer.delete()
logger.info("Отмена")
def start(self):
self.loop.run_until_complete(self.start_sender(self.loop))
self.loop.run_forever()
@classmethod
def get_value(cls,
data: Optional[dict],
key: str,
default: Any = None) -> Any:
try:
value = data[key]
if value:
return value
elif default:
return default
else:
return value
except KeyError:
if default:
return default
else:
return None
def stop(self):
logger.info('Суецыд')
def test_5_specific_applicant_by_email_domain_has_dictionaries(self):
result = Applicant._5_specific_applicant_by_email_domain()
for applicant in result:
self.assertEqual(type(applicant).__name__, "dict")
def test_4_specific_applicant_by_first_name_has_dictionaries(self):
result = Applicant._4_specific_applicant_by_first_name()
for applicant in result:
self.assertEqual(type(applicant).__name__, "dict")
def test_5_specific_applicant_by_email_domain_values(self):
result = Applicant._5_specific_applicant_by_email_domain()
self.assertEqual(result[0], {'full_name': 'Jane Forbes'})
def create_applicants(self, scores, real_scores, ir_limit):
applicants = []
for i in range(0, self.size):
applicants.append(
Applicant(self, scores[i], real_scores[i], ir_limit))
return applicants
def test_4_specific_applicant_by_first_name_values(self):
result = Applicant._4_specific_applicant_by_first_name()
self.assertEqual(result[0], {'full_name': 'Carol Arnold'})