def contact(request):
if request.method == 'POST':
contact_form = ContactForm(request.POST or None)
if contact_form.is_valid():
data = contact_form.cleaned_data
city = data.get('city')
language = data.get('language')
email = data.get('email')
qs = Error.objects.filter(timestamp=dt.date.today())
if qs.exists():
err = qs.first()
data = err.data.get('user_data', [])
data.append({
'city': city,
'email': email,
'language': language
})
err.data['user_data'] = data
err.save()
else:
data = [{'city': city, 'email': email, 'language': language}]
Error(data=f"user_data:{data}").save()
messages.success(request, 'Данные отправлены администрации.')
return redirect('accounts:update')
else:
return redirect('accounts:update')
else:
return redirect('accounts:login')
def contact(request):
if request.method == 'POST':
contact_form = ContactForm(request.POST or None)
if contact_form.is_valid():
data = contact_form.cleaned_data
city = data.get('city')
language = data.get('language')
email = data.get('email')
qs = Error.objects.filter(timestamp=dt.date.today())
if qs.exists(): # если добавлена запись
err = qs.first()
data = err.data.get('user_data', [])# если данные пустые мы получаемпустой список
data.append({'city': city, 'email': email, 'language': language})# если данные есть они наполняются данными
err.data['user_data'] = data # и добавим новые данные перзапишем
err.save()
else:# если пусто, тогда создаем новый список
data = {'user_data': [
{'city': city, 'email': email, 'language': language}
]}
Error(data=data).save()
messages.success(request, 'Данные отправлены администрации.')
return redirect('accounts:update')
else:
return redirect('accounts:update')# если форма н евалидна
else:
return redirect('accounts:login')
loop = asyncio.get_event_loop()
tmp_tasks = [(func, data['url_data'][key], data['city'], data['language'])
for data in url_list for func, key in parsers]
# for data in url_list:
#
# for func, key in parsers:
# url = data['url_data'][key]
# j, e = func(url, city=data['city'], language=data['language'])
# jobs += j
# errors += e
if tmp_tasks:
tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
loop.run_until_complete(tasks)
loop.close()
for job in jobs:
v = Vacancy(**job)
try:
v.save()
except DatabaseError:
pass
if errors:
er = Error(data=errors).save()
#h = codecs.open('hh.txt', 'w', 'utf-8')
#h.write(str(jobs))
#h.close()
ten_days_ago = dt.date.today() - dt.timedelta(10)
Vacancy.objects.filter(timestamp__lte=ten_days_ago).delete()
# jobs.extend(j)
# errors.extend(e)
#
# pool = Pool(50)
# pool.map(ff1, tmp_tasks)
#
# pool.close()
# pool.join()
print(time.time() - start)
for job in jobs:
v = Vacancy(**job)
try:
v.save()
except DatabaseError:
pass
if errors:
qs = Error.objects.filter(timestamp=dt.date.today())
if qs.exists():
err = qs.first()
err.data.update({'errors': errors})
err.save()
else:
er = Error(data=f'errors: {errors}').save()
h = codecs.open('work.txt', 'w', 'utf-8')
h.write(str(jobs))
h.close()
url_dict = {(q['city_id'], q['profession_id']): q['url_data'] for q in qs}
urls = []
for pair in _setting:
tmp = {
'city': pair[0],
'profession': pair[1],
'url_data': url_dict[pair]
}
urls.append(tmp)
return urls
settings = get_settings()
url_list = get_urls(settings)
loop = asyncio.get_event_loop()
tmp_tasks = [(func, data['url_data'][key], data['city'], data['profession'])
for data in url_list for func, key in parsers]
tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
loop.run_until_complete(tasks)
loop.close()
for vacancy in vacancies:
try:
obj, created = Vacancy.objects.get_or_create(**vacancy)
except DatabaseError:
pass
if errors:
er = Error(data=errors)
err.data = json.dumps(data)
err.save()
# --------------------------------------------
# # 2 ============================================
# # for local server (db SQLite):
else:
err.data['errors'].extend(errors)
err.save()
# # ============================================
else:
# 1 -----------------------------------------
# for production server (db PostgreSQL)
err = Error(data=json.dumps({
'errors': errors,
'feedback': [],
})).save()
# --------------------------------------------
# # 2 ============================================
# # for local server (db SQLite):
# err = Error(data={'errors': errors, 'feedback': [],}).save()
# # ============================================
# ------------------------------------------------------------------
# if __name__ == '__main__':
# print(*jobs, sep='\n')
# print('count jobs -', len(jobs))
# print('errors -', errors)
# with codecs.open('work.txt', 'w', 'utf-8') as f:
# for data in url_list:
#
# for func, key in parsers:
# url = data['url_data'][key]
# j, e = func(url, city=data['city'], language=data['language'])
# jobs += j
# errors += e
if tmp_tasks:
tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
loop.run_until_complete(tasks)
loop.close()
for job in jobs:
v = Vacancy(**job)
try:
v.save()
except DatabaseError:
pass
if errors:
qs = Error.objects.filter(timestamp=dt.date.today())
if qs.exists():
err = qs.first()
err.data.update({'errors': errors})
err.save()
else:
er = Error(data={'errors': errors}).save()
# h = codecs.open('work.json', 'w', 'utf-8')
# h.write(str(jobs))
# h.close()
loop.close()
#print(time.time()-start)# 3 узнаем сколько времени ушло на ваполнениние запроса
for job in jobs:
v = Vacancy(**job) # раскрываем словарь,
try:
v.save() # и записвыаем в базу данных
except DatabaseError:
pass
if errors:
qs = Error.objects.filter(timestamp=dt.date.today())
if qs.exists(): # если кто ч утра записал
err = qs.first() # уникальность в один день одна запись
err.data.update({'errors': errors})
err.save()
else:
er = Error(data=f'errors:{errors}').save(
) # в словаре будет храниться весь массив ошибок которые были получены в этот день
# 52 урок закоментили
# h = codecs.open('work.txt', 'w', 'utf-8')# открываем в режиме записи и задаем кодировку 'utf-8'
# h.write(str(jobs))# записываем весь контент словарем
# h.close()
# jobs, errors = [], []
# for func, key in parsers:
# url = data['url_data'][key]
# j, e = func(url, city=data['city'], language=data['language'])
# jobs += j
# errors += e
def start():
print('Running')
start = time.time()
User = get_user_model()
parsers = ((tut_pars, 'tut_pars'), (bel_pars, 'bel_pars'))
jobs, errors = [], []
def get_settings():
qs = User.objects.filter(send_email=True).values()
print(qs)
settings_lst = set((q['city_id'], q['language_id']) for q in qs)
return settings_lst
def get_urls(_settings):
qs = Url.objects.all().values()
url_dct = {(q['city_id'], q['language_id']): q['url_data'] for q in qs}
urls = []
for pair in _settings:
if pair in url_dct:
tmp = {}
tmp['city'] = pair[0]
tmp['language'] = pair[1]
tmp['url_data'] = url_dct[pair]
urls.append(tmp)
return urls
async def main(value):
func, url, city, language = value
job, err = await loop.run_in_executor(None, func, url, city, language)
errors.extend(err)
jobs.extend(job)
settings = get_settings()
url_list = get_urls(settings)
#no async function
# for data in url_list:
# for func,key in parsers:
# url = data['url_data'][key]
# j,e = func(url, city=data['city'], language=data['language'])
# jobs += j
# errors += e
# h = codecs.open('pars.txt', 'w', 'utf-8')
# h.write(str(jobs))
# h.close()
loop = asyncio.get_event_loop() #loop forever?
tmp_tasks = [(func, data['url_data'][key], data['city'], data['language'])
for data in url_list for func, key in parsers]
if tmp_tasks:
tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
loop.run_until_complete(tasks)
# loop.close()
#no async
# print(time.time()-start) #33sek
# print(time.time()-start) #20 sek async
for job in jobs:
v = Vacancy(**job)
try:
v.save()
except DatabaseError:
pass
if errors:
qs = Error.objects.filter(timestamp=dt.date.today())
if qs.exists():
err = qs.first()
data = err.data
err.data.update({'errors': errors})
err.save()
else:
er = Error(data=f'errors:{errors}').save()
async def execute(task):
"""
Asynchronous execution of parser functions
"""
function, location, specialty, url = task
vacancy_log, error_log = await loop.run_in_executor(
None, function, location, specialty, url)
data_for_recording_vacancies.extend(vacancy_log)
data_for_recording_errors.extend(error_log)
loop = asyncio.get_event_loop()
instruction = [(function, data['url_json'][parser], data['location_id'],
data['specialty_id']) for data in url_list
for function, parser in parsers]
tasks = asyncio.wait([loop.create_task(execute(task)) for task in instruction])
loop.run_until_complete(tasks)
loop.close()
for record in data_for_recording_vacancies:
to_record = Vacancy(**record)
try:
to_record.save()
except DatabaseError:
pass
if data_for_recording_errors:
error_dump = Error(error_json=data_for_recording_errors)
error_dump.save()