コード例 #1
0
def contact(request):
    if request.method == 'POST':
        contact_form = ContactForm(request.POST or None)
        if contact_form.is_valid():
            data = contact_form.cleaned_data
            city = data.get('city')
            language = data.get('language')
            email = data.get('email')
            qs = Error.objects.filter(timestamp=dt.date.today())
            if qs.exists():
                err = qs.first()
                data = err.data.get('user_data', [])
                data.append({
                    'city': city,
                    'email': email,
                    'language': language
                })
                err.data['user_data'] = data
                err.save()
            else:
                data = [{'city': city, 'email': email, 'language': language}]
                Error(data=f"user_data:{data}").save()
            messages.success(request, 'Данные отправлены администрации.')
            return redirect('accounts:update')
        else:
            return redirect('accounts:update')
    else:
        return redirect('accounts:login')
コード例 #2
0
ファイル: views.py プロジェクト: drhtka/scripe_job
def contact(request):
    if request.method == 'POST':
        contact_form = ContactForm(request.POST or None)
        if contact_form.is_valid():
            data = contact_form.cleaned_data
            city = data.get('city')
            language = data.get('language')
            email = data.get('email')
            qs = Error.objects.filter(timestamp=dt.date.today())
            if qs.exists(): # если добавлена запись
                err = qs.first()
                data = err.data.get('user_data', [])# если данные пустые мы получаемпустой список
                data.append({'city': city, 'email': email, 'language': language})# если данные есть они наполняются данными
                err.data['user_data'] = data # и добавим новые данные перзапишем
                err.save()
            else:# если пусто, тогда создаем новый список
                data = {'user_data': [
                    {'city': city, 'email': email, 'language': language}
                ]}
                Error(data=data).save()
            messages.success(request, 'Данные отправлены администрации.')
            return redirect('accounts:update')
        else:
            return redirect('accounts:update')# если форма н евалидна
    else:
        return redirect('accounts:login')
コード例 #3
0
loop = asyncio.get_event_loop()
tmp_tasks = [(func, data['url_data'][key], data['city'], data['language'])
             for data in url_list for func, key in parsers]

# for data in url_list:
#
#     for func, key in parsers:
#         url = data['url_data'][key]
#         j, e = func(url, city=data['city'], language=data['language'])
#         jobs += j
#         errors += e
if tmp_tasks:
    tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
    loop.run_until_complete(tasks)
    loop.close()

for job in jobs:
    v = Vacancy(**job)
    try:
        v.save()
    except DatabaseError:
        pass
if errors:
    er = Error(data=errors).save()

#h = codecs.open('hh.txt', 'w', 'utf-8')
#h.write(str(jobs))
#h.close()
ten_days_ago = dt.date.today() - dt.timedelta(10)
Vacancy.objects.filter(timestamp__lte=ten_days_ago).delete()
コード例 #4
0
#     jobs.extend(j)
#     errors.extend(e)
#
# pool = Pool(50)
# pool.map(ff1, tmp_tasks)
#
# pool.close()
# pool.join()

print(time.time() - start)

for job in jobs:
    v = Vacancy(**job)
    try:
        v.save()
    except DatabaseError:
        pass

if errors:
    qs = Error.objects.filter(timestamp=dt.date.today())
    if qs.exists():
        err = qs.first()
        err.data.update({'errors': errors})
        err.save()
    else:
        er = Error(data=f'errors: {errors}').save()

h = codecs.open('work.txt', 'w', 'utf-8')
h.write(str(jobs))
h.close()
コード例 #5
0
ファイル: run_scrap.py プロジェクト: terek97/scraping_service
    url_dict = {(q['city_id'], q['profession_id']): q['url_data'] for q in qs}
    urls = []
    for pair in _setting:
        tmp = {
            'city': pair[0],
            'profession': pair[1],
            'url_data': url_dict[pair]
        }
        urls.append(tmp)
    return urls


settings = get_settings()
url_list = get_urls(settings)

loop = asyncio.get_event_loop()
tmp_tasks = [(func, data['url_data'][key], data['city'], data['profession'])
             for data in url_list for func, key in parsers]
tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
loop.run_until_complete(tasks)
loop.close()

for vacancy in vacancies:
    try:
        obj, created = Vacancy.objects.get_or_create(**vacancy)
    except DatabaseError:
        pass

if errors:
    er = Error(data=errors)
コード例 #6
0
            err.data = json.dumps(data)
            err.save()
        # --------------------------------------------

        # # 2 ============================================
        # # for local server (db SQLite):
        else:
            err.data['errors'].extend(errors)
            err.save()
        # #  ============================================

    else:
        # 1 -----------------------------------------
        # for production server (db PostgreSQL)
        err = Error(data=json.dumps({
            'errors': errors,
            'feedback': [],
        })).save()
        # --------------------------------------------

        # # 2 ============================================
        # # for local server (db SQLite):
        # err = Error(data={'errors': errors, 'feedback': [],}).save()
        # #  ============================================

# ------------------------------------------------------------------
# if __name__ == '__main__':
#     print(*jobs, sep='\n')
#     print('count jobs -', len(jobs))
#     print('errors -', errors)

# with codecs.open('work.txt', 'w', 'utf-8') as f:
コード例 #7
0
# for data in url_list:
#
#     for func, key in parsers:
#         url = data['url_data'][key]
#         j, e = func(url, city=data['city'], language=data['language'])
#         jobs += j
#         errors += e
if tmp_tasks:
    tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
    loop.run_until_complete(tasks)
    loop.close()

for job in jobs:
    v = Vacancy(**job)
    try:
        v.save()
    except DatabaseError:
        pass
if errors:
    qs = Error.objects.filter(timestamp=dt.date.today())
    if qs.exists():
        err = qs.first()
        err.data.update({'errors': errors})
        err.save()
    else:
        er = Error(data={'errors': errors}).save()

# h = codecs.open('work.json', 'w', 'utf-8')
# h.write(str(jobs))
# h.close()
コード例 #8
0
loop.close()
#print(time.time()-start)# 3 узнаем сколько времени ушло на ваполнениние запроса

for job in jobs:
    v = Vacancy(**job)  # раскрываем словарь,
    try:
        v.save()  # и записвыаем в базу данных
    except DatabaseError:
        pass

if errors:
    qs = Error.objects.filter(timestamp=dt.date.today())
    if qs.exists():  # если кто ч утра записал
        err = qs.first()  # уникальность в один день одна запись
        err.data.update({'errors': errors})
        err.save()
    else:
        er = Error(data=f'errors:{errors}').save(
        )  # в словаре будет храниться весь массив ошибок которые были получены в этот день
# 52 урок закоментили
# h = codecs.open('work.txt', 'w', 'utf-8')# открываем в режиме записи и задаем кодировку 'utf-8'
# h.write(str(jobs))# записываем весь контент словарем
# h.close()

# jobs, errors = [], []
# for func, key in parsers:
#     url = data['url_data'][key]
#     j, e = func(url, city=data['city'], language=data['language'])
#     jobs += j
#     errors += e
コード例 #9
0
def start():
    print('Running')
    start = time.time()

    User = get_user_model()

    parsers = ((tut_pars, 'tut_pars'), (bel_pars, 'bel_pars'))
    jobs, errors = [], []

    def get_settings():
        qs = User.objects.filter(send_email=True).values()
        print(qs)
        settings_lst = set((q['city_id'], q['language_id']) for q in qs)
        return settings_lst

    def get_urls(_settings):
        qs = Url.objects.all().values()

        url_dct = {(q['city_id'], q['language_id']): q['url_data'] for q in qs}

        urls = []
        for pair in _settings:
            if pair in url_dct:

                tmp = {}
                tmp['city'] = pair[0]

                tmp['language'] = pair[1]
                tmp['url_data'] = url_dct[pair]
                urls.append(tmp)
        return urls

    async def main(value):
        func, url, city, language = value
        job, err = await loop.run_in_executor(None, func, url, city, language)
        errors.extend(err)
        jobs.extend(job)

    settings = get_settings()
    url_list = get_urls(settings)

    #no async function

    # for data in url_list:
    #     for func,key in parsers:
    #         url = data['url_data'][key]

    #         j,e = func(url, city=data['city'], language=data['language'])
    #         jobs += j
    #         errors += e
    # h = codecs.open('pars.txt', 'w', 'utf-8')
    # h.write(str(jobs))
    # h.close()

    loop = asyncio.get_event_loop()  #loop forever?
    tmp_tasks = [(func, data['url_data'][key], data['city'], data['language'])
                 for data in url_list for func, key in parsers]
    if tmp_tasks:
        tasks = asyncio.wait([loop.create_task(main(f)) for f in tmp_tasks])
        loop.run_until_complete(tasks)
        # loop.close()

    #no async
    # print(time.time()-start) #33sek

    # print(time.time()-start) #20 sek async
    for job in jobs:
        v = Vacancy(**job)
        try:
            v.save()
        except DatabaseError:
            pass

    if errors:
        qs = Error.objects.filter(timestamp=dt.date.today())
        if qs.exists():
            err = qs.first()
            data = err.data
            err.data.update({'errors': errors})
            err.save()
        else:
            er = Error(data=f'errors:{errors}').save()
コード例 #10
0
async def execute(task):
    """
    Asynchronous execution of parser functions
    """
    function, location, specialty, url = task
    vacancy_log, error_log = await loop.run_in_executor(
        None, function, location, specialty, url)
    data_for_recording_vacancies.extend(vacancy_log)
    data_for_recording_errors.extend(error_log)


loop = asyncio.get_event_loop()

instruction = [(function, data['url_json'][parser], data['location_id'],
                data['specialty_id']) for data in url_list
               for function, parser in parsers]
tasks = asyncio.wait([loop.create_task(execute(task)) for task in instruction])
loop.run_until_complete(tasks)
loop.close()

for record in data_for_recording_vacancies:
    to_record = Vacancy(**record)
    try:
        to_record.save()
    except DatabaseError:
        pass

if data_for_recording_errors:
    error_dump = Error(error_json=data_for_recording_errors)
    error_dump.save()