import json
import os
import traceback
from bs4.element import Tag
from unidecode import unidecode
import monsters as m
from settings import set_logger
from utils import get_creature_paths, get_soup_with_cache
BASE = 'http://srd.nosolorol.com/DD5/'
logger = set_logger(__name__)
def extract_from_url(path: str):
link = BASE + path
try:
soup = get_soup_with_cache(BASE, path)
content: Tag = soup.select('#mainContent > div.row > div.col')[0]
name = m.extract_name(content)
logger.debug(f'{name}')
name_id = unidecode(name).lower()
name_id = name_id.replace(' ', '-').replace('(', '-').replace(')', '-')
fid = f'srd5-spanish.{name_id}'
alignment, size, type_, tags = m.extract_subtitle_info(content)
hit_points, hit_dice = m.extract_hit_points(content)
import asyncio
import aio_pika
import bcrypt
from settings import rabbit_connection, inbound_name, outbound_name
from settings import set_logger
current_loop = asyncio.get_event_loop()
aioauth_logger = set_logger('aioauth')
def get_hashed_password(plain_text_password):
return bcrypt.hashpw(plain_text_password, bcrypt.gensalt())
def check_password(plain_text_password, hashed_password):
return bcrypt.checkpw(plain_text_password, hashed_password)
async def set_connection(loop):
conn = await aio_pika.connect_robust(rabbit_connection, loop=loop)
ch = await conn.channel()
await ch.set_qos(prefetch_count=1)
inbound_q = await ch.declare_queue(inbound_name)
outbound_q = await ch.declare_queue(outbound_name)
outbound_exchange = await ch.declare_exchange('outbound_exchange',
aio_pika.ExchangeType.FANOUT)
await outbound_q.bind(outbound_exchange)
return conn, ch, inbound_q, outbound_q
from selenium.webdriver.remote.webdriver import WebDriver
from data.objects import Container
from selenium_helper import SeleniumHelper
from settings import set_logger
logger = set_logger()
instance = Container()
def initialize_env(func):
def wrapper(*args):
browser = SeleniumHelper().open_browser()
instance.browser = browser
func(*args)
return wrapper
def get_browser():
if hasattr(instance, 'browser') and isinstance(instance.browser,
WebDriver):
return instance.browser
else:
raise EnvironmentError("You must initialize business at first!")
import asyncio
import aio_pika
from os import getenv
from settings import set_logger
from settings import rabbit_connection, crawler_queue_name
from aioelasticsearch import Elasticsearch
current_loop = asyncio.get_event_loop()
logger = set_logger('crawler')
es = Elasticsearch([{'host': getenv('HOST_ES', 'localhost'), 'port': 9200}])
async def set_connection(loop):
conn = await aio_pika.connect_robust(rabbit_connection, loop=loop)
ch = await conn.channel()
await ch.set_qos(prefetch_count=1)
q = await ch.declare_queue(crawler_queue_name)
return conn, ch, q
async def collect_url(https, domain):
return "{}://{}".format('https' if https == 1 else 'http', domain)
import bcrypt
from os import getenv
from aioelasticsearch import Elasticsearch
from aiohttp import web
from urllib.parse import urlparse
from aioauth.interface import AuthMS
from crawler.interface import CrawlerMS
from settings import set_logger
auth_ms = AuthMS()
crawler_ms = CrawlerMS()
es = Elasticsearch([{'host': getenv('HOST_ES', 'localhost'), 'port': 9200}])
logger = set_logger('aioserver')
dsn = "dbname={} user={} password={} host= {}".format('asynctest',
'emirnavruzov',
'qwe123',
'db')
async def json_response(data, status=200):
return web.Response(text=json.dumps(data),
headers={'content-type': 'application/json'},
status=status)
def get_hashed_password(plain_text_password):
return bcrypt.hashpw(plain_text_password, bcrypt.gensalt())