async def parse(article, session, recursion_depth, from_page_id, limit_rd=1):
async with SEMA:
global URL
task = [asyncio.ensure_future(fetch(URL.format(article), session))]
response = await asyncio.gather(*task)
data = html.fromstring(response[0])
anchors = data.xpath('//a[@title]')
entities = []
for anchor in anchors:
href = unquote(anchor.attrib['href'])
# проверить, что урл начинается с /wiki, а также этот урл не повторяется в БД
if not is_valid_ref(
href, db_session
) or 'страница отсутствует' in anchor.attrib['title']:
continue
page = Page(url=URL.format(href), from_page_id=from_page_id)
db_session.add(page)
db_session.commit()
db_session.refresh(page)
entities.append((page.id, href))
# Если дошли до конца рекурсии, останавливаемся
if recursion_depth > limit_rd:
return
# Создаем новый ивент-луп под полученные ссылки, и также рекурсивно запускаем эту функцию под них
subtasks = []
for page_id, page_url in entities:
subtasks.append(
asyncio.ensure_future(
parse(page_url, session, recursion_depth + 1, page_id)))
await asyncio.gather(*subtasks)
async def parse(url, session):
async with SEMA:
task = [asyncio.ensure_future(fetch(url, session))]
response = await asyncio.gather(*task)
result = Result(data=response)
db_session.add(result)
db_session.commit()
db_session.refresh(result)
def add_order(user_id, date, status_id, delivery_id = None, total_price = None, assignee_id = None,
preferable_delivery_date = None, delivery_date=None, gift= None,
delivery_address = None, comment =None,order_number=None, discount=None):
order = Order(user_id, date, status_id, delivery_id, total_price, assignee_id, preferable_delivery_date,
delivery_date, gift,delivery_address,comment,order_number,discount)
db_session.add(order)
db_session.commit()
db_session.refresh(order)
return order.id
async def run(article, limit_rd):
global URL
async with ClientSession() as session:
page = Page(url=URL.format(article))
db_session.add(page)
db_session.commit()
db_session.refresh(page)
main_task = [
asyncio.ensure_future(
parse(article,
session,
0,
from_page_id=page.id,
limit_rd=limit_rd))
]
await asyncio.gather(*main_task)
def mutate(self, info, name, balance):
try:
#fetch min amount
minAmount = db_session.query(MinAmountModel).one()
# check balance
if balance >= minAmount.amount:
#create user if balance is greater
user = UserModel(name=name, balance=balance)
db_session.add(user)
db_session.commit()
db_session.refresh(user)
return ValidateAndAddUser(id=user.id,
name=user.name,
balance=user.balance)
else:
raise ValueError('balance too low, required atleast ' +
str(minAmount.amount))
except Exception as exc:
raise exc