class DraftInterface(ABC):
pick_type: t.Type[Pick]
passing_to: DraftInterface
class ConnectionException(Exception):
pass
def __init__(self, drafter: Drafter, draft: Draft, draft_seat: DraftSeat):
super().__init__()
self._drafter = drafter
self._draft = draft
self._draft_seat = draft_seat
self._pool = Cube()
self._messages: t.List[t.Mapping[str, t.Any]] = []
self._pick_counter = 0
self._booster_queue = Queue()
self._pick_queue = Queue()
self._out_queue = Queue()
self._current_booster: t.Optional[DraftBooster] = None
self._terminating = threading.Event()
self._booster_pusher = threading.Thread(target=self._draft_loop)
self._connect_lock = threading.Lock()
self._consumer: t.Optional[WebsocketConsumer] = None
@property
def messages(self) -> t.List[t.Mapping[str, t.Any]]:
return self._messages
@property
def pool(self) -> Cube:
return self._pool
def connect(self, consumer: WebsocketConsumer) -> None:
with self._connect_lock:
if self._consumer is not None:
raise self.ConnectionException('already connected')
self._consumer = consumer
def disconnect(self) -> None:
with self._connect_lock:
if self._consumer is None:
raise self.ConnectionException('no consumer connected')
self._consumer = None
def send_message(self, message_type: str, **kwargs) -> None:
self.out_queue.put({
'type': message_type,
**kwargs,
})
def send_error(self, error_type: str, **kwargs):
self.send_message('error', error_type=error_type, **kwargs)
@property
def booster_queue(self) -> Queue[DraftBooster]:
return self._booster_queue
@property
def booster_amount(self) -> int:
return self._booster_queue.qsize() + (1
if self._current_booster else 0)
def give_booster(self, booster: DraftBooster) -> None:
self._booster_queue.put(booster)
self._draft.broadcast_message(
'booster_amount_update',
drafter=self._drafter.user.pk,
queue_size=self.booster_amount,
)
@property
def pick_queue(self) -> Queue[Cubeable]:
return self._pick_queue
@property
def out_queue(self):
return self._out_queue
def receive_message(self, message: t.Any) -> None:
message_type = message.get('type')
if message_type == 'pick':
pick = message.get('pick')
if pick is None:
self.send_error('empty_pick')
return
try:
pick = RawStrategy(db).deserialize(self.pick_type, pick)
except SerializationException:
self.send_error('misconstrued_pick')
return
self._pick_queue.put(pick)
else:
self.send_error('unknown_message_type', message_type=message_type)
def start(self) -> None:
self.send_message(
'started',
**self._draft.serialize(),
)
self._booster_pusher.start()
def stop(self) -> None:
self._terminating.set()
@abstractmethod
def perform_pick(self, pick: Pick) -> bool:
pass
def _draft_loop(self) -> None:
while not self._terminating.is_set():
try:
booster = self._booster_queue.get(timeout=2)
except Empty:
continue
self._current_booster = booster
self.send_message('booster',
booster=RawStrategy.serialize(
self._current_booster))
while not self._terminating.is_set():
try:
pick = self._pick_queue.get(timeout=2)
except Empty:
continue
if not self.perform_pick(pick):
self.send_error(
'invalid_pick',
pick=pick.serialize(),
)
continue
self._pick_counter += 1
self.send_message(
'pick',
pick=pick.serialize(),
booster=RawStrategy.serialize(self._current_booster),
pick_number=self._pick_counter,
)
DraftPick.objects.create(
seat=self._draft_seat,
pack_number=self._draft.pack_counter,
pick_number=self._current_booster.pick_number,
global_pick_number=self._pick_counter - 1,
pack=self._current_booster,
pick=pick,
)
self._current_booster.pick_number += 1
if self._current_booster.cubeables:
self.passing_to.give_booster(self._current_booster)
else:
self._draft.booster_empty(self._current_booster)
self._current_booster = None
self._draft.broadcast_message(
'booster_amount_update',
drafter=self._drafter.user.pk,
queue_size=self.booster_amount,
)
break
class Nautilus:
def __init__(self, url, max_depth):
self.seed_url = url
self.result_urls = []
self.queue = Queue()
self.max_depth = int(max_depth)
self.cur_depth = 1
self.db = DBHelper()
self.logger = get_logger(__name__)
def is_file_link(self, url):
file_url_test = re.compile(r'^.*?\.(pdf|docx|doc|rtf|mobi|azw3|epub)$')
if file_url_test.match(url.lower()):
return True
else:
return False
def insert_links(self, file_url, parent_url):
with requests.get(parent_url, timeout=TIMEOUT) as html_response:
response_parse = BeautifulSoup(html_response.text, 'html.parser')
filename = unquote(file_url).split('/').pop()
[s.extract() for s in response_parse('script')]
[s.extract() for s in response_parse('style')]
content = response_parse.body.get_text() \
.replace(' ', '') \
.replace('\n', '') \
.replace('\r', '')
timestamp = str(int(round(time.time()) * 1000))
values = (file_url, filename, content, timestamp)
self.db.insert_item(values)
self.logger.info('GET: ' + filename + ' AT ' + file_url)
def resolve_links(self, item, parent_url):
link = item.get('href')
url_resolver = Urls(link)
if url_resolver.check_if_url() \
and url_resolver.inner_url(self.seed_url):
prefixed_url = url_resolver.prefix_url(self.seed_url, link)
if prefixed_url not in self.result_urls:
if self.is_file_link(prefixed_url):
self.insert_links(prefixed_url, parent_url)
else:
self.result_urls.append(prefixed_url)
self.queue.enqueue(prefixed_url)
self.logger.info('FETCH: ' + prefixed_url)
def get_url(self):
for i in range(0, len(self.queue.get())):
current_url = self.queue.dequeue()
time.sleep(1.5)
with requests.get(current_url, timeout=TIMEOUT) as html_response:
soup = BeautifulSoup(html_response.text, 'html.parser')
links = soup.find_all('a', {'href': True})
for item in links:
time.sleep(1.5)
self.resolve_links(item, current_url)
def bfs_traverse(self):
self.queue.enqueue(self.seed_url)
while self.cur_depth < self.max_depth:
try:
self.get_url()
self.cur_depth += 1
except Exception as e:
self.logger.error(str(e))
pass
def run(self):
self.logger.info('START BFS FROM: ' + self.seed_url)
time.sleep(1)
self.bfs_traverse()
