def loop_exception_hook(self, loop, context): """ We are having trouble using log.exception() with exc_info=True and seeing the stack trace, so we have a custom log.traceback() method for now. >>> self.log.exception(exc, exc_info=True, extra=extra) Not sure if we will keep it or not, since it might add some extra customization availabilities, but it works right now. """ log = logger.get('handle_exception', sync=True) log.debug('Handling Exception') # The only benefit including the frame has is that the filename # will not be in the logger, it will be in the last place before the # logger and this statement. try: log.traceback( context['exception'].__class__, context['exception'], context['exception'].__traceback__, ) except BlockingIOError: log.warning('Could Not Output Traceback due to Blocking IO') log.debug('Shutting Down in Exception Handler') loop.run_until_complete(shutdown(loop))
class ManagedProxyPool(SimpleProxyPool): __NAME__ = "Managed Proxy Pool" log = logger.get(__name__, __NAME__) async def put(self, proxy, evaluate=True, prepopulation=False): """ [x] NOTE: --------- We do not want to check confirmation based on the threshold and horizon, because that is the definition for confirmed proxies, which should not be put in the general pool. """ if evaluate: evaluation = proxy.evaluate_for_pool() if evaluation.passed: await super(ManagedProxyPool, self).put(proxy) else: # Do not log during prepopulation. if not prepopulation: if settings.logging.log_proxy_queue: self.log.debug(f'Cannot Add Proxy to {self.__NAME__}', extra={ 'other': str(evaluation) }) if proxy.confirmed(): self.log.warning('Removing Proxy That Was Confirmed from Pool', extra={ 'proxy': proxy, })
import os from plumbum.path import LocalPath from tortoise import fields from tortoise.models import Model from tortoise.exceptions import OperationalError, DoesNotExist from instattack import settings from instattack.lib import logger from instattack.lib.utils import stream_raw_data, read_raw_data from instattack.core.exceptions import DirExists, UserFileExists from instattack.core.passwords import password_gen log = logger.get(__name__, subname='User') class UserAttempt(Model): id = fields.IntField(pk=True) password = fields.CharField(max_length=100) user = fields.ForeignKeyField('models.User', related_name='attempts') last_attempt = fields.DatetimeField(null=True, auto_now=True) success = fields.BooleanField(default=False) num_attempts = fields.IntField(default=1) class Meta: unique_together = ('password', 'user')
import asyncio import contextlib from proxybroker import Broker from instattack import settings from instattack.lib import logger from instattack.core.models import Proxy log = logger.get(__name__, subname='Proxy Broker') class ProxyBroker(Broker): __name__ = 'Proxy Broker' def __init__(self, loop, limit=None): """ If we want the broker to run until we manually stop it, we still need to set a limit on initialization for the Proxy Broker package, so we set that arbitrarily high. [x] TODO: --------- The limit of the broker (i.e. the collect limit) needs to eventually be tied in with the pool in some way, and collection needs to be triggered when we start runnign low on prepopulated proxies. """ self.loop = loop self._stopped = False
class SimpleProxyPool(AbstractProxyPool): __NAME__ = "Simple Proxy Pool" log = logger.get(__name__, __NAME__)
import asyncio import re from instattack import settings from instattack.lib import logger from instattack.core.models import InstagramResult from instattack.core.exceptions import (InstagramResultError, HTTP_RESPONSE_ERRORS, HTTP_REQUEST_ERRORS) log = logger.get(__name__) class client: def __init__(self, loop, on_error, on_success): self.loop = loop self.on_error = on_error self.on_success = on_success def post(self, session, url, proxy, headers=None, data=None): return session.post( url, headers=headers, data=data, ssl=False, proxy=proxy.url # Only Http Proxies Are Supported by AioHTTP )
from termx.ext.utils import progress from instattack import settings from instattack.lib import logger from instattack.lib.utils import limit_as_completed from instattack.core.proxies import SimpleProxyPool, SimpleProxyManager from .base import AbstractRequestHandler from .client import train_client __all__ = ('TrainHandler', ) log = logger.get(__name__, 'Train Handler') class TrainHandler(AbstractRequestHandler): __name__ = 'Train Handler' __proxy_manager__ = SimpleProxyManager __proxy_pool__ = SimpleProxyPool __client__ = train_client async def train(self, limit=None, confirmed=False): try: results = await asyncio.gather( self._train(limit=limit), self.proxy_manager.start(limit=limit, confirmed=confirmed), )
class BrokeredProxyManager(SimpleProxyManager): __NAME__ = "Brokered Proxy Manager" log = logger.get(__name__, __NAME__) def __init__(self, *args, start_event): super(BrokeredProxyManager, self).__init__(*args) self.broker = ProxyBroker(self.loop) self.start_event = start_event async def stop(self): if self.broker._started: self.broker.stop() async def collect(self): """ Retrieves proxies from the broker and converts them to our Proxy model. The proxy is then evaluated as to whether or not it meets the standards specified and conditionally added to the pool. [x] TODO: --------- We need to move this to the appropriate place, most likely in the manager class. [!] IMPORTANT: ------------- We should maybe make this more dynamic, and add proxies from the broker when the pool drops below the limit. Figure out how to make the collect limit more dynamic, or at least calculated based on the prepopulated limit. >>> collect_limit = max(self._maxsize - self.qsize(), 0) """ self.log.debug('Collecting Proxies') count = 0 collect_limit = 10000 # Arbitrarily high for now. async for proxy, created in self.broker.collect(save=True): evaluation = proxy.evaluate_for_pool() if evaluation.passed: await self.put(proxy) # Set Start Event on First Proxy Retrieved from Broker if self.start_event and not self.start_event.is_set(): self.start_event.set() self.log.debug('Setting Start Event', extra={ 'other': 'Broker Started Sending Proxies' }) if collect_limit and count == collect_limit: break count += 1 async def start(self, limit=None, confirmed=False): """ Retrieves proxies from the queue that is populated from the Broker and then puts these proxies in the prioritized heapq pool. Prepopulates proxies if the flag is set to put proxies that we previously saved into the pool. [x] TODO: --------- We are eventually going to need the relationship between prepopulation and collection to be more dynamic and adjust, and collection to trigger if we are running low on proxies. """ await super(BrokeredProxyManager, self).start(limit=limit, confirmed=confirmed) if settings.proxies.pool.collect: # Pool will set start event when it starts collecting proxies. await self.pool.collect() else: if self.start_event.is_set(): raise ProxyPoolError('Start Event Already Set') self.start_event.set() self.log.debug('Setting Start Event', extra={ 'other': 'Proxy Pool Prepopulated' })
class SmartProxyManager(BrokeredProxyManager): __NAME__ = "Smart Proxy Manager" log = logger.get(__name__, __NAME__) def __init__(self, loop, pool_cls, start_event=None): super(SmartProxyManager, self).__init__(loop, pool_cls, start_event=start_event) self.lock = asyncio.Lock() self.confirmed = ConfirmedQueue(self.pool, self.lock) self.hold = HoldQueue(self.pool, self.lock) self.confirmed.hold = self.hold self.hold.confirmed = self.confirmed async def start(self, **kwargs): await super(SmartProxyManager, self).start(**kwargs) self.log.info(f"Prepopulated {self.confirmed.num_proxies} Confirmed Proxies") @property def num_proxies(self): return (super(SmartProxyManager, self).num_proxies + self.confirmed.num_proxies + self.hold.num_proxies) async def on_proxy_error(self, proxy, err): """ Callback for case when request notices a response error. """ await super(SmartProxyManager, self).on_proxy_error(proxy, err) await self.put(proxy) async def on_proxy_success(self, proxy): """ There is a chance that the proxy is already in the good queue... our overridden put method handles that. """ await super(SmartProxyManager, self).on_proxy_success(proxy) await self.put(proxy) async def get(self): """ [x] NOTE: -------- Logging for pulling proxies out of queues is done in the individual queues themselves. """ proxy = await self.confirmed.get() if proxy: return proxy proxy = await self.hold.get() if proxy: return proxy return await super(SmartProxyManager, self).get() async def put(self, proxy): """ Determines whether or not proxy should be put in the Confirmed Queue, the Hold Queue or back in the general pool. If the proxy is not supposed to be held or confirmed, evaluates whether or not the proxy meets the specified standards before putting in the general pool. - If `last_request` is None, that means the put method is being called from prepopoulation and the proxy should either be put in the General Pool or the Confirmed Qeuue. - If `last_request` is not None, and the proxy queue_id designates it is in Confirmed Queue or Hold Queue, have Sub Queue handle the rest of the put method. - If `last_request` is not None and the queue_id does not designate the Hold Queue or the Confirmed Queue, put in General Pool. [x] NOTE: -------- There may be an edge case where the last request error is a timeout error but enough time has passed for it to be immediately usable again -> this just means that it will be pulled from Hold Queue faster than it otherwise would though. """ last_request = proxy.requests(-1, active=True) if last_request is None: assert proxy.queue_id is None if proxy.confirmed(): await self.confirmed.raise_if_present(proxy) await self.confirmed.put(proxy, prepopulation=True) else: # Maybe Limit Evaluation if Proxy was Ever Confirmed? await super(SmartProxyManager, self).put(proxy, prepopulation=True) else: assert proxy.queue_id is not None if proxy.queue_id == 'confirmed': await self.put_from_confirmed(proxy) elif proxy.queue_id == 'hold': await self.put_from_hold(proxy) else: await self.put_from_pool(proxy) async def put_from_confirmed(self, proxy): """ Takes a proxy that was taken from the Confirmed Queue and determines how to handle it based on the new request appended to the proxy's history. Since we do not remove proxies from the Confirmed Queue until they fail, we do not need to call `self.confirmed.put(proxy)` to maintain it, we just don't take it out. [x] NOTE: --------- Proxies in the manager only move up, and not down (with the exception of a proxy moving from confirmed to held). Once a proxy leaves the pool, to either go into the Hold Queue or the Confirmed Queue, leaving either one of those queues is an indication that we don't want to use it again, so we do not put it back in the General Pool. [x] NOTE: --------- Because proxies in the ConfirmedQueue are used simultaneously by multiple threads at the same time, and a proxy from the Confirmed Queue is likely to cause subsequent successful responses, it is likely that the proxy is already in the ConfirmedQueue. There seems to be more problems with the Hold Queue and threading race conditions than with the Confirmed Queue. [x] TODO: -------- Remove sanity checks `raise_if_` once we are more confident in operation of the manager. """ # We are temporarily removing proxies from the Confirmed Queue to test out # the timeout issues. # await self.confirmed.raise_if_missing(proxy) await self.hold.warn_if_present(proxy) last_request = proxy.requests(-1, active=True) # [x] TODO: Figure out a better way of handling this situation. # This check has to be done first: proxy can be confirmed over a horizon # but still have a more recent timeout error. if last_request.was_timeout_error: # Since we are removing from Confirmed right now, we don't have to # move it, just to put it in. await self.hold.put(proxy) # await self.confirmed.move_to_hold(proxy) # Proxy Still Confirmed Over Horizon - Dont Move Out Yet elif proxy.confirmed(): # Temporary if not last_request.confirmed: pass # errs = proxy.errors_in_horizon() # if len(errs) != 0: # if config['instattack']['log.logging']['log_proxy_queue']: # self.log.debug(f'Maintaining Proxy in {self.__NAME__}', extra={ # 'proxy': proxy, # 'data': { # 'Num Errors': len(errs), # } # }) else: # Proxy No Longer Confirmed -> Discard by Removing # Temporarily Removing from Confirmed on Get # await self.confirmed.remove(proxy) pass async def put_from_hold(self, proxy): """ Takes a proxy that was taken from the Hold Queue and determines how to handle it based on the new request appended to the proxy's history. [x] NOTE: --------- Since the proxy is already in the Hold Queue, the second to last request should be a timeout error, otherwise it would not have been sent to the Hold Queue to begin with. [x] NOTE: --------- Proxies in the manager only move up, and not down (with the exception of a proxy moving from confirmed to held). - If a proxy is in the Hold Queue and times out, but then returns a confirmed request, we move back up to the Confirmed Queue. - If a proxy is in the Hold Queue and returns an error, or times out, we discard, not move back to the General Pool. [x] TODO: -------- Remove sanity checks `raise_if_` once we are more confident in operation of the manager. """ # await self.hold.raise_if_present(proxy) # Racee Condition - Another thread might be you to it. # # Don't know why this is failing. # await self.confirmed.raise_if_present(proxy) # [x] TODO: # This Keeps Failing - Only thing I can think of is a Race Condition? # We will log warning for now, hopefully find bug. last_last_request = proxy.requests(-2, active=True) if not last_last_request.was_timeout_error: e = ProxyPoolError( f"Second to Last Request Should be Timeout Error, " f"Not {last_last_request.error}" ) self.log.warning(e) last_request = proxy.requests(-1, active=True) # Request Confirmed - Move from Hold to Confirmed ^ if last_request.confirmed: # Why were we moving it? Proxy was removed from hold, it's not # in there anymore... await self.confirmed.put(proxy) # await self.hold.move_to_confirmed(proxy) # Another Timeout Error - Increment Timeout and Check Max elif last_request.was_timeout_error: if last_request.error == last_last_request.error: try: proxy.increment_timeout(last_request.error) # Proxy Maxes Out -> Discard # Should we maybe limit this to discarding only proxies that don't # have any recent confirmations? except ProxyMaxTimeoutError as e: self.log.info(e) proxy.reset_timeout(last_request.error) pass else: # Typical Race Conditions w Hold Queue await self.hold.safe_put(proxy) else: # Typical Race Conditions w Hold Queue await self.hold.safe_put(proxy) else: # Proxy No Longer Holdable -> Discard pass async def put_from_pool(self, proxy): """ Takes a proxy that is currently in the General Pool and determines how to handle it based on the new request appended to the proxy's history. This involves either: (1) Putting proxy in Confirmed Queue if it resulted in a successful response. (2) Putting proxy in Hold Queue if it resulted in a timeout error. (3) Putting back in pool. [x] TODO: -------- Since we do not return proxies from the Confirmed Queue or Hold Queue back to the General Pool, should we discard proxies that have errors after being taken out of General Pool? For the above, the evaluation will determine whether or not the proxy should stay in the General Pool, but this is a little counter-intuitive, since we don't apply that same evaluation logic to determine whether or not to keep the proxy after it fails in the Confirmed Queue or Hold Queue. [x] TODO: Race Condition -------- This is unusual: >>> await self.hold.warn_if_present(proxy) This means that a proxy in the pool is already in the Hold Queue - did some other thread already put it in there? Was it not fully removed from the Hold Queue? """ await self.hold.warn_if_present(proxy) await self.confirmed.raise_if_present(proxy) last_request = proxy.requests(-1, active=True) if last_request.confirmed: await self.confirmed.put(proxy) else: # [x] NOTE: There really shouldn't be any confirmed proxies in the # general pool unless the immediate last request was confirmed. Once # confirmed proxies leave the general pool, they stay out. if proxy.confirmed(): raise ProxyPoolError( f"Should Not be Confirmed Proxy in {self.pool.__NAME__}") if last_request.was_timeout_error: # Typical Race Conditions w Hold Queue await self.hold.safe_put(proxy) else: await super(SmartProxyManager, self).put(proxy)
class SimpleProxyManager(ProxyManagerInterface): __NAME__ = "Simple Proxy Manager" log = logger.get(__name__, __NAME__) def __init__(self, loop, pool_cls): self.loop = loop self.pool = pool_cls(loop) self.original_num_proxies = 0 async def start(self, limit=None, confirmed=False): """ Retrieves proxies from the queue that is populated from the Broker and then puts these proxies in the prioritized heapq pool. Prepopulates proxies if the flag is set to put proxies that we previously saved into the pool. [x] TODO: --------- We are eventually going to need the relationship between prepopulation and collection to be more dynamic and adjust, and collection to trigger if we are running low on proxies. """ self.log.debug('Prepopulating Proxies') filter = {} if confirmed: filter = {'confirmed': True} for proxy in await Proxy.filter(**filter).all(): if limit and self.num_proxies == limit: break await self.put(proxy) self.original_num_proxies += 1 if self.num_proxies == 0: self.log.error('No Proxies to Prepopulate') return self.log.info(f"Prepopulated {self.num_proxies} Proxies") async def stop(self): pass async def get(self): return await self.pool.get() async def put(self, proxy, **kwargs): await self.pool.put(proxy, **kwargs) async def on_proxy_error(self, proxy, exc): """ For training proxies, we only care about storing the state variables on the proxy model, and we do not need to put the proxy back in the pool, or a designated pool. """ if settings.logging.log_request_errors: self.log.error(exc, extra={'proxy': proxy}) req = ProxyRequest( error=exc.__subtype__, status_code=exc.status_code, ) proxy.add_failed_request(req) async def on_proxy_success(self, proxy): """ For training proxies, we only care about storing the state variables on the proxy model, and we do not need to put the proxy back in the pool, or a designated pool. """ req = ProxyRequest() proxy.add_successful_request(req)
import asyncio import aiohttp from instattack import settings from instattack.lib import logger from instattack.lib.utils import limit_as_completed, cancel_remaining_tasks from instattack.core.exceptions import TokenNotFound, PoolNoProxyError from instattack.core.proxies import SmartProxyManager, ManagedProxyPool from .base import AbstractRequestHandler from .client import instagram_client log = logger.get(__name__, 'Login Handler') class AbstractLoginHandler(AbstractRequestHandler): __client__ = instagram_client def __init__(self, *args, **kwargs): super(AbstractLoginHandler, self).__init__(*args, **kwargs) self.user = self.loop.user self.attempts_to_save = [] self._cookies = None self._token = None @property def token(self):
The above is problematic if there is a client disconnect. To remedy: (1) For client disconnections/fighting against Task cancellation I would recommend asyncio.shield. That's why it exists (2) In case user wants a way to control tasks in a more granular way, then I would recommend aiojobs (3) Ofc if user wants to execute background tasks (inside the same loop) I would also recommend aiojobs """ __all__ = ('AttackHandler', ) log = logger.get(__name__, 'Attack Handler') class AttackHandler(AbstractLoginHandler): __name__ = 'Attack Handler' __proxy_manager__ = SmartProxyManager __proxy_pool__ = ManagedProxyPool def __init__(self, *args, **kwargs): super(AttackHandler, self).__init__(*args, **kwargs) self.passwords = asyncio.Queue() self.num_passwords = 0 self.stop_event = asyncio.Event()
class ConfirmedQueue(ProxyQueue): __NAME__ = 'Confirmed Queue' log = logger.get(__name__, __NAME__) __queueid__ = 'confirmed' def __init__(self, pool, lock): super(ConfirmedQueue, self).__init__(lock) self.pool = pool self.rotating_index = 0 self.hold = None self.times_used = collections.Counter() self.mapped = {} def validate_for_queue(self, proxy): return True def raise_for_queue(self, proxy): """ Validates whether or not the proxy is allowed to be in the queue. For the Confirmed Queue, we do have to do this on get() and put(), since proxies are not completely removed from the queue in the get() method. """ if not proxy.confirmed(): raise ProxyPoolError(f"Found Unconfirmed Proxy in {self.__NAME__}") # THIS CAN HAPPEN # If one thread times out with a proxy, it will set it's value to a timeout # value before it is necessarily removed from the confirmed queue. # last_request = proxy.last_request(active=True) # if last_request and last_request.was_timeout_error: # raise ProxyPoolError(f"Found Holdable Proxy in {self.__NAME__}") async def get(self): """ Retrieves a proxy from the queue and removes it from the queue after a certain number of threads have retrieved it. This allows multiple threads access to the same proxy, but does not get stuck maxing out with timeout errors due to a large number of threads simultaneously using the same proxy. """ try: proxy = await self._get_proxy() except QueueEmpty as e: self.log.warning(e) return None else: self.raise_for_queue(proxy) # Temporarily Removing Proxies from Confirmed Queue: # Issue is that if a proxy from the confirmed queue raises a too many # requests exception, it is likely being used at the same time for # several requests, which causes it to timeout immediately. # await self.remove(proxy) return proxy def _delete_nth(self, n): self._queue.rotate(-n) self._queue.popleft() self._queue.rotate(n) def _get_nth(self, n): self._queue.rotate(-n) proxy = self._queue.popleft() self._queue.rotate(n) return proxy async def remove(self, proxy): """ [x] Note: --------- Because proxies in the ConfirmedQueue are used simultaneously by multiple threads at the same time (not the case for HeldQueue), it is possible that the proxy is already removed from the ConfirmedQueue by the time another thread determines it should be removed. """ async with self.lock: if proxy not in self._queue: raise ProxyPoolError( f'Cannot Remove Proxy from {self.__NAME__}', extra={'proxy': proxy}) ind = self._queue.index(proxy) self._delete_nth(ind) del self.times_used[proxy.id] del self.mapped[proxy.id] async def _get_proxy(self): """ [x] TODO: -------- There may be a smarter way to do this, that involves staggering the retrieval of the same proxy with asyncio.sleep() based on the number of times it was already pulled out. """ async with self.lock: least_common = self.times_used.most_common() if not least_common: raise QueueEmpty(self) proxy_id, count = least_common[:-2:-1][0] proxy = self.mapped[proxy_id] # As proxies are confirmed and put back in, this might cause issues, # since there may wind up being more proxies than the limit of giving # out. We will have to decrement the count when a proxy is put back # in. if count >= MAX_CONFIRMED_PROXIES: raise ProxyPoolError('Count %s exceeds %s.' % (count, MAX_CONFIRMED_PROXIES)) # As proxies are confirmed and put back in, if count == MAX_CONFIRMED_PROXIES - 1: await self.remove(proxy) if settings.logging.log_proxy_queue: self.log.debug( f'Returning & Removing Proxy from {self.__NAME__}', extra={ 'data': { 'Times Used': f"{self.times_used[proxy.id] or 0} (Last Allowed)", f'{self.__NAME__} Size': self.qsize(), }, 'proxy': proxy, }) else: if settings.logging.log_proxy_queue: self.log.debug(f'Returning Proxy from {self.__NAME__}', extra={ 'data': { 'Times Used': self.times_used[proxy.id] or 0, f'{self.__NAME__} Size': self.qsize(), }, 'proxy': proxy, }) self.times_used[proxy.id] += 1 return proxy async def move_to_hold(self, proxy): last_request = proxy.last_request(active=True) assert last_request.was_timeout_error await self.remove(proxy) await self.hold.put(proxy) if settings.logging.log_proxy_queue: self.log.debug( f'Moving Proxy from {self.__NAME__} to {self.hold.__NAME__}', extra={ 'data': { 'Last Request': last_request.error, f"{self.__NAME__} Size": self.qsize(), f"{self.hold.__NAME__} Size": self.hold.qsize(), }, 'proxy': proxy, }) async def put(self, proxy, prepopulation=False): """ [x] TODO: -------- Depending on treatment of Confirmed Queue when proxies exist in queue already, this might be able to moved to base class. [x] Note: --------- Because proxies in the ConfirmedQueue are used simultaneously by multiple threads at the same time, and a proxy from the ConfirmedQueue is likely to cause subsequent successful responses, it is likely that the proxy is already in the ConfirmedQueue. This means we have to check before we put in. """ async with self.lock: # This might happen a lot because confirmed proxies are not removed # from the queue! if await self.contains(proxy): raise ProxyPoolError(f'Cannot Add Proxy to {self.__NAME__}') if proxy.id in self.times_used: raise ProxyPoolError('Did not expect proxy to be in count.') # Have to initialize so that the _get_proxy() method can find it. self.times_used[proxy.id] = 0 self.mapped[proxy.id] = proxy await super(ConfirmedQueue, self).put(proxy, prepopulation=prepopulation)
class HoldQueue(ProxyQueue): __NAME__ = 'Hold Queue' log = logger.get(__name__, __NAME__) __queueid__ = 'hold' def __init__(self, pool, lock): super(HoldQueue, self).__init__(lock) self.pool = pool self.confirmed = None def validate_for_queue(self, proxy): return True def raise_for_queue(self, proxy): """ Validates whether or not the proxy is allowed to be in the queue. For the Hold Queue, we do not have to do this on get(), since we are removing the proxy in the put() method, vs. the Confirmed Queue, where the proxy stays in the Queue, so has to be validated on get() as well. """ # Proxy can be confirmed over horizon even if the most recent error is a # a timeout error, so we cannot do this: # >>> if proxy.confirmed_over_threshold_in_horizon(): # >>> raise ProxyPoolError("Hold Queue: Found Confirmed Proxy") # Most Recent Request Confirmed -> Always Should be in Confirmed Queue last_request = proxy.last_request(active=True) if last_request.confirmed: raise ProxyPoolError(f"Found Confirmed Proxy in {self.__NAME__}") if not last_request.was_timeout_error: raise ProxyPoolError( f"Found Non Holdable Proxy in {self.__NAME__}") async def get(self): async with self.lock: for proxy in self._queue: # This raise might be overkill? self.raise_for_queue(proxy) # Do we need to call self.recycle() at some point? Do we run # the risk of proxies getting stuck in here after their timeouts # have passed? last_request = proxy.last_request(active=True) self.log.critical( f'Checking if Hold Proxy OK {proxy.time_since_used} > {proxy.timeout(last_request.error)}?', extra={'proxy': proxy}) # noqa if proxy.time_since_used > proxy.timeout(last_request.error): # await self.remove(proxy) self.log.critical('Hold Proxy Ok') self._queue.remove(proxy) return proxy if len(self._queue) == 0: self.log.info(f'No Proxy Ready for Use in {self.__NAME__}') return None async def move_to_confirmed(self, proxy): await self.remove(proxy) await self.confirmed.put(proxy) if settings.logging.log_proxy_queue: self.log.debug( f'Moving Proxy from {self.__NAME__} to {self.confirmed.__NAME__}', extra={ 'data': { f"{self.__NAME__} Size": self.qsize(), f"{self.confirmed.__NAME__} Size": self.confirmed.qsize(), }, 'proxy': proxy, }) async def move_to_pool(self, proxy): await self.remove(proxy) await self.pool.put(proxy) if settings.logging.log_proxy_queue: self.log.debug( f'Moving Proxy from {self.__NAME__} to {self.pool.__NAME__}', extra={ 'data': { f"{self.__NAME__} Size": self.qsize(), f"{self.pool.__NAME__} Size": self.pool.qsize(), }, 'proxy': proxy, }) # async def recycle(self): # """ # Removes proxies from the hold that are no longer required to be in # hold. If the proxy has been confirmed to have a successful request, the # proxy is put in good, otherwise, the proxy is put back in the pool. # """ # async with self.lock: # for proxy in self._queue: # if proxy.hold(): # continue # # Should we use the historical confirmed value or just the last request # # confirmed value? # if proxy.confirmed: # await self.confirmed.put(proxy) # else: # await self.pool.put(proxy, evaluate=False) async def put(self, proxy): """ [x] TODO: -------- Depending on treatment of Confirmed Queue when proxies exist in queue already, this might be able to moved to base class. [x] Note: --------- Proxies in Hold Queue are not used by multiple threads simultaneously, so when one thread determines that the proxy should be put in the Hold Queue, it should not already be in there. """ async with self.lock: if await self.contains(proxy): raise ProxyPoolError(f'Cannot Add Proxy to {self.__NAME__}') await super(HoldQueue, self).put(proxy)
from dacite import from_dict import tortoise from tortoise import fields from tortoise.models import Model from instattack import settings from instattack.lib import logger from instattack.core.exceptions import ProxyMaxTimeoutError from .evaluation import evaluate from .mixins import ProxyMetrics, allow_exception_input log = logger.get(__name__, subname='Proxy') @dataclass class ProxyRequest: date: datetime = field(init=False) error: str = None status_code: int = None def __post_init__(self): self.date = datetime.now() @property def confirmed(self): return self.error is None