def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop, nstag=args.prefix)
self.jobs = {}
if self.args.log_dir:
os.makedirs(self.args.log_dir, exist_ok=True)
def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop)
self.locations = set(self.proxies.locations.keys())
self.output_dir = create_output_subdir(args.output)
self.jobs = {}
class RunProgramClient:
def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop, nstag=args.prefix)
self.jobs = {}
if self.args.log_dir:
os.makedirs(self.args.log_dir, exist_ok=True)
@asyncio.coroutine
def proxy_online(self, proxy):
self.jobs[proxy.loc] = \
self.loop.create_task(run_program_for_location(
proxy, self.args.program, self.args.log_dir or "."))
@asyncio.coroutine
def proxy_offline(self, proxy):
job = self.jobs.get(proxy.loc)
if job is not None:
del self.jobs[proxy.loc]
job.cancel()
# swallow cancellation exception
try:
yield from asyncio.wait_for(job)
except:
pass
@asyncio.coroutine
def run(self):
yield from self.proxies.run(self)
if self.jobs:
yield from asyncio.wait(self.jobs, loop=self.loop)
class TracerouteClient:
def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop)
self.locations = set(self.proxies.locations.keys())
self.output_dir = create_output_subdir(args.output)
self.jobs = {}
@asyncio.coroutine
def proxy_online(self, proxy):
self.jobs[proxy.loc] = \
self.loop.create_task(process_jobs_for_location(
proxy, proxy.loc, self.args.destinations, self.output_dir))
@asyncio.coroutine
def proxy_offline(self, proxy):
job = self.jobs.get(proxy.loc)
if job is not None:
del self.jobs[proxy.loc]
job.cancel()
# swallow cancellation exception
try: yield from asyncio.wait_for(job)
except: pass
@asyncio.coroutine
def run(self):
yield from self.proxies.run(self)
if self.jobs:
yield from asyncio.wait(self.jobs, loop=self.loop)
class RunProgramClient:
def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop, nstag=args.prefix)
self.jobs = {}
if self.args.log_dir:
os.makedirs(self.args.log_dir, exist_ok=True)
@asyncio.coroutine
def proxy_online(self, proxy):
self.jobs[proxy.loc] = \
self.loop.create_task(run_program_for_location(
proxy, self.args.program, self.args.log_dir or "."))
@asyncio.coroutine
def proxy_offline(self, proxy):
job = self.jobs.get(proxy.loc)
if job is not None:
del self.jobs[proxy.loc]
job.cancel()
# swallow cancellation exception
try: yield from asyncio.wait_for(job)
except: pass
@asyncio.coroutine
def run(self):
yield from self.proxies.run(self)
if self.jobs:
yield from asyncio.wait(self.jobs, loop=self.loop)
class TracerouteClient:
def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop)
self.locations = set(self.proxies.locations.keys())
self.output_dir = create_output_subdir(args.output)
self.jobs = {}
@asyncio.coroutine
def proxy_online(self, proxy):
self.jobs[proxy.loc] = \
self.loop.create_task(process_jobs_for_location(
proxy, proxy.loc, self.args.destinations, self.output_dir))
@asyncio.coroutine
def proxy_offline(self, proxy):
job = self.jobs.get(proxy.loc)
if job is not None:
del self.jobs[proxy.loc]
job.cancel()
# swallow cancellation exception
try:
yield from asyncio.wait_for(job)
except:
pass
@asyncio.coroutine
def run(self):
yield from self.proxies.run(self)
if self.jobs:
yield from asyncio.wait(self.jobs, loop=self.loop)
def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop)
self.locations = set(self.proxies.locations.keys())
self.output_dir = create_output_subdir(args.output)
self.jobs = {}
def __init__(self, args, loop=None):
if loop is None: loop = asyncio.get_event_loop()
self.args = args
self.loop = loop
self.proxies = ProxySet(args, loop=loop, nstag=args.prefix)
self.jobs = {}
if self.args.log_dir:
os.makedirs(self.args.log_dir, exist_ok=True)
def __init__(self, args):
self.args = args
self.loop = asyncio.get_event_loop()
self.global_bound = asyncio.Semaphore(args.total_workers)
self.workers = {}
self.active = {}
self.output_queue = asyncio.Queue()
self.drainer = self.loop.create_task(
output_queue_drainer(self.output_queue))
self.proxies = ProxySet(args,
nstag="cap",
loop=self.loop,
proxy_sort_key=self.proxy_sort_key)
run = 1
while True:
try:
path = os.path.join(self.args.output_dir, str(run))
os.makedirs(path)
self.output_dir = path
break
except FileExistsError:
run += 1
continue
with open(self.args.urls) as f:
urls = [l for l in (ll.strip() for ll in f) if l and l[0] != '#']
random.shuffle(urls)
urls = list(enumerate(urls))
self.workers = {
loc: CaptureWorker(self.output_dir, loc, urls[:], self.loop,
self.args.workers_per_loc, self.global_bound,
self.output_queue, self.args.quiet)
for loc in self.proxies.locations.keys()
}
def __init__(self, args):
self.args = args
self.loop = asyncio.get_event_loop()
self.global_bound = asyncio.Semaphore(args.total_workers)
self.workers = {}
self.active = {}
self.output_queue = asyncio.Queue()
self.drainer = self.loop.create_task(
output_queue_drainer(self.output_queue))
self.proxies = ProxySet(args,
nstag="cap",
loop=self.loop,
proxy_sort_key=self.proxy_sort_key)
run = 1
while True:
try:
path = os.path.join(self.args.output_dir, str(run))
os.makedirs(path)
self.output_dir = path
break
except FileExistsError:
run += 1
continue
with open(self.args.urls) as f:
urls = [l for l in (ll.strip() for ll in f)
if l and l[0] != '#']
random.shuffle(urls)
urls = list(enumerate(urls))
self.workers = {
loc: CaptureWorker(self.output_dir, loc, urls[:],
self.loop, self.args.workers_per_loc,
self.global_bound, self.output_queue,
self.args.quiet)
for loc in self.proxies.locations.keys()
}
class CaptureDispatcher:
def __init__(self, args):
self.args = args
self.loop = asyncio.get_event_loop()
self.global_bound = asyncio.Semaphore(args.total_workers)
self.workers = {}
self.active = {}
self.output_queue = asyncio.Queue()
self.drainer = self.loop.create_task(
output_queue_drainer(self.output_queue))
self.proxies = ProxySet(args,
nstag="cap",
loop=self.loop,
proxy_sort_key=self.proxy_sort_key)
run = 1
while True:
try:
path = os.path.join(self.args.output_dir, str(run))
os.makedirs(path)
self.output_dir = path
break
except FileExistsError:
run += 1
continue
with open(self.args.urls) as f:
urls = [l for l in (ll.strip() for ll in f)
if l and l[0] != '#']
random.shuffle(urls)
urls = list(enumerate(urls))
self.workers = {
loc: CaptureWorker(self.output_dir, loc, urls[:],
self.loop, self.args.workers_per_loc,
self.global_bound, self.output_queue)
for loc in self.proxies.locations.keys()
}
def proxy_sort_key(self, loc, method):
# Consider locales with more work to do first.
# Consider locales whose proxy is 'direct' first.
# Consider locales named 'us' first.
# As a final tie breaker use alphabetical order of locale name.
return (-len(self.workers[loc].urls),
method != 'direct',
loc != 'us',
loc)
@asyncio.coroutine
def run(self):
yield from self.proxies.run(self)
if self.active:
yield from asyncio.wait(self.active, loop=self.loop)
yield from self.output_queue.put(None)
yield from asyncio.wait_for(self.drainer, None)
@asyncio.coroutine
def proxy_online(self, proxy):
self.active[proxy.loc] = \
self.loop.create_task(self.workers[proxy.loc].run(proxy))
@asyncio.coroutine
def proxy_offline(self, proxy):
job = self.active.get(proxy.loc)
if job is not None:
del self.active[proxy.loc]
job.cancel()
# swallow the cancellation exception
try: yield from asyncio.wait_for(job, None)
except: pass
class CaptureDispatcher:
def __init__(self, args):
self.args = args
self.loop = asyncio.get_event_loop()
self.workers = {}
self.active = {}
self.output_queue = asyncio.Queue()
self.drainer = self.loop.create_task(
output_queue_drainer(self.output_queue))
self.proxies = ProxySet(args,
nstag="cap",
loop=self.loop,
proxy_sort_key=self.proxy_sort_key)
run = 1
while True:
try:
path = os.path.join(self.args.output_dir, str(run))
os.makedirs(path)
self.output_dir = path
break
except FileExistsError:
run += 1
continue
with open(self.args.urls) as f:
urls = [l for l in (ll.strip() for ll in f) if l and l[0] != '#']
random.shuffle(urls)
urls = list(enumerate(urls))
self.workers = {
loc: CaptureWorker(self.output_dir, loc, urls[:], self.loop,
self.args.workers_per_loc, self.output_queue,
self.args.quiet)
for loc in self.proxies.locations.keys()
}
def proxy_sort_key(self, loc, method):
# Consider locales with more work to do first.
# Consider locales whose proxy is 'direct' first.
# Consider locales named 'us' first.
# As a final tie breaker use alphabetical order of locale name.
return (-len(self.workers[loc].urls), method != 'direct', loc != 'us',
loc)
@asyncio.coroutine
def run(self):
with BrowserManager(...) as bmgr:
self.bmgr = bmgr
yield from self.proxies.run(self)
if self.active:
yield from asyncio.wait(self.active, loop=self.loop)
yield from self.output_queue.put(None)
yield from asyncio.wait_for(self.drainer, None)
@asyncio.coroutine
def proxy_online(self, proxy):
self.active[proxy.loc] = self.loop.create_task(
self.workers[proxy.loc].run(self.bmgr, proxy))
@asyncio.coroutine
def proxy_offline(self, proxy):
job = self.active.get(proxy.loc)
if job is not None:
del self.active[proxy.loc]
job.cancel()
# swallow the cancellation exception
try:
yield from asyncio.wait_for(job, None)
except:
pass