def tor_bootstrap() -> None:
"""Bootstrap wrapper for Tor.
The function will bootstrap the Tor proxy. It will retry for
:data:`~darc.proxy.tor.TOR_RETRY` times in case of failure.
Also, it will **NOT** re-bootstrap the proxy as is guaranteed by
:data:`~darc.proxy.tor._TOR_BS_FLAG`.
Warns:
TorBootstrapFailed: If failed to bootstrap Tor proxy.
See Also:
* :func:`darc.proxy.tor._tor_bootstrap`
* :data:`darc.proxy.tor.TOR_RETRY`
* :data:`darc.proxy.tor._TOR_BS_FLAG`
"""
# don't re-bootstrap
if _TOR_BS_FLAG:
return
logger.info('-*- Tor Bootstrap -*-')
for _ in range(TOR_RETRY + 1):
try:
_tor_bootstrap()
break
except Exception:
if DEBUG:
logger.ptb('[Error bootstraping Tor proxy]')
logger.pexc(LOG_WARNING,
category=TorBootstrapFailed,
line='tor_bootstrap()')
logger.pline(LOG_INFO, logger.horizon)
def zeronet_bootstrap() -> None:
"""Bootstrap wrapper for ZeroNet.
The function will bootstrap the ZeroNet proxy. It will retry for
:data:`~darc.proxy.zeronet.ZERONET_RETRY` times in case of failure.
Also, it will **NOT** re-bootstrap the proxy as is guaranteed by
:data:`~darc.proxy.zeronet._ZERONET_BS_FLAG`.
Warns:
ZeroNetBootstrapFailed: If failed to bootstrap ZeroNet proxy.
Raises:
:exc:`UnsupportedPlatform`: If the system is not supported, i.e. not macOS or Linux.
See Also:
* :func:`darc.proxy.zeronet._zeronet_bootstrap`
* :data:`darc.proxy.zeronet.ZERONET_RETRY`
* :data:`darc.proxy.zeronet._ZERONET_BS_FLAG`
"""
# don't re-bootstrap
if _ZERONET_BS_FLAG:
return
logger.info('-*- ZeroNet Bootstrap -*-')
for _ in range(ZERONET_RETRY+1):
try:
_zeronet_bootstrap()
break
except Exception:
if DEBUG:
logger.ptb('[Error bootstraping ZeroNet proxy]')
logger.pexc(LOG_WARNING, category=ZeroNetBootstrapFailed, line='zeronet_bootstrap()')
logger.pline(LOG_INFO, logger.horizon)
def launch_freenet() -> 'Popen[bytes]':
"""Launch Freenet process.
See Also:
This function mocks the behaviour of :func:`stem.process.launch_tor`.
"""
pidfile = os.path.join(FREENET_PATH, 'Freenet.pid')
with contextlib.suppress(OSError):
os.remove(pidfile)
zeronet_process = None
try:
zeronet_process = subprocess.Popen( # pylint: disable=consider-using-with # nosec
_FREENET_ARGS,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
def timeout_handlet(signum: 'Optional[Union[int, Signals]]' = None,
frame: 'Optional[FrameType]' = None) -> 'NoReturn':
raise OSError('reached a %i second timeout without success' %
BS_WAIT)
signal.signal(signal.SIGALRM, timeout_handlet)
signal.setitimer(signal.ITIMER_REAL, BS_WAIT)
while True:
init_line = cast('IO[bytes]',
zeronet_process.stdout).readline().decode(
'utf-8', 'replace').strip()
logger.pline(LOG_VERBOSE, init_line)
if not init_line:
if (code :=
zeronet_process.returncode) is not None and code == 0:
return zeronet_process
raise OSError(f'Process terminated: Timed out [{code}]')
if os.path.exists(pidfile):
pid = getpid(pidfile)
time.sleep(1) # wait a little bit
if psutil.pid_exists(pid):
return zeronet_process
except BaseException:
if zeronet_process is not None:
zeronet_process.kill() # don't leave a lingering process
zeronet_process.wait()
raise
finally:
signal.alarm(0) # stop alarm
def freenet_bootstrap() -> None:
"""Bootstrap wrapper for Freenet.
The function will bootstrap the Freenet proxy. It will retry for
:data:`~darc.proxy.freenet.FREENET_RETRY` times in case of failure.
Also, it will **NOT** re-bootstrap the proxy as is guaranteed by
:data:`~darc.proxy.freenet._FREENET_BS_FLAG`.
Warns:
FreenetBootstrapFailed: If failed to bootstrap Freenet proxy.
Raises:
:exc:`UnsupportedPlatform`: If the system is not supported, i.e. not macOS or Linux.
See Also:
* :func:`darc.proxy.freenet._freenet_bootstrap`
* :data:`darc.proxy.freenet.FREENET_RETRY`
* :data:`darc.proxy.freenet._FREENET_BS_FLAG`
"""
if _unsupported:
raise UnsupportedPlatform(f'unsupported system: {platform.system()}')
# don't re-bootstrap
if _FREENET_BS_FLAG:
return
logger.info('-*- Freenet Bootstrap -*-')
for _ in range(FREENET_RETRY + 1):
try:
_freenet_bootstrap()
break
except Exception:
if DEBUG:
logger.ptb('[Error bootstraping Freenet proxy]')
logger.pexc(LOG_WARNING,
category=FreenetBootstrapFailed,
line='freenet_bootstrap()')
logger.pline(LOG_INFO, logger.horizon)
def launch_i2p() -> 'Popen[bytes]':
"""Launch I2P process.
See Also:
This function mocks the behaviour of :func:`stem.process.launch_tor`.
"""
i2p_process = None
try:
i2p_process = subprocess.Popen( # pylint: disable=consider-using-with # nosec
_I2P_ARGS, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
)
def timeout_handlet(signum: 'Optional[Union[int, Signals]]' = None,
frame: 'Optional[FrameType]' = None) -> 'NoReturn':
raise OSError('reached a %i second timeout without success' % BS_WAIT)
signal.signal(signal.SIGALRM, timeout_handlet)
signal.setitimer(signal.ITIMER_REAL, BS_WAIT)
while True:
init_line = cast(
'IO[bytes]', i2p_process.stdout
).readline().decode('utf-8', 'replace').strip()
logger.pline(LOG_VERBOSE, init_line)
if not init_line:
raise OSError('Process terminated: Timed out')
if 'running: PID:' in init_line:
return i2p_process
if 'I2P Service is already running.' in init_line:
return i2p_process
except BaseException:
if i2p_process is not None:
i2p_process.kill() # don't leave a lingering process
i2p_process.wait()
raise
finally:
signal.alarm(0) # stop alarm
def main(argv: 'Optional[List[str]]' = None) -> int:
"""Entrypoint.
Args:
argv: Optional command line arguments.
Returns:
Exit code.
"""
parser = get_parser()
args = parser.parse_args(argv)
pid = os.getpid()
with open(PATH_ID, 'w') as file:
print(pid, file=file)
# wait for Redis
if _WAIT_REDIS:
if not FLAG_DB:
_redis_command('set', 'darc', pid)
if FLAG_DB:
while True:
try:
with DB:
_db_operation(DB.create_tables, [
HostnameQueueModel,
RequestsQueueModel,
SeleniumQueueModel,
])
except Exception:
logger.pexc(LOG_WARNING,
category=DatabaseOperaionFailed,
line='DB.create_tables([HostnameQueueModel, ...]')
continue
break
if SAVE_DB:
while True:
try:
with DB_WEB:
_db_operation(DB_WEB.create_tables, [
HostnameModel,
URLModel,
URLThroughModel,
RobotsModel,
SitemapModel,
HostsModel,
RequestsModel,
RequestsHistoryModel,
SeleniumModel,
])
except Exception:
logger.pexc(LOG_WARNING,
category=DatabaseOperaionFailed,
line='DB.create_tables([HostnameModel, ...]')
continue
break
logger.debug('-*- Initialisation -*-')
if DEBUG and not FLAG_DB:
# nuke the db
_redis_command('delete', 'queue_hostname')
_redis_command('delete', 'queue_requests')
_redis_command('delete', 'queue_selenium')
link_list = []
for link in filter(
None, map(lambda s: s.strip(),
args.link)): # type: ignore[name-defined,var-annotated]
logger.pline(LOG_DEBUG, link)
link_list.append(link)
if args.file is not None:
for path in args.file:
with open(path) as file:
for line in filter(None, map(lambda s: s.strip(), file)):
if line.startswith('#'):
continue
logger.pline(LOG_DEBUG, line)
link_list.append(line)
# write to database
link_pool = [parse_link(link, backref=None) for link in link_list]
save_requests(link_pool, score=0, nx=True)
logger.pline(LOG_DEBUG, logger.horizon)
# init link file
if not os.path.isfile(PATH_LN):
with open(PATH_LN, 'w') as file:
print('proxy,scheme,host,hash,link', file=file)
try:
process(args.type)
except BaseException:
traceback.print_exc()
_exit()
return 0
if _system in ['Linux', 'Darwin']:
_FREENET_ARGS = [
'su', '-', DARC_USER,
os.path.join(FREENET_PATH, 'run.sh'), 'start'
]
else:
_unsupported = True
_FREENET_ARGS = []
else:
_FREENET_ARGS = [os.path.join(FREENET_PATH, 'run.sh'), 'start']
_FREENET_ARGS.extend(FREENET_ARGS)
if _unsupported:
if DEBUG:
logger.debug('-*- FREENET PROXY -*-')
logger.pline(LOG_ERROR, 'unsupported system: %s', platform.system())
logger.pline(LOG_DEBUG, logger.horizon)
else:
logger.plog(LOG_DEBUG, '-*- FREENET PROXY -*-', object=_FREENET_ARGS)
def launch_freenet() -> 'Popen[bytes]':
"""Launch Freenet process.
See Also:
This function mocks the behaviour of :func:`stem.process.launch_tor`.
"""
pidfile = os.path.join(FREENET_PATH, 'Freenet.pid')
with contextlib.suppress(OSError):
os.remove(pidfile)