def load_requests(check: bool = CHECK) -> 'List[Link]':
"""Load link from the :mod:`requests` database.
Args:
check: If perform checks on loaded links,
default to :data:`~darc.const.CHECK`.
Returns:
List of loaded links from the :mod:`requests` database.
Note:
At runtime, the function will load links with maximum number
at :data:`~darc.db.MAX_POOL` to limit the memory usage.
See Also:
* :func:`darc.db._load_requests_db`
* :func:`darc.db._load_requests_redis`
"""
if FLAG_DB:
with database.connection_context():
try:
link_pool = _load_requests_db()
except Exception:
logger.pexc(LOG_WARNING,
category=DatabaseOperaionFailed,
line='_load_requests_db()')
link_pool = []
else:
link_pool = _load_requests_redis()
if check:
link_pool = _check(link_pool)
logger.plog(LOG_VERBOSE,
'-*- [REQUESTS] LINK POOL -*-',
object=sorted(link.url for link in link_pool))
return link_pool
ZERONET_RETRY = int(os.getenv('ZERONET_RETRY', '3'))
# ZeroNet project path
ZERONET_PATH = os.getenv('ZERONET_PATH', '/usr/local/src/zeronet')
# manage ZeroNet through darc?
_MNG_ZERONET = bool(int(os.getenv('DARC_ZERONET', '1')))
# ZeroNet bootstrapped flag
_ZERONET_BS_FLAG = not _MNG_ZERONET
# ZeroNet daemon process
_ZERONET_PROC = None
# ZeroNet bootstrap args
_ZERONET_ARGS = [os.path.join(ZERONET_PATH, 'ZeroNet.sh'), 'main']
_ZERONET_ARGS.extend(ZERONET_ARGS)
logger.plog(LOG_DEBUG, '-*- ZERONET PROXY -*-', object=_ZERONET_ARGS)
def launch_zeronet() -> 'Popen[bytes]':
"""Launch ZeroNet process.
See Also:
This function mocks the behaviour of :func:`stem.process.launch_tor`.
"""
zeronet_process = None
try:
zeronet_process = subprocess.Popen( # pylint: disable=consider-using-with # nosec
_ZERONET_ARGS, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
)
sys.exit('please specify a non-root user as DARC_USER')
# data storage
PATH_DB = os.path.abspath(os.getenv('PATH_DATA', 'data'))
PATH_MISC = os.path.join(PATH_DB, 'misc')
os.makedirs(PATH_MISC, exist_ok=True)
# link file mapping
PATH_LN = os.path.join(PATH_DB, 'link.csv')
# PID file
PATH_ID = os.path.join(PATH_DB, 'darc.pid')
# extract link pattern
_LINK_WHITE_LIST = json.loads(os.getenv('LINK_WHITE_LIST', '[]'))
logger.plog(LOG_DEBUG, '-*- LINK WHITE LIST -*-', object=_LINK_WHITE_LIST)
LINK_WHITE_LIST = [re.compile(link, re.IGNORECASE) for link in _LINK_WHITE_LIST]
del _LINK_WHITE_LIST
# link black list
_LINK_BLACK_LIST = json.loads(os.getenv('LINK_BLACK_LIST', '[]'))
logger.plog(LOG_DEBUG, '-*- LINK BLACK LIST -*-', object=_LINK_BLACK_LIST)
LINK_BLACK_LIST = [re.compile(link, re.IGNORECASE) for link in _LINK_BLACK_LIST]
del _LINK_BLACK_LIST
# link fallback value
LINK_FALLBACK = bool(int(os.getenv('LINK_FALLBACK', '0')))
# content type white list
_MIME_WHITE_LIST = json.loads(os.getenv('MIME_WHITE_LIST', '[]'))
logger.plog(LOG_DEBUG, '-*- MIME WHITE LIST -*-', object=_MIME_WHITE_LIST)
# use stem to manage Tor?
_MNG_TOR = bool(int(os.getenv('DARC_TOR', '1')))
# Tor bootstrapped flag
_TOR_BS_FLAG = not _MNG_TOR # only if Tor managed through stem
# Tor controller
_TOR_CTRL = None # type: Optional[Controller]
# Tor daemon process
_TOR_PROC = None # type: Optional[Popen[bytes]]
# Tor bootstrap config
_TOR_CONFIG = {
'SocksPort': TOR_PORT,
'ControlPort': TOR_CTRL,
}
_TOR_CONFIG.update(TOR_CFG)
logger.plog(LOG_DEBUG, '-*- TOR PROXY -*-', object=_TOR_CONFIG)
def renew_tor_session() -> None:
"""Renew Tor session."""
global _TOR_CTRL # pylint: disable=global-statement
try:
# Tor controller process
if _TOR_CTRL is None:
_TOR_CTRL = stem.control.Controller.from_port(port=int(TOR_CTRL))
_TOR_CTRL.authenticate(TOR_PASS)
_TOR_CTRL.signal(stem.Signal.NEWNYM) # pylint: disable=no-member
except Exception:
logger.pexc(
LOG_WARNING,
os.path.join(FREENET_PATH, 'run.sh'), 'start'
]
else:
_unsupported = True
_FREENET_ARGS = []
else:
_FREENET_ARGS = [os.path.join(FREENET_PATH, 'run.sh'), 'start']
_FREENET_ARGS.extend(FREENET_ARGS)
if _unsupported:
if DEBUG:
logger.debug('-*- FREENET PROXY -*-')
logger.pline(LOG_ERROR, 'unsupported system: %s', platform.system())
logger.pline(LOG_DEBUG, logger.horizon)
else:
logger.plog(LOG_DEBUG, '-*- FREENET PROXY -*-', object=_FREENET_ARGS)
def launch_freenet() -> 'Popen[bytes]':
"""Launch Freenet process.
See Also:
This function mocks the behaviour of :func:`stem.process.launch_tor`.
"""
pidfile = os.path.join(FREENET_PATH, 'Freenet.pid')
with contextlib.suppress(OSError):
os.remove(pidfile)
zeronet_process = None
try: