def Main():
"""Main IRC loop."""
networks = CONFIG.get("networks", type="list")
log.setup_logger()
LOG.info("Starting %s" % version.version_string())
LOG.info("Connecting to IRC Networks: %s" % ", ".join(networks))
procs = []
for network in networks:
proc = Process(network)
proc.start()
procs.append(proc)
try:
while True:
time.sleep(1)
for proc in procs:
if not proc.is_alive():
procs.remove(proc)
if not procs:
LOG.info("No longer connected to any networks, shutting down")
sys.exit(0)
except KeyboardInterrupt:
LOG.info("Caught KeyboardInterrupt, shutting down")
def Main():
"""Main IRC loop."""
networks = CONFIG.get("networks", type="list")
log.setup_logger()
LOG.info("Starting %s" % version.version_string())
LOG.info("Connecting to IRC Networks: %s" % ", ".join(networks))
procs = []
for network in networks:
proc = Process(network)
proc.start()
procs.append(proc)
try:
while True:
time.sleep(1)
for proc in procs:
if not proc.is_alive():
procs.remove(proc)
if not procs:
LOG.info("No longer connected to any networks, shutting down")
sys.exit(0)
except KeyboardInterrupt:
LOG.info("Caught KeyboardInterrupt, shutting down")
def wrapper(self, *, config, name='monitor', **kwargs):
self.name = name or f'{config["algorithm"]}'
self._root_dir = root_dir = config['root_dir']
self._model_name = model_name = config['model_name'] or 'baseline'
self._writer = setup_tensorboard(root_dir, model_name)
tf.summary.experimental.set_step(0)
self._logger = setup_logger(root_dir, model_name)
init_fn(self, config=config, **kwargs)
import re
import json
import hashlib
from urllib.parse import urlparse
from core.colors import green, end
from core.requester import requester
from core.utils import deJSON, js_extractor, handle_anchor, getVar, updateVar
from core.log import setup_logger
logger = setup_logger(__name__)
def is_defined(o):
return o is not None
def scan(data, extractor, definitions, matcher=None):
matcher = matcher or _simple_match
detected = []
for component in definitions:
extractors = definitions[component].get("extractors",
None).get(extractor, None)
if (not is_defined(extractors)):
continue
for i in extractors:
match = matcher(i, data)
if (match):
detected.append({
"version": match,
"component": component,
def __init__(self, url_list):
self.URL_LIST = url_list
self.RESULT = {}
self.LOGGER = setup_logger(__name__)
def __init__(self):
self.LOGGER = setup_logger(__name__)
def wrapper(self, *, name=None, config, models, env, **kwargs):
"""
Args:
name: Agent's name
config: configuration for agent,
should be read from config.yaml
models: a dict of models
kwargs: optional arguments for each specific agent
"""
""" For the basic configuration, see config.yaml in algo/*/ """
config_attr(self, config)
# name is used in stdout/stderr as the agent's identifier
# while model_name is used for logging and checkpoint
# e.g., all workers share the same name, but with differnt model_names
self.name = name or config["algorithm"]
self._model_name = self._model_name or 'baseline'
self._dtype = global_policy().compute_dtype
self.model = models
# track models and optimizers for Checkpoint
self._ckpt_models = {}
for name_, model in models.items():
setattr(self, name_, model)
if isinstance(model, tf.Module) or isinstance(model, tf.Variable):
self._ckpt_models[name_] = model
self._env_step = tf.Variable(0, trainable=False, dtype=tf.int64)
self._train_step = tf.Variable(0, trainable=False, dtype=tf.int64)
self.env_step = 0
self.train_step = 0
if config.get('writer', True):
self._writer = setup_tensorboard(self._root_dir, self._model_name)
tf.summary.experimental.set_step(0)
# Agent initialization
init_fn(self, env=env, **kwargs)
# save optimizers
for k, v in vars(self).items():
if isinstance(v, Optimizer):
self._ckpt_models[k[1:]] = v
logger.info(f'ckpt models: {self._ckpt_models}')
self.print_construction_complete()
if config.get('display_var', True):
display_model_var_info(self._ckpt_models)
if config.get('save_code', True):
save_code(self._root_dir, self._model_name)
self._ckpt, self._ckpt_path, self._ckpt_manager = \
setup_checkpoint(self._ckpt_models, self._root_dir,
self._model_name, self._env_step, self._train_step)
self.restore()
# to save stats to files, specify `logger: True` in config.yaml
self._logger = setup_logger(
config.get('logger', True) and self._root_dir, self._model_name)
