def setUp(self):
"""
Create a temporary working directory and a virtual environment where
pip-accel can be tested in isolation (starting with an empty download
cache, source index and binary index and no installed modules) and make
sure pip and pip-accel use the directory. Also creates the directories
for the download cache, the source index and the binary index (normally
this is done from pip_accel.main).
"""
coloredlogs.install(level=logging.DEBUG)
# Create a temporary working directory.
self.working_directory = tempfile.mkdtemp()
self.download_cache = os.path.join(self.working_directory, 'download-cache')
# Create a temporary build directory.
self.build_directory = os.path.join(self.working_directory, 'build')
# Create a temporary virtual environment.
self.virtual_environment = os.path.join(self.working_directory, 'environment')
python = 'python%i.%i' % (sys.version_info[0], sys.version_info[1])
assert os.system('virtualenv --python=%s %s' % (pipes.quote(python), pipes.quote(self.virtual_environment))) == 0
# Make sure pip-accel uses the pip in the temporary virtual environment.
os.environ['PATH'] = '%s:%s' % (os.path.join(self.virtual_environment, 'bin'), os.environ['PATH'])
os.environ['VIRTUAL_ENV'] = self.virtual_environment
# Make pip and pip-accel use the temporary working directory.
os.environ['PIP_DOWNLOAD_CACHE'] = self.download_cache
os.environ['PIP_ACCEL_CACHE'] = self.working_directory
def _configure_logging():
"""
Configures python logging to using KWIVER / SPROKIT environment variables
SeeAlso:
kwiver/vital/logger: logic for the vital logger
"""
# Use the C++ logging level by default, but allow python to be different
cxx_level = os.environ.get('KWIVER_DEFAULT_LOG_LEVEL', 'DEBUG')
py_level = os.environ.get('KWIVER_PYTHON_DEFAULT_LOG_LEVEL', cxx_level)
# Option to colorize the python logs (must pip install coloredlogs)
truthy_values = {'true', 'on', 'yes', '1'}
use_color_env = os.environ.get('KWIVER_PYTHON_COLOREDLOGS', 'false')
# Default options
use_color = use_color_env.strip().lower() in truthy_values
level = getattr(logging, py_level.upper())
# Match KWIVER's log prefix: date time level file(lineno)
logfmt = '%(asctime)s.%(msecs)03d %(levelname)s %(name)s(%(lineno)d): %(message)s'
datefmt = '%Y-%m-%d %H:%M:%S'
# Maybe use file based configs in the future?
if use_color:
import coloredlogs
# The colorscheme can be controlled by several environment variables
# https://coloredlogs.readthedocs.io/en/latest/#environment-variables
coloredlogs.install(level=level, fmt=logfmt, datefmt=datefmt)
else:
logging.basicConfig(format=logfmt, level=level, datefmt=datefmt)
def main():
coloredlogs.install(level=logging.INFO, show_hostname=False)
publisher = zmq_context.socket(zmq.PUB)
publisher.bind('tcp://0.0.0.0:8888')
poller = zmq.Poller()
poller.register(publisher, zmq.POLLOUT)
logger.info("listening on tcp://0.0.0.0:8888")
MAX = 200
index = 0
while True:
if index > MAX:
url = 'stop'
else:
url = generate_name()
index += 1
try:
socks = dict(poller.poll(0.3))
if publisher in socks and socks[publisher] == zmq.POLLOUT:
logger.info("sending data %s", url)
publisher.send(b" ".join([b'status', bytes(url)]))
if url == 'stop':
break
else:
time.sleep(.13)
except KeyboardInterrupt:
publisher.close()
zmq_context.term()
raise SystemExit
def main():
"""The command line interface for the ``pip-accel`` program."""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
if 'install' not in arguments:
# This will not return.
os.execvp('pip', ['pip'] + arguments)
else:
arguments = [arg for arg in arguments if arg != 'install']
# Initialize logging output.
coloredlogs.install()
# Adjust verbosity based on -v, -q, --verbose, --quiet options.
for argument in list(arguments):
if match_option(argument, '-v', '--verbose'):
coloredlogs.increase_verbosity()
elif match_option(argument, '-q', '--quiet'):
coloredlogs.decrease_verbosity()
# Perform the requested action(s).
try:
accelerator = PipAccelerator(Config())
accelerator.install_from_arguments(arguments)
except Exception:
logger.exception("Caught unhandled exception!")
sys.exit(1)
def main():
import argparse
import coloredlogs
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--config", "-c", help="file to load configuration from",
default="merc.conf")
parser.add_argument("--verbose", "-v", help="enable verbose (debug) logging",
action="store_true", default=False)
args = parser.parse_args()
coloredlogs.install(level=logging.DEBUG if args.verbose else logging.INFO)
logging.getLogger("asyncio").setLevel(logging.WARN)
try:
app = Application(args.config)
app.start()
except config.ParseError as e:
logger.fatal('Could not load configuration file, aborting.')
logger.fatal(e)
except Exception as e:
logger.fatal('Could not initialize merc, aborting.')
logger.fatal(e)
def setUpModule():
"""
Prepare the test suite.
This function does two things:
1. Sets up verbose logging to the terminal. When a test fails the logging
output can help to perform a post-mortem analysis of the failure in
question (even when its hard to reproduce locally). This is especially
useful when debugging remote test failures, whether they happened on
Travis CI or a user's local system.
2. Creates temporary directories where the pip download cache and the
pip-accel binary cache are located. Isolating the pip-accel binary cache
from the user's system is meant to ensure that the tests are as
independent from the user's system as possible. The function
:py:func:`tearDownModule` is responsible for cleaning up the temporary
directory after the test suite finishes.
"""
# Initialize verbose logging to the terminal.
coloredlogs.install()
coloredlogs.increase_verbosity()
# Create temporary directories to store the pip download cache and
# pip-accel's binary cache, to make sure these tests run isolated from the
# rest of the system.
os.environ['PIP_DOWNLOAD_CACHE'] = create_temporary_directory()
os.environ['PIP_ACCEL_CACHE'] = create_temporary_directory()
def test_plain_text_output_format(self):
"""Inspect the plain text output of coloredlogs."""
logger = VerboseLogger(random_string(25))
stream = StringIO()
install(level=logging.NOTSET, logger=logger, stream=stream)
# Test that filtering on severity works.
logger.setLevel(logging.INFO)
logger.debug("No one should see this message.")
assert len(stream.getvalue().strip()) == 0
# Test that the default output format looks okay in plain text.
logger.setLevel(logging.NOTSET)
for method, severity in ((logger.debug, 'DEBUG'),
(logger.info, 'INFO'),
(logger.verbose, 'VERBOSE'),
(logger.warning, 'WARNING'),
(logger.error, 'ERROR'),
(logger.critical, 'CRITICAL')):
# Prepare the text.
text = "This is a message with severity %r." % severity.lower()
# Log the message with the given severity.
method(text)
# Get the line of output generated by the handler.
output = stream.getvalue()
lines = output.splitlines()
last_line = lines[-1]
assert text in last_line
assert severity in last_line
assert PLAIN_TEXT_PATTERN.match(last_line)
def pre_run(self):
# Hack around http://bugs.python.org/issue9253 ?
if "--version" in sys.argv:
import pkg_resources
version = pkg_resources.get_distribution("elasticluster").version
print("elasticluster version %s" % version)
sys.exit(0)
cli.app.CommandLineApp.pre_run(self)
# print *all* Python warnings through the logging subsystem
warnings.resetwarnings()
warnings.simplefilter('once')
utils.redirect_warnings(logger='gc3.elasticluster')
# Set verbosity level
loglevel = max(1, logging.WARNING - 10 * max(0, self.params.verbose))
coloredlogs.install(logger=log, level=loglevel)
# In debug mode, avoid forking
if self.params.verbose > 3:
log.DO_NOT_FORK = True
log.raiseExceptions = True
if not os.path.isdir(self.params.storage):
# We do not create *all* the parents, but we do create the
# directory if we can.
try:
os.makedirs(self.params.storage)
except OSError as ex:
sys.stderr.write("Unable to create storage directory: "
"%s\n" % (str(ex)))
sys.exit(1)
# If no configuration file was specified and default does not exists...
if not os.path.isfile(self.params.config):
if self.params.config == self.default_configuration_file:
# Copy the default configuration file to the user's home
if not os.path.exists(os.path.dirname(self.params.config)):
os.mkdir(os.path.dirname(self.params.config))
template = resource_filename(
'elasticluster', 'share/etc/config.template')
log.warning("Deploying default configuration file to %s.",
self.params.config)
shutil.copyfile(template, self.params.config)
else:
# Exit if supplied configuration file does not exists.
if not os.path.isfile(self.params.config):
sys.stderr.write(
"Unable to read configuration file `%s`.\n" %
self.params.config)
sys.exit(1)
assert self.params.func, ("No subcommand defined in `ElastiCluster.setup()")
try:
self.params.func.pre_run()
except (RuntimeError, ConfigurationError) as ex:
sys.stderr.write(str(ex).strip())
sys.stderr.write('\n')
sys.exit(1)
def create_app() -> Callable[[Any, Any], Any]:
''' Create a WSGI compatible App object. '''
validate_config()
coloredlogs.install(fmt='[%(asctime)-15s] %(name)s %(message)s')
logging.getLogger('elasticsearch').disabled = True
if config.config['debug']:
logging.getLogger('szurubooru').setLevel(logging.INFO)
if config.config['show_sql']:
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
purge_thread = threading.Thread(target=purge_old_uploads)
purge_thread.daemon = True
purge_thread.start()
try:
posts.populate_reverse_search()
db.session.commit()
except errors.ThirdPartyError:
pass
rest.errors.handle(errors.AuthError, _on_auth_error)
rest.errors.handle(errors.ValidationError, _on_validation_error)
rest.errors.handle(errors.SearchError, _on_search_error)
rest.errors.handle(errors.IntegrityError, _on_integrity_error)
rest.errors.handle(errors.NotFoundError, _on_not_found_error)
rest.errors.handle(errors.ProcessingError, _on_processing_error)
rest.errors.handle(errors.ThirdPartyError, _on_third_party_error)
rest.errors.handle(sa.orm.exc.StaleDataError, _on_stale_data_error)
return rest.application
def setUp(self):
"""Enable logging to the terminal and prepare a temporary package cache."""
coloredlogs.install()
coloredlogs.set_level(logging.DEBUG)
self.db_directory = tempfile.mkdtemp()
self.load_package_cache()
os.environ['DPT_FORCE_ENTROPY'] = 'yes'
def __init__(self, workspace=None):
"""
Initialize the Validator.
A workspace may be provided for an easy parameter configuration,
such as location and extension of descriptors, verbosity level, etc.
:param workspace: SONATA workspace object
"""
self._workspace = workspace
self._syntax = True
self._integrity = True
self._topology = True
# create "virtual" workspace if not provided (don't actually create
# file structure)
if not self._workspace:
self._workspace = Workspace('.', log_level='info')
# load configurations from workspace
self._dext = self._workspace.default_descriptor_extension
self._dpath = '.'
self._log_level = self._workspace.log_level
# configure logs
coloredlogs.install(level=self._log_level)
# descriptors storage
self._storage = DescriptorStorage()
# syntax validation
self._schema_validator = SchemaValidator(self._workspace)
# wrapper to count number of errors and warnings
log.error = CountCalls(log.error)
log.warning = CountCalls(log.warning)
def cli(debug):
log_level = logging.INFO
requests_cache.install_cache('fr_cache', expire_after=60*60*24*3) # 3 days
if debug:
log_level = logging.DEBUG
sys.excepthook = lambda t, v, tb: ipdb.post_mortem(tb)
coloredlogs.install(level=log_level, fmt="%(levelname)s %(message)s")
def set(self, debug=None, info=None, timings=None, dry=None, quiet=None, verbosity=None, colors=None, log=None):
self.log = log if log is not None else os.getenv(MB_LOG, str(DEFAULT_LOG))
self.quiet = quiet if quiet is not None else lib.str2bool(os.getenv(MB_QUIET, str(DEFAULT_QUIET)))
self.debug = debug if debug is not None else lib.str2bool(os.getenv(MB_DEBUG, str(DEFAULT_DEBUG)))
self.info = info if info is not None else lib.str2bool(os.getenv(MB_INFO, str(DEFAULT_INFO)))
self.timings = timings if timings is not None else lib.str2bool(os.getenv(MB_TIMINGS, str(DEFAULT_TIMINGS)))
self.dry = dry if dry is not None else lib.str2bool(os.getenv(MB_DRY, str(DEFAULT_DRY)))
self.colors = colors if colors is not None else lib.str2bool(os.getenv(MB_COLORS, str(DEFAULT_COLORS)))
self.verbosity = verbosity if verbosity is not None else os.getenv(MB_VERBOSITY, DEFAULT_VERBOSITY)
if self.timings or self.info:
self.verbosity = 'info'
self.quiet = True
if self.debug:
self.verbosity = 'debug'
self.quiet = True
self.level = verbosities[self.verbosity]
logging.basicConfig(level=self.level, format=self.fmt)
if self.colors:
coloredlogs.install(level=self.level, fmt=self.fmt)
if self.log is not None:
if check_file_writable(self.log):
fh = logging.FileHandler(self.log)
fh.setLevel(logging.DEBUG)
logging.getLogger().addHandler(fh)
else:
logger.warning('No permission to write to %s for current user %s', self.log, current_user)
logger.debug(self)
def configure(self, syntax=None, integrity=None, topology=None,
dpath=None, dext=None, debug=False):
"""
Configure parameters for validation. It is recommended to call this
function before performing a validation.
:param syntax: specifies whether to validate syntax
:param integrity: specifies whether to validate integrity
:param topology: specifies whether to validate network topology
:param dpath: directory to search for function descriptors (VNFDs)
:param dext: extension of descriptor files (default: 'yml')
:param debug: increase verbosity level of logger
"""
# assign parameters
if syntax is not None:
self._syntax = syntax
if integrity is not None:
self._integrity = integrity
if topology is not None:
self._topology = topology
if dext is not None:
self._dext = dext
if dpath is not None:
self._dpath = dpath
if debug:
coloredlogs.install(level='debug')
def query_logger():
if pytest.config.option.verbose > 0:
import logging
import coloredlogs
coloredlogs.install(fmt='[%(asctime)-15s] %(name)s %(message)s', isatty=True)
logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
def main():
"""The command line interface for the ``pip-accel`` program."""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
if 'install' not in arguments:
# This will not return.
os.execvp('pip', ['pip'] + arguments)
else:
arguments = [arg for arg in arguments if arg != 'install']
# Initialize logging output.
coloredlogs.install()
# Adjust verbosity based on -v, -q, --verbose, --quiet options.
for argument in list(arguments):
if match_option(argument, '-v', '--verbose'):
coloredlogs.increase_verbosity()
elif match_option(argument, '-q', '--quiet'):
coloredlogs.decrease_verbosity()
# Perform the requested action(s).
try:
accelerator = PipAccelerator(Config())
accelerator.install_from_arguments(arguments)
except NothingToDoError as e:
# Don't print a traceback for this (it's not very user friendly) and
# exit with status zero to stay compatible with pip. For more details
# please refer to https://github.com/paylogic/pip-accel/issues/47.
logger.warning("%s", e)
sys.exit(0)
except Exception:
logger.exception("Caught unhandled exception!")
sys.exit(1)
def test_program_name_filter(self):
"""Make sure :func:`install()` integrates with :class:`~coloredlogs.ProgramNameFilter()`."""
install(fmt='%(programname)s')
with CaptureOutput() as capturer:
logging.info("A truly insignificant message ..")
output = capturer.get_text()
assert find_program_name() in output
def main():
description = """This program takes as input a set of OSSOS measurements and adjusts the astrometric and photometric
entries to be consistent with the current best estimate for the astrometric and photometric calibrations.
"""
parser = argparse.ArgumentParser(description=description)
parser.add_argument('ast_file', help="An MPC file to update.")
parser.add_argument('--discovery', help="Only process the discovery images.", action='store_true', default=False)
parser.add_argument('--result_base_name', help="base name for remeasurement results (defaults to basename of input)",
default=None)
parser.add_argument('--skip-mags', action="store_true", help="Recompute magnitudes.", default=False)
parser.add_argument('--skip-centroids', action="store_true", help="Recompute centroids.", default=False)
parser.add_argument('--compare-orbits', action='store_true', help="Compute/Compare pre and post remeasure orbits?", default=False)
parser.add_argument('--debug', action='store_true')
args = parser.parse_args()
level = logging.INFO
if args.debug:
level = logging.DEBUG
import coloredlogs
logger = logging.getLogger('update_astrom')
coloredlogs.install(level=level)
if args.result_base_name is None:
base_name = os.path.splitext(os.path.basename(args.ast_file))[0]
else:
base_name = args.result_base_name
run(args.ast_file, base_name,
skip_discovery=not args.discovery,
skip_mags=args.skip_mags,
skip_centroids=args.skip_centroids,
compare_orbits=args.compare_orbits)
def app_main():
args = docopt.docopt(__doc__, version=__version__)
debug = args["-d"]
if debug:
log_level = logging.DEBUG
else:
log_level = [logging.ERROR, logging.INFO, logging.DEBUG][args["-v"]]
try:
import coloredlogs
coloredlogs.install(
level=log_level, stream=stderr, datefmt=DATEFMT, fmt=LOGFMT
)
except ImportError:
_LOGGER.debug("no colored logs. pip install coloredlogs?")
logging.basicConfig(
level=log_level, stream=stderr, datefmt=DATEFMT, format=LOGFMT
)
logging.captureWarnings(debug)
if debug:
_LOGGER.info("Debug is on")
try:
asyncio.run(main(args), debug=debug) # pylint: disable=no-member
except KeyboardInterrupt:
exit()
def configure_logging(args):
level = logging.DEBUG if args['--debug'] else logging.ERROR
coloredlogs.install(
level=level,
show_timestamps=False,
show_hostname=False
)
def setUpModule():
"""
Prepare the test suite.
Sets up logging to the terminal. When a test fails the logging output can
help to perform a post-mortem analysis of the failure in question (even
when its hard to reproduce locally). This is especially useful when
debugging remote test failures, whether they happened on Travis CI or a
user's local system.
Also makes sure that the Apache web server is installed and running because
this is required to run the test suite.
"""
# Set up logging to the terminal.
coloredlogs.install(level=logging.DEBUG)
# Make sure Apache is installed and configured.
try:
manager = ApacheManager()
manager.fetch_status_page(manager.text_status_url)
except Exception as e:
logger.exception("Failed to connect to local Apache server!")
raise Exception(compact("""
Please make sure the Apache web server is installed and configured
(running) before you run this test suite because this test suite
tests the actual integration with Apache (it doesn't use mocking)
and so requires Apache to be installed, configured and running.
Swallowed exception: {message} ({type})
""", message=e, type=type(e)))
def __init__(self):
# make sure the config file has correct permissions (0600)
self._check_config_chmod()
coloredlogs.install(show_hostname=False, show_name=False, show_severity=False)
logging.info('Reading configuration..')
self.config = ConfigParser.ConfigParser(allow_no_value=True)
try:
self.config.readfp(open(os.path.dirname(os.path.realpath(__file__)) + '/CloneRow.cfg'))
except IOError:
logging.error('You have not setup a CloneRow.cfg file for your requirements')
logging.info('take a look at CloneRow.example.cfg')
logging.info('https://github.com/lathonez/mysql-clone-row#configuration')
sys.exit(3)
self.source = {
'alias': None,
'connection': None,
'db_name': None,
'row': {}
}
self.target = {
'alias': None,
'backup': None,
'connection': None,
'db_name': None,
'new_insert': False,
'row': {}
}
self.database = {
'table': None,
'column': None,
'filter': None,
'ignore_columns': [],
'deltas': {}
}
def main():
"""Entry point of the program."""
# Logging
coloredlogs.install(level='INFO')
# Connect to Rabbit
rabbit = Client('hms_irc', settings.RABBIT_EXCHANGE,
settings.RABBIT_ROUTING_KEYS)
rabbit.connect(settings.RABBIT_HOST)
rabbit_thread = Thread(target=rabbit.start_consuming)
rabbit_thread.setDaemon(True) # To kill the thread when main is gone
# IRC bot settings
bot = MyBot(settings.IRC_CHAN, settings.IRC_NAME, settings.IRC_SERVER)
bot.rabbit = rabbit
# Add callbacks
rabbit.listeners.append(bot.handle_rabbit_msg)
def chan_joined():
"""Callback that will start the RabbitMQ receive thread."""
if not rabbit_thread.is_alive():
get_logger().info('Starting RabbitMQ consume thread...')
rabbit_thread.start()
else:
get_logger().warning("Chan joined but RabbitMQ thread is alive")
bot.join_callback = chan_joined
# Create restart thread
def restart_periodic():
while True:
time.sleep(3600)
bot.reconnect_if_disconnected()
restart_thread = Thread(target=restart_periodic)
restart_thread.setDaemon(True)
restart_thread.start()
# Start IRC thread that will start Rabbit thread using callback
try:
get_logger().info('Starting IRC bot...')
bot.start()
except KeyboardInterrupt:
get_logger().critical("Got a KeyboardInterrupt")
get_logger().info("Disconnecting from Rabbit")
# Beautiful RabbiMQ shutdown attempt
rabbit.stop_consuming()
rabbit.disconnect()
# Beautiful IRC shutdown
get_logger().info("Disconnecting from IRC")
bot.die(msg="got a KeyboardInterrupt in my face! >_<")
# Exit and kill daemon thread
sys.exit(0)
def __init__(self, prog, plugin):
os.chdir(os.path.dirname(__file__))
# parse input arguments
parser = argparse.ArgumentParser(
prog=__file__,
description="Task runner for SCAPL plugin {}.".format(plugin),
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument("-c", "--config", dest="config", help="task configuration dictionary")
parser.add_argument("-p", "--param", dest="param", help="plugin's specific parameters")
parser.add_argument("-t", "--task", dest="task", help="task identifier for logging purpose")
parser.add_argument(
"-v", dest="verbose", action="count", default=0, help="verbose level [default: 0 (critical)]"
)
args = parser.parse_args()
args.config, args.param = eval(args.config), eval(args.param)
# configure logging and get the root logger
args.verbose = args.config["LOG_LEVEL_MAPPING"][min(max(args.config["LOG_LEVEL_MAPPING"].keys()), args.verbose)]
logging.basicConfig(format="%(name)s - %(asctime)s [%(levelname)s] %(message)s", level=args.verbose)
self.logger = logging.getLogger(args.task)
if colored_logs_present:
coloredlogs.install(args.verbose)
# set arguments as attributes
for arg in vars(args):
setattr(self, arg, getattr(args, arg))
def main():
"""Command line interface for the ``dwim`` program."""
from dwim import DEFAULT_PROFILE, dwim
# Initialize logging to the terminal.
coloredlogs.install()
# Define the command line option defaults.
profile_script = DEFAULT_PROFILE
# Parse the command line arguments.
try:
options, _ = getopt.getopt(sys.argv[1:], 'c:vqh', [
'config=', 'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-c', '--config'):
profile_script = value
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
except Exception as e:
warning("Error: Failed to parse command line arguments! (%s)", e)
sys.exit(1)
# Execute the requested action(s).
try:
dwim(profile_script)
except Exception:
logger.exception("Caught a fatal exception! Terminating ..")
sys.exit(1)
def setupLogger():
if settings.logfile is not None:
fmt = '%(asctime)s [%(levelname)s] %(message)s'
date_fmt = '%Y-%m-%d %H:%M'
try:
consoleFormatter = coloredlogs.ColoredFormatter(fmt, date_fmt)
except NameError:
consoleFormatter = logging.Formatter(fmt, date_fmt)
consoleHandler = logging.StreamHandler()
consoleHandler.setLevel(settings.loglevel_stderr)
consoleHandler.setFormatter(consoleFormatter)
fileFormatter = logging.Formatter(fmt, date_fmt)
fileHandler = logging.FileHandler(settings.logfile)
fileHandler.setLevel(settings.loglevel_file)
fileHandler.setFormatter(fileFormatter)
logging.getLogger().setLevel(0)
logging.getLogger().addHandler(fileHandler)
logging.getLogger().addHandler(consoleHandler)
else:
try:
coloredlogs.install(fmt=fmt, datefmt=date_fmt, level=settings.loglevel_stderr)
except NameError:
logging.basicConfig(format=fmt, datefmt=date_fmt, level=settings.loglevel_stderr)
def main():
"""Command line interface for the ``negotiator-guest`` program."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
list_commands = False
execute_command = None
start_daemon = False
timeout = DEFAULT_TIMEOUT
character_device = None
try:
options, arguments = getopt.getopt(sys.argv[1:], 'le:dt:c:vqh', [
'list-commands', 'execute=', 'daemon', 'timeout=',
'character-device=', 'verbose', 'quiet', 'help'
])
for option, value in options:
if option in ('-l', '--list-commands'):
list_commands = True
elif option in ('-e', '--execute'):
execute_command = value
elif option in ('-d', '--daemon'):
start_daemon = True
elif option in ('-t', '--timeout'):
timeout = int(value)
elif option in ('-c', '--character-device'):
character_device = value
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
if not (list_commands or execute_command or start_daemon):
usage(__doc__)
sys.exit(0)
except Exception:
warning("Error: Failed to parse command line arguments!")
sys.exit(1)
# Start the guest daemon.
try:
if not character_device:
channel_name = HOST_TO_GUEST_CHANNEL_NAME if start_daemon else GUEST_TO_HOST_CHANNEL_NAME
character_device = find_character_device(channel_name)
ga = GuestAgent(character_device)
if start_daemon:
ga.enter_main_loop()
elif list_commands:
with TimeOut(timeout):
print('\n'.join(ga.call_remote_method('list_commands')))
elif execute_command:
with TimeOut(timeout):
timer = Timer()
output = ga.call_remote_method('execute', *shlex.split(execute_command), capture=True)
logger.debug("Took %s to execute remote command.", timer)
print(output.rstrip())
except Exception:
logger.exception("Caught a fatal exception! Terminating ..")
sys.exit(1)
def test_colorama_enabled(self):
"""Test that colorama is enabled (through mocking)."""
init_function = MagicMock()
with mocked_colorama_module(init_function):
# Configure logging to the terminal.
coloredlogs.install()
# Ensure that our mock method was called.
assert init_function.called
def main(log_level):
"""
Pilot Jenkins from the command line.
"""
coloredlogs.install(
fmt='%(levelname)s %(message)s',
level=log_level
)
def main():
args = parse_args()
loglevel = logging.INFO if args.verbose is None else logging.DEBUG
logging.basicConfig(level=loglevel)
coloredlogs.install()
import_new_features(args.node_eui, args.exclude)
import math
import base64
import utils
import coloredlogs
import time
import threading
from queue import Queue, Empty as QEmpty
import input_obj
import gzip
import gzipinputstream
from datetime import datetime
from newline_reader import NewlineReader, NewlineIterator
logger = logging.getLogger(__name__)
coloredlogs.install(level=logging.DEBUG)
class EcoRecode(object):
"""
Processing censys http eco system dataset.
Recodes dataset to regular snapshots. Generates json, dataset for classification.
Script is designed to run on a big server with huge amount of RAM.
One-pass, all in ram method.
In case of optimisation is needed:
- download big cert file to the disk, ungzip
- do a sort on the fingerprints, disk sort
- host file, one by one, download, disk sort, find fingerprints in the main big cert file.
def build_act_with_param_noise(make_obs_ph, q_func, num_actions, scope="deepq", reuse=None, param_noise_filter_func=None):
"""Creates the act function with support for parameter space noise exploration (https://arxiv.org/abs/1706.01905):
Parameters
----------
make_obs_ph: str -> tf.placeholder or TfInput
a function that take a name and creates a placeholder of input with that name
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
num_actions: int
number of actions.
scope: str or VariableScope
optional scope for variable_scope.
reuse: bool or None
whether or not the variables should be reused. To be able to reuse the scope must be given.
param_noise_filter_func: tf.Variable -> bool
function that decides whether or not a variable should be perturbed. Only applicable
if param_noise is True. If set to None, default_param_noise_filter is used by default.
Returns
-------
act: (tf.Variable, bool, float, bool, float, bool) -> tf.Variable
function to select and action given observation.
` See the top of the file for details.
"""
logger = logging.getLogger()
coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s')
logger.setLevel(logging.DEBUG)
if param_noise_filter_func is None:
param_noise_filter_func = default_param_noise_filter
with tf.variable_scope(scope, reuse=reuse):
observations_ph = make_obs_ph("observation")
stochastic_ph = tf.placeholder(tf.bool, (), name="stochastic")
update_eps_ph = tf.placeholder(tf.float32, (), name="update_eps")
update_param_noise_threshold_ph = tf.placeholder(tf.float32, (), name="update_param_noise_threshold")
update_param_noise_scale_ph = tf.placeholder(tf.bool, (), name="update_param_noise_scale")
reset_ph = tf.placeholder(tf.bool, (), name="reset")
eps = tf.get_variable("eps", (), initializer=tf.constant_initializer(0))
param_noise_scale = tf.get_variable("param_noise_scale", (), initializer=tf.constant_initializer(0.01), trainable=False)
param_noise_threshold = tf.get_variable("param_noise_threshold", (), initializer=tf.constant_initializer(0.05), trainable=False)
# Unmodified Q.
q_values = q_func(observations_ph.get(), num_actions, scope="q_func")
# Perturbable Q used for the actual rollout.
q_values_perturbed = q_func(observations_ph.get(), num_actions, scope="perturbed_q_func")
# We have to wrap this code into a function due to the way tf.cond() works. See
# https://stackoverflow.com/questions/37063952/confused-by-the-behavior-of-tf-cond for
# a more detailed discussion.
def perturb_vars(original_scope, perturbed_scope):
all_vars = scope_vars(absolute_scope_name(original_scope))
all_perturbed_vars = scope_vars(absolute_scope_name(perturbed_scope))
assert len(all_vars) == len(all_perturbed_vars)
perturb_ops = []
for var, perturbed_var in zip(all_vars, all_perturbed_vars):
if param_noise_filter_func(perturbed_var):
# Perturb this variable.
op = tf.assign(perturbed_var, var + tf.random_normal(shape=tf.shape(var), mean=0., stddev=param_noise_scale))
else:
# Do not perturb, just assign.
op = tf.assign(perturbed_var, var)
perturb_ops.append(op)
assert len(perturb_ops) == len(all_vars)
return tf.group(*perturb_ops)
# Set up functionality to re-compute `param_noise_scale`. This perturbs yet another copy
# of the network and measures the effect of that perturbation in action space. If the perturbation
# is too big, reduce scale of perturbation, otherwise increase.
q_values_adaptive = q_func(observations_ph.get(), num_actions, scope="adaptive_q_func")
perturb_for_adaption = perturb_vars(original_scope="q_func", perturbed_scope="adaptive_q_func")
kl = tf.reduce_sum(tf.nn.softmax(q_values) * (tf.log(tf.nn.softmax(q_values)) - tf.log(tf.nn.softmax(q_values_adaptive))), axis=-1)
mean_kl = tf.reduce_mean(kl)
def update_scale():
with tf.control_dependencies([perturb_for_adaption]):
update_scale_expr = tf.cond(mean_kl < param_noise_threshold,
lambda: param_noise_scale.assign(param_noise_scale * 1.01),
lambda: param_noise_scale.assign(param_noise_scale / 1.01),
)
return update_scale_expr
# Functionality to update the threshold for parameter space noise.
update_param_noise_threshold_expr = param_noise_threshold.assign(tf.cond(update_param_noise_threshold_ph >= 0,
lambda: update_param_noise_threshold_ph, lambda: param_noise_threshold))
# Put everything together.
deterministic_actions = tf.argmax(q_values_perturbed, axis=1)
batch_size = tf.shape(observations_ph.get())[0]
random_actions = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=num_actions, dtype=tf.int64)
chose_random = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps
stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)
output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)
update_eps_expr = eps.assign(tf.cond(update_eps_ph >= 0, lambda: update_eps_ph, lambda: eps))
updates = [
update_eps_expr,
tf.cond(reset_ph, lambda: perturb_vars(original_scope="q_func", perturbed_scope="perturbed_q_func"), lambda: tf.group(*[])),
tf.cond(update_param_noise_scale_ph, lambda: update_scale(), lambda: tf.Variable(0., trainable=False)),
update_param_noise_threshold_expr,
]
_act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph, reset_ph, update_param_noise_threshold_ph, update_param_noise_scale_ph],
outputs=output_actions,
givens={update_eps_ph: -1.0, stochastic_ph: True, reset_ph: False, update_param_noise_threshold_ph: False, update_param_noise_scale_ph: False},
updates=updates)
def act(ob, reset=False, update_param_noise_threshold=False, update_param_noise_scale=False, stochastic=True, update_eps=-1):
return _act(ob, stochastic, update_eps, reset, update_param_noise_threshold, update_param_noise_scale)
return act
def build_act(make_obs_ph, q_func, num_actions, scope="deepq", reuse=None):
"""Creates the act function:
Parameters
----------
make_obs_ph: str -> tf.placeholder or TfInput
a function that take a name and creates a placeholder of input with that name
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
num_actions: int
number of actions.
scope: str or VariableScope
optional scope for variable_scope.
reuse: bool or None
whether or not the variables should be reused. To be able to reuse the scope must be given.
Returns
-------
(
act: (tf.Variable, bool, float) -> tf.Variable
function to select and action given observation.
` See the top of the file for details.
"""
logger = logging.getLogger()
coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s')
logger.setLevel(logging.DEBUG)
with tf.variable_scope(scope, reuse=reuse):
observations_ph = make_obs_ph("observation")
stochastic_ph = tf.placeholder(tf.bool, (), name="stochastic")
update_eps_ph = tf.placeholder(tf.float32, (), name="update_eps")
eps = tf.get_variable("eps", (), initializer=tf.constant_initializer(0))
q_values = q_func(observations_ph.get(), num_actions, scope="q_func")
deterministic_actions = tf.argmax(q_values, axis=1)
batch_size = tf.shape(observations_ph.get())[0]
random_actions = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=num_actions, dtype=tf.int64)
chose_random = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps
stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)
output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)
update_eps_expr = eps.assign(tf.cond(update_eps_ph >= 0, lambda: update_eps_ph, lambda: eps))
# _act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph],
# outputs=output_actions,
# givens={update_eps_ph: -1.0, stochastic_ph: True},
# updates=[update_eps_expr])
# def act(ob, stochastic=True, update_eps=-1):
# return _act(ob, stochastic, update_eps)
# return act
_act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph],
outputs=(output_actions, q_values),
givens={update_eps_ph: -1.0, stochastic_ph: True},
updates=[update_eps_expr])
def act(ob, stochastic=True, update_eps=-1):
return _act(ob, stochastic, update_eps)
return act
def happi_cli(args):
args = parser.parse_args(args)
# Logging Level handling
if args.verbose:
shown_logger = logging.getLogger()
level = "DEBUG"
else:
shown_logger = logging.getLogger('happi')
level = "INFO"
coloredlogs.install(level=level,
logger=shown_logger,
fmt='[%(asctime)s] - %(levelname)s - %(message)s')
logger.debug("Set logging level of %r to %r", shown_logger.name, level)
# Version endpoint
if args.version:
print(f'Happi: Version {happi.__version__} from {happi.__file__}')
return
logger.debug('Command line arguments: %r' % args)
client = happi.client.Client.from_config(cfg=args.path)
logger.debug("Happi client: %r" % client)
logger.debug('Happi command: %r' % args.cmd)
if args.cmd == 'search':
logger.debug("We're in the search block")
# Get search criteria into dictionary for use by client
client_args = {}
for user_arg in args.search_criteria:
if '=' in user_arg:
criteria, value = user_arg.split('=', 1)
else:
criteria = 'name'
value = user_arg
if criteria in client_args:
logger.error(
'Received duplicate search criteria %s=%r (was %r)',
criteria, value, client_args[criteria])
return
if value.replace('.', '').isnumeric():
logger.debug('Changed %s to float', value)
value = float(value)
client_args[criteria] = value
results = client.search(**client_args)
if results:
for res in results:
res.device.show_info()
return results
else:
logger.error('No devices found')
elif args.cmd == 'add':
logger.debug('Starting interactive add')
registry = happi.containers.registry
if args.clone:
clone_source = client.find_device(name=args.clone)
# Must use the same container if cloning
response = registry.entry_from_class(clone_source.__class__)
else:
# Keep Device at registry for backwards compatibility but filter
# it out of new devices options
options = os.linesep.join(
[k for k, _ in registry.items() if k != "Device"])
logger.info(
'Please select a container, or press enter for generic '
'Ophyd Device container: %s%s', os.linesep, options)
response = input()
if response and response not in registry:
logger.info('Invalid device container f{response}')
return
elif not response:
response = 'OphydItem'
container = registry[response]
kwargs = {}
for info in container.entry_info:
valid_value = False
while not valid_value:
if args.clone:
default = getattr(clone_source, info.key)
else:
default = info.default
logger.info(f'Enter value for {info.key}, default={default}, '
f'enforce={info.enforce}')
item_value = input()
if not item_value:
if info.optional or args.clone:
logger.info(f'Selecting default value {default}')
item_value = default
else:
logger.info('Not an optional field!')
continue
try:
info.enforce_value(item_value)
valid_value = True
kwargs[info.key] = item_value
except Exception:
logger.info(f'Invalid value {item_value}')
device = client.create_device(container, **kwargs)
logger.info('Please confirm the following info is correct:')
device.show_info()
ok = input('y/N\n')
if 'y' in ok:
logger.info('Adding device')
device.save()
else:
logger.info('Aborting')
elif args.cmd == 'edit':
logger.debug('Starting edit block')
device = client.find_device(name=args.name)
for edit in args.edits:
field, value = edit.split('=', 1)
logger.info(f'Setting {args.name}.{field} = {value}')
setattr(device, field, value)
device.save()
device.show_info()
elif args.cmd == 'load':
logger.debug('Starting load block')
logger.info(f'Creating shell with devices {args.device_names}')
devices = {}
for name in args.device_names:
devices[name] = client.load_device(name=name)
start_ipython(argv=['--quick'], user_ns=devices)
def build_train(make_obs_ph, q_func, num_actions, optimizer, grad_norm_clipping=None, gamma=1.0,
double_q=True, scope="deepq", reuse=None, param_noise=False, param_noise_filter_func=None):
"""Creates the train function:
Parameters
----------
make_obs_ph: str -> tf.placeholder or TfInput
a function that takes a name and creates a placeholder of input with that name
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
num_actions: int
number of actions
reuse: bool
whether or not to reuse the graph variables
optimizer: tf.train.Optimizer
optimizer to use for the Q-learning objective.
grad_norm_clipping: float or None
clip gradient norms to this value. If None no clipping is performed.
gamma: float
discount rate.
double_q: bool
if true will use Double Q Learning (https://arxiv.org/abs/1509.06461).
In general it is a good idea to keep it enabled.
scope: str or VariableScope
optional scope for variable_scope.
reuse: bool or None
whether or not the variables should be reused. To be able to reuse the scope must be given.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
param_noise_filter_func: tf.Variable -> bool
function that decides whether or not a variable should be perturbed. Only applicable
if param_noise is True. If set to None, default_param_noise_filter is used by default.
Returns
-------
act: (tf.Variable, bool, float) -> tf.Variable
function to select and action given observation.
` See the top of the file for details.
train: (object, np.array, np.array, object, np.array, np.array) -> np.array
optimize the error in Bellman's equation.
` See the top of the file for details.
update_target: () -> ()
copy the parameters from optimized Q function to the target Q function.
` See the top of the file for details.
debug: {str: function}
a bunch of functions to print debug data like q_values.
"""
logger = logging.getLogger()
coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s')
logger.setLevel(logging.DEBUG)
#logger.info("Inside build train function in build_graph")
if param_noise:
#logger.info("Inside build train function in build_graph and there is param noise")
act_f = build_act_with_param_noise(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse,
param_noise_filter_func=param_noise_filter_func)
else:
#logger.info("Inside build train function in build_graph and there is NOT param noise")
act_f = build_act(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse)
with tf.variable_scope(scope, reuse=reuse):
# set up placeholders
#logger.info("Inside build train function in build_graph and setting up placeholders")
obs_t_input = make_obs_ph("obs_t")
act_t_ph = tf.placeholder(tf.int32, [None], name="action")
rew_t_ph = tf.placeholder(tf.float32, [None], name="reward")
obs_tp1_input = make_obs_ph("obs_tp1")
done_mask_ph = tf.placeholder(tf.float32, [None], name="done")
importance_weights_ph = tf.placeholder(tf.float32, [None], name="weight")
# q network evaluation
#logger.info("Inside build train function in build_graph and performing q eval")
q_t = q_func(obs_t_input.get(), num_actions, scope="q_func", reuse=True) # reuse parameters from act
q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=tf.get_variable_scope().name + "/q_func")
# target q network evalution
#logger.info("Inside build train function in build_graph and performing target q eval")
q_tp1 = q_func(obs_tp1_input.get(), num_actions, scope="target_q_func")
target_q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=tf.get_variable_scope().name + "/target_q_func")
# q scores for actions which we know were selected in the given state.
#logger.info("Inside build train function in build_graph and collecting q scores")
q_t_selected = tf.reduce_sum(q_t * tf.one_hot(act_t_ph, num_actions), 1)
# compute estimate of best possible value starting from state at t + 1
if double_q:
#logger.info("Inside build train function in build_graph and double q is yes")
q_tp1_using_online_net = q_func(obs_tp1_input.get(), num_actions, scope="q_func", reuse=True)
q_tp1_best_using_online_net = tf.argmax(q_tp1_using_online_net, 1)
q_tp1_best = tf.reduce_sum(q_tp1 * tf.one_hot(q_tp1_best_using_online_net, num_actions), 1)
else:
#logger.info("Inside build train function in build_graph and double q is no")
q_tp1_best = tf.reduce_max(q_tp1, 1)
#logger.info("Inside build train function in build_graph and exited double q check")
q_tp1_best_masked = (1.0 - done_mask_ph) * q_tp1_best
# compute RHS of bellman equation
#logger.info("Inside build train function in build_graph and computing RHS of bellman equation")
q_t_selected_target = rew_t_ph + gamma * q_tp1_best_masked
# compute the error (potentially clipped)
#logger.info("Inside build train function in build_graph and computing the error")
td_error = q_t_selected - tf.stop_gradient(q_t_selected_target)
errors = U.huber_loss(td_error)
weighted_error = tf.reduce_mean(importance_weights_ph * errors)
# compute optimization op (potentially with gradient clipping)
#logger.info("Inside build train function in build_graph and compute optimization op")
if grad_norm_clipping is not None:
#logger.info("Inside build train function in build_graph and grad norm clipping is yes")
gradients = optimizer.compute_gradients(weighted_error, var_list=q_func_vars)
for i, (grad, var) in enumerate(gradients):
if grad is not None:
gradients[i] = (tf.clip_by_norm(grad, grad_norm_clipping), var)
optimize_expr = optimizer.apply_gradients(gradients)
else:
#logger.info("Inside build train function in build_graph and grad norm clipping is no")
optimize_expr = optimizer.minimize(weighted_error, var_list=q_func_vars)
#logger.info("Inside build train function in build_graph and exited grad norm clipping check")
# update_target_fn will be called periodically to copy Q network to target Q network
update_target_expr = []
for var, var_target in zip(sorted(q_func_vars, key=lambda v: v.name),
sorted(target_q_func_vars, key=lambda v: v.name)):
update_target_expr.append(var_target.assign(var))
update_target_expr = tf.group(*update_target_expr)
# Create callable functions
train = U.function(
inputs=[
obs_t_input,
act_t_ph,
rew_t_ph,
obs_tp1_input,
done_mask_ph,
importance_weights_ph
],
outputs=td_error,
updates=[optimize_expr]
)
update_target = U.function([], [], updates=[update_target_expr])
q_values = U.function([obs_t_input], q_t)
#logger.info("Inside build train function in build_graph and about to return")
return act_f, train, update_target, {'q_values': q_values}
def log_config():
coloredlogs.install()
logging.basicConfig()
LOGGER.setLevel(logging.DEBUG)
def main():
parser = argparse.ArgumentParser(
description="Security analysis of Ethereum smart contracts")
parser.add_argument("solidity_file", nargs="*")
commands = parser.add_argument_group("commands")
commands.add_argument("-g",
"--graph",
help="generate a control flow graph")
commands.add_argument(
"-V",
"--version",
action="store_true",
help="print the Mythril version number and exit",
)
commands.add_argument(
"-x",
"--fire-lasers",
action="store_true",
help="detect vulnerabilities, use with -c, -a or solidity file(s)",
)
commands.add_argument(
"-t",
"--truffle",
action="store_true",
help="analyze a truffle project (run from project dir)",
)
commands.add_argument("-d",
"--disassemble",
action="store_true",
help="print disassembly")
commands.add_argument(
"-j",
"--statespace-json",
help="dumps the statespace json",
metavar="OUTPUT_FILE",
)
inputs = parser.add_argument_group("input arguments")
inputs.add_argument(
"-c",
"--code",
help='hex-encoded bytecode string ("6060604052...")',
metavar="BYTECODE",
)
inputs.add_argument(
"-f",
"--codefile",
help="file containing hex-encoded bytecode string",
metavar="BYTECODEFILE",
type=argparse.FileType("r"),
)
inputs.add_argument(
"-a",
"--address",
help="pull contract from the blockchain",
metavar="CONTRACT_ADDRESS",
)
inputs.add_argument(
"-l",
"--dynld",
action="store_true",
help="auto-load dependencies from the blockchain",
)
outputs = parser.add_argument_group("output formats")
outputs.add_argument(
"-o",
"--outform",
choices=["text", "markdown", "json"],
default="text",
help="report output format",
metavar="<text/markdown/json>",
)
outputs.add_argument(
"--verbose-report",
action="store_true",
help="Include debugging information in report",
)
database = parser.add_argument_group("local contracts database")
database.add_argument("-s",
"--search",
help="search the contract database",
metavar="EXPRESSION")
database.add_argument(
"--leveldb-dir",
help="specify leveldb directory for search or direct access operations",
metavar="LEVELDB_PATH",
)
utilities = parser.add_argument_group("utilities")
utilities.add_argument("--hash",
help="calculate function signature hash",
metavar="SIGNATURE")
utilities.add_argument(
"--storage",
help="read state variables from storage index, use with -a",
metavar="INDEX,NUM_SLOTS,[array] / mapping,INDEX,[KEY1, KEY2...]",
)
utilities.add_argument(
"--solv",
help=
"specify solidity compiler version. If not present, will try to install it (Experimental)",
metavar="SOLV",
)
utilities.add_argument(
"--contract-hash-to-address",
help="returns corresponding address for a contract address hash",
metavar="SHA3_TO_LOOK_FOR",
)
options = parser.add_argument_group("options")
options.add_argument(
"-m",
"--modules",
help="Comma-separated list of security analysis modules",
metavar="MODULES",
)
options.add_argument(
"--max-depth",
type=int,
default=22,
help="Maximum recursion depth for symbolic execution",
)
options.add_argument(
"--strategy",
choices=["dfs", "bfs", "naive-random", "weighted-random"],
default="dfs",
help="Symbolic execution strategy",
)
options.add_argument(
"--max-transaction-count",
type=int,
default=3,
help="Maximum number of transactions issued by laser",
)
options.add_argument(
"--execution-timeout",
type=int,
default=600,
help="The amount of seconds to spend on symbolic execution",
)
options.add_argument(
"--create-timeout",
type=int,
default=10,
help="The amount of seconds to spend on "
"the initial contract creation",
)
options.add_argument("--solc-args", help="Extra arguments for solc")
options.add_argument("--phrack",
action="store_true",
help="Phrack-style call graph")
options.add_argument("--enable-physics",
action="store_true",
help="enable graph physics simulation")
options.add_argument("-v",
type=int,
help="log level (0-2)",
metavar="LOG_LEVEL")
options.add_argument(
"-q",
"--query-signature",
action="store_true",
help="Lookup function signatures through www.4byte.directory",
)
rpc = parser.add_argument_group("RPC options")
rpc.add_argument("-i",
action="store_true",
help="Preset: Infura Node service (Mainnet)")
rpc.add_argument(
"--rpc",
help="custom RPC settings",
metavar="HOST:PORT / ganache / infura-[network_name]",
)
rpc.add_argument("--rpctls",
type=bool,
default=False,
help="RPC connection over TLS")
# Get config values
args = parser.parse_args()
if args.version:
if args.outform == "json":
print(json.dumps({"version_str": VERSION}))
else:
print("Mythril version {}".format(VERSION))
sys.exit()
# Parse cmdline args
if not (args.search or args.hash or args.disassemble or args.graph
or args.fire_lasers or args.storage or args.truffle
or args.statespace_json or args.contract_hash_to_address):
parser.print_help()
sys.exit()
if args.v:
if 0 <= args.v < 3:
coloredlogs.install(
fmt="%(name)s[%(process)d] %(levelname)s %(message)s",
level=[logging.NOTSET, logging.INFO, logging.DEBUG][args.v],
)
else:
exit_with_error(
args.outform,
"Invalid -v value, you can find valid values in usage")
if args.query_signature:
if sigs.ethereum_input_decoder == None:
exit_with_error(
args.outform,
"The --query-signature function requires the python package ethereum-input-decoder",
)
# -- commands --
if args.hash:
print(Mythril.hash_for_function_signature(args.hash))
sys.exit()
try:
# the mythril object should be our main interface
# infura = None, rpc = None, rpctls = None
# solc_args = None, dynld = None, max_recursion_depth = 12):
mythril = Mythril(
solv=args.solv,
dynld=args.dynld,
solc_args=args.solc_args,
enable_online_lookup=args.query_signature,
)
if args.dynld and not (args.rpc or args.i):
mythril.set_api_from_config_path()
if args.address:
# Establish RPC connection if necessary
if args.i:
mythril.set_api_rpc_infura()
elif args.rpc:
mythril.set_api_rpc(rpc=args.rpc, rpctls=args.rpctls)
elif not args.dynld:
mythril.set_api_rpc_localhost()
elif args.search or args.contract_hash_to_address:
# Open LevelDB if necessary
mythril.set_api_leveldb(mythril.leveldb_dir if not args.leveldb_dir
else args.leveldb_dir)
if args.search:
# Database search ops
mythril.search_db(args.search)
sys.exit()
if args.contract_hash_to_address:
# search corresponding address
try:
mythril.contract_hash_to_address(args.contract_hash_to_address)
except AddressNotFoundError:
print("Address not found.")
sys.exit()
if args.truffle:
try:
# not really pythonic atm. needs refactoring
mythril.analyze_truffle_project(args)
except FileNotFoundError:
print(
"Build directory not found. Make sure that you start the analysis from the project root, and that 'truffle compile' has executed successfully."
)
sys.exit()
# Load / compile input contracts
address = None
if args.code:
# Load from bytecode
address, _ = mythril.load_from_bytecode(args.code)
elif args.codefile:
bytecode = "".join(
[l.strip() for l in args.codefile if len(l.strip()) > 0])
address, _ = mythril.load_from_bytecode(bytecode)
elif args.address:
# Get bytecode from a contract address
address, _ = mythril.load_from_address(args.address)
elif args.solidity_file:
# Compile Solidity source file(s)
if args.graph and len(args.solidity_file) > 1:
exit_with_error(
args.outform,
"Cannot generate call graphs from multiple input files. Please do it one at a time.",
)
address, _ = mythril.load_from_solidity(
args.solidity_file) # list of files
else:
exit_with_error(
args.outform,
"No input bytecode. Please provide EVM code via -c BYTECODE, -a ADDRESS, or -i SOLIDITY_FILES",
)
# Commands
if args.storage:
if not args.address:
exit_with_error(
args.outform,
"To read storage, provide the address of a deployed contract with the -a option.",
)
storage = mythril.get_state_variable_from_storage(
address=address,
params=[a.strip() for a in args.storage.strip().split(",")],
)
print(storage)
elif args.disassemble:
easm_text = mythril.contracts[0].get_easm(
) # or mythril.disassemble(mythril.contracts[0])
sys.stdout.write(easm_text)
elif args.graph or args.fire_lasers:
if not mythril.contracts:
exit_with_error(
args.outform,
"input files do not contain any valid contracts")
if args.graph:
html = mythril.graph_html(
strategy=args.strategy,
contract=mythril.contracts[0],
address=address,
enable_physics=args.enable_physics,
phrackify=args.phrack,
max_depth=args.max_depth,
execution_timeout=args.execution_timeout,
create_timeout=args.create_timeout,
)
try:
with open(args.graph, "w") as f:
f.write(html)
except Exception as e:
exit_with_error(args.outform,
"Error saving graph: " + str(e))
else:
report = mythril.fire_lasers(
strategy=args.strategy,
address=address,
modules=[
m.strip() for m in args.modules.strip().split(",")
] if args.modules else [],
verbose_report=args.verbose_report,
max_depth=args.max_depth,
execution_timeout=args.execution_timeout,
create_timeout=args.create_timeout,
max_transaction_count=args.max_transaction_count,
)
outputs = {
"json": report.as_json(),
"text": report.as_text(),
"markdown": report.as_markdown(),
}
print(outputs[args.outform])
elif args.statespace_json:
if not mythril.contracts:
exit_with_error(
args.outform,
"input files do not contain any valid contracts")
statespace = mythril.dump_statespace(
strategy=args.strategy,
contract=mythril.contracts[0],
address=address,
max_depth=args.max_depth,
execution_timeout=args.execution_timeout,
create_timeout=args.create_timeout,
)
try:
with open(args.statespace_json, "w") as f:
json.dump(statespace, f)
except Exception as e:
exit_with_error(args.outform, "Error saving json: " + str(e))
else:
parser.print_help()
except CriticalError as ce:
exit_with_error(args.outform, str(ce))
from app import app_controller
import filestore
from magic_defines import *
import sd_material_ui
from sd_material_ui import Snackbar
import coloredlogs, logging
from sd_material_ui import Snackbar
from magic_defines import *
from utils import *
from autolink import Redirect
from localstorage_writer import LocalStorageWriter
from localstorage_reader import LocalStorageReader
import coloredlogs, logging
logger = logging.getLogger(__name__)
coloredlogs.install(level='DEBUG', logger=logger)
import gettext
zh = gettext.translation("user_profile", locale_d(), languages=["zh_CN"])
zh.install(True)
_ = zh.gettext
def gen_id(name):
# user module as name prefix
s_id = g_id(__name__, name)
return s_id
def generate_field_user_id(user):
field_row = html.Div(
def main():
parser = argparse.ArgumentParser(
prog='nrfsec',
description=
'perform security related tasks on nRF51 targets through SWD interface',
usage='nrfsec <command> [<args>]')
parser.add_argument(
'-f',
'--frequency',
help='frequency to run the SWD interface (default 4 MHz)',
type=int,
default=4000000)
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='increase output verbosity',
)
subparsers = parser.add_subparsers(title='supported subcommands',
dest='command')
# parser - info
parser_info = subparsers.add_parser('info',
help='display chip information')
parser_info.add_argument(
'-d',
'--delay',
type=int,
help=
'delay after reset before reading info (to initialize RAM and peripherals)'
)
parser_info.add_argument(
'-m',
'--maxsearch',
help=
'Maximum space to search for gadgets, starting at the reset vector',
default='0x1000')
# parser - erase
parser_erase = subparsers.add_parser('erase',
help='perform a complete erase')
parser_erase.add_argument(
'-m',
'--maxsearch',
help=
'Maximum space to search for gadgets, starting at the reset vector',
default='0x1000')
# parser - restore
parser_restore = subparsers.add_parser(
'restore', help='restore specific images to an unlocked chip')
parser_restore.add_argument('-u',
'--uicr',
help='UICR image file to restore',
default=os.getcwd() +
'/fw/0x10001000_0x10001400_UICR.bin')
parser_restore.add_argument('-r',
'--rom',
help='ROM image file to restore',
default=os.getcwd() +
'/fw/0x00000000_0x00040000_ROM.bin')
parser_restore.add_argument('-s',
'--skipverify',
action='store_true',
help='skip image verification steps')
# parser - read
parser_read = subparsers.add_parser('read',
help='read memory contents to outfile')
parser_read.add_argument('-o',
'--outfile',
help='File to store memory contents',
default='dump.bin')
parser_read.add_argument(
'-m',
'--maxsearch',
help=
'Maximum space to search for gadgets, starting at the reset vector',
default='0x1000')
parser_read.add_argument('--skipverify',
action='store_true',
help='skip image verification steps')
parser_read.add_argument(
'-d',
'--delay',
type=int,
help='Delay time after boot before reading memory')
address_group = parser_read.add_mutually_exclusive_group()
address_group.add_argument('-s',
'--start',
help='address to begin reading memory')
parser_read.add_argument('-e',
'--end',
help='address to stop reading memory')
address_group.add_argument(
'-a',
'--all',
action='store_true',
help=
'Dump all memory as discovered in FICR register starting @ 0x00000000')
# parser - unlock
parser_unlock = subparsers.add_parser('unlock',
help='unlock the device if locked')
parser_unlock.add_argument(
'-m',
'--maxsearch',
help=
'Maximum space to search for gadgets, starting at the reset vector',
default='0x1000')
parser_unlock.add_argument(
'-d',
'--directory',
help='Directory to store the recovered firmware',
default=os.getcwd() + '/fw')
parser_unlock.add_argument('--skipverify',
action='store_true',
help='skip image verification steps')
# parser - lock
subparsers.add_parser('lock', help='lock the device if unlocked')
args = parser.parse_args()
# logging setup
if args.verbose:
coloredlogs.install(level='DEBUG',
datefmt='%H:%M:%S',
fmt='%(asctime)s: %(levelname)s - %(message)s')
else:
coloredlogs.install(level='INFO',
datefmt='%H:%M:%S',
fmt='%(asctime)s: %(levelname)s - %(message)s')
sub_commands = ['info', 'erase', 'restore', 'read', 'unlock', 'lock']
if args.command not in sub_commands:
logger.info('please specify a subcommand: {}'.format(sub_commands))
exit()
# connect to ST-Link, required for all commands
logger.info('Connecting to debug interface @ {}Hz'.format(args.frequency))
dev = dbg.DeviceInterface(args.frequency)
# commands unaffected by lock state
if args.command == 'erase':
if dev.EraseAll():
logger.info('target erased')
exit()
else:
logger.info('unable to erase target')
if dev.isLocked():
logger.debug('target memory is locked')
if args.command == 'lock':
logger.info('target is already locked')
exit()
if args.command == 'restore':
logger.info('target is locked, perform unlock before restoring')
exit()
r_gadget = dev.SearchReadGadget(int(args.maxsearch, 0))
if r_gadget:
if args.command == 'info':
if args.delay:
dev.Delay(args.delay)
dev.Dumpinfo(r_gadget)
utils.printChipInfo(dev.chip_info)
if args.verbose:
print('Factory Information Configuration Registers (FICR)')
utils.printHexDump(dev.ficr, nordic.NRF_FICR_BASE)
print('User Information Configuration Registers (UICR)')
utils.printHexDump(dev.uicr, nordic.NRF_UICR_BASE)
if 0x3FFFFFFF > int(dev.chip_info['ECB -> ECBDATAPTR'],
0) > 0x20000000:
utils.printECB(dev.GetECB(r_gadget))
if 0x20000000 < int(dev.chip_info['CCM -> CNFPTR'],
0) < 0x3FFFFFFF:
pass
if args.command == 'read':
if args.delay:
dev.Delay(args.delay)
if args.all:
dev.Dumpinfo(r_gadget)
dev.ReadAll(os.getcwd() + '/fw', args.skipverify, r_gadget)
elif args.start is not None and args.end is not None:
if int(args.start, 0) < int(args.end, 0):
dev.DumpToFile(args.outfile, int(args.start, 0),
int(args.end, 0), r_gadget)
else:
logger.error(
'End address cannot begin before start address')
exit()
else:
logger.error(
'You must supply a start and end address to read')
exit()
if args.command == 'unlock':
dev.Dumpinfo(r_gadget)
if dev.VolatileUnlock(r_gadget, args.directory,
args.skipverify):
logger.info('Unlocking procedure was successful')
dev.cortex_m.reset()
exit()
else:
logger.info('Unlocking procedure was unsuccessful')
exit()
else:
logger.info('no read gadget found, exitting')
exit()
# unlocked
else:
logger.debug('target memory is unlocked')
if args.command == 'lock':
dev.LockTarget()
if args.command == 'unlock':
logger.info(
'target is already unlocked proceed with other commands')
exit()
if args.command == 'restore':
if not os.path.isfile(args.rom):
logger.error(
'ROM file {} not found\n'
'specify the -u and -r filename for the UICR and ROM files respectivily'
.format(args.rom))
exit()
if not os.path.isfile(args.uicr):
logger.error(
'UICR file {} not found:\n'
'specify the -u and -r filename for the UICR and ROM files respectivily'
.format(args.uicr))
exit()
utils.UnlockUICRImage(args.uicr)
dev.RestoreImage(args.rom, nordic.ROM_START)
if not args.skipverify:
if not dev.VerifyImage(args.rom, nordic.ROM_START,
os.path.getsize(args.rom)):
logger.info(
'failed to verify writing {} to the target'.format(
args.rom))
exit()
dev.RestoreImage(args.uicr, nordic.NRF_UICR_BASE)
if not args.skipverify:
if not dev.VerifyImage(args.uicr, nordic.NRF_UICR_BASE,
os.path.getsize(args.uicr)):
logger.info(
'failed to verify writing {} to the target'.format(
args.uicr))
exit()
logger.info('Target memory has been restored')
dev.cortex_m.nodebug()
exit()
if args.command == 'info':
dev.Dumpinfo(args.delay)
utils.printChipInfo(dev.chip_info)
if args.verbose:
print('Factory Information Configuration Registers (FICR)')
utils.printHexDump(dev.ficr, nordic.NRF_FICR_BASE)
print('User Information Configuration Registers (UICR)')
utils.printHexDump(dev.uicr, nordic.NRF_UICR_BASE)
# these values must point to RAM or they are uninitialized
if 0x20000000 < int(dev.chip_info['ECB -> ECBDATAPTR'],
0) < 0x3FFFFFFF:
(dev.GetECB())
if 0x20000000 < int(dev.chip_info['CCM -> CNFPTR'],
0) < 0x3FFFFFFF:
pass
if args.command == 'read':
if args.delay:
dev.Delay(args.delay)
if args.all:
dev.Dumpinfo()
dev.ReadAll(os.getcwd() + '/fw', args.skipverify)
elif args.start is not None and args.end is not None:
if int(args.start, 0) < int(args.end, 0):
dev.DumpToFile(args.outfile, int(args.start, 0),
int(args.end, 0))
else:
logger.error(
'End address cannot begin before start address')
exit()
else:
logger.error('You must supply a start and end address to read')
exit()
import os
import sys
from etree_patch import ET, patch_et
import logging
level = logging.DEBUG
level = logging.ERROR
try:
import coloredlogs
coloredlogs.DEFAULT_LOG_FORMAT = '%(levelname)-8s %(name)-15s %(message)s'
coloredlogs.install()
coloredlogs.set_level(level)
except Exception:
logging.basicConfig(level=level)
gir_nsmap = {'glib': 'http://www.gtk.org/introspection/glib/1.0', 'c': 'http://www.gtk.org/introspection/c/1.0', None: 'http://www.gtk.org/introspection/core/1.0'}
ns_c = "{%s}" % (gir_nsmap['c'],)
ns_gir = "{%s}" % (gir_nsmap[None],)
ns_glib = "{%s}" % (gir_nsmap['glib'],)
patch_et(gir_nsmap)
class NO_DEFAULT (object):
pass
_squelched = set()
def squelch (f, msg, *args):
if args: msg = msg % args
if msg in _squelched: return
_squelched.add(msg)
if len(_squelched) > 2000: _squelched.clear()
f(msg)
def _init_logger(self):
coloredlogs.install()
logging.getLogger("requests").setLevel(logging.WARNING)
logger = logging.getLogger('passme')
logger.setLevel(logging.INFO)
self.logger = logger
from collections import OrderedDict
if util.find_spec("coloredlogs"):
import coloredlogs
if util.find_spec("humanize"):
import humanize
if sys.version_info[0] < 3:
raise Exception("run_fpga_task script must be using Python 3")
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
# Configure logging system
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
LOG_FORMAT = "%(levelname)5s (%(threadName)15s) - %(message)s"
if util.find_spec("coloredlogs"):
coloredlogs.install(level='INFO', stream=sys.stdout, fmt=LOG_FORMAT)
else:
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format=LOG_FORMAT)
logger = logging.getLogger('OpenFPGA_Task_logs')
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
# Read commandline arguments
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
parser = argparse.ArgumentParser()
parser.add_argument('tasks', nargs='+')
parser.add_argument('--maxthreads',
type=int,
default=2,
help="Number of fpga_flow threads to run default = 2," +
import csv
# Locals
dirname = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
sys.path.append(dirname)
from src.models.methods.metadata import get_parameter_metadata
from src.models.icestupaClass import Icestupa
from src.utils.settings import config
from src.utils import setup_logger
if __name__ == "__main__":
logger = logging.getLogger(__name__)
coloredlogs.install(
fmt="%(funcName)s %(levelname)s %(message)s",
level=logging.INFO,
logger=logger,
)
# answers = dict(
# # location="Schwarzsee 2019",
# location="Guttannen 2020",
# # location="Gangles 2021",
# )
locations = [
"Guttannen 2021", "Guttannen 2020", "Schwarzsee 2019", "Gangles 2021"
]
filenames = []
for location in locations:
# Initialise icestupa object
import coloredlogs
import numpy as np
import torch
import torch.nn as nn
import torchtext.vocab
from tqdm.auto import tqdm
import table
import table.Models
import table.modules
from table.Models import CopyGenerator, LayCoAttention, ParserModel, QCoAttention, RNNEncoder, SeqDecoder
from table.modules.Embeddings import PartUpdateEmbedding
logger = logging.getLogger(__name__)
coloredlogs.install(level='DEBUG')
def load_orig_glove(emb_file: str) -> dict:
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
if os.path.isfile(emb_file + ".pickle"):
logger.info(" * load glove from pickle")
emb = pickle.load(open(emb_file + ".pickle", "rb"))
else:
logger.info(" * load glove from txt, dumping to pickle")
emb = dict(
get_coefs(*o.split(" ")) for o in open(emb_file, encoding='latin'))
pickle.dump(emb, open(emb_file + ".pickle", "wb"))
# make condor scratch directory
scratch = make_condor_scratch_dir(args.traverse_root)
# make condor file
condorpath = make_condor_file(
scratch,
args.previous_traverse,
args.traverse_root,
args.cpus,
args.memory,
args.chunk_size,
args.exclude,
args.fast_forward,
args.accounting_group,
args.path_to_virtualenv,
)
# Execute
if args.dryrun:
logging.error(
f"Script Aborted: Condor job not submitted ({condorpath}).")
else:
cmd = f"condor_submit {condorpath}"
logging.info(cmd)
subprocess.check_call(cmd.split(), cwd=scratch)
if __name__ == "__main__":
coloredlogs.install(level="DEBUG")
main()
def initLogging(self):
self.logger = logging.getLogger(self.__class__.__name__)
if 'logLevel' in self.params:
coloredlogs.install(level=self.params['logLevel'])
# -*- coding: utf-8 -*-
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import random
import logging
import coloredlogs
coloredlogs.install(level=logging.INFO)
import bson
from tqdm import tqdm
from data.utils import encode_dict_list
def products_iter(input_bson_path, product_ids):
with open(input_bson_path, 'rb') as reader:
data = bson.decode_file_iter(reader)
for i, prod in tqdm(enumerate(data), unit='products', total=args.num_products, disable=True):
prod_id = prod.get('_id')
if prod_id in product_ids:
yield prod
def main(args):
if os.path.exists(args.save_train_bson):
raise FileExistsError(args.save_train_bson)
if os.path.exists(args.save_val_bson):
raise FileExistsError(args.save_val_bson)
from botaclan.constants import (
LOG_LEVEL,
LOG_ASYNCIO_LEVEL,
LOG_DISCORD_LEVEL,
LOG_WEBSOCKETS_LEVEL,
)
from importlib_metadata import version
import coloredlogs
import logging
import sys
__version__ = version(__package__)
root_log = logging.getLogger()
root_log.setLevel(LOG_LEVEL)
coloredlogs.install(logger=root_log, stream=sys.stdout, level=LOG_LEVEL)
logging.Formatter("%(asctime)s | %(name)s | %(levelname)s | %(message)s")
logging.getLogger("asyncio").setLevel(LOG_ASYNCIO_LEVEL)
logging.getLogger("discord").setLevel(LOG_DISCORD_LEVEL)
logging.getLogger("websockets").setLevel(LOG_WEBSOCKETS_LEVEL)
logging.getLogger(__name__)
def main():
coloredlogs.install()
coloredlogs.install(fmt='%(asctime)s %(message)s')
logging.info('Price-notifier bot just started')
report()
def main(
argsl: Optional[List[str]] = None,
args: Optional[argparse.Namespace] = None,
job_order_object: Optional[CWLObjectType] = None,
stdin: IO[Any] = sys.stdin,
stdout: Optional[Union[TextIO, StreamWriter]] = None,
stderr: IO[Any] = sys.stderr,
versionfunc: Callable[[], str] = versionstring,
logger_handler: Optional[logging.Handler] = None,
custom_schema_callback: Optional[Callable[[], None]] = None,
executor: Optional[JobExecutor] = None,
loadingContext: Optional[LoadingContext] = None,
runtimeContext: Optional[RuntimeContext] = None,
input_required: bool = True,
) -> int:
if not stdout: # force UTF-8 even if the console is configured differently
if hasattr(sys.stdout,
"encoding") and sys.stdout.encoding.upper() not in (
"UTF-8",
"UTF8",
):
if hasattr(sys.stdout, "detach"):
stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
else:
stdout = getwriter("utf-8")(sys.stdout) # type: ignore
else:
stdout = sys.stdout
_logger.removeHandler(defaultStreamHandler)
stderr_handler = logger_handler
if stderr_handler is not None:
_logger.addHandler(stderr_handler)
else:
coloredlogs.install(logger=_logger, stream=stderr)
stderr_handler = _logger.handlers[-1]
workflowobj = None
prov_log_handler = None # type: Optional[logging.StreamHandler]
try:
if args is None:
if argsl is None:
argsl = sys.argv[1:]
addl = [] # type: List[str]
if "CWLTOOL_OPTIONS" in os.environ:
addl = os.environ["CWLTOOL_OPTIONS"].split(" ")
parser = arg_parser()
argcomplete.autocomplete(parser)
args = parser.parse_args(addl + argsl)
if args.record_container_id:
if not args.cidfile_dir:
args.cidfile_dir = os.getcwd()
del args.record_container_id
if runtimeContext is None:
runtimeContext = RuntimeContext(vars(args))
else:
runtimeContext = runtimeContext.copy()
# If caller parsed its own arguments, it may not include every
# cwltool option, so fill in defaults to avoid crashing when
# dereferencing them in args.
for key, val in get_default_args().items():
if not hasattr(args, key):
setattr(args, key, val)
configure_logging(args, stderr_handler, runtimeContext)
if args.version:
print(versionfunc())
return 0
_logger.info(versionfunc())
if args.print_supported_versions:
print("\n".join(supported_cwl_versions(args.enable_dev)))
return 0
if not args.workflow:
if os.path.isfile("CWLFile"):
args.workflow = "CWLFile"
else:
_logger.error(
"CWL document required, no input file was provided")
parser.print_help()
return 1
if args.ga4gh_tool_registries:
ga4gh_tool_registries[:] = args.ga4gh_tool_registries
if not args.enable_ga4gh_tool_registry:
del ga4gh_tool_registries[:]
if args.mpi_config_file is not None:
runtimeContext.mpi_config = MpiConfig.load(args.mpi_config_file)
setup_schema(args, custom_schema_callback)
if args.provenance:
if argsl is None:
raise Exception("argsl cannot be None")
if setup_provenance(args, argsl, runtimeContext) is not None:
return 1
loadingContext = setup_loadingContext(loadingContext, runtimeContext,
args)
uri, tool_file_uri = resolve_tool_uri(
args.workflow,
resolver=loadingContext.resolver,
fetcher_constructor=loadingContext.fetcher_constructor,
)
try_again_msg = ("" if args.debug else
", try again with --debug for more information")
try:
job_order_object, input_basedir, jobloader = load_job_order(
args,
stdin,
loadingContext.fetcher_constructor,
loadingContext.overrides_list,
tool_file_uri,
)
if args.overrides:
loadingContext.overrides_list.extend(
load_overrides(file_uri(os.path.abspath(args.overrides)),
tool_file_uri))
loadingContext, workflowobj, uri = fetch_document(
uri, loadingContext)
if args.print_deps and loadingContext.loader:
printdeps(workflowobj, loadingContext.loader, stdout,
args.relative_deps, uri)
return 0
loadingContext, uri = resolve_and_validate_document(
loadingContext,
workflowobj,
uri,
preprocess_only=(args.print_pre or args.pack),
skip_schemas=args.skip_schemas,
)
if loadingContext.loader is None:
raise Exception("Impossible code path.")
processobj, metadata = loadingContext.loader.resolve_ref(uri)
processobj = cast(CommentedMap, processobj)
if args.pack:
stdout.write(print_pack(loadingContext, uri))
return 0
if args.provenance and runtimeContext.research_obj:
# Can't really be combined with args.pack at same time
runtimeContext.research_obj.packed_workflow(
print_pack(loadingContext, uri))
if args.print_pre:
stdout.write(
json_dumps(processobj,
indent=4,
sort_keys=True,
separators=(",", ": ")))
return 0
tool = make_tool(uri, loadingContext)
if args.make_template:
make_template(tool)
return 0
if args.validate:
print(f"{args.workflow} is valid CWL.")
return 0
if args.print_rdf:
stdout.write(
printrdf(tool, loadingContext.loader.ctx,
args.rdf_serializer))
return 0
if args.print_dot:
printdot(tool, loadingContext.loader.ctx, stdout)
return 0
if args.print_targets:
for f in ("outputs", "steps", "inputs"):
if tool.tool[f]:
_logger.info("%s%s targets:", f[0].upper(), f[1:-1])
stdout.write(" " + "\n ".join(
[shortname(t["id"]) for t in tool.tool[f]]) + "\n")
return 0
if args.target:
ctool = choose_target(args, tool, loadingContext)
if ctool is None:
return 1
else:
tool = ctool
elif args.single_step:
ctool = choose_step(args, tool, loadingContext)
if ctool is None:
return 1
else:
tool = ctool
if args.print_subgraph:
if "name" in tool.tool:
del tool.tool["name"]
stdout.write(
json_dumps(tool.tool,
indent=4,
sort_keys=True,
separators=(",", ": ")))
return 0
except (ValidationException) as exc:
_logger.error("Tool definition failed validation:\n%s",
str(exc),
exc_info=args.debug)
return 1
except (RuntimeError, WorkflowException) as exc:
_logger.error(
"Tool definition failed initialization:\n%s",
str(exc),
exc_info=args.debug,
)
return 1
except Exception as exc:
_logger.error(
"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s",
try_again_msg,
str(exc) if not args.debug else "",
exc_info=args.debug,
)
return 1
if isinstance(tool, int):
return tool
# If on MacOS platform, TMPDIR must be set to be under one of the
# shared volumes in Docker for Mac
# More info: https://dockstore.org/docs/faq
if sys.platform == "darwin":
default_mac_path = "/private/tmp/docker_tmp"
if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX:
runtimeContext.tmp_outdir_prefix = default_mac_path
if runtimeContext.tmpdir_prefix == DEFAULT_TMP_PREFIX:
runtimeContext.tmpdir_prefix = default_mac_path
if check_working_directories(runtimeContext) is not None:
return 1
if args.cachedir:
if args.move_outputs == "move":
runtimeContext.move_outputs = "copy"
runtimeContext.tmp_outdir_prefix = args.cachedir
runtimeContext.secret_store = getdefault(runtimeContext.secret_store,
SecretStore())
runtimeContext.make_fs_access = getdefault(
runtimeContext.make_fs_access, StdFsAccess)
if not executor:
if args.parallel:
temp_executor = MultithreadedJobExecutor()
runtimeContext.select_resources = temp_executor.select_resources
real_executor = temp_executor # type: JobExecutor
else:
real_executor = SingleJobExecutor()
else:
real_executor = executor
try:
runtimeContext.basedir = input_basedir
if isinstance(tool, ProcessGenerator):
tfjob_order = {} # type: CWLObjectType
if loadingContext.jobdefaults:
tfjob_order.update(loadingContext.jobdefaults)
if job_order_object:
tfjob_order.update(job_order_object)
tfout, tfstatus = real_executor(tool.embedded_tool,
tfjob_order, runtimeContext)
if not tfout or tfstatus != "success":
raise WorkflowException(
"ProcessGenerator failed to generate workflow")
tool, job_order_object = tool.result(tfjob_order, tfout,
runtimeContext)
if not job_order_object:
job_order_object = None
try:
initialized_job_order_object = init_job_order(
job_order_object,
args,
tool,
jobloader,
stdout,
print_input_deps=args.print_input_deps,
relative_deps=args.relative_deps,
make_fs_access=runtimeContext.make_fs_access,
input_basedir=input_basedir,
secret_store=runtimeContext.secret_store,
input_required=input_required,
)
except SystemExit as err:
return err.code
del args.workflow
del args.job_order
conf_file = getattr(args,
"beta_dependency_resolvers_configuration",
None) # str
use_conda_dependencies = getattr(args, "beta_conda_dependencies",
None) # str
if conf_file or use_conda_dependencies:
runtimeContext.job_script_provider = DependenciesConfiguration(
args)
else:
runtimeContext.find_default_container = functools.partial(
find_default_container,
default_container=runtimeContext.default_container,
use_biocontainers=args.beta_use_biocontainers,
)
(out, status) = real_executor(tool,
initialized_job_order_object,
runtimeContext,
logger=_logger)
if out is not None:
if runtimeContext.research_obj is not None:
runtimeContext.research_obj.create_job(out, True)
def remove_at_id(doc: CWLObjectType) -> None:
for key in list(doc.keys()):
if key == "@id":
del doc[key]
else:
value = doc[key]
if isinstance(value, MutableMapping):
remove_at_id(value)
elif isinstance(value, MutableSequence):
for entry in value:
if isinstance(entry, MutableMapping):
remove_at_id(entry)
remove_at_id(out)
visit_class(
out,
("File", ),
functools.partial(add_sizes,
runtimeContext.make_fs_access("")),
)
def loc_to_path(obj: CWLObjectType) -> None:
for field in ("path", "nameext", "nameroot", "dirname"):
if field in obj:
del obj[field]
if cast(str, obj["location"]).startswith("file://"):
obj["path"] = uri_file_path(cast(str, obj["location"]))
visit_class(out, ("File", "Directory"), loc_to_path)
# Unsetting the Generation from final output object
visit_class(out, ("File", ),
MutationManager().unset_generation)
if isinstance(out, str):
stdout.write(out)
else:
stdout.write(json_dumps(out, indent=4, ensure_ascii=False))
stdout.write("\n")
if hasattr(stdout, "flush"):
stdout.flush()
if status != "success":
_logger.warning("Final process status is %s", status)
return 1
_logger.info("Final process status is %s", status)
return 0
except (ValidationException) as exc:
_logger.error("Input object failed validation:\n%s",
str(exc),
exc_info=args.debug)
return 1
except UnsupportedRequirement as exc:
_logger.error(
"Workflow or tool uses unsupported feature:\n%s",
str(exc),
exc_info=args.debug,
)
return 33
except WorkflowException as exc:
_logger.error(
"Workflow error%s:\n%s",
try_again_msg,
strip_dup_lineno(str(exc)),
exc_info=args.debug,
)
return 1
except Exception as exc: # pylint: disable=broad-except
_logger.error(
"Unhandled error%s:\n %s",
try_again_msg,
str(exc),
exc_info=args.debug,
)
return 1
finally:
if (args and runtimeContext and runtimeContext.research_obj
and workflowobj and loadingContext):
research_obj = runtimeContext.research_obj
if loadingContext.loader is not None:
research_obj.generate_snapshot(
prov_deps(workflowobj, loadingContext.loader, uri))
else:
_logger.warning("Unable to generate provenance snapshot "
" due to missing loadingContext.loader.")
if prov_log_handler is not None:
# Stop logging so we won't half-log adding ourself to RO
_logger.debug("[provenance] Closing provenance log file %s",
prov_log_handler)
_logger.removeHandler(prov_log_handler)
# Ensure last log lines are written out
prov_log_handler.flush()
# Underlying WritableBagFile will add the tagfile to the manifest
prov_log_handler.stream.close()
prov_log_handler.close()
research_obj.close(args.provenance)
_logger.removeHandler(stderr_handler)
_logger.addHandler(defaultStreamHandler)
from datetime import datetime
from tests.settings import SUMMARIES_BASE_DIR, TESTS_BASE_DIR
from src.orbitaz.utils.filesystem import (
find_or_create_target_directory, )
import pytest
from orbitaz.utils.fixtures_handler import FixtureHandler
import shutil
from pathlib import Path
# ______________________________________________________________________________
# /////////////////////////////////////////////////////////////////////// LOGGER
import coloredlogs, logging
logger = logging.getLogger(__name__)
coloredlogs.install(level="DEBUG", logger=logger)
# ______________________________________________________________________________
# ///////////////////////////////////////////////////////////////////////// CONF
def pytest_configure(config) -> None:
"""
This pytest built-in fixture function allows to set pytest terminal commands programatically on runtime.
This specific implementation configures timestamps for excel summaries and takes care of storing excel test summaries at the right folder.
In the course of that, it applies a recursive search function called 'get_or_create_target_directory' to find or make a path for the excel reports.
"""
# We try to find a summary directory for the current year and month
target_directory_path = (SUMMARIES_BASE_DIR /
def run(init: bool, server: str, country_code: str, country: str, area: str,
tcp: bool, daemon: bool, max_load: int, top_servers: int, pings: str,
kill: bool, kill_flush: bool, update: bool, list_servers: bool,
force_fw_rules: bool, p2p: bool, dedicated: bool, double_vpn: bool,
tor_over_vpn: bool, anti_ddos: bool, netflix: bool, test: bool,
internally_allowed: List, skip_dns_patch: bool, silent: bool,
nvram: str, openvpn_options: str, location: float) -> bool:
if init:
initialise(log_folder)
fieldstyles = {
'asctime': {
'color': 'green'
},
'hostname': {
'color': 'magenta'
},
'levelname': {
'color': 'black',
'bold': True
},
'name': {
'color': 'blue'
},
'programname': {
'color': 'cyan'
},
}
levelstyles = {
'spam': {
'color': 'green',
'faint': True
},
'debug': {
'color': 'green',
'bold': True
},
'verbose': {
'color': 'blue',
'bold': True
},
'info': {},
'notice': {
'color': 'magenta',
'bold': True
},
'warning': {
'color': 'yellow',
'bold': True
},
'success': {
'color': 'green',
'bold': True
},
'error': {
'color': 'red',
'bold': True
},
'critical': {
'color': 'white',
'background': 'red',
'bold': True
}
}
logger.addHandler(logging.StreamHandler())
# if log folder doesnt exist, exit, "--init" creates it
if not os.path.exists(log_folder):
logger.error(
"Please initialise first by running 'sudo openpyn --init', then start using 'openpyn' without sudo"
)
return 1
# Add another rotating handler to log to .log files
# fix permissions if needed
for attempt in range(2):
try:
file_handler = logging.handlers.TimedRotatingFileHandler(
log_folder + '/openpyn.log', when='W0', interval=4)
file_handler_formatter = logging.Formatter(log_format)
file_handler.setFormatter(file_handler_formatter)
logger.addHandler(file_handler)
except PermissionError:
root.verify_root_access(
"Root access needed to set permissions of {}/openpyn.log".
format(log_folder))
subprocess.run(
"sudo chmod 777 {}/openpyn.log".format(log_folder).split())
subprocess.run(
"sudo chmod 777 {}/openpyn-notifications.log".format(
log_folder).split())
else:
break
# In this case only log messages originating from this logger will show up on the terminal.
coloredlogs.install(level="verbose",
logger=logger,
fmt=log_format,
level_styles=levelstyles,
field_styles=fieldstyles)
stats = True
if sys.__stdin__.isatty():
logger.debug("Interactive")
else:
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.setLevel(logging.WARNING)
logger.debug("Non-Interactive")
stats = False
port = "udp"
if tcp:
port = "tcp"
detected_os = sys.platform
if detected_os == "linux":
if subprocess.check_output(["/bin/uname", "-o"]).decode(
sys.stdout.encoding).strip() == "ASUSWRT-Merlin":
force_fw_rules = False
silent = True
skip_dns_patch = True
if openvpn_options:
openvpn_options += " " + "--syslog openpyn"
else:
openvpn_options = "--syslog openpyn"
logger.debug(openvpn_options)
elif os.path.exists("/etc/openwrt_release"):
force_fw_rules = False
silent = True
skip_dns_patch = True
nvram = None
else:
nvram = None
elif detected_os == "win32":
logger.error("Are you even a l33t mate? Try GNU/Linux")
return 1
# check if dependencies are installed
if shutil.which("openvpn") is None or shutil.which(
"wget") is None or shutil.which("unzip") is None:
# In case of Debian Sid where "openvpn" is only in root's PATH, don't error out
try:
root_access = root.verify_root_access(
"Sudo credentials required to check if 'openvpn' is available in root's PATH"
)
if root_access is False:
root.obtain_root_access()
subprocess.check_output(["sudo", "which", "wget"])
subprocess.check_output(["sudo", "which", "unzip"])
# subprocess.check_output(["sudo", "which", "openvpn"])
except subprocess.CalledProcessError:
logger.error("Please Install 'openvpn' 'wget' 'unzip' first")
return 1
elif daemon:
if detected_os != "linux":
logger.error("Daemon mode is only available in GNU/Linux distros")
return 1
if not root.verify_running_as_root():
logger.error("Please run '--daemon' or '-d' mode with sudo")
return 1
openpyn_options = ""
# if only positional argument used
if country_code is None and server is None:
country_code = country # consider the positional arg e.g "us" same as "-c us"
# if either "-c" or positional arg f.e "au" is present
if country_code:
if len(country_code) > 2: # full country name
# get the country_code from the full name
country_code = api.get_country_code(full_name=country_code)
country_code = country_code.lower()
openpyn_options += country_code
elif server:
openpyn_options += " --server " + server
if area:
openpyn_options += " --area " + area
if tcp:
openpyn_options += " --tcp"
if max_load:
openpyn_options += " --max-load " + str(max_load)
if top_servers:
openpyn_options += " --top-servers " + str(top_servers)
if pings:
openpyn_options += " --pings " + str(pings)
if force_fw_rules:
openpyn_options += " --force-fw-rules"
if p2p:
openpyn_options += " --p2p"
if dedicated:
openpyn_options += " --dedicated"
if double_vpn:
openpyn_options += " --double"
if tor_over_vpn:
openpyn_options += " --tor"
if anti_ddos:
openpyn_options += " --anti-ddos"
if netflix:
openpyn_options += " --netflix"
if test:
openpyn_options += " --test"
if internally_allowed:
open_ports = ""
for port_number in internally_allowed:
open_ports += " " + port_number
openpyn_options += " --allow" + open_ports
if skip_dns_patch:
openpyn_options += " --skip-dns-patch"
if nvram:
openpyn_options += " --nvram " + str(nvram)
if openvpn_options:
openpyn_options += " --openvpn-options '" + openvpn_options + "'"
# logger.debug(openpyn_options)
if subprocess.check_output(["/bin/uname", "-o"]).decode(
sys.stdout.encoding).strip() == "ASUSWRT-Merlin":
initd.update_service(openpyn_options, run=True)
elif os.path.exists("/etc/openwrt_release"):
initd.update_service(openpyn_options, run=True)
else:
systemd.update_service(openpyn_options, run=True)
return 0
elif kill:
logger.warning("Killing the running processes")
kill_management_client()
kill_vpn_processes() # don't touch iptable rules
kill_openpyn_process()
elif kill_flush:
firewall.clear_fw_rules() # also clear iptable rules
# if --allow present, allow those ports internally
logger.info("Re-enabling ipv6")
firewall.manage_ipv6(disable=False)
if internally_allowed:
network_interfaces = get_network_interfaces()
firewall.internally_allow_ports(network_interfaces,
internally_allowed)
kill_management_client()
kill_vpn_processes()
kill_openpyn_process()
elif update:
update_config_files()
# a hack to list all countries and their codes when no arg supplied with "-l"
elif list_servers != 'nope': # means "-l" supplied
if list_servers is None: # no arg given with "-l"
if p2p or dedicated or double_vpn or tor_over_vpn or anti_ddos or netflix:
# show the special servers in all countries
display_servers(list_servers="all",
port=port,
area=area,
p2p=p2p,
dedicated=dedicated,
double_vpn=double_vpn,
tor_over_vpn=tor_over_vpn,
anti_ddos=anti_ddos,
netflix=netflix,
location=location)
else:
api.list_all_countries()
# if a country code is supplied give details about that country only.
else:
# if full name of the country supplied get country_code
if len(list_servers) > 2:
list_servers = api.get_country_code(full_name=list_servers)
display_servers(list_servers=list_servers,
port=port,
area=area,
p2p=p2p,
dedicated=dedicated,
double_vpn=double_vpn,
tor_over_vpn=tor_over_vpn,
anti_ddos=anti_ddos,
netflix=netflix,
location=location)
# only clear/touch FW Rules if "-f" used
elif force_fw_rules:
firewall.clear_fw_rules()
# check if OpenVPN config files exist if not download them.
check_config_files()
# if only positional argument used
if country_code is None and server is None:
country_code = country # consider the positional arg e.g "us" same as "-c us"
# if either "-c" or positional arg f.e "au" is present
if country_code:
# ask for and store credentials if not present, skip if "--test"
if not test:
if credentials.check_credentials() is False:
credentials.save_credentials()
if len(country_code) > 2: # full country name
# get the country_code from the full name
country_code = api.get_country_code(full_name=country_code)
country_code = country_code.lower()
# keep trying to connect to new servers
for tries in range(3): # pylint: disable=W0612
better_servers_list = find_better_servers(
country_code, area, max_load, top_servers, tcp, p2p, dedicated,
double_vpn, tor_over_vpn, anti_ddos, netflix, location, stats)
pinged_servers_list = ping_servers(better_servers_list, pings,
stats)
chosen_servers = choose_best_servers(pinged_servers_list, stats)
# connect to chosen_servers, if one fails go to next
for aserver in chosen_servers:
if stats:
print(
Style.BRIGHT + Fore.BLUE +
"Out of the Best Available Servers, Chose",
(Fore.GREEN + aserver + Fore.BLUE) + "\n")
# if "-f" used apply firewall rules
if force_fw_rules:
network_interfaces = get_network_interfaces()
vpn_server_ip = get_vpn_server_ip(aserver, port)
firewall.apply_fw_rules(network_interfaces, vpn_server_ip,
skip_dns_patch)
if internally_allowed:
firewall.internally_allow_ports(
network_interfaces, internally_allowed)
if nvram:
# TODO return 0 on success else 1 in asus.run()
asus.run(aserver, country_code, nvram, "All", "adaptive",
"Strict", tcp, test)
logger.success("SAVED SERVER " + aserver + " ON PORT " +
port + " TO NVRAM")
return (connect(aserver, port, silent, test, skip_dns_patch,
openvpn_options))
elif server:
# ask for and store credentials if not present, skip if "--test"
if not test:
if credentials.check_credentials() is False:
credentials.save_credentials()
server = server.lower()
# if "-f" used apply firewall rules
if force_fw_rules:
network_interfaces = get_network_interfaces()
vpn_server_ip = get_vpn_server_ip(server, port)
firewall.apply_fw_rules(network_interfaces, vpn_server_ip,
skip_dns_patch)
if internally_allowed:
firewall.internally_allow_ports(network_interfaces,
internally_allowed)
if nvram:
asus.run(server, country_code, nvram, "All", "adaptive", "Strict",
tcp, test)
logger.success("SAVED SERVER " + server + " ON PORT " + port +
" TO NVRAM")
return 0
for i in range(20): # pylint: disable=W0612
return (connect(server, port, silent, test, skip_dns_patch,
openvpn_options))
else:
logger.info(
'To see usage options type: "openpyn -h" or "openpyn --help"')
return 0 # if everything went ok
import re
import signal
import sys
import tempfile
import threading
import time
import traceback
from typing import List, Optional
import coloredlogs
import yaml
from dnsrobocert.core import background, certbot, config, legacy, utils
LOGGER = logging.getLogger(__name__)
coloredlogs.install(logger=LOGGER)
def _process_config(
config_path: str,
directory_path: str,
runtime_config_path: str,
lock: threading.Lock,
):
dnsrobocert_config = config.load(config_path)
if not dnsrobocert_config:
return
if dnsrobocert_config.get("draft"):
LOGGER.info(
import logging
import os
import random
import time
import coloredlogs
from pmdarima import auto_arima
from matplotlib import gridspec, pyplot as plt
from data.convertData import interpolate_cgm, convert
from data.readData import read_data
import pandas as pd
coloredlogs.install(
level='INFO',
fmt=
'%(asctime)s %(filename)s[%(lineno)d]:%(funcName)s %(levelname)s %(message)s'
)
path = os.getenv('T1DPATH', '../')
filename = path + "data/csv/data_17_1-6.csv"
test = 60 * 3
time_periods = [600, 2400, 3600, 6000]
def prep_csv():
data = read_data(filename)
data = convert(data)
data = interpolate_cgm(data)
cgm = data['cgmValue']
cgm.to_csv(path + 'data/csv/cgm_17_1-6.csv')
parser = argparse.ArgumentParser()
parser.add_argument('-i', dest='inventory', required=True,
help='Path to inventory file')
parser.add_argument('-v', dest='verbose', action='store_true', help='Verbose output')
args = parser.parse_args()
log = logging.getLogger(__name__)
if args.verbose:
log.setLevel(logging.DEBUG)
level = 'DEBUG'
else:
log.setLevel(logging.INFO)
level = 'INFO'
coloredlogs.install(level=level)
if __name__ == '__main__':
with open(args.inventory, 'r') as inventory_file:
inventory = yaml.load(inventory_file)
log.info('Parse K8S config')
k8s_config = k8s.K8SConfig(**inventory['k8s'])
log.info('Parse docker config')
docker_config = k8s.DockerConfig(**inventory['docker'])
log.info('Parse nodes')
nodes = utils.parse_nodes(inventory['nodes'])
utils.parse_roles(**inventory['nodes'])
log.info('Install {} for nodes with label {}'.format(' '.join(constants.PACKAGES), 'all'))
utils.install_pkgs(constants.PACKAGES, 'all')
utils.create_repo('docker.list', constants.DOCKER_REPO)
log.info('Install {} for nodes with label {}'.format(' '.join(constants.DOCKER_PACKAGES), 'docker'))
configuration repos for a production release.
"""
import argparse
import logging
from pathlib import Path
import coloredlogs
import hglib
import verboselogs
import yaml
NAME = "tag_release"
COMMIT_AUTHOR = f"SalishSeaNowcast.release_mgmt.{NAME}"
verboselogs.install()
logger = logging.getLogger(NAME)
coloredlogs.install(
fmt="%(asctime)s %(hostname)s %(name)s %(levelname)s %(message)s")
def main():
parsed_args = _command_line_interface()
repos = _load_repos_list(parsed_args.repos_file)
for repo in repos:
_tag_repo(repo, parsed_args.tag)
def _command_line_interface():
parser = argparse.ArgumentParser(description="""
Tag NEMO code and model configuration repos for a production release.
""")
parser.prog = f"python -m release_mgmt.{NAME}"
parser.add_argument(
def __init__(self, config, external_logger=None):
"""
Initialize the model, read config file, start and end date, and logging
"""
# read the config file and store
if isinstance(config, str):
if not os.path.isfile(config):
raise Exception(
'Configuration file does not exist --> {}'.format(config))
configFile = config
# Read in the original users config
ucfg = get_user_config(config, modules='smrf')
elif isinstance(config, UserConfig):
ucfg = config
configFile = config.filename
else:
raise Exception('Config passed to SMRF is neither file name nor '
' UserConfig instance')
# start logging
if external_logger == None:
if 'log_level' in ucfg.cfg['system']:
loglevel = ucfg.cfg['system']['log_level'].upper()
else:
loglevel = 'INFO'
numeric_level = getattr(logging, loglevel, None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
# setup the logging
logfile = None
if ucfg.cfg['system']['log_file'] != None:
logfile = ucfg.cfg['system']['log_file']
if not os.path.isabs(logfile):
logfile = abspath(
join(dirname(configFile),
ucfg.cfg['system']['log_file']))
if not os.path.isdir(dirname(logfile)):
os.makedirs(dirname(logfile))
if not os.path.isfile(logfile):
with open(logfile, 'w+') as f:
f.close()
fmt = '%(levelname)s:%(name)s:%(message)s'
if logfile is not None:
logging.basicConfig(filename=logfile,
level=numeric_level,
filemode='w+',
format=fmt)
else:
logging.basicConfig(level=numeric_level)
coloredlogs.install(level=numeric_level, fmt=fmt)
self._loglevel = numeric_level
self._logger = logging.getLogger(__name__)
else:
self._logger = external_logger
# add the title
title = self.title(2)
for line in title:
self._logger.info(line)
out = ucfg.cfg['output']['out_location']
# Make the tmp and output directories if they do not exist
makeable_dirs = [out, join(out, 'tmp')]
for path in makeable_dirs:
if not os.path.isdir(path):
try:
self._logger.info("Directory does not exist, Creating:\n{}"
"".format(path))
os.makedirs(path)
except OSError as e:
raise e
self.temp_dir = path
# Check the user config file for errors and report issues if any
self._logger.info("Checking config file for issues...")
warnings, errors = check_config(ucfg)
print_config_report(warnings, errors, logger=self._logger)
self.ucfg = ucfg
self.config = self.ucfg.cfg
# Exit SMRF if config file has errors
if len(errors) > 0:
self._logger.error("Errors in the config file. See configuration"
" status report above.")
sys.exit()
# Write the config file to the output dir no matter where the project is
full_config_out = abspath(join(out, 'config.ini'))
self._logger.info("Writing config file with full options.")
generate_config(self.ucfg, full_config_out)
# Process the system variables
for k, v in self.config['system'].items():
setattr(self, k, v)
os.environ['WORKDIR'] = self.temp_dir
# Get the time section utils
self.start_date = pd.to_datetime(self.config['time']['start_date'])
self.end_date = pd.to_datetime(self.config['time']['end_date'])
# Get the timesetps correctly in the time zone
d = data.mysql_data.date_range(
self.start_date, self.end_date,
timedelta(minutes=int(self.config['time']['time_step'])))
tzinfo = pytz.timezone(self.config['time']['time_zone'])
self.date_time = [di.replace(tzinfo=tzinfo) for di in d]
self.time_steps = len(self.date_time)
# need to align date time
if 'date_method_start_decay' in self.config['albedo'].keys():
self.config['albedo']['date_method_start_decay'] = \
self.config['albedo']['date_method_start_decay'].replace(tzinfo=tzinfo)
self.config['albedo']['date_method_end_decay'] = \
self.config['albedo']['date_method_end_decay'].replace(tzinfo=tzinfo)
# if a gridded dataset will be used
self.gridded = False
self.forecast_flag = False
if 'gridded' in self.config:
self.gridded = True
if self.config['gridded']['data_type'] in [
'hrrr_netcdf', 'hrrr_grib'
]:
self.forecast_flag = self.config['gridded'][
'hrrr_forecast_flag']
# hours from start of day
self.day_hour = self.start_date - pd.to_datetime(
d[0].strftime("%Y%m%d"))
self.day_hour = int(self.day_hour / np.timedelta64(1, 'h'))
if ((self.start_date > datetime.now() and not self.gridded)
or (self.end_date > datetime.now() and not self.gridded)):
raise ValueError("A date set in the future can only be used with"
" WRF generated data!")
self.distribute = {}
if self.config['system']['qotw']:
self._logger.info(getqotw())
# Initialize the distribute dict
self._logger.info('Started SMRF --> %s' % datetime.now())
self._logger.info('Model start --> %s' % self.start_date)
self._logger.info('Model end --> %s' % self.end_date)
self._logger.info('Number of time steps --> %i' % self.time_steps)
from bs4 import BeautifulSoup
import argparse
import yaml
import coloredlogs
import logging
import os
import datetime
from pushbullet import Pushbullet
import shelve
import schedule
import time
import re
import sys
# Setup logging
coloredlogs.install(level="INFO")
log = logging.getLogger("main")
class LuxMedSniper:
LUXMED_LOGIN_URL = 'https://portalpacjenta.luxmed.pl/PatientPortal/Account/LogIn'
LUXMED_LOGOUT_URL = 'https://portalpacjenta.luxmed.pl/PatientPortal/Account/LogOn'
MAIN_PAGE_URL = 'https://portalpacjenta.luxmed.pl/PatientPortal'
REQUEST_RESERVATION_URL = 'https://portalpacjenta.luxmed.pl/PatientPortal/Reservations/Reservation/PartialSearch'
VISITS_PAGE_URL = 'https://portalpacjenta.luxmed.pl/PatientPortal/Reservations/Visits/GetVisitsAjax?source=0&fromDate={FromDate}&toDate={ToDate}' # &toDate=14-03-2020&fromDate=14-01-2020
def __init__(self, configuration_file="luxmedSniper.yaml"):
self.log = logging.getLogger("LuxMedSniper")
self.log.info("LuxMedSniper logger initialized")
# Open configuration file
import cairosvg
import os
from shutil import copyfile
import logging, coloredlogs
logging.basicConfig(
format='%(asctime)s %(levelname)-8s %(name)s : %(message)s',
level=logging.INFO,
datefmt='%Y-%m-%d %H:%M:%S')
log = logging.getLogger("convert")
coloredlogs.install(level=os.getenv("LOG_LEVEL", "INFO"))
source_dir = "picons-source/build-source/logos/"
output_dir = "picons"
for subdir, dirs, files in os.walk(source_dir):
log.info("Found {} files in {}. Converting...".format(
len(files), source_dir))
for file in files:
raw_name = file.replace(".default.svg", "")
raw_name = raw_name.replace(".default.png", "")
raw_name = raw_name.replace(".default.png", "")
raw_name = raw_name.split("-")[0]
output_file = "{}/{}.png".format(output_dir, raw_name)
if file.endswith("png"):
log.debug("Already a png {}. saving to {}...".format(
file, output_file))
copyfile("picons-source/build-source/logos/{}".format(file),
output_file)
import openpyxl
import argparse
import os
import logging
import coloredlogs
argparser = argparse.ArgumentParser()
argparser.add_argument('dir', help='存有Excel的目录')
argparser.add_argument('col', help='列名（例如`E`）')
argparser.add_argument('-s', '--skip-rows', type=int, default=0, help='跳过行数')
argparser.add_argument('-v', '--verbose', action='store_true', help='打印调试日志')
args = argparser.parse_args()
coloredlogs.install(level=logging.DEBUG if args.verbose else logging.INFO)
col_index = ord(args.col.lower()) - ord('a')
total_char_count = 0
for filename in os.listdir(args.dir):
if not filename.lower().endswith('.xlsx') or filename.startswith('~$'):
logging.info(f'跳过文件：{filename}')
continue
logging.debug(f'打开：{filename}')
wb = openpyxl.open(os.path.join(args.dir, filename))
for ws in wb:
logging.debug(f'工作簿：{ws.title}')
rows = list(ws.iter_rows())
if len(rows) <= args.skip_rows:
