def main(name=None):
if name in (None, __name__):
if len(sys.argv) != 2:
print("Usage: {} DECISION_TASK_ID".format(sys.argv[0]), file=sys.stderr)
sys.exit(1)
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(
fmt="%(asctime)s %(levelname)8s - %(message)s", datefmt="%Y-%m-%dT%H:%M:%S"
)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
log.addHandler(handler)
makedirs("build")
orig_dir = os.getcwd()
context = Values()
gnupghome = os.path.join(os.getcwd(), 'gpg')
os.chmod(gnupghome, 0o700)
context.gpg = gnupg.GPG(gpgbinary='gpg2', gnupghome=gnupghome)
context.gpg.encoding = 'utf-8'
credentials = {
'credentials': {
'clientId': os.environ["TASKCLUSTER_CLIENT_ID"],
'accessToken': os.environ["TASKCLUSTER_ACCESS_TOKEN"],
}
}
loop = asyncio.get_event_loop()
try:
os.chdir("build")
with aiohttp.ClientSession() as context.session:
context.queue = Queue(credentials, session=context.session)
context.decision_task_id = sys.argv[1]
loop.run_until_complete(async_main(context))
finally:
os.chdir(orig_dir)
loop.close()
def read_cmd_options():
from optparse import OptionParser, Values
tornado.options.define('distribution', default=None,
help='distribution of the time between events')
tornado.options.define('limit', default=0, type=int,
help='number of events to generate')
tornado.options.define('eventlog', default=False,
help='dump event log',
type=bool)
tornado.options.define('timestamp', default=False,
help='add an X-Float-Timestamp header to events',
type=bool)
tornado.options.define('bayeux', default=False,
help='use the Bayeux protocol for publishing',
type=bool)
tornado.options.define('timescale', default=1.0,
help='accelerate time by this factor',
type=float)
tornado.options.define('delay', default=2.0,
help='initial delay, in seconds',
type=float)
remaining = tornado.options.parse_command_line()
options = Values()
if len(remaining) >= 2:
options.filename = remaining[0]
options.server_urls = remaining[1:]
else:
OptionParser().error('At least one file name and one '
'server URL required')
return options
def loadDaemon():
''' Load the daemon configuration file.
'''
args = hlm_args.args()
configFile = args.daemonConfig
if configFile == None:
configFile = hlm_paths.daemonConfig()
config = configparser.SafeConfigParser()
try:
config.read(configFile)
except SystemExit:
raise
except BaseException as exc:
raise FatalError(_('Can\'t load the daemon configuration file {0}: {1}').format(quote(configFile), exc))
try:
if config.sections() != ['daemon']:
raise Exception(_('only the section {0} is allowed in this file.').format('[daemon]'))
options = config.options('daemon')
_checkAlienDirectives(['credentials', 'user', 'group', 'ping_site', 'ping_interval'], options, 'daemon')
# "credentials" accepts a credentials.conf file path
if args.daemonCredentials == None:
credentials = None
if config.has_option('daemon', 'credentials'):
credentials = config.get('daemon', 'credentials')
else:
credentials = hlm_paths.credentialsConfig()
args.daemonCredentials = credentials
# "user" accepts an account name
user = None
if config.has_option('daemon', 'user'):
user = config.get('daemon', 'user')
# "group" accepts a group name
group = None
if config.has_option('daemon', 'group'):
group = config.get('daemon', 'group')
# "pingSite" requires an http:// website URL
pingSite = _mandatoryDirective('ping_site', options, 'daemon', config.get)
if not pingSite.startswith('http://'):
raise Exception(_('ping_site {0} is is incorrect (must start with {1}), exiting.').format(quote(pingSite), quote('http://')))
pingInterval = _mandatoryDirective('ping_interval', options, 'daemon', config.getint)
if pingInterval < _minimumPingInterval:
if __WARNING__: logWarning(_('Daemon configuration file {0}: ping_interval is way too low ({1} seconds), forcing it to {2} seconds.').format(quote(configFile), pingInterval, _minimumPingInterval))
pingInterval = _minimumPingInterval
# Wrap configuration into a single Values() object
configCredentials = Values()
configCredentials.user = user
configCredentials.group = group
configCredentials.pingSite = pingSite
configCredentials.pingInterval = pingInterval
if __DEBUG__: logDebug('Daemon configuration has been loaded from {0}.'.format(configFile))
return configCredentials
except SystemExit:
raise
except BaseException as exc:
raise FatalError(_('Incorrect daemon configuration file {0}: {1}').format(quote(configFile), exc))
def test_start(self,mock_parse_args,mock_delete_raw_logs):
a = Values()
a.start = "2001/11/12"
a.end = None
a.debug = None
a.path= None
mock_parse_args.return_value = (a,'')
mock_delete_raw_logs.return_value = (True,{'dirsdeleted':range(0,10)})
assert logclean() == 0, "Bad return value"
mock_delete_raw_logs.return_value = (False,{'dirsdeleted':range(0,10), 'dirserrors':range(0,20)})
assert logclean() == -1 ,"Bad return value"
def read_cmd_options():
from optparse import OptionParser, Values
tornado.options.define('eventlog', default=False,
help='dump event log',
type=bool)
remaining = tornado.options.parse_command_line()
options = Values()
if len(remaining) >= 1:
options.stream_urls = remaining
else:
OptionParser().error('At least one source stream URL required')
return options
def read_cmd_options():
from optparse import OptionParser, Values
tornado.options.define('eventlog', default=False,
help='dump event log',
type=bool)
tornado.options.define('timescale', default=40000.0,
help='accelerate time by this factor',
type=float)
remaining = tornado.options.parse_command_line()
options = Values()
if len(remaining) >= 1:
options.server_urls = remaining
else:
OptionParser.error('At least one server URL required')
return options
def read_cmd_options():
from optparse import OptionParser, Values
tornado.options.define('distribution', default='exp(5)',
help='distribution of the time between events')
tornado.options.define('limit', default=0, type=int,
help='number of events to generate')
tornado.options.define('eventlog', default=False,
help='dump event log',
type=bool)
remaining = tornado.options.parse_command_line()
options = Values()
if len(remaining) >= 1:
options.server_urls = remaining
else:
OptionParser().error('At least one server URL required')
return options
def get_opts(docstr, meta={}, version=None, argv=None):
"""
Get docopt dict, and set argv and flags from get_optvalues.
"""
if not argv:
import sys
argv = sys.argv[1:]
pattern, collected = docopt(docstr, argv, version=version,
return_spec=True)
opts = Values()
opts.argv = argv
parsed = pattern.flat() + collected
#assert not ( 'argv' in opts or 'flags' in opts or 'args' in opts),\
# "Dont use 'argv', 'flags' or 'args'. "
opts.cmds, opts.flags, opts.args = get_optvalues(parsed, meta)
return opts
def testPlottingMore():
"""Test the plotting module"""
opt = Values()
opt.imgDir = "tests/img/"
opt.jsonDir = "tests/json/"
if not os.path.exists(opt.jsonDir):
os.makedirs(opt.jsonDir)
if not os.path.exists(opt.imgDir):
os.makedirs(opt.imgDir)
testPlotter = plotter(opt)
dic = {'TTT':10,'AAA':5}
testHistPlotter = histPlotter(dic,opt,filename="testHist")
testHistPlotter.plot()
dic = {'AAA':{'Expected':1,'Observed':3},'TTT':{'Expected':10,'Observed':30}}
testMultiHistPlotter = multiHistPlotter(dic,opt,filename="testMultiHist")
testMultiHistPlotter.plot()
def _GetDefaultOption(self, set_10_days_ago, query='steps'):
ops = Values()
ops.query = query
if set_10_days_ago:
ops.interval_value = 10
ops.interval_unit = 'days'
else:
ops.interval_value = None
ops.email_entries = ['comments']
ops.project_name = 'chromium'
ops.query_title = 'query title'
ops.max_comments = None
return ops
def _GetDefaultOption(self, set_10_days_ago, query="steps"):
ops = Values()
ops.query = query
if set_10_days_ago:
ops.interval_value = 10
ops.interval_unit = "days"
else:
ops.interval_value = None
ops.email_entries = ["comments"]
ops.project_name = "chromium"
ops.query_title = "query title"
ops.max_comments = None
return ops
def read_cmd_options():
from optparse import OptionParser, Values
tornado.options.define('eventlog', default=False,
help='dump event log',
type=bool)
tornado.options.define('noparse', default=False,
help='quick client that does not parse events',
type=bool)
tornado.options.define('reconnect', default=False,
help='reconnect disconnected clients',
type=bool)
remaining = tornado.options.parse_command_line()
options = Values()
if len(remaining) == 2:
options.stream_url = remaining[0]
options.num_clients = int(remaining[1])
else:
OptionParser().error('A source stream URL required')
return options
def read_cmd_options():
from optparse import OptionParser, Values
tornado.options.define('label', default=None,
help='define a client label',
type=str)
tornado.options.define('missing', default=False,
help=('retrieve missing events '
'(requires a client label)'),
type=bool)
tornado.options.define('deflate', default=True,
help='Accept compressed data with deflate',
type=bool)
remaining = tornado.options.parse_command_line()
options = Values()
if len(remaining) >= 1:
options.stream_urls = remaining
else:
OptionParser().error('At least one source stream URL required')
return options
def read_cmd_options():
from optparse import Values, OptionParser
tornado.options.define('port', default=8888, help='run on the given port',
type=int)
tornado.options.define('aggregatorid', default=None,
help='aggregator id', type=str)
tornado.options.define('buffer', default=None, help='event buffer time (s)',
type=float)
tornado.options.define('eventlog', default=False,
help='dump event log',
type=bool)
tornado.options.define('autostop', default=False,
help='stop the server when the source finishes',
type=bool)
remaining = tornado.options.parse_command_line()
options = Values()
if len(remaining) >= 1:
options.stream_urls = remaining
else:
OptionParser().error('At least one source stream URL required')
return options
def add(self,typl,what):
# ~~> cast already prepared becasue .add is always called
if typl == '': return
# ~~> bundling of the layers becasue .add is always called
if what["xref"] not in self.obdata.keys():
obdata = Values()
obdata.type = ''
obdata.unit = []
obdata.support = []
obdata.function = []
obdata.values = []
self.obdata.update({ what["xref"]:obdata })
# ~~> file references
if what['file'] in self.object.keys(): return
# ~~> unexplored teritory
if 'sortie' in typl.lower():
self.object.update({ what['file']:Sortie(what['file']) })
elif 'csv' in typl.lower():
self.object.update({ what['file']:CSV(what['file']) })
elif 'tif' in typl.lower() or 'jpg' in typl.lower() \
or 'gif' in typl.lower() or 'png' in typl.lower() \
or 'bmp' in typl.lower():
self.object.update({ what['file']:'image' })
# ~~> SELAFIN file
elif 'SELAFIN' in typl.upper() or 'slf' in typl.lower():
slf = castSELAFIN(what['file'])
slf.setKDTree()
slf.setMPLTri()
self.object.update({ what['file']:slf })
else: # TODO: raise exception
print '... do not know how to extract from this format: ' + typl
sys.exit(1)
def test_valid_rsync_includes_returns_correct_list(tmp_flow_config):
"""Test that the rsync includes in the correct """
reg = 'rsynctest'
flow_file = tmp_flow_config(
reg, """
[scheduling]
initial cycle point = 2020-01-01
[[dependencies]]
graph = "blah => deeblah"
[scheduler]
install = dir/, dir2/, file1, file2
allow implicit tasks = True
""")
config = WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
rsync_includes = WorkflowConfig.get_validated_rsync_includes(config)
assert rsync_includes == ['dir/', 'dir2/', 'file1', 'file2']
def test_git_clone(self, mock_shell):
parsed_options = {
'url': 'git://puppet/project.git',
'path': 'puppet/modules'
}
options = Values(defaults=parsed_options)
# test
builder.git_clone(options)
# validation
mock_shell.assert_called_with('git clone --recursive %s' % options.url)
self.assertEqual(mock_shell.call_count, 1)
self.assertEqual(options.path, 'puppet/modules')
def process_include_files(infile,
outfile,
inpath='',
catalogpath=None,
fixtypenames=None,
no_collect_includes=False,
no_redefine_groups=False):
"""The root/main function"""
load_catalog(catalogpath)
options = Values({
'force': False,
'fixtypenames': fixtypenames,
'no_collect_includes': no_collect_includes,
'no_redefine_groups': no_redefine_groups,
})
doc, ns_dict, schema_ns_dict, rename_data = prep_schema_doc(
infile, outfile, inpath, options)
return doc, ns_dict, schema_ns_dict, rename_data
def __init__(self):
self.options = {
'verbosity': 0,
'ask_pass': False,
'private_key_file': None,
'remote_user': None,
'connection': 'smart',
'timeout': 10,
'ssh_common_args': '',
'sftp_extra_args': '',
'scp_extra_args': '',
'ssh_extra_args': '',
'force_handlers': False,
'flush_cache': None,
'become': False,
'become_method': 'sudo',
'become_user': None,
'become_ask_pass': False,
'tags': ['all'],
'skip_tags': [],
'check': False,
'syntax': None,
'diff': False,
'inventory': ansible_host,
'listhosts': None,
'subset': None,
'extra_vars': [],
'ask_vault_pass': False,
'vault_password_files': [],
'vault_ids': [],
'forks': 5,
'module_path': None,
'listtasks': None,
'listtags': None,
'step': None,
'start_at_task': None,
'args': ['fake']}
self.ops = Values(self.options)
self.loader = DataLoader()
self.passwords = dict()
self.results_callback = ResultsCollector()
self.inventory = InventoryManager(loader=self.loader, sources=[self.options['inventory']])
self.variable_manager = VariableManager(loader=self.loader, inventory=self.inventory)
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
config = get_config(parse_args=False)
# Setup the correct options so the agent will use the forwarder
opts, args = Values({
'autorestart': False,
'dd_url': None,
'use_forwarder': True,
'disabled_dd': False
}), []
agentConfig = get_config(parse_args=False, options=opts)
self.hostname = get_hostname(agentConfig)
# Watchdog for Windows
self._collector_heartbeat, self._collector_send_heartbeat = multiprocessing.Pipe(
False)
self._collector_failed_heartbeats = 0
self._max_failed_heartbeats = \
MAX_FAILED_HEARTBEATS * agentConfig['check_freq'] / SERVICE_SLEEP_INTERVAL
# Watch JMXFetch restarts
self._MAX_JMXFETCH_RESTARTS = 3
self._count_jmxfetch_restarts = 0
# Keep a list of running processes so we can start/end as needed.
# Processes will start started in order and stopped in reverse order.
self.procs = {
'forwarder':
ProcessWatchDog("forwarder", DDForwarder(config, self.hostname)),
'collector':
ProcessWatchDog(
"collector",
DDAgent(agentConfig,
self.hostname,
heartbeat=self._collector_send_heartbeat)),
'dogstatsd':
ProcessWatchDog("dogstatsd",
DogstatsdProcess(config, self.hostname)),
'jmxfetch':
ProcessWatchDog("jmxfetch", JMXFetchProcess(config, self.hostname),
3),
}
def setUp(self):
super(TestUnitKafkaConsumer, self).setUp()
bot_options = Values()
bot_options.kafkatopic = 'test_topic'
bot_options.kafkagroup = 'test_group'
bot_options.broker = 'test_broker'
bot_options.username = '******'
self.kafkaHandler = KafkaHandler(options=bot_options)
self.kafkaHandler.handle_create_post = MagicMock()
self.kafkaHandler.on_event_handler = MagicMock()
self.KafkaMessagesConsumer = KafkaMessagesConsumer(
bot_options, self.kafkaHandler, Mock())
def _register_attributes(self, app, args):
# FIXME: there is no better lib function than this snippet
schema_file = app.get_share_file('schema')
if os.path.exists(schema_file):
self.log('Registering schema %s' % schema_file)
lo, pos = self._get_ldap_connection(args)
with self._get_password_file(args) as password_file:
create_recursive_container(
'cn=ldapschema,cn=univention,%s' % ucr_get('ldap/base'),
lo, pos)
if app.automatic_schema_creation:
schema_obj = UniventionLDAPSchema(ucr_instance())
userdn = self._get_userdn(args)
udm_passthrough_options = [
'--binddn', userdn, '--bindpwdfile', password_file
]
opts = Values()
opts.packagename = 'appcenter-app-%s' % app.id
opts.packageversion = app.version
opts.ucsversionstart = None
opts.ucsversionend = None
os.environ['UNIVENTION_APP_IDENTIFIER'] = app.id
try:
schema_obj.register(schema_file, opts,
udm_passthrough_options)
except SystemExit as exc:
if exc.code == 4:
self.warn(
'A newer version of %s has already been registered. Skipping...'
% schema_file)
else:
raise RegisterSchemaFailed(exc.code)
else:
if not schema_obj.wait_for_activation():
raise RegisterSchemaFileFailed(schema_file)
finally:
if 'UNIVENTION_APP_IDENTIFIER' in os.environ:
del os.environ['UNIVENTION_APP_IDENTIFIER']
# and this is what should be there after one line of lib.register_schema(schema_file)
app = app.get_app_cache_obj().copy(
locale='en').find_by_component_id(app.component_id)
attributes, __, options = get_extended_attributes(app)
for option in options:
self.log('Registering option %s' % option.name)
create_extended_option(option, app, lo, pos)
if attributes:
for i, attribute in enumerate(attributes):
self.log('Registering attribute %s' % attribute.name)
create_extended_attribute(attribute, app, i + 1, lo,
pos)
def test_invalid_custom_output_msg(tmp_flow_config: Callable):
"""Test invalid output message (colon not allowed)."""
reg = 'invalid_output'
flow_file = tmp_flow_config(
reg, """
[scheduling]
[[graph]]
R1 = "foo:x => bar"
[runtime]
[[bar]]
[[foo]]
[[[outputs]]]
x = "the quick: brown fox"
""")
with pytest.raises(WorkflowConfigError) as cm:
WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
assert ('Invalid message trigger "[runtime][foo][outputs]x = '
'the quick: brown fox"') in str(cm.value)
def construct_options_dict(options: Values) -> Dict[str, Any]:
'''
Returns a dictionary whose keys are strings with the attribute names of
options and whose values are the option value corresponding to the name.
Arguments
---------
options: Values
Read options
Returns
-------
Dict[str, Any]: values in dict format
'''
# Just in case Values have items in dir by default, we use set difference
# so that they are not included
names = set(dir(options)) - set(dir(Values()))
values = map(lambda x: getattr(options, x), names)
return {k: v for k, v in zip(names, values) if v is not None}
def meta_config(linter):
"""
Return the config as `optparse.Values` (linter.config) for all checkers
:param linter:
:return:
>>> meta = meta_config(get_linter())
>>> jobs = meta.jobs
>>> jobs in list(range(1, 5))
True
"""
from optparse import Values
meta = Values(linter.config.__dict__)
for checker in linter.prepare_checkers():
meta.__dict__.update(checker.config.__dict__)
return meta
def test_expanded_paths(self, mock_parse):
parsed_options = {
'working_dir': '~/',
'output_dir': '~/',
'path': None,
'branch': None,
'tag': None
}
mock_parse.return_value = (Values(defaults=parsed_options), [])
# test
options = builder.get_options()
# validation
self.assertEqual(options.working_dir, os.path.expanduser('~/'))
self.assertEqual(options.output_dir, os.path.expanduser('~/'))
def numpydiff(ref, new,
recursive=False,
exclude=exclude_default,
include=['*'],
format='bin_v3d',
show_file=False,
variables=None,
verbosity=0):
'''Python interface to numpydiff
See OptionParser for arguments info.
'''
options = Values()
options.recursive = recursive
options.exclude = exclude
options.include = include
options.format = format
options.variables = variables
options.verbosity = verbosity
options.show_file = show_file
return diff(ref, new, options)
class ApplicationContextTest(unittest.TestCase):
sut = ApplicationContext(
ScriptSettings(Values({
'project': 'project',
'env': 'system-test',
'build': 510,
'grab_envs': 'false',
'sanity_test_timeout_sec': 120
}),
workdir='.jtalks'))
def test_nexus_is_created(self):
nexus = self.sut.old_nexus()
self.assertEquals(nexus.build_number, 510)
def test_tomcat_is_created(self):
tomcat = self.sut.tomcat()
# can't use assertIsInstance() because on CentOS we use old mock lib
self.assertTrue(isinstance(tomcat, Tomcat))
def test_build_manifest(self, mock_chdir, mock_digest, mock_open, mock_getsize, mock_listdir):
parsed_options = {
'output_dir': '/tmp/output',
}
options = Values(defaults=parsed_options)
files = [
'file_1%s' % builder.ARCHIVE_SUFFIX,
'file_2%s' % builder.ARCHIVE_SUFFIX,
'file_3', # not a module
]
digests = ['hash_1', 'hash_2']
mock_digest.side_effect = digests
mock_listdir.return_value = files
mock_getsize.side_effect = [10, 20]
manifest = []
mock_fp = Mock()
mock_fp.write = manifest.append
mock_fp.__enter__ = Mock(return_value=mock_fp)
mock_fp.__exit__ = Mock()
mock_open.return_value = mock_fp
# test
builder.build_manifest(options)
# validation
_manifest = [
'file_1.tar.gz',
',hash_1',
',10\n',
'file_2.tar.gz',
',hash_2',
',20\n'
]
mock_open.assert_called_with('PULP_MANIFEST', 'w+')
mock_chdir.assert_called_with(os.getcwd())
self.assertEqual(manifest, _manifest)
def test_validate_both_branch_and_tag(self, mock_parse, mock_stdout, mock_exit):
parsed_options = {
'working_dir': None,
'output_dir': None,
'path': None,
'branch': 'br',
'tag': 'tg'
}
mock_parse.return_value = (Values(defaults=parsed_options), [])
# test
builder.get_options()
# validation
mock_stdout.assert_any_call(builder.BAD_BRANCH_AND_TAG)
mock_exit.assert_called_with(os.EX_USAGE)
def test_clean(self, mock_shell):
parsed_options = {
'url': 'git@test:foo/bar.git',
'working_dir': '/tmp/working',
'clean': True
}
options = Values(defaults=parsed_options)
# test
builder.clean(options)
# validation
url = urlparse(options.url)
path = os.path.basename(url.path)
path = os.path.splitext(path)[0]
path = os.path.join(options.working_dir, path)
mock_shell.assert_called_with('rm -rf %s' % path)
def minify_script(patches=None, keep_report=True, show_diff=False):
"""minifies createstubs.py
Args:
patches ([PathLike], optional): List of paths to patches to apply.
Defaults to None.
keep_report (bool, optional): Keeps single report line in createstubs
Defautls to True.
show_diff (bool, optional): Print diff from edits. Defaults to False.
Returns:
str: minified source text
"""
patches = patches or []
edits = [
("comment", "print"),
("comment", "import logging"),
("comment", "self._log ="),
("comment", "self._log.debug"),
("comment", "self._log.warning"),
("comment", "self._log.info"),
("comment", "self._log.error"),
]
if keep_report:
report = ('rprint', ('self._log.info("Stub module: {:<20} to file:'
' {:<55} mem:{:>5}".'
'format(module_name, file_name, m1))'))
clean = (
'rprint',
'self._log.info("Clean/remove files in folder: {}".format(path))')
edits.insert(0, report)
edits.insert(1, clean)
minopts = Values({'tabs': False})
with SCRIPT.open('r') as f:
content = f.read()
for path in patches:
path = Path(path)
content = apply_patch(content, path.read_text())
content = edit_lines(content, edits, show_diff=show_diff)
tokens = token_utils.listified_tokenizer(content)
source = minification.minify(tokens, minopts)
return source
def test_family_inheritance_and_quotes(fam_txt: str, mock_glbl_cfg: Callable,
tmp_flow_config: Callable) -> None:
"""Test that inheritance does not ignore items, if not all quoted.
For example:
inherit = 'MAINFAM<major, minor>', SOMEFAM
inherit = 'BIGFAM', SOMEFAM
See bug #2700 for more/
"""
mock_glbl_cfg(
'cylc.flow.platforms.glbl_cfg', '''
[platforms]
[[localhost]]
hosts = localhost
''')
reg = 'test'
file_path = tmp_flow_config(
reg, f'''
[scheduler]
allow implicit tasks = True
[task parameters]
major = 1..5
minor = 10..20
[scheduling]
[[graph]]
R1 = """hello => MAINFAM<major, minor>
hello => SOMEFAM"""
[runtime]
[[root]]
script = true
[[MAINFAM<major, minor>]]
[[SOMEFAM]]
[[ goodbye_0<major, minor> ]]
inherit = 'MAINFAM<major, minor>', {fam_txt}
''')
config = WorkflowConfig(reg, file_path, template_vars={}, options=Values())
assert ('goodbye_0_major1_minor10'
in config.runtime['descendants']['MAINFAM_major1_minor10'])
assert ('goodbye_0_major1_minor10'
in config.runtime['descendants']['SOMEFAM'])
def run(self):
"""
Runs the builder
"""
# parse command line parameters
parser = self.buildParser()
(opts, args) = parser.parse_args()
if len(args) == 0:
parser.error("incorrect number of arguments")
# download only
if opts.download:
if len(args) > 1 and opts.targetName:
parser.error(
"option --targetName only allowed for a single dictionary"
" but %d dictionaries given" % len(args))
for dictionary in args:
downloader = getDownloader(dictionary, quiet=opts.quiet)
downloader.download(targetName=opts.targetName,
targetPath=opts.targetPath)
return True
options = dict([
(option, getattr(opts, option)) for option in dir(opts)
if not hasattr(Values(), option) and getattr(opts, option) != None
])
installer = DictionaryInstaller(quiet=opts.quiet)
for dictionary in args:
try:
installer.install(dictionary, **options)
except OperationalError as e:
if not opts.quiet:
warn("Error writing to database: %s" % e)
if not opts.local:
warn("Try choosing --local for installing into HOME")
return False
return True
def get_parsed_args():
parser = OptionParser()
parser.add_option('-A',
'--autorestart',
action='store_true',
default=False,
dest='autorestart')
parser.add_option('-d',
'--dd_url',
action='store',
default=None,
dest='dd_url')
parser.add_option('-u',
'--use-local-forwarder',
action='store_true',
default=False,
dest='use_forwarder')
parser.add_option('-v',
'--verbose',
action='store_true',
default=False,
dest='verbose',
help='Print out stacktraces for errors in checks')
parser.add_option('-p',
'--profile',
action='store_true',
default=False,
dest='profile',
help='Enable Developer Mode')
try:
options, args = parser.parse_args()
except SystemExit:
# Ignore parse errors
options, args = Values({
'autorestart': False,
'dd_url': None,
'use_forwarder': False,
'verbose': False,
'profile': False
}), []
return options, args
def __init__(self, values=None):
if values:
self.defaults.update(values.__dict__) # Ensure that context has default values for all unset variables
super(self.__class__, self).__init__(vars(Values(self.defaults)))
if self.masterDataDirectory:
self.master_datadir = self.masterDataDirectory
else:
self.master_datadir = gp.get_masterdatadir()
self.master_port = self.get_master_port()
if self.local_dump_prefix:
self.dump_prefix = self.local_dump_prefix + "_"
else:
self.dump_prefix = ""
if not self.include_dump_tables: self.include_dump_tables = []
if not self.exclude_dump_tables: self.exclude_dump_tables = []
if not self.output_options: self.output_options = []
if not self.dump_schema: self.dump_schema = []
if not self.exclude_dump_schema: self.exclude_dump_schema = []
def test_git_checkout_tag(self, mock_shell):
parsed_options = {
'origin': 'git://',
'branch': None,
'tag': 'mytag',
}
options = Values(defaults=parsed_options)
# test
builder.git_checkout(options)
# validation
mock_shell.assert_any_call('git fetch')
mock_shell.assert_any_call('git fetch --tags')
mock_shell.assert_any_call('git checkout %s' % options.tag)
self.assertEqual(mock_shell.call_count, 3)
def test_validate_path(self, mock_parse, mock_stdout, mock_exit):
parsed_options = {
'url': None,
'working_dir': None,
'output_dir': None,
'path': '/',
'branch': None,
'tag': None
}
mock_parse.return_value = (Values(defaults=parsed_options), [])
# test
builder.get_options()
# validation
mock_stdout.assert_any_call(builder.BAD_PATH)
mock_exit.assert_called_with(os.EX_USAGE)
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
config = get_config(parse_args=False)
# Setup the correct options so the agent will use the forwarder
opts, args = Values({'clean': False, 'disabled_dd': False}), []
agentConfig = get_config(parse_args=False, options=opts)
self.restart_interval = \
int(agentConfig.get('autorestart_interval', RESTART_INTERVAL))
log.info("Autorestarting the collector ever %s seconds" %
self.restart_interval)
# Keep a list of running processes so we can start/end as needed.
# Processes will start started in order and stopped in reverse order.
self.procs = {
'monasca-forwarder': MonascaForwarder(config),
'monasca-collector': MonascaCollector(agentConfig),
'monasca-statsd': MonascaStatsd(config),
}
def get_parsed_args():
parser = OptionParser()
parser.add_option('-c',
'--clean',
action='store_true',
default=False,
dest='clean')
parser.add_option('-v',
'--verbose',
action='store_true',
default=False,
dest='verbose',
help='Print out stacktraces for errors in checks')
try:
options, args = parser.parse_args()
except SystemExit:
# Ignore parse errors
options, args = Values({'clean': False}), []
return options, args
def get_default_options(options, non_default_options):
"""
Given a dictionary of options containing the defaults optparse has filled
in, and a dictionary of options containing only options parsed from the
command line, returns a dictionary containing the default options that
remain after removing the default options that were overridden by the
options passed on the command line.
Mathematically, this returns a dictionary with
default_options.keys = options.keys() \ non_default_options.keys()
where \ is the set difference operator.
The value of a key present in default_options is the value of the same key
in options.
"""
options_dict = vars(options)
non_default_options_dict = vars(non_default_options)
default_options = Values(
dict((k, options_dict[k]) for k in options_dict
if k not in non_default_options_dict))
return default_options
def get_parsed_args():
parser = OptionParser()
parser.add_option('-d',
'--dd_url',
action='store',
default=None,
dest='dd_url')
parser.add_option('-c',
'--clean',
action='store_true',
default=False,
dest='clean')
parser.add_option('-u',
'--use-local-forwarder',
action='store_true',
default=False,
dest='use_forwarder')
parser.add_option('-n',
'--disable-dd',
action='store_true',
default=False,
dest="disable_dd")
parser.add_option('-v',
'--verbose',
action='store_true',
default=False,
dest='verbose',
help='Print out stacktraces for errors in checks')
try:
options, args = parser.parse_args()
except SystemExit:
# Ignore parse errors
options, args = Values({
'dd_url': None,
'clean': False,
'use_forwarder': False,
'disable_dd': False,
'use_forwarder': False
}), []
return options, args
def test_errata(self):
with open("Tests/quote1.json", encoding='utf-8') as f:
errata = json.load(f)
if not os.path.exists("Temp"):
os.mkdir("Temp")
state = {"text": "./Tests", "html": "./Temp", "ossPath": "css"}
options = Values(defaults={'search': False})
doc = apply_errata(errata, options, state)
templates = Templates(
os.path.join(os.path.dirname(__file__), "Template"))
doc.apply(True, templates)
self.assertEqual(doc.InlineCount, 0)
self.assertEqual(doc.SectionCount, 1)
self.assertEqual(doc.EndnoteCount, 0)
self.assertTrue(
compare_file("./Temp/RFC8275.html", "./Results/quote1.html", True))
def test_parse_file_one_error():
target_path = os.path.join(tempfile.gettempdir(), str(uuid4()))
os.makedirs(target_path)
cwd = os.getcwd()
os.chdir(target_path)
try:
rp = ReportSVGBadge(
Values({
'output_file': None,
'image': '123.svg',
'format': 'svg'
}))
with open('lala.py', 'w') as f:
f.write('print(123)\n')
f.write('print(123)\n')
with open('lulu.py', 'w') as f:
f.write('print(123)\n')
f.write('print(123)\n')
f.write('print(123)\n')
f.write('print(123)\n')
rp.beginning('lala.py')
rp.finished('lala.py')
rp.beginning('lulu.py')
rp.handle(Violation('F101', 'lulu.py', 123, 123, 123, 123))
rp.finished('lulu.py')
rp.stop()
with open("123.svg") as f:
el = xmltodict.parse(f.read())
assert el['svg']['g'][1]['text'][3]['#text'] == '67%'
finally:
os.chdir(cwd)
rmtree(target_path)
def __call__(self, filename, lines):
"""
### &doc_id postprocess:DoxypypyPostProcess_call
The main bridge between DoxyTH and Doxypypy. Re-creates the doxypypy variables and gives it to the AstWalker
Args:
filename: The file_name name
lines: The file_name lines
"""
from doxypypy.doxypypy import AstWalker
from os import sep
from optparse import Values
from .langs import ascii_encode
# Replace all the lines by a normal \n at the end
try:
lines = [f"{line.rstrip()}\n" for line in lines]
except UnicodeEncodeError:
lines = [f"{ascii_encode(line).rstrip()}\n" for line in lines]
# Define options
options = {'autobrief': True, 'autocode': True, 'topLevelNamespace': None, 'tablength': 4, 'debug': None}
# All the code below is extracted from doxypypy (with a few edits to fit in with doxyth)
full_path_namespace = filename.replace(sep, '.')[:-3]
# Use any provided top-level namespace argument to trim off excess.
real_namespace = full_path_namespace
if options['topLevelNamespace']:
namespace_start = full_path_namespace.find(options['topLevelNamespace'])
if namespace_start >= 0:
real_namespace = full_path_namespace[namespace_start:]
options['fullPathNamespace'] = real_namespace
ast_walker = AstWalker(lines, Values(defaults=options), filename)
ast_walker.parseLines()
# Output the modified source.
print(ast_walker.getLines())
def run(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.logger.info('Binding to %s:%s', self.bind_ip, self.bind_port)
s.bind((self.bind_ip, self.bind_port))
s.listen(20)
while True:
client, address = s.accept()
self.logger.debug('Accepting connection from %s:%s', *address)
client_file = client.makefile('rb+', 1)
json_data = client_file.read()
conf = Values(json.loads(json_data))
self.logger.info(
'Starting run for %s:%s [put/get/del_concurrency: %s/%s/%s, '
'num_objects: %s, num_gets: %s]', address[0], address[1],
conf.put_concurrency, conf.get_concurrency,
conf.del_concurrency, conf.num_objects, conf.num_gets)
logger = logging.getLogger('bench-server')
level = logging.DEBUG if conf.log_level.lower() == 'debug' \
else logging.INFO
logger.setLevel(level)
loghandler = logging.StreamHandler(client_file)
logformat = logging.Formatter(
'%(server)s %(asctime)s %(levelname)s %(message)s')
loghandler.setFormatter(logformat)
logger.addHandler(loghandler)
logger = LogAdapter(logger, 'swift-bench-server')
controller = BenchController(logger, conf)
try:
controller.run()
except socket.error:
logger.warning('Socket error', exc_info=1)
logger.logger.removeHandler(loghandler)
client_file.close()
client.close()
self.logger.info('...bench run completed; waiting for next run.')
def build_options(query=''):
options = Values()
options.ensure_value('query', query)
options.ensure_value('type', 'video')
options.ensure_value('maxResults', 50)
options.ensure_value('regionCode', 'jp')
options.ensure_value('order', 'viewCount')
return options
def _build_opts_from_dict(val):
opts = Values()
for key in val.keys():
opts.__dict__[key] = val[key]
return opts
def testComplexErrorSim():
"""Test the read error simulation"""
N= 500
randomLeftFlank = "".join([random.choice(getAlphabet()) for _ in range(N)])
randomRightFlank = "".join([random.choice(getAlphabet()) for _ in range(N)])
seq = randomLeftFlank+'AGTATACCTCGCATCGATCGATCG' +randomRightFlank# len 12
ref = "".join([random.choice(getAlphabet()) for _ in range(100000)])+seq + "".join([random.choice(getAlphabet()) for _ in range(100000)])
refRecord=SeqRecord(Seq(ref),'Chromosome dna:chromosome chromosome:ASM19595v1:Chromosome:1:4411532:1','','')
record=SeqRecord(Seq(seq),'st=%s'%(100000),'','')
opt = Values()
opt.snpFreq = 0.1
rep = [0.1]*N
opt.refPos = 100000
errorSim = complexError(record,opt,id = 'st=%s'%(100000),baseErrorProb = rep+[0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep)
assert_equal(errorSim.errorProb,rep+[0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep)
errorSim.snp(pos=N,rl='T')
assert_equal(errorSim.errorProb,rep+[0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep)
assert_equal( str(errorSim.seq) , randomLeftFlank+'TGTATACCTCGCATCGATCGATCG'+randomRightFlank)
errorSim.ins(pos=N+3,rl='CCC')
assert_equal(errorSim.errorProb,rep+[0.1,0.15,0.1,0.15,0.1,0.1,0.1,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep)
assert_equal( str(errorSim.seq) , randomLeftFlank+'TGTACCCTACCTCGCATCGATCGATCG'+randomRightFlank)
errorSim.deletion(pos=N+17,dlen=5)
assert_equal( str(errorSim.seq) , randomLeftFlank+'TGTACCCTACCTCGCATGATCG'+randomRightFlank)
## Write these to a fasta file
refFilename = 'tests/ref.fa'
readFilename = 'tests/read.fq'
writeFasta(filename = refFilename,seqList = [refRecord])
writeFastq(filename = readFilename,seqList = [errorSim.record])
## Align ref to read
align.refIndex(file=refFilename)
# ## Align reads to the reference
samfileName = readFilename + '.sam'
aligned = align.align(reference=refFilename, read_file=readFilename,stdout=samfileName)
reader = errorReader(samfile=samfileName,ref=ref)
errorList = []
for error in reader:
assert_equal(error.read.cigarstring,'M%sI3M%sD5M%s'%(N+4,10,N+5))
errorList.append(error)
assert_equal(len(errorList),3)
error1 = errorList[0]
assert_equal(error1.isSnp,True)
assert_equal(error1.errorType,'SNP')
assert_equal(error1.true,'A')
assert_equal(error1.emission,'T')
assert_equal(error1.after(3),'GTA')
assert_equal(error1.readPos,N)
assert_equal(error1.refPos,100000+N)
error2 = errorList[1]
assert_equal(error2.isIndel,True)
assert_equal(error2.isInsertion,True)
assert_equal(error2.errorType,'Insertion')
assert_equal(error2.true,'')
assert_equal(error2.emission,'CCC')
assert_equal(error2.after(3),'TAC')
assert_equal(error2.before(3),'GTA')
assert_equal(error2.readPos,N+3)
assert_equal(error2.refPos,100000+3+N)
error3 = errorList[2]
assert_equal(error3.isIndel,True)
assert_equal(error3.isDeletion,True)
assert_equal(error3.errorType,'Deletion')
assert_equal(error3.true,'CGATC')
assert_equal(error3.emission,'')
assert_equal(error3.after(3),'GAT')
assert_equal(error3.before(3),'CAT')
assert_equal(error3.readPos,N+17)
assert_equal(error3.refPos,100000+17+N)
assert_equal(error1.mappedCorrectly,True)
assert_equal(error1.mappedDist,0)
## Test counter
opt.maxKmerLength = 4
opt.dbName = 'proto_errTEST'
opt.observedErrorDBName = 'errors'
opt.simulatedErrorDBName = 'simulatedErrors'
errorCounter = counter(ref=ref,opt=opt,errorList=errorList,makeDB=True)
assert_equal(errorCounter.probKmer('A') + errorCounter.probKmer('T') +
errorCounter.probKmer('C') +errorCounter.probKmer('G') ,1)
count,errorQueryList = errorCounter.getCount(returnList=True)
assert_equal(count,3)
assert_equal(errorCounter.getCount(kmerBefore='GTA'),1)
assert_equal(errorCounter.getCount(kmerBefore='CAT'),1)
assert_equal(errorCounter.getCount(kmerAfter='TAC'),1)
assert_equal(errorCounter.getCount(kmerAfter='GAT'),1)
def generateGraph(self, filename, suggested_pathways=[], compound_data=None, gene_data=None, protein_data=None, format='svg'):
# Build options-like structure for generation of graph
# (compatibility with command line version, we need to fake it)
options = Values()
options._update_loose({
'file': None,
#'pathways': self.config.Read('/Pathways/Show'),
#'not_pathways':'',
'show_all': False, # self.config.ReadBool('/Pathways/ShowAll'),
'search': '',
'cluster_by': self.config.get('/App/ClusterBy'),
'show_enzymes': self.config.get('/App/ShowEnzymes'), # self.config.ReadBool('/App/ShowEnzymes'),
'show_secondary': self.config.get('/App/Show2nd'),
'show_molecular': self.config.get('/App/ShowMolecular'),
'show_network_analysis': self.config.get('/App/ShowAnalysis'),
'show_gibbs': self.config.get('/App/ShowGibbs'),
'highlightpathways': self.config.get('/App/HighlightPathways'),
'highlightregions': self.config.get('/App/HighlightRegions'),
'splines': 'true',
'focus': False,
'show_pathway_links': self.config.get('/Pathways/ShowLinks'),
# Always except when saving the file
'output': format,
})
#pathway_ids = self.config.value('/Pathways/Show').split(',')
if suggested_pathways:
pathway_ids = [p.id for p in suggested_pathways.entities[1]]
else:
pathway_ids = []
# Add the manually Shown pathways
pathway_ids_show = self.config.get('/Pathways/Show')
pathway_ids.extend(pathway_ids_show)
# Now remove the Hide pathways
pathway_ids_hide = self.config.get('/Pathways/Hide')
pathway_ids = [p for p in pathway_ids if p not in pathway_ids_hide]
# Convert pathways_ids to pathways
pathways = [self.m.db.pathway(pid) for pid in pathway_ids if self.m.db.pathway(pid) is not None]
if pathway_ids == []:
return None
if compound_data or gene_data or protein_data:
# Generate independent scales
node_colors = {}
for dsi in compound_data, gene_data, protein_data:
if dsi == None:
continue
#if self.m.data.analysis_timecourse:
# # Generate the multiple views
# tps = sorted( self.m.data.analysis_timecourse.keys(), key=int )
# # Insert counter variable into the filename
# filename = self.get_filename_with_counter(filename)
# print "Generate timecourse..."
# for tp in tps:
# print "%s" % tp
# graph = generator( pathways, options, self.m.db, analysis=self.m.data.analysis_timecourse[ tp ]) #, layout=self.layout)
# graph.write(filename % tp, format=options.output, prog='neato')
# return tps
#else:
print("Generate map for single control:test...")
# Build analysis lookup dict; we want a single color for each metabolite
mini, maxi = min(abs(np.median(dsi.data)), 0), max(abs(np.median(dsi.data)), 0)
mini, maxi = -1.0, +1.0 # Fudge; need an intelligent way to determine (2*median? 2*mean?)
scale = utils.calculate_scale([mini, 0, maxi], [9, 1], out=np.around) # rdbu9 scale
for n, m in enumerate(dsi.entities[1]):
if m is not None:
ecol = utils.calculate_rdbu9_color(scale, dsi.data[0, n])
#print xref, ecol
if ecol is not None:
node_colors[m.id] = ecol
graph = generator(pathways, options, self.m.db, analysis=node_colors) # , layout=self.layout)
self.status.emit('waiting')
self.progress.emit(0.5)
graph.write(filename, format=options.output, prog='neato')
return None
else:
graph = generator(pathways, options, self.m.db) # , layout=self.layout)
self.status.emit('waiting')
self.progress.emit(0.5)
graph.write(filename, format=options.output, prog='neato')
return None
def get_plugin_config_options():
options = Values()
options.websockets_port = config.config.getint('server',
'websockets_port')
options.cherrypy_port = config.config.getint('server',
'cherrypy_port')
options.proxy_port = config.config.getint('server', 'proxy_port')
options.session_timeout = config.config.getint('server',
'session_timeout')
options.test = config.config.get('server', 'test')
if options.test == 'None':
options.test = None
options.environment = config.config.get('server', 'environment')
options.server_root = config.config.get('server', 'server_root')
options.max_body_size = config.config.get('server', 'max_body_size')
options.log_dir = config.config.get('logging', 'log_dir')
options.log_level = config.config.get('logging', 'log_level')
return options
def __init__(self, *args, **kwargs):
Values.__init__(self, *args, **kwargs)
self._config_files = []
def copyValues(values):
if not values:
return None
new_values = Values()
new_values.__dict__ = copy(values.__dict__)
return new_values
def Calc(calc, outfile, NoDataValue=None, type=None, format=None, creation_options=None, allBands='', overwrite=False, debug=False, quiet=False, **input_files):
""" Perform raster calculations with numpy syntax.
Use any basic arithmetic supported by numpy arrays such as +-*\ along with logical
operators such as >. Note that all files must have the same dimensions, but no projection checking is performed.
Keyword arguments:
[A-Z]: input files
[A_band - Z_band]: band to use for respective input file
Examples:
add two files together:
Calc("A+B", A="input1.tif", B="input2.tif", outfile="result.tif")
average of two layers:
Calc(calc="(A+B)/2", A="input1.tif", B="input2.tif", outfile="result.tif")
set values of zero and below to null:
Calc(calc="A*(A>0)", A="input.tif", A_band=2, outfile="result.tif", NoDataValue=0)
"""
opts = Values()
opts.input_files = input_files
opts.calc = calc
opts.outF = outfile
opts.NoDataValue = NoDataValue
opts.type = type
opts.format = format
opts.creation_options = [] if creation_options is None else creation_options
opts.allBands = allBands
opts.overwrite = overwrite
opts.debug = debug
opts.quiet = quiet
doit(opts, None)
