def main(name=None): if name in (None, __name__): if len(sys.argv) != 2: print("Usage: {} DECISION_TASK_ID".format(sys.argv[0]), file=sys.stderr) sys.exit(1) log.setLevel(logging.DEBUG) formatter = logging.Formatter( fmt="%(asctime)s %(levelname)8s - %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" ) handler = logging.StreamHandler() handler.setFormatter(formatter) log.addHandler(handler) makedirs("build") orig_dir = os.getcwd() context = Values() gnupghome = os.path.join(os.getcwd(), 'gpg') os.chmod(gnupghome, 0o700) context.gpg = gnupg.GPG(gpgbinary='gpg2', gnupghome=gnupghome) context.gpg.encoding = 'utf-8' credentials = { 'credentials': { 'clientId': os.environ["TASKCLUSTER_CLIENT_ID"], 'accessToken': os.environ["TASKCLUSTER_ACCESS_TOKEN"], } } loop = asyncio.get_event_loop() try: os.chdir("build") with aiohttp.ClientSession() as context.session: context.queue = Queue(credentials, session=context.session) context.decision_task_id = sys.argv[1] loop.run_until_complete(async_main(context)) finally: os.chdir(orig_dir) loop.close()
def read_cmd_options(): from optparse import OptionParser, Values tornado.options.define('distribution', default=None, help='distribution of the time between events') tornado.options.define('limit', default=0, type=int, help='number of events to generate') tornado.options.define('eventlog', default=False, help='dump event log', type=bool) tornado.options.define('timestamp', default=False, help='add an X-Float-Timestamp header to events', type=bool) tornado.options.define('bayeux', default=False, help='use the Bayeux protocol for publishing', type=bool) tornado.options.define('timescale', default=1.0, help='accelerate time by this factor', type=float) tornado.options.define('delay', default=2.0, help='initial delay, in seconds', type=float) remaining = tornado.options.parse_command_line() options = Values() if len(remaining) >= 2: options.filename = remaining[0] options.server_urls = remaining[1:] else: OptionParser().error('At least one file name and one ' 'server URL required') return options
def loadDaemon(): ''' Load the daemon configuration file. ''' args = hlm_args.args() configFile = args.daemonConfig if configFile == None: configFile = hlm_paths.daemonConfig() config = configparser.SafeConfigParser() try: config.read(configFile) except SystemExit: raise except BaseException as exc: raise FatalError(_('Can\'t load the daemon configuration file {0}: {1}').format(quote(configFile), exc)) try: if config.sections() != ['daemon']: raise Exception(_('only the section {0} is allowed in this file.').format('[daemon]')) options = config.options('daemon') _checkAlienDirectives(['credentials', 'user', 'group', 'ping_site', 'ping_interval'], options, 'daemon') # "credentials" accepts a credentials.conf file path if args.daemonCredentials == None: credentials = None if config.has_option('daemon', 'credentials'): credentials = config.get('daemon', 'credentials') else: credentials = hlm_paths.credentialsConfig() args.daemonCredentials = credentials # "user" accepts an account name user = None if config.has_option('daemon', 'user'): user = config.get('daemon', 'user') # "group" accepts a group name group = None if config.has_option('daemon', 'group'): group = config.get('daemon', 'group') # "pingSite" requires an http:// website URL pingSite = _mandatoryDirective('ping_site', options, 'daemon', config.get) if not pingSite.startswith('http://'): raise Exception(_('ping_site {0} is is incorrect (must start with {1}), exiting.').format(quote(pingSite), quote('http://'))) pingInterval = _mandatoryDirective('ping_interval', options, 'daemon', config.getint) if pingInterval < _minimumPingInterval: if __WARNING__: logWarning(_('Daemon configuration file {0}: ping_interval is way too low ({1} seconds), forcing it to {2} seconds.').format(quote(configFile), pingInterval, _minimumPingInterval)) pingInterval = _minimumPingInterval # Wrap configuration into a single Values() object configCredentials = Values() configCredentials.user = user configCredentials.group = group configCredentials.pingSite = pingSite configCredentials.pingInterval = pingInterval if __DEBUG__: logDebug('Daemon configuration has been loaded from {0}.'.format(configFile)) return configCredentials except SystemExit: raise except BaseException as exc: raise FatalError(_('Incorrect daemon configuration file {0}: {1}').format(quote(configFile), exc))
def test_start(self,mock_parse_args,mock_delete_raw_logs): a = Values() a.start = "2001/11/12" a.end = None a.debug = None a.path= None mock_parse_args.return_value = (a,'') mock_delete_raw_logs.return_value = (True,{'dirsdeleted':range(0,10)}) assert logclean() == 0, "Bad return value" mock_delete_raw_logs.return_value = (False,{'dirsdeleted':range(0,10), 'dirserrors':range(0,20)}) assert logclean() == -1 ,"Bad return value"
def read_cmd_options(): from optparse import OptionParser, Values tornado.options.define('eventlog', default=False, help='dump event log', type=bool) remaining = tornado.options.parse_command_line() options = Values() if len(remaining) >= 1: options.stream_urls = remaining else: OptionParser().error('At least one source stream URL required') return options
def read_cmd_options(): from optparse import OptionParser, Values tornado.options.define('eventlog', default=False, help='dump event log', type=bool) tornado.options.define('timescale', default=40000.0, help='accelerate time by this factor', type=float) remaining = tornado.options.parse_command_line() options = Values() if len(remaining) >= 1: options.server_urls = remaining else: OptionParser.error('At least one server URL required') return options
def read_cmd_options(): from optparse import OptionParser, Values tornado.options.define('distribution', default='exp(5)', help='distribution of the time between events') tornado.options.define('limit', default=0, type=int, help='number of events to generate') tornado.options.define('eventlog', default=False, help='dump event log', type=bool) remaining = tornado.options.parse_command_line() options = Values() if len(remaining) >= 1: options.server_urls = remaining else: OptionParser().error('At least one server URL required') return options
def get_opts(docstr, meta={}, version=None, argv=None): """ Get docopt dict, and set argv and flags from get_optvalues. """ if not argv: import sys argv = sys.argv[1:] pattern, collected = docopt(docstr, argv, version=version, return_spec=True) opts = Values() opts.argv = argv parsed = pattern.flat() + collected #assert not ( 'argv' in opts or 'flags' in opts or 'args' in opts),\ # "Dont use 'argv', 'flags' or 'args'. " opts.cmds, opts.flags, opts.args = get_optvalues(parsed, meta) return opts
def testPlottingMore(): """Test the plotting module""" opt = Values() opt.imgDir = "tests/img/" opt.jsonDir = "tests/json/" if not os.path.exists(opt.jsonDir): os.makedirs(opt.jsonDir) if not os.path.exists(opt.imgDir): os.makedirs(opt.imgDir) testPlotter = plotter(opt) dic = {'TTT':10,'AAA':5} testHistPlotter = histPlotter(dic,opt,filename="testHist") testHistPlotter.plot() dic = {'AAA':{'Expected':1,'Observed':3},'TTT':{'Expected':10,'Observed':30}} testMultiHistPlotter = multiHistPlotter(dic,opt,filename="testMultiHist") testMultiHistPlotter.plot()
def _GetDefaultOption(self, set_10_days_ago, query='steps'): ops = Values() ops.query = query if set_10_days_ago: ops.interval_value = 10 ops.interval_unit = 'days' else: ops.interval_value = None ops.email_entries = ['comments'] ops.project_name = 'chromium' ops.query_title = 'query title' ops.max_comments = None return ops
def _GetDefaultOption(self, set_10_days_ago, query="steps"): ops = Values() ops.query = query if set_10_days_ago: ops.interval_value = 10 ops.interval_unit = "days" else: ops.interval_value = None ops.email_entries = ["comments"] ops.project_name = "chromium" ops.query_title = "query title" ops.max_comments = None return ops
def read_cmd_options(): from optparse import OptionParser, Values tornado.options.define('eventlog', default=False, help='dump event log', type=bool) tornado.options.define('noparse', default=False, help='quick client that does not parse events', type=bool) tornado.options.define('reconnect', default=False, help='reconnect disconnected clients', type=bool) remaining = tornado.options.parse_command_line() options = Values() if len(remaining) == 2: options.stream_url = remaining[0] options.num_clients = int(remaining[1]) else: OptionParser().error('A source stream URL required') return options
def read_cmd_options(): from optparse import OptionParser, Values tornado.options.define('label', default=None, help='define a client label', type=str) tornado.options.define('missing', default=False, help=('retrieve missing events ' '(requires a client label)'), type=bool) tornado.options.define('deflate', default=True, help='Accept compressed data with deflate', type=bool) remaining = tornado.options.parse_command_line() options = Values() if len(remaining) >= 1: options.stream_urls = remaining else: OptionParser().error('At least one source stream URL required') return options
def read_cmd_options(): from optparse import Values, OptionParser tornado.options.define('port', default=8888, help='run on the given port', type=int) tornado.options.define('aggregatorid', default=None, help='aggregator id', type=str) tornado.options.define('buffer', default=None, help='event buffer time (s)', type=float) tornado.options.define('eventlog', default=False, help='dump event log', type=bool) tornado.options.define('autostop', default=False, help='stop the server when the source finishes', type=bool) remaining = tornado.options.parse_command_line() options = Values() if len(remaining) >= 1: options.stream_urls = remaining else: OptionParser().error('At least one source stream URL required') return options
def add(self,typl,what): # ~~> cast already prepared becasue .add is always called if typl == '': return # ~~> bundling of the layers becasue .add is always called if what["xref"] not in self.obdata.keys(): obdata = Values() obdata.type = '' obdata.unit = [] obdata.support = [] obdata.function = [] obdata.values = [] self.obdata.update({ what["xref"]:obdata }) # ~~> file references if what['file'] in self.object.keys(): return # ~~> unexplored teritory if 'sortie' in typl.lower(): self.object.update({ what['file']:Sortie(what['file']) }) elif 'csv' in typl.lower(): self.object.update({ what['file']:CSV(what['file']) }) elif 'tif' in typl.lower() or 'jpg' in typl.lower() \ or 'gif' in typl.lower() or 'png' in typl.lower() \ or 'bmp' in typl.lower(): self.object.update({ what['file']:'image' }) # ~~> SELAFIN file elif 'SELAFIN' in typl.upper() or 'slf' in typl.lower(): slf = castSELAFIN(what['file']) slf.setKDTree() slf.setMPLTri() self.object.update({ what['file']:slf }) else: # TODO: raise exception print '... do not know how to extract from this format: ' + typl sys.exit(1)
def test_valid_rsync_includes_returns_correct_list(tmp_flow_config): """Test that the rsync includes in the correct """ reg = 'rsynctest' flow_file = tmp_flow_config( reg, """ [scheduling] initial cycle point = 2020-01-01 [[dependencies]] graph = "blah => deeblah" [scheduler] install = dir/, dir2/, file1, file2 allow implicit tasks = True """) config = WorkflowConfig(workflow=reg, fpath=flow_file, options=Values()) rsync_includes = WorkflowConfig.get_validated_rsync_includes(config) assert rsync_includes == ['dir/', 'dir2/', 'file1', 'file2']
def test_git_clone(self, mock_shell): parsed_options = { 'url': 'git://puppet/project.git', 'path': 'puppet/modules' } options = Values(defaults=parsed_options) # test builder.git_clone(options) # validation mock_shell.assert_called_with('git clone --recursive %s' % options.url) self.assertEqual(mock_shell.call_count, 1) self.assertEqual(options.path, 'puppet/modules')
def process_include_files(infile, outfile, inpath='', catalogpath=None, fixtypenames=None, no_collect_includes=False, no_redefine_groups=False): """The root/main function""" load_catalog(catalogpath) options = Values({ 'force': False, 'fixtypenames': fixtypenames, 'no_collect_includes': no_collect_includes, 'no_redefine_groups': no_redefine_groups, }) doc, ns_dict, schema_ns_dict, rename_data = prep_schema_doc( infile, outfile, inpath, options) return doc, ns_dict, schema_ns_dict, rename_data
def __init__(self): self.options = { 'verbosity': 0, 'ask_pass': False, 'private_key_file': None, 'remote_user': None, 'connection': 'smart', 'timeout': 10, 'ssh_common_args': '', 'sftp_extra_args': '', 'scp_extra_args': '', 'ssh_extra_args': '', 'force_handlers': False, 'flush_cache': None, 'become': False, 'become_method': 'sudo', 'become_user': None, 'become_ask_pass': False, 'tags': ['all'], 'skip_tags': [], 'check': False, 'syntax': None, 'diff': False, 'inventory': ansible_host, 'listhosts': None, 'subset': None, 'extra_vars': [], 'ask_vault_pass': False, 'vault_password_files': [], 'vault_ids': [], 'forks': 5, 'module_path': None, 'listtasks': None, 'listtags': None, 'step': None, 'start_at_task': None, 'args': ['fake']} self.ops = Values(self.options) self.loader = DataLoader() self.passwords = dict() self.results_callback = ResultsCollector() self.inventory = InventoryManager(loader=self.loader, sources=[self.options['inventory']]) self.variable_manager = VariableManager(loader=self.loader, inventory=self.inventory)
def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) config = get_config(parse_args=False) # Setup the correct options so the agent will use the forwarder opts, args = Values({ 'autorestart': False, 'dd_url': None, 'use_forwarder': True, 'disabled_dd': False }), [] agentConfig = get_config(parse_args=False, options=opts) self.hostname = get_hostname(agentConfig) # Watchdog for Windows self._collector_heartbeat, self._collector_send_heartbeat = multiprocessing.Pipe( False) self._collector_failed_heartbeats = 0 self._max_failed_heartbeats = \ MAX_FAILED_HEARTBEATS * agentConfig['check_freq'] / SERVICE_SLEEP_INTERVAL # Watch JMXFetch restarts self._MAX_JMXFETCH_RESTARTS = 3 self._count_jmxfetch_restarts = 0 # Keep a list of running processes so we can start/end as needed. # Processes will start started in order and stopped in reverse order. self.procs = { 'forwarder': ProcessWatchDog("forwarder", DDForwarder(config, self.hostname)), 'collector': ProcessWatchDog( "collector", DDAgent(agentConfig, self.hostname, heartbeat=self._collector_send_heartbeat)), 'dogstatsd': ProcessWatchDog("dogstatsd", DogstatsdProcess(config, self.hostname)), 'jmxfetch': ProcessWatchDog("jmxfetch", JMXFetchProcess(config, self.hostname), 3), }
def setUp(self): super(TestUnitKafkaConsumer, self).setUp() bot_options = Values() bot_options.kafkatopic = 'test_topic' bot_options.kafkagroup = 'test_group' bot_options.broker = 'test_broker' bot_options.username = '******' self.kafkaHandler = KafkaHandler(options=bot_options) self.kafkaHandler.handle_create_post = MagicMock() self.kafkaHandler.on_event_handler = MagicMock() self.KafkaMessagesConsumer = KafkaMessagesConsumer( bot_options, self.kafkaHandler, Mock())
def _register_attributes(self, app, args): # FIXME: there is no better lib function than this snippet schema_file = app.get_share_file('schema') if os.path.exists(schema_file): self.log('Registering schema %s' % schema_file) lo, pos = self._get_ldap_connection(args) with self._get_password_file(args) as password_file: create_recursive_container( 'cn=ldapschema,cn=univention,%s' % ucr_get('ldap/base'), lo, pos) if app.automatic_schema_creation: schema_obj = UniventionLDAPSchema(ucr_instance()) userdn = self._get_userdn(args) udm_passthrough_options = [ '--binddn', userdn, '--bindpwdfile', password_file ] opts = Values() opts.packagename = 'appcenter-app-%s' % app.id opts.packageversion = app.version opts.ucsversionstart = None opts.ucsversionend = None os.environ['UNIVENTION_APP_IDENTIFIER'] = app.id try: schema_obj.register(schema_file, opts, udm_passthrough_options) except SystemExit as exc: if exc.code == 4: self.warn( 'A newer version of %s has already been registered. Skipping...' % schema_file) else: raise RegisterSchemaFailed(exc.code) else: if not schema_obj.wait_for_activation(): raise RegisterSchemaFileFailed(schema_file) finally: if 'UNIVENTION_APP_IDENTIFIER' in os.environ: del os.environ['UNIVENTION_APP_IDENTIFIER'] # and this is what should be there after one line of lib.register_schema(schema_file) app = app.get_app_cache_obj().copy( locale='en').find_by_component_id(app.component_id) attributes, __, options = get_extended_attributes(app) for option in options: self.log('Registering option %s' % option.name) create_extended_option(option, app, lo, pos) if attributes: for i, attribute in enumerate(attributes): self.log('Registering attribute %s' % attribute.name) create_extended_attribute(attribute, app, i + 1, lo, pos)
def test_invalid_custom_output_msg(tmp_flow_config: Callable): """Test invalid output message (colon not allowed).""" reg = 'invalid_output' flow_file = tmp_flow_config( reg, """ [scheduling] [[graph]] R1 = "foo:x => bar" [runtime] [[bar]] [[foo]] [[[outputs]]] x = "the quick: brown fox" """) with pytest.raises(WorkflowConfigError) as cm: WorkflowConfig(workflow=reg, fpath=flow_file, options=Values()) assert ('Invalid message trigger "[runtime][foo][outputs]x = ' 'the quick: brown fox"') in str(cm.value)
def construct_options_dict(options: Values) -> Dict[str, Any]: ''' Returns a dictionary whose keys are strings with the attribute names of options and whose values are the option value corresponding to the name. Arguments --------- options: Values Read options Returns ------- Dict[str, Any]: values in dict format ''' # Just in case Values have items in dir by default, we use set difference # so that they are not included names = set(dir(options)) - set(dir(Values())) values = map(lambda x: getattr(options, x), names) return {k: v for k, v in zip(names, values) if v is not None}
def meta_config(linter): """ Return the config as `optparse.Values` (linter.config) for all checkers :param linter: :return: >>> meta = meta_config(get_linter()) >>> jobs = meta.jobs >>> jobs in list(range(1, 5)) True """ from optparse import Values meta = Values(linter.config.__dict__) for checker in linter.prepare_checkers(): meta.__dict__.update(checker.config.__dict__) return meta
def test_expanded_paths(self, mock_parse): parsed_options = { 'working_dir': '~/', 'output_dir': '~/', 'path': None, 'branch': None, 'tag': None } mock_parse.return_value = (Values(defaults=parsed_options), []) # test options = builder.get_options() # validation self.assertEqual(options.working_dir, os.path.expanduser('~/')) self.assertEqual(options.output_dir, os.path.expanduser('~/'))
def numpydiff(ref, new, recursive=False, exclude=exclude_default, include=['*'], format='bin_v3d', show_file=False, variables=None, verbosity=0): '''Python interface to numpydiff See OptionParser for arguments info. ''' options = Values() options.recursive = recursive options.exclude = exclude options.include = include options.format = format options.variables = variables options.verbosity = verbosity options.show_file = show_file return diff(ref, new, options)
class ApplicationContextTest(unittest.TestCase): sut = ApplicationContext( ScriptSettings(Values({ 'project': 'project', 'env': 'system-test', 'build': 510, 'grab_envs': 'false', 'sanity_test_timeout_sec': 120 }), workdir='.jtalks')) def test_nexus_is_created(self): nexus = self.sut.old_nexus() self.assertEquals(nexus.build_number, 510) def test_tomcat_is_created(self): tomcat = self.sut.tomcat() # can't use assertIsInstance() because on CentOS we use old mock lib self.assertTrue(isinstance(tomcat, Tomcat))
def test_build_manifest(self, mock_chdir, mock_digest, mock_open, mock_getsize, mock_listdir): parsed_options = { 'output_dir': '/tmp/output', } options = Values(defaults=parsed_options) files = [ 'file_1%s' % builder.ARCHIVE_SUFFIX, 'file_2%s' % builder.ARCHIVE_SUFFIX, 'file_3', # not a module ] digests = ['hash_1', 'hash_2'] mock_digest.side_effect = digests mock_listdir.return_value = files mock_getsize.side_effect = [10, 20] manifest = [] mock_fp = Mock() mock_fp.write = manifest.append mock_fp.__enter__ = Mock(return_value=mock_fp) mock_fp.__exit__ = Mock() mock_open.return_value = mock_fp # test builder.build_manifest(options) # validation _manifest = [ 'file_1.tar.gz', ',hash_1', ',10\n', 'file_2.tar.gz', ',hash_2', ',20\n' ] mock_open.assert_called_with('PULP_MANIFEST', 'w+') mock_chdir.assert_called_with(os.getcwd()) self.assertEqual(manifest, _manifest)
def test_validate_both_branch_and_tag(self, mock_parse, mock_stdout, mock_exit): parsed_options = { 'working_dir': None, 'output_dir': None, 'path': None, 'branch': 'br', 'tag': 'tg' } mock_parse.return_value = (Values(defaults=parsed_options), []) # test builder.get_options() # validation mock_stdout.assert_any_call(builder.BAD_BRANCH_AND_TAG) mock_exit.assert_called_with(os.EX_USAGE)
def test_clean(self, mock_shell): parsed_options = { 'url': 'git@test:foo/bar.git', 'working_dir': '/tmp/working', 'clean': True } options = Values(defaults=parsed_options) # test builder.clean(options) # validation url = urlparse(options.url) path = os.path.basename(url.path) path = os.path.splitext(path)[0] path = os.path.join(options.working_dir, path) mock_shell.assert_called_with('rm -rf %s' % path)
def minify_script(patches=None, keep_report=True, show_diff=False): """minifies createstubs.py Args: patches ([PathLike], optional): List of paths to patches to apply. Defaults to None. keep_report (bool, optional): Keeps single report line in createstubs Defautls to True. show_diff (bool, optional): Print diff from edits. Defaults to False. Returns: str: minified source text """ patches = patches or [] edits = [ ("comment", "print"), ("comment", "import logging"), ("comment", "self._log ="), ("comment", "self._log.debug"), ("comment", "self._log.warning"), ("comment", "self._log.info"), ("comment", "self._log.error"), ] if keep_report: report = ('rprint', ('self._log.info("Stub module: {:<20} to file:' ' {:<55} mem:{:>5}".' 'format(module_name, file_name, m1))')) clean = ( 'rprint', 'self._log.info("Clean/remove files in folder: {}".format(path))') edits.insert(0, report) edits.insert(1, clean) minopts = Values({'tabs': False}) with SCRIPT.open('r') as f: content = f.read() for path in patches: path = Path(path) content = apply_patch(content, path.read_text()) content = edit_lines(content, edits, show_diff=show_diff) tokens = token_utils.listified_tokenizer(content) source = minification.minify(tokens, minopts) return source
def test_family_inheritance_and_quotes(fam_txt: str, mock_glbl_cfg: Callable, tmp_flow_config: Callable) -> None: """Test that inheritance does not ignore items, if not all quoted. For example: inherit = 'MAINFAM<major, minor>', SOMEFAM inherit = 'BIGFAM', SOMEFAM See bug #2700 for more/ """ mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', ''' [platforms] [[localhost]] hosts = localhost ''') reg = 'test' file_path = tmp_flow_config( reg, f''' [scheduler] allow implicit tasks = True [task parameters] major = 1..5 minor = 10..20 [scheduling] [[graph]] R1 = """hello => MAINFAM<major, minor> hello => SOMEFAM""" [runtime] [[root]] script = true [[MAINFAM<major, minor>]] [[SOMEFAM]] [[ goodbye_0<major, minor> ]] inherit = 'MAINFAM<major, minor>', {fam_txt} ''') config = WorkflowConfig(reg, file_path, template_vars={}, options=Values()) assert ('goodbye_0_major1_minor10' in config.runtime['descendants']['MAINFAM_major1_minor10']) assert ('goodbye_0_major1_minor10' in config.runtime['descendants']['SOMEFAM'])
def run(self): """ Runs the builder """ # parse command line parameters parser = self.buildParser() (opts, args) = parser.parse_args() if len(args) == 0: parser.error("incorrect number of arguments") # download only if opts.download: if len(args) > 1 and opts.targetName: parser.error( "option --targetName only allowed for a single dictionary" " but %d dictionaries given" % len(args)) for dictionary in args: downloader = getDownloader(dictionary, quiet=opts.quiet) downloader.download(targetName=opts.targetName, targetPath=opts.targetPath) return True options = dict([ (option, getattr(opts, option)) for option in dir(opts) if not hasattr(Values(), option) and getattr(opts, option) != None ]) installer = DictionaryInstaller(quiet=opts.quiet) for dictionary in args: try: installer.install(dictionary, **options) except OperationalError as e: if not opts.quiet: warn("Error writing to database: %s" % e) if not opts.local: warn("Try choosing --local for installing into HOME") return False return True
def get_parsed_args(): parser = OptionParser() parser.add_option('-A', '--autorestart', action='store_true', default=False, dest='autorestart') parser.add_option('-d', '--dd_url', action='store', default=None, dest='dd_url') parser.add_option('-u', '--use-local-forwarder', action='store_true', default=False, dest='use_forwarder') parser.add_option('-v', '--verbose', action='store_true', default=False, dest='verbose', help='Print out stacktraces for errors in checks') parser.add_option('-p', '--profile', action='store_true', default=False, dest='profile', help='Enable Developer Mode') try: options, args = parser.parse_args() except SystemExit: # Ignore parse errors options, args = Values({ 'autorestart': False, 'dd_url': None, 'use_forwarder': False, 'verbose': False, 'profile': False }), [] return options, args
def __init__(self, values=None): if values: self.defaults.update(values.__dict__) # Ensure that context has default values for all unset variables super(self.__class__, self).__init__(vars(Values(self.defaults))) if self.masterDataDirectory: self.master_datadir = self.masterDataDirectory else: self.master_datadir = gp.get_masterdatadir() self.master_port = self.get_master_port() if self.local_dump_prefix: self.dump_prefix = self.local_dump_prefix + "_" else: self.dump_prefix = "" if not self.include_dump_tables: self.include_dump_tables = [] if not self.exclude_dump_tables: self.exclude_dump_tables = [] if not self.output_options: self.output_options = [] if not self.dump_schema: self.dump_schema = [] if not self.exclude_dump_schema: self.exclude_dump_schema = []
def test_git_checkout_tag(self, mock_shell): parsed_options = { 'origin': 'git://', 'branch': None, 'tag': 'mytag', } options = Values(defaults=parsed_options) # test builder.git_checkout(options) # validation mock_shell.assert_any_call('git fetch') mock_shell.assert_any_call('git fetch --tags') mock_shell.assert_any_call('git checkout %s' % options.tag) self.assertEqual(mock_shell.call_count, 3)
def test_validate_path(self, mock_parse, mock_stdout, mock_exit): parsed_options = { 'url': None, 'working_dir': None, 'output_dir': None, 'path': '/', 'branch': None, 'tag': None } mock_parse.return_value = (Values(defaults=parsed_options), []) # test builder.get_options() # validation mock_stdout.assert_any_call(builder.BAD_PATH) mock_exit.assert_called_with(os.EX_USAGE)
def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) config = get_config(parse_args=False) # Setup the correct options so the agent will use the forwarder opts, args = Values({'clean': False, 'disabled_dd': False}), [] agentConfig = get_config(parse_args=False, options=opts) self.restart_interval = \ int(agentConfig.get('autorestart_interval', RESTART_INTERVAL)) log.info("Autorestarting the collector ever %s seconds" % self.restart_interval) # Keep a list of running processes so we can start/end as needed. # Processes will start started in order and stopped in reverse order. self.procs = { 'monasca-forwarder': MonascaForwarder(config), 'monasca-collector': MonascaCollector(agentConfig), 'monasca-statsd': MonascaStatsd(config), }
def get_parsed_args(): parser = OptionParser() parser.add_option('-c', '--clean', action='store_true', default=False, dest='clean') parser.add_option('-v', '--verbose', action='store_true', default=False, dest='verbose', help='Print out stacktraces for errors in checks') try: options, args = parser.parse_args() except SystemExit: # Ignore parse errors options, args = Values({'clean': False}), [] return options, args
def get_default_options(options, non_default_options): """ Given a dictionary of options containing the defaults optparse has filled in, and a dictionary of options containing only options parsed from the command line, returns a dictionary containing the default options that remain after removing the default options that were overridden by the options passed on the command line. Mathematically, this returns a dictionary with default_options.keys = options.keys() \ non_default_options.keys() where \ is the set difference operator. The value of a key present in default_options is the value of the same key in options. """ options_dict = vars(options) non_default_options_dict = vars(non_default_options) default_options = Values( dict((k, options_dict[k]) for k in options_dict if k not in non_default_options_dict)) return default_options
def get_parsed_args(): parser = OptionParser() parser.add_option('-d', '--dd_url', action='store', default=None, dest='dd_url') parser.add_option('-c', '--clean', action='store_true', default=False, dest='clean') parser.add_option('-u', '--use-local-forwarder', action='store_true', default=False, dest='use_forwarder') parser.add_option('-n', '--disable-dd', action='store_true', default=False, dest="disable_dd") parser.add_option('-v', '--verbose', action='store_true', default=False, dest='verbose', help='Print out stacktraces for errors in checks') try: options, args = parser.parse_args() except SystemExit: # Ignore parse errors options, args = Values({ 'dd_url': None, 'clean': False, 'use_forwarder': False, 'disable_dd': False, 'use_forwarder': False }), [] return options, args
def test_errata(self): with open("Tests/quote1.json", encoding='utf-8') as f: errata = json.load(f) if not os.path.exists("Temp"): os.mkdir("Temp") state = {"text": "./Tests", "html": "./Temp", "ossPath": "css"} options = Values(defaults={'search': False}) doc = apply_errata(errata, options, state) templates = Templates( os.path.join(os.path.dirname(__file__), "Template")) doc.apply(True, templates) self.assertEqual(doc.InlineCount, 0) self.assertEqual(doc.SectionCount, 1) self.assertEqual(doc.EndnoteCount, 0) self.assertTrue( compare_file("./Temp/RFC8275.html", "./Results/quote1.html", True))
def test_parse_file_one_error(): target_path = os.path.join(tempfile.gettempdir(), str(uuid4())) os.makedirs(target_path) cwd = os.getcwd() os.chdir(target_path) try: rp = ReportSVGBadge( Values({ 'output_file': None, 'image': '123.svg', 'format': 'svg' })) with open('lala.py', 'w') as f: f.write('print(123)\n') f.write('print(123)\n') with open('lulu.py', 'w') as f: f.write('print(123)\n') f.write('print(123)\n') f.write('print(123)\n') f.write('print(123)\n') rp.beginning('lala.py') rp.finished('lala.py') rp.beginning('lulu.py') rp.handle(Violation('F101', 'lulu.py', 123, 123, 123, 123)) rp.finished('lulu.py') rp.stop() with open("123.svg") as f: el = xmltodict.parse(f.read()) assert el['svg']['g'][1]['text'][3]['#text'] == '67%' finally: os.chdir(cwd) rmtree(target_path)
def __call__(self, filename, lines): """ ### &doc_id postprocess:DoxypypyPostProcess_call The main bridge between DoxyTH and Doxypypy. Re-creates the doxypypy variables and gives it to the AstWalker Args: filename: The file_name name lines: The file_name lines """ from doxypypy.doxypypy import AstWalker from os import sep from optparse import Values from .langs import ascii_encode # Replace all the lines by a normal \n at the end try: lines = [f"{line.rstrip()}\n" for line in lines] except UnicodeEncodeError: lines = [f"{ascii_encode(line).rstrip()}\n" for line in lines] # Define options options = {'autobrief': True, 'autocode': True, 'topLevelNamespace': None, 'tablength': 4, 'debug': None} # All the code below is extracted from doxypypy (with a few edits to fit in with doxyth) full_path_namespace = filename.replace(sep, '.')[:-3] # Use any provided top-level namespace argument to trim off excess. real_namespace = full_path_namespace if options['topLevelNamespace']: namespace_start = full_path_namespace.find(options['topLevelNamespace']) if namespace_start >= 0: real_namespace = full_path_namespace[namespace_start:] options['fullPathNamespace'] = real_namespace ast_walker = AstWalker(lines, Values(defaults=options), filename) ast_walker.parseLines() # Output the modified source. print(ast_walker.getLines())
def run(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.logger.info('Binding to %s:%s', self.bind_ip, self.bind_port) s.bind((self.bind_ip, self.bind_port)) s.listen(20) while True: client, address = s.accept() self.logger.debug('Accepting connection from %s:%s', *address) client_file = client.makefile('rb+', 1) json_data = client_file.read() conf = Values(json.loads(json_data)) self.logger.info( 'Starting run for %s:%s [put/get/del_concurrency: %s/%s/%s, ' 'num_objects: %s, num_gets: %s]', address[0], address[1], conf.put_concurrency, conf.get_concurrency, conf.del_concurrency, conf.num_objects, conf.num_gets) logger = logging.getLogger('bench-server') level = logging.DEBUG if conf.log_level.lower() == 'debug' \ else logging.INFO logger.setLevel(level) loghandler = logging.StreamHandler(client_file) logformat = logging.Formatter( '%(server)s %(asctime)s %(levelname)s %(message)s') loghandler.setFormatter(logformat) logger.addHandler(loghandler) logger = LogAdapter(logger, 'swift-bench-server') controller = BenchController(logger, conf) try: controller.run() except socket.error: logger.warning('Socket error', exc_info=1) logger.logger.removeHandler(loghandler) client_file.close() client.close() self.logger.info('...bench run completed; waiting for next run.')
def build_options(query=''): options = Values() options.ensure_value('query', query) options.ensure_value('type', 'video') options.ensure_value('maxResults', 50) options.ensure_value('regionCode', 'jp') options.ensure_value('order', 'viewCount') return options
def _build_opts_from_dict(val): opts = Values() for key in val.keys(): opts.__dict__[key] = val[key] return opts
def testComplexErrorSim(): """Test the read error simulation""" N= 500 randomLeftFlank = "".join([random.choice(getAlphabet()) for _ in range(N)]) randomRightFlank = "".join([random.choice(getAlphabet()) for _ in range(N)]) seq = randomLeftFlank+'AGTATACCTCGCATCGATCGATCG' +randomRightFlank# len 12 ref = "".join([random.choice(getAlphabet()) for _ in range(100000)])+seq + "".join([random.choice(getAlphabet()) for _ in range(100000)]) refRecord=SeqRecord(Seq(ref),'Chromosome dna:chromosome chromosome:ASM19595v1:Chromosome:1:4411532:1','','') record=SeqRecord(Seq(seq),'st=%s'%(100000),'','') opt = Values() opt.snpFreq = 0.1 rep = [0.1]*N opt.refPos = 100000 errorSim = complexError(record,opt,id = 'st=%s'%(100000),baseErrorProb = rep+[0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep) assert_equal(errorSim.errorProb,rep+[0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep) errorSim.snp(pos=N,rl='T') assert_equal(errorSim.errorProb,rep+[0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep) assert_equal( str(errorSim.seq) , randomLeftFlank+'TGTATACCTCGCATCGATCGATCG'+randomRightFlank) errorSim.ins(pos=N+3,rl='CCC') assert_equal(errorSim.errorProb,rep+[0.1,0.15,0.1,0.15,0.1,0.1,0.1,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15,0.1,0.15]+rep) assert_equal( str(errorSim.seq) , randomLeftFlank+'TGTACCCTACCTCGCATCGATCGATCG'+randomRightFlank) errorSim.deletion(pos=N+17,dlen=5) assert_equal( str(errorSim.seq) , randomLeftFlank+'TGTACCCTACCTCGCATGATCG'+randomRightFlank) ## Write these to a fasta file refFilename = 'tests/ref.fa' readFilename = 'tests/read.fq' writeFasta(filename = refFilename,seqList = [refRecord]) writeFastq(filename = readFilename,seqList = [errorSim.record]) ## Align ref to read align.refIndex(file=refFilename) # ## Align reads to the reference samfileName = readFilename + '.sam' aligned = align.align(reference=refFilename, read_file=readFilename,stdout=samfileName) reader = errorReader(samfile=samfileName,ref=ref) errorList = [] for error in reader: assert_equal(error.read.cigarstring,'M%sI3M%sD5M%s'%(N+4,10,N+5)) errorList.append(error) assert_equal(len(errorList),3) error1 = errorList[0] assert_equal(error1.isSnp,True) assert_equal(error1.errorType,'SNP') assert_equal(error1.true,'A') assert_equal(error1.emission,'T') assert_equal(error1.after(3),'GTA') assert_equal(error1.readPos,N) assert_equal(error1.refPos,100000+N) error2 = errorList[1] assert_equal(error2.isIndel,True) assert_equal(error2.isInsertion,True) assert_equal(error2.errorType,'Insertion') assert_equal(error2.true,'') assert_equal(error2.emission,'CCC') assert_equal(error2.after(3),'TAC') assert_equal(error2.before(3),'GTA') assert_equal(error2.readPos,N+3) assert_equal(error2.refPos,100000+3+N) error3 = errorList[2] assert_equal(error3.isIndel,True) assert_equal(error3.isDeletion,True) assert_equal(error3.errorType,'Deletion') assert_equal(error3.true,'CGATC') assert_equal(error3.emission,'') assert_equal(error3.after(3),'GAT') assert_equal(error3.before(3),'CAT') assert_equal(error3.readPos,N+17) assert_equal(error3.refPos,100000+17+N) assert_equal(error1.mappedCorrectly,True) assert_equal(error1.mappedDist,0) ## Test counter opt.maxKmerLength = 4 opt.dbName = 'proto_errTEST' opt.observedErrorDBName = 'errors' opt.simulatedErrorDBName = 'simulatedErrors' errorCounter = counter(ref=ref,opt=opt,errorList=errorList,makeDB=True) assert_equal(errorCounter.probKmer('A') + errorCounter.probKmer('T') + errorCounter.probKmer('C') +errorCounter.probKmer('G') ,1) count,errorQueryList = errorCounter.getCount(returnList=True) assert_equal(count,3) assert_equal(errorCounter.getCount(kmerBefore='GTA'),1) assert_equal(errorCounter.getCount(kmerBefore='CAT'),1) assert_equal(errorCounter.getCount(kmerAfter='TAC'),1) assert_equal(errorCounter.getCount(kmerAfter='GAT'),1)
def generateGraph(self, filename, suggested_pathways=[], compound_data=None, gene_data=None, protein_data=None, format='svg'): # Build options-like structure for generation of graph # (compatibility with command line version, we need to fake it) options = Values() options._update_loose({ 'file': None, #'pathways': self.config.Read('/Pathways/Show'), #'not_pathways':'', 'show_all': False, # self.config.ReadBool('/Pathways/ShowAll'), 'search': '', 'cluster_by': self.config.get('/App/ClusterBy'), 'show_enzymes': self.config.get('/App/ShowEnzymes'), # self.config.ReadBool('/App/ShowEnzymes'), 'show_secondary': self.config.get('/App/Show2nd'), 'show_molecular': self.config.get('/App/ShowMolecular'), 'show_network_analysis': self.config.get('/App/ShowAnalysis'), 'show_gibbs': self.config.get('/App/ShowGibbs'), 'highlightpathways': self.config.get('/App/HighlightPathways'), 'highlightregions': self.config.get('/App/HighlightRegions'), 'splines': 'true', 'focus': False, 'show_pathway_links': self.config.get('/Pathways/ShowLinks'), # Always except when saving the file 'output': format, }) #pathway_ids = self.config.value('/Pathways/Show').split(',') if suggested_pathways: pathway_ids = [p.id for p in suggested_pathways.entities[1]] else: pathway_ids = [] # Add the manually Shown pathways pathway_ids_show = self.config.get('/Pathways/Show') pathway_ids.extend(pathway_ids_show) # Now remove the Hide pathways pathway_ids_hide = self.config.get('/Pathways/Hide') pathway_ids = [p for p in pathway_ids if p not in pathway_ids_hide] # Convert pathways_ids to pathways pathways = [self.m.db.pathway(pid) for pid in pathway_ids if self.m.db.pathway(pid) is not None] if pathway_ids == []: return None if compound_data or gene_data or protein_data: # Generate independent scales node_colors = {} for dsi in compound_data, gene_data, protein_data: if dsi == None: continue #if self.m.data.analysis_timecourse: # # Generate the multiple views # tps = sorted( self.m.data.analysis_timecourse.keys(), key=int ) # # Insert counter variable into the filename # filename = self.get_filename_with_counter(filename) # print "Generate timecourse..." # for tp in tps: # print "%s" % tp # graph = generator( pathways, options, self.m.db, analysis=self.m.data.analysis_timecourse[ tp ]) #, layout=self.layout) # graph.write(filename % tp, format=options.output, prog='neato') # return tps #else: print("Generate map for single control:test...") # Build analysis lookup dict; we want a single color for each metabolite mini, maxi = min(abs(np.median(dsi.data)), 0), max(abs(np.median(dsi.data)), 0) mini, maxi = -1.0, +1.0 # Fudge; need an intelligent way to determine (2*median? 2*mean?) scale = utils.calculate_scale([mini, 0, maxi], [9, 1], out=np.around) # rdbu9 scale for n, m in enumerate(dsi.entities[1]): if m is not None: ecol = utils.calculate_rdbu9_color(scale, dsi.data[0, n]) #print xref, ecol if ecol is not None: node_colors[m.id] = ecol graph = generator(pathways, options, self.m.db, analysis=node_colors) # , layout=self.layout) self.status.emit('waiting') self.progress.emit(0.5) graph.write(filename, format=options.output, prog='neato') return None else: graph = generator(pathways, options, self.m.db) # , layout=self.layout) self.status.emit('waiting') self.progress.emit(0.5) graph.write(filename, format=options.output, prog='neato') return None
def get_plugin_config_options(): options = Values() options.websockets_port = config.config.getint('server', 'websockets_port') options.cherrypy_port = config.config.getint('server', 'cherrypy_port') options.proxy_port = config.config.getint('server', 'proxy_port') options.session_timeout = config.config.getint('server', 'session_timeout') options.test = config.config.get('server', 'test') if options.test == 'None': options.test = None options.environment = config.config.get('server', 'environment') options.server_root = config.config.get('server', 'server_root') options.max_body_size = config.config.get('server', 'max_body_size') options.log_dir = config.config.get('logging', 'log_dir') options.log_level = config.config.get('logging', 'log_level') return options
def __init__(self, *args, **kwargs): Values.__init__(self, *args, **kwargs) self._config_files = []
def copyValues(values): if not values: return None new_values = Values() new_values.__dict__ = copy(values.__dict__) return new_values
def Calc(calc, outfile, NoDataValue=None, type=None, format=None, creation_options=None, allBands='', overwrite=False, debug=False, quiet=False, **input_files): """ Perform raster calculations with numpy syntax. Use any basic arithmetic supported by numpy arrays such as +-*\ along with logical operators such as >. Note that all files must have the same dimensions, but no projection checking is performed. Keyword arguments: [A-Z]: input files [A_band - Z_band]: band to use for respective input file Examples: add two files together: Calc("A+B", A="input1.tif", B="input2.tif", outfile="result.tif") average of two layers: Calc(calc="(A+B)/2", A="input1.tif", B="input2.tif", outfile="result.tif") set values of zero and below to null: Calc(calc="A*(A>0)", A="input.tif", A_band=2, outfile="result.tif", NoDataValue=0) """ opts = Values() opts.input_files = input_files opts.calc = calc opts.outF = outfile opts.NoDataValue = NoDataValue opts.type = type opts.format = format opts.creation_options = [] if creation_options is None else creation_options opts.allBands = allBands opts.overwrite = overwrite opts.debug = debug opts.quiet = quiet doit(opts, None)