def reset_thread(): global orig_mod_cfg global shutit global STATUS # Start with a fresh shutit object shutit = shutit_global.shutit = shutit_global.init() # This has already happened but we have to do it again on top of our new # shutit object util.parse_args(shutit.cfg) # The rest of the loading from shutit_main util.load_configs(shutit) shutit_main.shutit_module_init(shutit) shutit_main.conn_container(shutit) # Some hacks for server mode shutit.cfg['build']['build_log'] = StringIO.StringIO() shutit.cfg['build']['interactive'] = 0 STATUS['cid'] = shutit.cfg['container']['container_id'] for mid in shutit.shutit_map: orig_mod_cfg[mid] = STATUS['cfg'][mid] = shutit.cfg[mid] # Add in core sections for mid in ['repository']: orig_mod_cfg[mid] = STATUS['cfg'][mid] = shutit.cfg[mid] # Make sure that orig_mod_cfg can be updated seperately to # STATUS and shutit.cfg (which remain linked), as it will hold # our overrides orig_mod_cfg = copy.deepcopy(orig_mod_cfg) update_modules([], None) STATUS['resetting'] = False
def test_parse_args_good(self): args = util.parse_args("") self.assertEqual([], args) args = util.parse_args(None) self.assertEqual([], args) args = util.parse_args('a b "c d"') self.assertEqual(["a", "b", "c d"], args)
def test_parse_args_open_quote(self): output = "*** No closing quotation" result = util.parse_args('"open') self.assertIsNone(result) self.assertEqual(output, self.redirect.getvalue().rstrip()) self.reset_redirect() util.parse_args("'still open") self.assertEqual(output, self.redirect.getvalue().rstrip()) self.reset_redirect() util.parse_args("this also counts as open'") self.assertEqual(output, self.redirect.getvalue().rstrip())
def expand_macro(self, formatter, name, content): args, kw = util.parse_args(content) prefix = args and args[0] or None format = kw.get('format', '') minsize = max(int(kw.get('min', 2)), 2) depth = int(kw.get('depth', -1)) start = prefix and prefix.count('/') or 0 wiki = formatter.wiki pages = sorted([page for page in wiki.get_pages(prefix) \ if 'WIKI_VIEW' in formatter.perm('wiki', page)]) pagelangs = {} for page in pages: name, lang = util.split_lang(page, '') langs = pagelangs.get(name, []) if lang not in langs: langs.append(lang) pagelangs[name] = langs pages = sorted(pagelangs.keys()) # collection of default pages upages, spages = self.split_pages(pages) def format_page_name(page, split=False): try: # for trac 0.11 return wiki.format_page_name(page, split=split) except: # for trac 0.10 if split: return self.PAGE_SPLIT_RE.sub(r"\1 \2", page) return page def split(page): if format != 'group': return [format_page_name(page)] else: return self.SPLIT_RE.split(format_page_name(page, split=True)) # Group by Wiki word and/or Wiki hierarchy upages, spages = [[(split(page), page) for page in pages if depth < 0 or depth >= page.count('/') - start] for pages in (upages, spages)] def split_in_groups(group): """Return list of pagename or (key, sublist) elements""" groups = [] for key, subgrp in groupby(group, lambda (k,p): k and k[0] or ''): subgrp = [(k[1:],p) for k,p in subgrp] if key and len(subgrp) >= minsize: sublist = split_in_groups(sorted(subgrp)) if len(sublist) == 1: elt = (key+sublist[0][0], sublist[0][1]) else: elt = (key, sublist) groups.append(elt) else: for elt in subgrp: groups.append(elt[1]) return groups
def reset_thread(): global ORIG_MOD_CFG global shutit # Start with a fresh shutit object shutit = shutit_global.shutit = shutit_global.init() # This has already happened but we have to do it again on top of our new # shutit object util.parse_args(shutit) shutit.cfg['build']['interactive'] = 0 # The rest of the loading from shutit_main util.load_configs(shutit) util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'setup.py')) util.load_shutit_modules(shutit) shutit_main.init_shutit_map(shutit) shutit_main.config_collection(shutit) # Here we can set the starting image. if STATUS['image_tag'] != '': shutit.cfg['target']['docker_image'] = STATUS['image_tag'] else: STATUS['image_tag'] = shutit.cfg['target']['docker_image'] shutit_main.conn_target(shutit) shutit_main.config_collection_for_built(shutit) # Some hacks for server mode shutit.cfg['build']['build_log'] = StringIO.StringIO() shutit.cfg['build']['interactive'] = 0 STATUS['cid'] = shutit.cfg['target']['container_id'] for module_id in shutit.shutit_map: ORIG_MOD_CFG[module_id] = STATUS['cfg'][module_id] = shutit.cfg[module_id] # Add in core sections for module_id in ['repository', 'target']: ORIG_MOD_CFG[module_id] = STATUS['cfg'][module_id] = shutit.cfg[module_id] # Make sure that ORIG_MOD_CFG can be updated seperately to # STATUS and shutit.cfg (which remain linked), as it will hold # our overrides ORIG_MOD_CFG = copy.deepcopy(ORIG_MOD_CFG) update_modules([], None) STATUS['resetting'] = False
def reset_thread(): global orig_mod_cfg global shutit global STATUS # Start with a fresh shutit object shutit = shutit_global.shutit = shutit_global.init() # This has already happened but we have to do it again on top of our new # shutit object util.parse_args(shutit.cfg) # The rest of the loading from shutit_main util.load_configs(shutit) shutit_main.shutit_module_init(shutit) # Here we can set the starting image. if STATUS["image_tag"] != "": shutit.cfg["container"]["docker_image"] = STATUS["image_tag"] else: STATUS["image_tag"] = shutit.cfg["container"]["docker_image"] shutit_main.conn_container(shutit) # Some hacks for server mode shutit.cfg["build"]["build_log"] = StringIO.StringIO() shutit.cfg["build"]["interactive"] = 0 STATUS["cid"] = shutit.cfg["container"]["container_id"] for mid in shutit.shutit_map: orig_mod_cfg[mid] = STATUS["cfg"][mid] = shutit.cfg[mid] # Add in core sections for mid in ["repository", "container"]: orig_mod_cfg[mid] = STATUS["cfg"][mid] = shutit.cfg[mid] # Make sure that orig_mod_cfg can be updated seperately to # STATUS and shutit.cfg (which remain linked), as it will hold # our overrides orig_mod_cfg = copy.deepcopy(orig_mod_cfg) update_modules([], None) STATUS["resetting"] = False
def main(): # some basic settings # Expected Folder Structure: base_dir/instrument/target/raw/night/*.fits.gz base_dir = "./Test" mask_dir = "./Test/UVES/HD132205" start_logging() if len(sys.argv) > 1: instrument, target, steps_to_take = parse_args() else: # Manual settings # Instrument instrument = "UVES" # target star target = "HD132205" # Which parts of the reduction to perform steps_to_take = [ #"bias", #"flat", #"orders", "norm_flat", "wavecal", "science", ] # load configuration for the current instrument with open("settings_%s.json" % instrument) as f: config = json.load(f) # TODO: Test settings config["plot"] = False config["manual"] = True modes = config["modes"][1:2] # Search the available days dates = join(base_dir, instrument, target, "raw", "????-??-??") dates = glob.glob(dates) dates = [r + os.sep for r in dates if os.path.isdir(r)] logging.info("Instrument: %s", instrument) logging.info("Target: %s", target) for night in dates: night = os.path.basename(night[:-1]) logging.info("Observation Date: %s", night) for mode in modes: logging.info("Instrument Mode: %s", mode) run_steps(base_dir, mask_dir, target, instrument, mode, night, config, steps=steps_to_take)
def mark_or_unmark(self, args, mark=True): args = util.parse_args(args) if not args: print('*** Transaction number(s) required') return if args[0].lower() == 'all': args = [] for key, thing in self.current_listing.iteritems(): if ((mark and not thing.is_pending()) or (not mark and thing.is_pending())): args.append(key) at_least_one_success = False messages = '' for num in args: if num not in self.current_listing: messages += 'Transaction not found: {}\n'.format(num) continue thing = self.current_listing[num] if mark and thing.is_pending(): messages += 'Already marked pending: ' + num + '\n' continue elif not mark and not thing.is_pending(): messages += "Not marked; can't unmark: " + num + '\n' continue if mark: self.current_listing[num].set_pending() self.total_pending += thing.rec_amount else: thing.set_uncleared() self.total_pending -= thing.rec_amount at_least_one_success = True if at_least_one_success: self.ledgerfile.write_file() self.list_transactions() if messages: print(messages, end='')
def main(): opts, args = parse_args() fname = opts.filename or DEFAULT_SETTING_FILE if not os.path.exists(fname): raise SystemExit("%s is not found." % (fname,)) parser = ConfigParser.SafeConfigParser() if not parser.read(fname): raise SystemExit("%s has no section." % (fname,)) if not args: raise SystemExit("No target was given.") target = args[0] try: ck = parser.get(target, "consumer_key") secret = parser.get(target, "consumer_secret") if not (ck and secret): msg = ("No consumer token was found.", "Check 'consumer_key' and 'consumer_secret' on %s." % (target)) raise SystemExit("\n".join(msg)) except ConfigParser.NoOptionError, e: raise SystemExit(e.message)
def expand_macro(self, formatter, name, args): """Expand wildcard page spec to non-suffixed page names. """ args, kw = util.parse_args(args) wiki = WikiSystem(self.env) newargs = [] for arg in args: newargs.append(arg) arg = arg.strip() if arg.endswith('*'): newargs.pop() # expand wildcard as indivisual pages with removing # suffixed pages. prefix = arg[:-1] pages = [] for page in wiki.get_pages(prefix): name, lang = util.split_lang(page) if name not in pages: pages.append(name) newargs += pages # reconstruct 'args' argument and call ancestor args = ','.join(newargs + ['%s=%s' % pair for pair in kw.items()]) return BaseTOCMacro.expand_macro(self, formatter, name, args)
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if sys.stdout.isatty() and spawn.find_executable('shutit') is None: setup_shutit_path() shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(shutit) if cfg['action']['skeleton']: util.create_skeleton(shutit) cfg['build']['completed'] = True return if cfg['action']['serve']: import shutit_srv cfg['build']['interactive'] = 0 revert_dir = os.getcwd() os.chdir(sys.path[0]) shutit_srv.start() os.chdir(revert_dir) return util.load_configs(shutit) shutit_module_init(shutit) conn_target(shutit) errs = [] errs.extend(check_deps(shutit)) if cfg['action']['show_config']: # Show dependency graph digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' shutit.cfg['build']['depgraph'] = digraph digraph_all = 'digraph depgraph {\n' digraph_all = digraph_all + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() ]) digraph_all = digraph_all + '\n}' shutit.cfg['build']['depgraph_all'] = digraph_all shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True) shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) shutit.log('\n\n' + digraph, force_stdout=True) shutit.log('\n================================================================================\n' + digraph, force_stdout=True) shutit.log('\nAbove is the digraph for this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_target(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a target ' + 'called ' + shutit.cfg['target']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the target\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True
def main(): deploy_path = os.environ.get('ZETTEN') if not deploy_path: exit('ERROR - The ZETTEN env variable must be set!') if not os.path.isdir(deploy_path): exit('ERROR - Directory set by ZETTEN does not exist: '+deploy_path) config_path = join(deploy_path, "conf/zetten.props") if not isfile(config_path): exit('ERROR - A config file does not exist at '+config_path) hosts_dir = join(deploy_path, "conf/hosts/") # parse command line args retval = parse_args(hosts_dir) if not retval: print "Invalid command line arguments. For help, use 'zetten -h'" sys.exit(1) (opts, action, args) = retval hosts_path = join(hosts_dir, opts.cluster) config = DeployConfig(deploy_path, config_path, hosts_path, opts.cluster) config.verify_config(action) if action == 'launch': conn = get_ec2_conn(config) launch_cluster(conn, config) elif action == 'status': conn = get_ec2_conn(config) nodes = get_cluster(conn, config, ['running']) print "Found {0} nodes in {1} cluster".format(len(nodes), config.cluster_name) for node in nodes: print " ", node.tags.get('Name', 'UNKNOWN_NAME'), node.id, node.private_ip_address, node.ip_address elif action == 'sync': sync_cluster(config) elif action == 'setup': setup_cluster(config) elif action == 'config': if opts.property == 'all': config.print_all() else: config.print_property(opts.property) elif action == 'ssh': wait_until_proxy_ready(config) fwd = '' if config.has_option('general', 'proxy_socks_port'): fwd = "-D "+config.get('general', 'proxy_socks_port') ssh_command = "ssh -C -A -o 'StrictHostKeyChecking no' {fwd} {usr}@{ldr}".format(usr=config.get('general', 'cluster_user'), ldr=config.proxy_public_ip(), fwd=fwd) print "Logging into proxy using: {0}".format(ssh_command) retcode = subprocess.call(ssh_command, shell=True) check_code(retcode, ssh_command) elif action == 'wipe': if not isfile(hosts_path): exit("Hosts file does not exist for cluster: "+hosts_path) print "Killing all processes and wiping data from {0} cluster".format(config.cluster_name) execute_playbook(config, "wipe.yml") elif action == 'run': app = opts.application repo = config.get('apps', app + '_repo') branch = config.get('apps', app + '_branch') command = config.get('apps', app + '_command') run_args = "{0} {1} {2} {3}".format(app, repo, branch, command) if opts.app_args: run_args = "{0} {1}".format(run_args, opts.app_args) basedir = config.get('general', 'cluster_basedir') exec_on_proxy_verified(config, "{0}/apps/run.sh {1}".format(basedir, run_args), opts='-t') elif action == 'terminate': conn = get_ec2_conn(config) nodes = get_active_cluster(conn, config) if len(nodes) == 0: exit("No nodes running in {0} cluster to terminate".format(config.cluster_name)) print "The following {0} nodes in {1} cluster will be terminated:".format(len(nodes), config.cluster_name) for node in nodes: print " ", node.tags.get('Name', 'UNKNOWN_NAME'), node.id, node.private_ip_address, node.ip_address response = raw_input("Do you want to continue? (y/n) ") if response == "y": for node in nodes: node.terminate() print "Terminated instances" if isfile(hosts_path): os.remove(hosts_path) print "Removed hosts file at ",hosts_path else: print "Aborted termination" else: print 'ERROR - Unknown action:', action
# Copyright (c) 2007-2009 The PyAMF Project. # See LICENSE.txt for details. """ Echo test client. You can use this example with the echo test server. @since: 0.1.0 """ import sys from util import parse_args, new_client options = parse_args(sys.argv[1:]) service = options[0].service client = new_client('Echo Test', options[0], service) print client('Hello World')
pass def main(stdscr): pass if __name__ == '__main__': args = parse_args()
def list_transactions(self, args=None): args = util.parse_args(args) if args and args[0].lower() == 'all': show_all = True else: show_all = False self.current_listing = {} count = 0 for thing in self.open_transactions: if thing.thing_date > self.ending_date \ and not show_all \ and not thing.is_pending(): continue if not self.current_listing: print() # only print one buffer line at top if items count += 1 self.current_listing[str(count)] = thing count_str = '{:-4}.'.format(count) print( '{number} {date} {amount} {status:1} {payee} ' '{code:>7}'.format( number=util.get_cyan_text(count_str), date=thing.get_date_string(), code=thing.transaction_code, payee=util.get_cyan_text( thing.payee, column_width=40 ), amount=util.get_colored_amount( thing.rec_amount, 10 ), status=thing.rec_status ) ) end_date = util.get_cyan_text( util.get_date_string(self.ending_date) ) if self.ending_balance is None: end_balance = util.get_cyan_text('(not set)') else: end_balance = util.get_colored_amount(self.ending_balance) print( '\nending date: {end_date} ending balance: {end_balance} ' 'cleared: {cleared}'.format( end_date=end_date, end_balance=end_balance, cleared=util.get_colored_amount(self.total_cleared) ) ) if self.ending_balance is not None: print('to zero: {}'.format( util.get_colored_amount(self.get_zero_candidate()) )) print()
util.conf_io() argparser = argparse.ArgumentParser(description='Shows the difference between the indices ElasticSearch.') argparser.add_argument("source", metavar='source', nargs=1, help='URL source ElasticSearch in format [http://]name_host.com[:9200]/name_index[/doc_type]') argparser.add_argument("destination", metavar='destination', nargs=1, help='URL destination ElasticSearch in format [http://]name_host.com[:9200]/name_index[/doc_type]') argparser.add_argument("--query-sour", dest='query_sour', help="Query for source ElasticSearch") argparser.add_argument("--query-dest", dest='query_dest', help="Query for destination ElasticSearch") argparser.add_argument("-g", "--group-code", dest='group_code', help='Group code for query, not working with keys --query-sour and --query-dest', type=int) argparser.add_argument("-m", "--meld", dest='meld', help="Use meld for show difference", action='store_true') argparser.add_argument("-f", "--first", dest='first', help="Show only the first N rows of differences, default 10", type=int, default=10) args = argparser.parse_args() logger = logging.getLogger('elasticsearch') logger.addHandler(logging.NullHandler()) util.parse_args(args) def main(): dest_ids = set() try: for hit in args.documents_destination: index2 = hit['_index'] id2=hit['_id'] type2=hit['_type'] dest_ids.add((index2, type2, id2)) except: pass change = False difference = 0
""" from sslim import sslim_train from util import tsv_to_matrix, make_compatible, save_matrix from util.metrics import compute_precision_as_an_oracle from util.recommender import slim_recommender from util import parse_args, normalize_values import simplejson as json def main(train_file, user_sideinformation_file, test_file, normalize): A = tsv_to_matrix(train_file) B = tsv_to_matrix(user_sideinformation_file) if normalize: B = normalize_values(B) A, B = make_compatible(A, B) W = sslim_train(A, B) save_matrix(W, 'sslim_oracle_wmatrix.tsv') recommendations = slim_recommender(A, W) precisions = compute_precision_as_an_oracle(recommendations, test_file) return precisions args = parse_args(side_information=True) precisions = main(args.train, args.side_information, args.test, args.normalize) open(args.output, 'w').write(json.dumps(precisions))
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(cfg) if cfg['action']['skeleton']: util.create_skeleton(shutit) return if cfg['action']['serve']: import shutit_srv shutit_srv.start() return util.load_configs(shutit) shutit_module_init(shutit) conn_container(shutit) errs = [] errs.extend(check_deps(shutit)) # Show dependency graph if cfg['action']['show_depgraph']: digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['build'] ]) digraph = digraph + '\n}' shutit.log(digraph, force_stdout=True) return # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_container(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a container ' + 'called ' + shutit.cfg['container']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the container\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31')
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if sys.stdout.isatty(): if spawn.find_executable('shutit') is None: # try the current directory, the .. directory, or the ../shutit directory, the ~/shutit pwd = os.getcwd() path_to_shutit = '' for d in ('.','..','~','~/shutit'): if os.path.isfile(os.path.expanduser(d) + '/shutit'): path_to_shutit = d + '/shutit' res = util.util_raw_input(prompt='shutit appears not to be on your path - would you like me to add it to your ~/.bashrc (Y/n)? ') if res not in ('n','N'): bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: #http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch myfile.write('export PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit == '': while True: res = util.util_raw_input(prompt='shutit appears not to be on your path - please input the path to your shutit dir\n') if os.path.isfile(os.path.expanduser(res) + '/shutit'): path_to_shutit = res + '/shutit' bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: myfile.write('\nexport PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit != '': util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n') sys.exit() shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(cfg) if cfg['action']['skeleton']: util.create_skeleton(shutit) return if cfg['action']['serve']: import shutit_srv shutit_srv.start() return util.load_configs(shutit) shutit_module_init(shutit) conn_container(shutit) errs = [] errs.extend(check_deps(shutit)) # Show dependency graph if cfg['action']['show_depgraph']: digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' shutit.log(digraph, force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_container(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a container ' + 'called ' + shutit.cfg['container']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the container\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True
#!/usr/bin/env python """ Run the server for the app. Note: Run as root on production, and user locally. """ from getpass import getuser from app import APP from util import parse_args parse_args() if getuser() == "root": APP.run(debug=False, host="172.31.11.17", port=80) else: APP.run(debug=True)
argparser.add_argument("--query-sour", dest='query_sour', help="Query for source ElasticSearch") argparser.add_argument("--query-dest", dest='query_dest', help="Query for destination ElasticSearch") argparser.add_argument("-g", "--group-code", dest='group_code', help='Group code for query, work only without key --query-sour and --query-dest', type=int) args = argparser.parse_args() logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0]) formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(name)s %(message)s', datefmt="%Y-%m-%d %H:%M:%S %Z") ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) logger.addHandler(ch) logger.setLevel(logging.DEBUG) logger_elasticsearch = logging.getLogger('elasticsearch') logger_elasticsearch.addHandler(ch) util.parse_args(args, logger) def gen(): dest_ids = set() try: for hit in args.documents_destination: index2 = hit['_index'] id2=hit['_id'] type2=hit['_type'] dest_ids.add((index2, type2, id2)) except Exception as e: logger.error(e) for hit in args.documents_source: source_index = hit['_index'] destination_index = args.translate.trans(source_index) source_id = hit['_id']
def serve(): options = docopt(__doc__, version='{} {}'.format(config.name, config.version)) util.parse_args(options, app) app.run(debug=True)
if resp.getcode() == 200 or resp.getcode() == 201: print("Master User created successfully.") elif resp.getcode() == 409 or resp.getcode() == 412: print("Master User already exists!") else: print("Creating master user failed.") def main(couch_params, main_db, avatar_db, master_params, wikifeat_home): # Set up credentials credentials = util.get_credentials(couch_params.adminuser, couch_params.adminpass) global gh, ph gh = util.get_headers(credentials) ph = util.put_headers(credentials) global wf_dir wf_dir = wikifeat_home # Establish a connection to couchdb conn = util.get_connection(couch_params.use_ssl, couch_params.host, couch_params.port) conn.connect() setup_main_db(conn, main_db) setup_user_db(conn) setup_avatar_db(conn, avatar_db) if master_params.skip_master is False: create_master_user(conn, master_params) if __name__ == "__main__": print(welcome_text) args = util.parse_args() main(util.CouchParameters(args), args.main_db, args.avatar_db, util.MasterUserParameters(args), args.wikifeat_home)
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - list_configs - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(shutit) if cfg['action']['skeleton']: util.create_skeleton(shutit) cfg['build']['completed'] = True return if cfg['action']['serve']: import shutit_srv cfg['build']['interactive'] = 0 revert_dir = os.getcwd() os.chdir(sys.path[0]) shutit_srv.start() os.chdir(revert_dir) return util.load_configs(shutit) # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if util.determine_interactive() and spawn.find_executable('shutit') is None: setup_shutit_path(cfg) util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'setup.py')) util.load_shutit_modules(shutit) if cfg['action']['list_modules']: util.list_modules(shutit) sys.exit(0) init_shutit_map(shutit) config_collection(shutit) conn_target(shutit) errs = [] errs.extend(check_deps(shutit)) if cfg['action']['list_deps']: # Show dependency graph digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' f = file(cfg['build']['log_config_path'] + '/digraph.txt','w') f.write(digraph) f.close() digraph_all = 'digraph depgraph {\n' digraph_all = digraph_all + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() ]) digraph_all = digraph_all + '\n}' f = file(cfg['build']['log_config_path'] + '/digraph_all.txt','w') f.write(digraph_all) f.close() shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True) shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) shutit.log('\n\n' + digraph, force_stdout=True) shutit.log('\n================================================================================\n' + digraph, force_stdout=True) shutit.log('\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) # Exit now sys.exit(0) # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['list_configs'] or cfg['build']['debug']: shutit.log(util.print_config(cfg, history=cfg['list_configs']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True f = file(cfg['build']['log_config_path'] + '/cfg.txt','w') f.write(util.print_config(cfg, history=cfg['list_configs']['cfghistory'])) f.close() shutit.log('================================================================================', force_stdout=True) shutit.log('Config details placed in: ' + cfg['build']['log_config_path'], force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('To render the digraph of this build into an image run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph.gv ' + cfg['build']['log_config_path'] + '/digraph.txt && dot -Tpdf -o digraph.pdf ' + cfg['build']['log_config_path'] + '/digraph.gv\n\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph_all.gv ' + cfg['build']['log_config_path'] + '/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' + cfg['build']['log_config_path'] + '/digraph_all.gv\n\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('\nConfiguration details have been written to the folder: ' + cfg['build']['log_config_path'] + '\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) if cfg['action']['list_configs']: return # Check for conflicts now. errs.extend(check_conflicts(shutit)) # Cache the results of check_ready at the start. errs.extend(check_ready(shutit, throw_error=False)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_target(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if cfg['build']['build_log']: shutit.cfg['build']['report_final_messages'] += "Build log file: " + cfg['host']['logfile'] # Show final report messages (ie messages to show after standard report). if shutit.cfg['build']['report_final_messages'] != '': shutit.log(shutit.cfg['build']['report_final_messages'], prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a target ' + 'called ' + shutit.cfg['target']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/build.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh. These may need ' + 'tweaking for your particular environment, eg sudo\n\n' + 'You can inspect the details of the build in the target image\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True