def reset_thread(): global orig_mod_cfg global shutit global STATUS # Start with a fresh shutit object shutit = shutit_global.shutit = shutit_global.init() # This has already happened but we have to do it again on top of our new # shutit object util.parse_args(shutit.cfg) # The rest of the loading from shutit_main util.load_configs(shutit) shutit_main.shutit_module_init(shutit) shutit_main.conn_container(shutit) # Some hacks for server mode shutit.cfg['build']['build_log'] = StringIO.StringIO() shutit.cfg['build']['interactive'] = 0 STATUS['cid'] = shutit.cfg['container']['container_id'] for mid in shutit.shutit_map: orig_mod_cfg[mid] = STATUS['cfg'][mid] = shutit.cfg[mid] # Add in core sections for mid in ['repository']: orig_mod_cfg[mid] = STATUS['cfg'][mid] = shutit.cfg[mid] # Make sure that orig_mod_cfg can be updated seperately to # STATUS and shutit.cfg (which remain linked), as it will hold # our overrides orig_mod_cfg = copy.deepcopy(orig_mod_cfg) update_modules([], None) STATUS['resetting'] = False
def test_parse_args_good(self):
args = util.parse_args("")
self.assertEqual([], args)
args = util.parse_args(None)
self.assertEqual([], args)
args = util.parse_args('a b "c d"')
self.assertEqual(["a", "b", "c d"], args)
def test_parse_args_open_quote(self):
output = "*** No closing quotation"
result = util.parse_args('"open')
self.assertIsNone(result)
self.assertEqual(output, self.redirect.getvalue().rstrip())
self.reset_redirect()
util.parse_args("'still open")
self.assertEqual(output, self.redirect.getvalue().rstrip())
self.reset_redirect()
util.parse_args("this also counts as open'")
self.assertEqual(output, self.redirect.getvalue().rstrip())
def expand_macro(self, formatter, name, content):
args, kw = util.parse_args(content)
prefix = args and args[0] or None
format = kw.get('format', '')
minsize = max(int(kw.get('min', 2)), 2)
depth = int(kw.get('depth', -1))
start = prefix and prefix.count('/') or 0
wiki = formatter.wiki
pages = sorted([page for page in wiki.get_pages(prefix) \
if 'WIKI_VIEW' in formatter.perm('wiki', page)])
pagelangs = {}
for page in pages:
name, lang = util.split_lang(page, '')
langs = pagelangs.get(name, [])
if lang not in langs:
langs.append(lang)
pagelangs[name] = langs
pages = sorted(pagelangs.keys()) # collection of default pages
upages, spages = self.split_pages(pages)
def format_page_name(page, split=False):
try:
# for trac 0.11
return wiki.format_page_name(page, split=split)
except:
# for trac 0.10
if split:
return self.PAGE_SPLIT_RE.sub(r"\1 \2", page)
return page
def split(page):
if format != 'group':
return [format_page_name(page)]
else:
return self.SPLIT_RE.split(format_page_name(page, split=True))
# Group by Wiki word and/or Wiki hierarchy
upages, spages = [[(split(page), page) for page in pages
if depth < 0 or depth >= page.count('/') - start]
for pages in (upages, spages)]
def split_in_groups(group):
"""Return list of pagename or (key, sublist) elements"""
groups = []
for key, subgrp in groupby(group, lambda (k,p): k and k[0] or ''):
subgrp = [(k[1:],p) for k,p in subgrp]
if key and len(subgrp) >= minsize:
sublist = split_in_groups(sorted(subgrp))
if len(sublist) == 1:
elt = (key+sublist[0][0], sublist[0][1])
else:
elt = (key, sublist)
groups.append(elt)
else:
for elt in subgrp:
groups.append(elt[1])
return groups
def reset_thread(): global ORIG_MOD_CFG global shutit # Start with a fresh shutit object shutit = shutit_global.shutit = shutit_global.init() # This has already happened but we have to do it again on top of our new # shutit object util.parse_args(shutit) shutit.cfg['build']['interactive'] = 0 # The rest of the loading from shutit_main util.load_configs(shutit) util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'setup.py')) util.load_shutit_modules(shutit) shutit_main.init_shutit_map(shutit) shutit_main.config_collection(shutit) # Here we can set the starting image. if STATUS['image_tag'] != '': shutit.cfg['target']['docker_image'] = STATUS['image_tag'] else: STATUS['image_tag'] = shutit.cfg['target']['docker_image'] shutit_main.conn_target(shutit) shutit_main.config_collection_for_built(shutit) # Some hacks for server mode shutit.cfg['build']['build_log'] = StringIO.StringIO() shutit.cfg['build']['interactive'] = 0 STATUS['cid'] = shutit.cfg['target']['container_id'] for module_id in shutit.shutit_map: ORIG_MOD_CFG[module_id] = STATUS['cfg'][module_id] = shutit.cfg[module_id] # Add in core sections for module_id in ['repository', 'target']: ORIG_MOD_CFG[module_id] = STATUS['cfg'][module_id] = shutit.cfg[module_id] # Make sure that ORIG_MOD_CFG can be updated seperately to # STATUS and shutit.cfg (which remain linked), as it will hold # our overrides ORIG_MOD_CFG = copy.deepcopy(ORIG_MOD_CFG) update_modules([], None) STATUS['resetting'] = False
def reset_thread():
global orig_mod_cfg
global shutit
global STATUS
# Start with a fresh shutit object
shutit = shutit_global.shutit = shutit_global.init()
# This has already happened but we have to do it again on top of our new
# shutit object
util.parse_args(shutit.cfg)
# The rest of the loading from shutit_main
util.load_configs(shutit)
shutit_main.shutit_module_init(shutit)
# Here we can set the starting image.
if STATUS["image_tag"] != "":
shutit.cfg["container"]["docker_image"] = STATUS["image_tag"]
else:
STATUS["image_tag"] = shutit.cfg["container"]["docker_image"]
shutit_main.conn_container(shutit)
# Some hacks for server mode
shutit.cfg["build"]["build_log"] = StringIO.StringIO()
shutit.cfg["build"]["interactive"] = 0
STATUS["cid"] = shutit.cfg["container"]["container_id"]
for mid in shutit.shutit_map:
orig_mod_cfg[mid] = STATUS["cfg"][mid] = shutit.cfg[mid]
# Add in core sections
for mid in ["repository", "container"]:
orig_mod_cfg[mid] = STATUS["cfg"][mid] = shutit.cfg[mid]
# Make sure that orig_mod_cfg can be updated seperately to
# STATUS and shutit.cfg (which remain linked), as it will hold
# our overrides
orig_mod_cfg = copy.deepcopy(orig_mod_cfg)
update_modules([], None)
STATUS["resetting"] = False
def main():
# some basic settings
# Expected Folder Structure: base_dir/instrument/target/raw/night/*.fits.gz
base_dir = "./Test"
mask_dir = "./Test/UVES/HD132205"
start_logging()
if len(sys.argv) > 1:
instrument, target, steps_to_take = parse_args()
else:
# Manual settings
# Instrument
instrument = "UVES"
# target star
target = "HD132205"
# Which parts of the reduction to perform
steps_to_take = [
#"bias",
#"flat",
#"orders",
"norm_flat",
"wavecal",
"science",
]
# load configuration for the current instrument
with open("settings_%s.json" % instrument) as f:
config = json.load(f)
# TODO: Test settings
config["plot"] = False
config["manual"] = True
modes = config["modes"][1:2]
# Search the available days
dates = join(base_dir, instrument, target, "raw", "????-??-??")
dates = glob.glob(dates)
dates = [r + os.sep for r in dates if os.path.isdir(r)]
logging.info("Instrument: %s", instrument)
logging.info("Target: %s", target)
for night in dates:
night = os.path.basename(night[:-1])
logging.info("Observation Date: %s", night)
for mode in modes:
logging.info("Instrument Mode: %s", mode)
run_steps(base_dir, mask_dir, target, instrument, mode, night, config, steps=steps_to_take)
def mark_or_unmark(self, args, mark=True):
args = util.parse_args(args)
if not args:
print('*** Transaction number(s) required')
return
if args[0].lower() == 'all':
args = []
for key, thing in self.current_listing.iteritems():
if ((mark and not thing.is_pending())
or (not mark and thing.is_pending())):
args.append(key)
at_least_one_success = False
messages = ''
for num in args:
if num not in self.current_listing:
messages += 'Transaction not found: {}\n'.format(num)
continue
thing = self.current_listing[num]
if mark and thing.is_pending():
messages += 'Already marked pending: ' + num + '\n'
continue
elif not mark and not thing.is_pending():
messages += "Not marked; can't unmark: " + num + '\n'
continue
if mark:
self.current_listing[num].set_pending()
self.total_pending += thing.rec_amount
else:
thing.set_uncleared()
self.total_pending -= thing.rec_amount
at_least_one_success = True
if at_least_one_success:
self.ledgerfile.write_file()
self.list_transactions()
if messages:
print(messages, end='')
def main():
opts, args = parse_args()
fname = opts.filename or DEFAULT_SETTING_FILE
if not os.path.exists(fname):
raise SystemExit("%s is not found." % (fname,))
parser = ConfigParser.SafeConfigParser()
if not parser.read(fname):
raise SystemExit("%s has no section." % (fname,))
if not args:
raise SystemExit("No target was given.")
target = args[0]
try:
ck = parser.get(target, "consumer_key")
secret = parser.get(target, "consumer_secret")
if not (ck and secret):
msg = ("No consumer token was found.", "Check 'consumer_key' and 'consumer_secret' on %s." % (target))
raise SystemExit("\n".join(msg))
except ConfigParser.NoOptionError, e:
raise SystemExit(e.message)
def expand_macro(self, formatter, name, args):
"""Expand wildcard page spec to non-suffixed page names.
"""
args, kw = util.parse_args(args)
wiki = WikiSystem(self.env)
newargs = []
for arg in args:
newargs.append(arg)
arg = arg.strip()
if arg.endswith('*'):
newargs.pop()
# expand wildcard as indivisual pages with removing
# suffixed pages.
prefix = arg[:-1]
pages = []
for page in wiki.get_pages(prefix):
name, lang = util.split_lang(page)
if name not in pages:
pages.append(name)
newargs += pages
# reconstruct 'args' argument and call ancestor
args = ','.join(newargs + ['%s=%s' % pair for pair in kw.items()])
return BaseTOCMacro.expand_macro(self, formatter, name, args)
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- sc - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if sys.stdout.isatty() and spawn.find_executable('shutit') is None:
setup_shutit_path()
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(shutit)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
cfg['build']['completed'] = True
return
if cfg['action']['serve']:
import shutit_srv
cfg['build']['interactive'] = 0
revert_dir = os.getcwd()
os.chdir(sys.path[0])
shutit_srv.start()
os.chdir(revert_dir)
return
util.load_configs(shutit)
shutit_module_init(shutit)
conn_target(shutit)
errs = []
errs.extend(check_deps(shutit))
if cfg['action']['show_config']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
shutit.cfg['build']['depgraph'] = digraph
digraph_all = 'digraph depgraph {\n'
digraph_all = digraph_all + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
])
digraph_all = digraph_all + '\n}'
shutit.cfg['build']['depgraph_all'] = digraph_all
shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
shutit.log('\n\n' + digraph, force_stdout=True)
shutit.log('\n================================================================================\n' + digraph, force_stdout=True)
shutit.log('\nAbove is the digraph for this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['show_config']:
shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
errs.extend(check_ready(shutit))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a target ' +
'called ' + shutit.cfg['target']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/default.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' +
'default test runner in test.sh\n\n' +
'You can inspect the details of the build in the target\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
# Mark the build as completed
shutit.cfg['build']['completed'] = True
def main():
deploy_path = os.environ.get('ZETTEN')
if not deploy_path:
exit('ERROR - The ZETTEN env variable must be set!')
if not os.path.isdir(deploy_path):
exit('ERROR - Directory set by ZETTEN does not exist: '+deploy_path)
config_path = join(deploy_path, "conf/zetten.props")
if not isfile(config_path):
exit('ERROR - A config file does not exist at '+config_path)
hosts_dir = join(deploy_path, "conf/hosts/")
# parse command line args
retval = parse_args(hosts_dir)
if not retval:
print "Invalid command line arguments. For help, use 'zetten -h'"
sys.exit(1)
(opts, action, args) = retval
hosts_path = join(hosts_dir, opts.cluster)
config = DeployConfig(deploy_path, config_path, hosts_path, opts.cluster)
config.verify_config(action)
if action == 'launch':
conn = get_ec2_conn(config)
launch_cluster(conn, config)
elif action == 'status':
conn = get_ec2_conn(config)
nodes = get_cluster(conn, config, ['running'])
print "Found {0} nodes in {1} cluster".format(len(nodes), config.cluster_name)
for node in nodes:
print " ", node.tags.get('Name', 'UNKNOWN_NAME'), node.id, node.private_ip_address, node.ip_address
elif action == 'sync':
sync_cluster(config)
elif action == 'setup':
setup_cluster(config)
elif action == 'config':
if opts.property == 'all':
config.print_all()
else:
config.print_property(opts.property)
elif action == 'ssh':
wait_until_proxy_ready(config)
fwd = ''
if config.has_option('general', 'proxy_socks_port'):
fwd = "-D "+config.get('general', 'proxy_socks_port')
ssh_command = "ssh -C -A -o 'StrictHostKeyChecking no' {fwd} {usr}@{ldr}".format(usr=config.get('general', 'cluster_user'),
ldr=config.proxy_public_ip(), fwd=fwd)
print "Logging into proxy using: {0}".format(ssh_command)
retcode = subprocess.call(ssh_command, shell=True)
check_code(retcode, ssh_command)
elif action == 'wipe':
if not isfile(hosts_path):
exit("Hosts file does not exist for cluster: "+hosts_path)
print "Killing all processes and wiping data from {0} cluster".format(config.cluster_name)
execute_playbook(config, "wipe.yml")
elif action == 'run':
app = opts.application
repo = config.get('apps', app + '_repo')
branch = config.get('apps', app + '_branch')
command = config.get('apps', app + '_command')
run_args = "{0} {1} {2} {3}".format(app, repo, branch, command)
if opts.app_args:
run_args = "{0} {1}".format(run_args, opts.app_args)
basedir = config.get('general', 'cluster_basedir')
exec_on_proxy_verified(config, "{0}/apps/run.sh {1}".format(basedir, run_args), opts='-t')
elif action == 'terminate':
conn = get_ec2_conn(config)
nodes = get_active_cluster(conn, config)
if len(nodes) == 0:
exit("No nodes running in {0} cluster to terminate".format(config.cluster_name))
print "The following {0} nodes in {1} cluster will be terminated:".format(len(nodes), config.cluster_name)
for node in nodes:
print " ", node.tags.get('Name', 'UNKNOWN_NAME'), node.id, node.private_ip_address, node.ip_address
response = raw_input("Do you want to continue? (y/n) ")
if response == "y":
for node in nodes:
node.terminate()
print "Terminated instances"
if isfile(hosts_path):
os.remove(hosts_path)
print "Removed hosts file at ",hosts_path
else:
print "Aborted termination"
else:
print 'ERROR - Unknown action:', action
# Copyright (c) 2007-2009 The PyAMF Project.
# See LICENSE.txt for details.
"""
Echo test client.
You can use this example with the echo test server.
@since: 0.1.0
"""
import sys
from util import parse_args, new_client
options = parse_args(sys.argv[1:])
service = options[0].service
client = new_client('Echo Test', options[0], service)
print client('Hello World')
pass
def main(stdscr):
pass
if __name__ == '__main__':
args = parse_args()
def list_transactions(self, args=None):
args = util.parse_args(args)
if args and args[0].lower() == 'all':
show_all = True
else:
show_all = False
self.current_listing = {}
count = 0
for thing in self.open_transactions:
if thing.thing_date > self.ending_date \
and not show_all \
and not thing.is_pending():
continue
if not self.current_listing:
print() # only print one buffer line at top if items
count += 1
self.current_listing[str(count)] = thing
count_str = '{:-4}.'.format(count)
print(
'{number} {date} {amount} {status:1} {payee} '
'{code:>7}'.format(
number=util.get_cyan_text(count_str),
date=thing.get_date_string(),
code=thing.transaction_code,
payee=util.get_cyan_text(
thing.payee,
column_width=40
),
amount=util.get_colored_amount(
thing.rec_amount,
10
),
status=thing.rec_status
)
)
end_date = util.get_cyan_text(
util.get_date_string(self.ending_date)
)
if self.ending_balance is None:
end_balance = util.get_cyan_text('(not set)')
else:
end_balance = util.get_colored_amount(self.ending_balance)
print(
'\nending date: {end_date} ending balance: {end_balance} '
'cleared: {cleared}'.format(
end_date=end_date,
end_balance=end_balance,
cleared=util.get_colored_amount(self.total_cleared)
)
)
if self.ending_balance is not None:
print('to zero: {}'.format(
util.get_colored_amount(self.get_zero_candidate())
))
print()
util.conf_io()
argparser = argparse.ArgumentParser(description='Shows the difference between the indices ElasticSearch.')
argparser.add_argument("source", metavar='source', nargs=1, help='URL source ElasticSearch in format [http://]name_host.com[:9200]/name_index[/doc_type]')
argparser.add_argument("destination", metavar='destination', nargs=1, help='URL destination ElasticSearch in format [http://]name_host.com[:9200]/name_index[/doc_type]')
argparser.add_argument("--query-sour", dest='query_sour', help="Query for source ElasticSearch")
argparser.add_argument("--query-dest", dest='query_dest', help="Query for destination ElasticSearch")
argparser.add_argument("-g", "--group-code", dest='group_code', help='Group code for query, not working with keys --query-sour and --query-dest', type=int)
argparser.add_argument("-m", "--meld", dest='meld', help="Use meld for show difference", action='store_true')
argparser.add_argument("-f", "--first", dest='first', help="Show only the first N rows of differences, default 10", type=int, default=10)
args = argparser.parse_args()
logger = logging.getLogger('elasticsearch')
logger.addHandler(logging.NullHandler())
util.parse_args(args)
def main():
dest_ids = set()
try:
for hit in args.documents_destination:
index2 = hit['_index']
id2=hit['_id']
type2=hit['_type']
dest_ids.add((index2, type2, id2))
except:
pass
change = False
difference = 0
"""
from sslim import sslim_train
from util import tsv_to_matrix, make_compatible, save_matrix
from util.metrics import compute_precision_as_an_oracle
from util.recommender import slim_recommender
from util import parse_args, normalize_values
import simplejson as json
def main(train_file, user_sideinformation_file, test_file, normalize):
A = tsv_to_matrix(train_file)
B = tsv_to_matrix(user_sideinformation_file)
if normalize:
B = normalize_values(B)
A, B = make_compatible(A, B)
W = sslim_train(A, B)
save_matrix(W, 'sslim_oracle_wmatrix.tsv')
recommendations = slim_recommender(A, W)
precisions = compute_precision_as_an_oracle(recommendations, test_file)
return precisions
args = parse_args(side_information=True)
precisions = main(args.train, args.side_information, args.test, args.normalize)
open(args.output, 'w').write(json.dumps(precisions))
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- sc - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(cfg)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
return
if cfg['action']['serve']:
import shutit_srv
shutit_srv.start()
return
util.load_configs(shutit)
shutit_module_init(shutit)
conn_container(shutit)
errs = []
errs.extend(check_deps(shutit))
# Show dependency graph
if cfg['action']['show_depgraph']:
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['build']
])
digraph = digraph + '\n}'
shutit.log(digraph, force_stdout=True)
return
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['show_config']:
shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']),
force_stdout=True)
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
errs.extend(check_ready(shutit))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_container(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a container ' +
'called ' + shutit.cfg['container']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/default.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' +
'default test runner in test.sh\n\n' +
'You can inspect the details of the build in the container\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- sc - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if sys.stdout.isatty():
if spawn.find_executable('shutit') is None:
# try the current directory, the .. directory, or the ../shutit directory, the ~/shutit
pwd = os.getcwd()
path_to_shutit = ''
for d in ('.','..','~','~/shutit'):
if os.path.isfile(os.path.expanduser(d) + '/shutit'):
path_to_shutit = d + '/shutit'
res = util.util_raw_input(prompt='shutit appears not to be on your path - would you like me to add it to your ~/.bashrc (Y/n)? ')
if res not in ('n','N'):
bashrc = os.path.expanduser('~/') + '.bashrc'
if os.path.isfile(bashrc):
with open(bashrc, "a") as myfile:
#http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch
myfile.write('export PATH="$PATH:' + path_to_shutit + '"\n')
break
if path_to_shutit == '':
while True:
res = util.util_raw_input(prompt='shutit appears not to be on your path - please input the path to your shutit dir\n')
if os.path.isfile(os.path.expanduser(res) + '/shutit'):
path_to_shutit = res + '/shutit'
bashrc = os.path.expanduser('~/') + '.bashrc'
if os.path.isfile(bashrc):
with open(bashrc, "a") as myfile:
myfile.write('\nexport PATH="$PATH:' + path_to_shutit + '"\n')
break
if path_to_shutit != '':
util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n')
sys.exit()
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(cfg)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
return
if cfg['action']['serve']:
import shutit_srv
shutit_srv.start()
return
util.load_configs(shutit)
shutit_module_init(shutit)
conn_container(shutit)
errs = []
errs.extend(check_deps(shutit))
# Show dependency graph
if cfg['action']['show_depgraph']:
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
shutit.log(digraph, force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['show_config']:
shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
errs.extend(check_ready(shutit))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_container(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a container ' +
'called ' + shutit.cfg['container']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/default.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' +
'default test runner in test.sh\n\n' +
'You can inspect the details of the build in the container\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
# Mark the build as completed
shutit.cfg['build']['completed'] = True
#!/usr/bin/env python
"""
Run the server for the app.
Note: Run as root on production, and user locally.
"""
from getpass import getuser
from app import APP
from util import parse_args
parse_args()
if getuser() == "root":
APP.run(debug=False, host="172.31.11.17", port=80)
else:
APP.run(debug=True)
argparser.add_argument("--query-sour", dest='query_sour', help="Query for source ElasticSearch")
argparser.add_argument("--query-dest", dest='query_dest', help="Query for destination ElasticSearch")
argparser.add_argument("-g", "--group-code", dest='group_code', help='Group code for query, work only without key --query-sour and --query-dest', type=int)
args = argparser.parse_args()
logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(name)s %(message)s', datefmt="%Y-%m-%d %H:%M:%S %Z")
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)
logger_elasticsearch = logging.getLogger('elasticsearch')
logger_elasticsearch.addHandler(ch)
util.parse_args(args, logger)
def gen():
dest_ids = set()
try:
for hit in args.documents_destination:
index2 = hit['_index']
id2=hit['_id']
type2=hit['_type']
dest_ids.add((index2, type2, id2))
except Exception as e:
logger.error(e)
for hit in args.documents_source:
source_index = hit['_index']
destination_index = args.translate.trans(source_index)
source_id = hit['_id']
def serve():
options = docopt(__doc__, version='{} {}'.format(config.name, config.version))
util.parse_args(options, app)
app.run(debug=True)
if resp.getcode() == 200 or resp.getcode() == 201:
print("Master User created successfully.")
elif resp.getcode() == 409 or resp.getcode() == 412:
print("Master User already exists!")
else:
print("Creating master user failed.")
def main(couch_params, main_db, avatar_db, master_params, wikifeat_home):
# Set up credentials
credentials = util.get_credentials(couch_params.adminuser, couch_params.adminpass)
global gh, ph
gh = util.get_headers(credentials)
ph = util.put_headers(credentials)
global wf_dir
wf_dir = wikifeat_home
# Establish a connection to couchdb
conn = util.get_connection(couch_params.use_ssl, couch_params.host, couch_params.port)
conn.connect()
setup_main_db(conn, main_db)
setup_user_db(conn)
setup_avatar_db(conn, avatar_db)
if master_params.skip_master is False:
create_master_user(conn, master_params)
if __name__ == "__main__":
print(welcome_text)
args = util.parse_args()
main(util.CouchParameters(args), args.main_db, args.avatar_db, util.MasterUserParameters(args), args.wikifeat_home)
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(shutit)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
cfg['build']['completed'] = True
return
if cfg['action']['serve']:
import shutit_srv
cfg['build']['interactive'] = 0
revert_dir = os.getcwd()
os.chdir(sys.path[0])
shutit_srv.start()
os.chdir(revert_dir)
return
util.load_configs(shutit)
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if util.determine_interactive() and spawn.find_executable('shutit') is None:
setup_shutit_path(cfg)
util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'setup.py'))
util.load_shutit_modules(shutit)
if cfg['action']['list_modules']:
util.list_modules(shutit)
sys.exit(0)
init_shutit_map(shutit)
config_collection(shutit)
conn_target(shutit)
errs = []
errs.extend(check_deps(shutit))
if cfg['action']['list_deps']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph.txt','w')
f.write(digraph)
f.close()
digraph_all = 'digraph depgraph {\n'
digraph_all = digraph_all + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
])
digraph_all = digraph_all + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph_all.txt','w')
f.write(digraph_all)
f.close()
shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
shutit.log('\n\n' + digraph, force_stdout=True)
shutit.log('\n================================================================================\n' + digraph, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
# Exit now
sys.exit(0)
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['list_configs'] or cfg['build']['debug']:
shutit.log(util.print_config(cfg, history=cfg['list_configs']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
f = file(cfg['build']['log_config_path'] + '/cfg.txt','w')
f.write(util.print_config(cfg, history=cfg['list_configs']['cfghistory']))
f.close()
shutit.log('================================================================================', force_stdout=True)
shutit.log('Config details placed in: ' + cfg['build']['log_config_path'], force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('To render the digraph of this build into an image run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph.gv ' + cfg['build']['log_config_path'] + '/digraph.txt && dot -Tpdf -o digraph.pdf ' + cfg['build']['log_config_path'] + '/digraph.gv\n\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph_all.gv ' + cfg['build']['log_config_path'] + '/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' + cfg['build']['log_config_path'] + '/digraph_all.gv\n\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('\nConfiguration details have been written to the folder: ' + cfg['build']['log_config_path'] + '\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
if cfg['action']['list_configs']:
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(shutit, throw_error=False))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if cfg['build']['build_log']:
shutit.cfg['build']['report_final_messages'] += "Build log file: " + cfg['host']['logfile']
# Show final report messages (ie messages to show after standard report).
if shutit.cfg['build']['report_final_messages'] != '':
shutit.log(shutit.cfg['build']['report_final_messages'], prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a target ' +
'called ' + shutit.cfg['target']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/build.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh. These may need ' +
'tweaking for your particular environment, eg sudo\n\n' +
'You can inspect the details of the build in the target image\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
# Mark the build as completed
shutit.cfg['build']['completed'] = True
