def _readable(self):
"""The readable parsed article"""
if not self.candidates:
logger.info("No candidates found in document.")
return self._handle_no_candidates()
# right now we return the highest scoring candidate content
best_candidates = sorted(
(c for c in self.candidates.values()),
key=attrgetter("content_score"), reverse=True)
printer = PrettyPrinter(indent=2)
logger.debug(printer.pformat(best_candidates))
# since we have several candidates, check the winner's siblings
# for extra content
winner = best_candidates[0]
updated_winner = check_siblings(winner, self.candidates)
updated_winner.node = prep_article(updated_winner.node)
if updated_winner.node is not None:
dom = build_base_document(
updated_winner.node, self._return_fragment)
else:
logger.info(
'Had candidates but failed to find a cleaned winning DOM.')
dom = self._handle_no_candidates()
return self._remove_orphans(dom.get_element_by_id("readabilityBody"))
def main():
pp = PrettyPrinter(indent=4)
slack = Slacker(SLACK_AUTH_TOKEN)
emails_logins = create_login_dictionary()
logins_passwords = create_password_dictionary()
# get a list of all member ids
response = slack.channels.info(slack.channels.get_channel_id('cp3402'))
members = response.body['channel']['members']
# get relevant details for just those members
users = get_member_details(slack, members)
# pp.pprint(users)
contacted = []
for user_id, email in users.items():
try:
login = emails_logins[email]
# print(user_id, "Your MySQL username and database on the ditwebtsv.jcu.edu.au server is {} and your password is {}".format(login, logins_passwords[login]))
message = "Your MySQL username and database on the ditwebtsv.jcu.edu.au server is {} and your password is {}".format(login, logins_passwords[login])
# print(message)
# Send Slack message
slack.chat.post_message(user_id, message, as_user=True)
contacted.append((email, login, user_id))
except KeyError:
print("Skipping", email)
pp.pprint(contacted)
def printKey(key, XmlMap):
if XmlMap.has_key(key):
PP = PrettyPrinter()
print "%s:" % key
PP.pprint(XmlMap[key])
else:
print "entry %s not found" % key
def get_resource(self, arguments):
""" Gets the resource requested in the arguments """
user_id = arguments['--user_id']
object_id = arguments['--object_id']
start_date = arguments['--start_date']
end_date = arguments['--end_date']
detail = arguments['--detail']
misfit = Misfit(self.client_id, self.client_secret, self.access_token,
user_id)
if arguments['profile']:
result = misfit.profile(object_id)
elif arguments['device']:
result = misfit.device(object_id)
elif arguments['goal']:
result = misfit.goal(start_date, end_date, object_id)
elif arguments['summary']:
result = misfit.summary(start_date, end_date, detail, object_id)
elif arguments['session']:
result = misfit.session(start_date, end_date, object_id)
elif arguments['sleep']:
result = misfit.sleep(start_date, end_date, object_id)
pp = PrettyPrinter(indent=4)
if isinstance(result, list):
pp.pprint([res.data for res in result])
else:
pp.pprint(result.data)
def get_nice_schema(collection):
"""stupid proxy for stdlib pprint"""
schema = get_schema(collection)
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=2, width=16)
nice = pp.pformat(schema)
return nice
def _return(self, req, template, content_type='text/html'):
""" Wrap the return so that things are processed by Stan
"""
if req.args.has_key('hdfdump'):
# FIXME: the administrator should probably be able to disable HDF
# dumps
from pprint import PrettyPrinter
outstream = StringIO()
pp = PrettyPrinter(stream=outstream)
pp.pprint(req.standata)
content_type = 'text/plain'
data = outstream.getvalue()
outstream.close()
else:
ct = content_type.split('/')[0]
data = self._render(req.standata, template)
req.send_response(200)
req.send_header('Cache-control', 'must-revalidate')
req.send_header('Expires', 'Fri, 01 Jan 1999 00:00:00 GMT')
req.send_header('Content-Type', content_type + ';charset=utf-8')
req.send_header('Content-Length', len(data))
req.end_headers()
if req.method != 'HEAD':
req.write(data)
pass
def main(args=None):
parser = ArgumentParser(usage="Usage: %(prog)s [options]", description="query datacatalog")
parser.add_argument("-H","--host",dest='host',help="hostname of influxdb instance")
parser.add_argument("-u","--user",dest="user",help="username")
parser.add_argument("-p","--password",dest="pw",help="password")
parser.add_argument("-P","--port",dest="port",type=int,default=8086,help="influxdb ingest port")
parser.add_argument("-n","--dbname", dest="dbname",help="name of DB to store data in.")
parser.add_argument("-d", "--dry", dest="dry", action="store_true", default=False, help="do not report results to grafana")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False, help="verbose mode")
opts = parser.parse_args(args)
json_bdy = []
for site in batchSites:
status_dict = {key: 0 for key in statii}
stats = JobInstance.objects.filter(site=site).item_frequencies("status")
status_dict.update(stats)
for stat, freq in status_dict.iteritems():
json_bdy.append(__makeEntry__(stat, site, freq))
print 'found %i measurements to add'%len(json_bdy)
pp = PrettyPrinter(indent=2)
if opts.verbose: pp.pprint(json_bdy)
if opts.dry:
return
if influxdb:
client = InfluxDBClient(opts.host,opts.port,opts.user,opts.pw,opts.dbname)
client.create_database(opts.dbname)
ret = client.write_points(json_bdy)
if not ret:
try:
raise Exception("Could not write points to DB")
except Exception:
print_exc()
sys_exit(int(ret))
def run_grid_search(pipeline, parameters) -> dict:
from pprint import PrettyPrinter
pp = PrettyPrinter()
print("parameters:\n ", end='')
pp.pprint(parameters)
grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, cv=num_cv_folds)
grid_search.fit( unzip(train_data,0), unzip(train_data,1) )
prediction = grid_search.predict(unzip(train_data,0))
print(classification_report(unzip(train_data,1), prediction, target_names=model_names))
best_parameters = grid_search.best_estimator_.get_params()
print("Best score : %0.3f" % grid_search.best_score_)
ret = {}
print("Best parameter:")
try:
for param_name in sorted(parameters.keys()):
print(" %s: %r" % (param_name, best_parameters[param_name]))
ret[param_name] = (best_parameters[param_name],)
except AttributeError:
for param_name in sorted(parameters[0].keys()):
print(" %s: %r" % (param_name, best_parameters[param_name]))
ret[param_name] = (best_parameters[param_name],)
print("")
return ret
def _readable(self):
"""The readable parsed article"""
doc = self.orig.html
# cleaning doesn't return, just wipes in place
html_cleaner(doc)
doc = drop_tag(doc, "noscript", "iframe")
doc = transform_misused_divs_into_paragraphs(doc)
candidates, should_drop = find_candidates(doc)
if candidates:
LOG.debug("Candidates found:")
pp = PrettyPrinter(indent=2)
# right now we return the highest scoring candidate content
by_score = sorted([c for c in candidates.values()], key=attrgetter("content_score"), reverse=True)
LOG.debug(pp.pformat(by_score))
# since we have several candidates, check the winner's siblings
# for extra content
winner = by_score[0]
LOG.debug("Selected winning node: " + str(winner))
updated_winner = check_siblings(winner, candidates)
LOG.debug("Begin final prep of article")
updated_winner.node = prep_article(updated_winner.node)
doc = build_base_document(updated_winner.node, self.fragment)
else:
LOG.warning("No candidates found: using document.")
LOG.debug("Begin final prep of article")
# since we've not found a good candidate we're should help this
# cleanup by removing the should_drop we spotted.
[n.drop_tree() for n in should_drop]
doc = prep_article(doc)
doc = build_base_document(doc, self.fragment)
return doc
def dump(self):
"""
A debug function to display the cache contents.
"""
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=4)
pp.pprint(self)
def scrape(self, soup_jobs: list) -> list:
"""
In goes a bunch of html files (in the form of beautiful soups),
out comes serial data, i.e. dictionaries of field name/value
pairs, where values are raw strings.
:param soup_jobs: Stamped(Stamp, BeautifulSoup)
:return: Stamped(Stamp, (job_details, addresses))
"""
assert soup_jobs is not None, 'Argument cannot be None.'
serial_jobs = list()
for i, soup_job in enumerate(soup_jobs):
self.stamp = soup_job.stamp
job_details, addresses = self._scrape_job(soup_job)
serial_job = Stamped(soup_job.stamp, (job_details, addresses))
serial_jobs.append(serial_job)
if DEBUG:
print('Scraped {n}/{N}: {date}-uuid-{uuid}.html'
.format(date=str(soup_job.stamp.date),
uuid=soup_job.stamp.uuid,
N=len(soup_jobs),
n=i+1))
pp = PrettyPrinter()
pp.pprint(job_details)
pp.pprint(addresses)
return serial_jobs
def __process_article(self, article, print_out=False):
'''
Process an individual article as necessary
article - individual article dictionary to look at
return the article if usable or None
'''
# debug items
if (print_out):
pp = PrettyPrinter(indent = 2)
pp.pprint(article)
# process into numbers
for item in self.__LABELS:
target = self.__values[item]
article_item = article[item]
# ignore bad articles
if (not article_item):
return None
# assign new index if it doesn't exist
if (not target.has_key(article_item)):
length = len(target)
target[article_item] = length
return article
def dump(ctx):
bld = ctx.exec_dict['bld']
suite = bld.env.orch_suite
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=2)
pp.pprint(suite)
sys.exit(0)
def test_config_load3():
""" test config many shards with many replicas """
print test_config_load3.__name__
test_config_file = BytesIO()
# non ordered
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs3')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs3')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs3')
test_config_file.seek(0)
# config file processing
config = configparser.ConfigParser()
config.read_file(test_config_file)
pp = PrettyPrinter()
all_settings = load_mongo_replicas_from_setting(config,
'mongo-oplog')
pp.pprint(all_settings)
assert(3 == len(all_settings.keys()))
assert(sorted(all_settings.keys()) == \
sorted(['mongo-oplog-shard1',
'mongo-oplog-shard2',
'mongo-oplog-shard3']))
def _readable(self):
"""The readable parsed article"""
if self.candidates:
LOG.debug('Candidates found:')
pp = PrettyPrinter(indent=2)
# cleanup by removing the should_drop we spotted.
[n.drop_tree() for n in self._should_drop
if n.getparent() is not None]
# right now we return the highest scoring candidate content
by_score = sorted([c for c in self.candidates.values()],
key=attrgetter('content_score'), reverse=True)
LOG.debug(pp.pformat(by_score))
# since we have several candidates, check the winner's siblings
# for extra content
winner = by_score[0]
LOG.debug('Selected winning node: ' + str(winner))
updated_winner = check_siblings(winner, self.candidates)
LOG.debug('Begin final prep of article')
updated_winner.node = prep_article(updated_winner.node)
if updated_winner.node is not None:
doc = build_base_document(updated_winner.node, self.fragment)
else:
LOG.warning('Had candidates but failed to find a cleaned winning doc.')
doc = self._handle_no_candidates()
else:
LOG.warning('No candidates found: using document.')
LOG.debug('Begin final prep of article')
doc = self._handle_no_candidates()
return doc
def dprint(object, stream=None, indent=1, width=80, depth=None):
"""
A small addition to pprint that converts any Django model objects to dictionaries so they print prettier.
h3. Example usage
>>> from toolbox.dprint import dprint
>>> from app.models import Dummy
>>> dprint(Dummy.objects.all().latest())
{'first_name': u'Ben',
'last_name': u'Welsh',
'city': u'Los Angeles',
'slug': u'ben-welsh',
"""
# Catch any singleton Django model object that might get passed in
if getattr(object, '__metaclass__', None):
if object.__metaclass__.__name__ == 'ModelBase':
# Convert it to a dictionary
object = object.__dict__
# Catch any Django QuerySets that might get passed in
elif isinstance(object, QuerySet):
# Convert it to a list of dictionaries
object = [i.__dict__ for i in object]
# Pass everything through pprint in the typical way
printer = PrettyPrinter(stream=stream, indent=indent, width=width, depth=depth)
printer.pprint(object)
def addInteraction(self, label, paramsAndValues):
self.info[label]=paramsAndValues
f=file(self.databaseFile, 'w')
pp=PrettyPrinter(stream=f)
pp.pprint(self.info)
# print >>f, self.info
f.close()
class Action(object):
def __init__(self, config, stream):
self.program = config.pop(0)
self.action = config.pop(0)
self.arguments = config
self.stream = stream
self.writer = csv.writer(self.stream, delimiter=' ')
self.pp = PrettyPrinter(stream=self.stream)
def start(self):
if self.action in ['display', 'zip']:
print("#" ,end='', file=self.stream)
print(*self.arguments, file=self.stream)
print("", file=self.stream)
def __call__(self, result):
getattr(self, self.action)(result, *self.arguments)
def report(self, result):
print(result, file=self.stream)
def inspect(self, result):
self.pp.pprint(result.hash())
def display(self, result, *cols):
self.writer.writerow(map(lambda x: x if x is not None else 'None',
[result.datum(col) for col in cols]))
result.files.clear()
def name(self, result):
print(result.name, file=self.stream)
def accept(self, result):
acceptance = open(os.path.join(result.path, 'acceptance.txt'), 'w')
acceptance.write("Accepted: OK")
acceptance.close()
def reject(self, result):
acceptance = open(os.path.join(result.path,'acceptance.txt'), 'w')
acceptance.write("Accepted: NO")
acceptance.close()
def delete(self, result):
shutil.rmtree(result.path, ignore_errors=True)
def cat(self, result, *files):
for pattern in files:
for fn in glob.iglob(os.path.join(result.path, pattern)):
with file(fn) as f:
content = f.read()
print(content, file=self.stream)
def zip(self, result, *cols):
print("#%s" % result.name, file=self.stream)
self.writer.writerows(
zip(*[result.datum(col) for col in cols])
)
print("\n", file=self.stream)
def status():
""" returns current status of the instance """
config = get_config()
pp = PrettyPrinter(indent=4)
pp.pprint(config)
if has_state():
state = load_state()
pp.pprint(state)
def _format(self, object, stream, indent, allowance, context, level):
"""
Recursive part of the formatting
"""
try:
PrettyPrinter._format(self, object, stream, indent, allowance, context, level)
except Exception as e:
stream.write(_format_exception(e))
def pprint(tree):
"""
Utility function to pretty print the history tree of a piece
:param tree: History tree of a piece
:return: None
"""
p = PrettyPrinter(indent=2)
p.pprint(tree)
def __ls(models=None, yamldata=None, yamlfile=None, encoding=None, model=None,
output = sys.stdout,
yaml_mark_subref = YAML_MARK_SUBREF,
yaml_file_header = YAML_FILE_HEADER,
yaml_data_header = YAML_DATA_HEADER,
datastore_header = DATASTORE_HEADER,
raw_data_format = RAW_DATA_FORMAT,
raw_data_format_options = None,
model_instance_format = MODEL_INSTANCE_FORMAT,
**options):
if not output:
output = StringIO()
if yamldata is not None or yamlfile is not None:
if raw_data_format_options is None:
raw_data_format_options = dict(RAW_DATA_FORMAT_OPTIONS)
if options:
raw_data_format_options.update(options)
raw_data_pp = PrettyPrinter(stream=output, **raw_data_format_options)
if models is not None:
for m in models:
if datastore_header:
output.write(datastore_header % dict(model=m.__name__))
for i in m.all():
if model_instance_format: output.write(model_instance_format % i)
if yamldata is not None:
if yaml_data_header:
output.write(yaml_data_header % dict(model=model and model.__name__ or '-'))
for d in yaml_records(yamldata, encoding=encoding, yaml_mark_subref=yaml_mark_subref):
if raw_data_format: output.write(raw_data_format % raw_data_pp.pformat(d))
if model:
i = model(**d)
if model_instance_format: output.write(model_instance_format % i)
if yamlfile is not None:
if yaml_file_header:
output.write(yaml_file_header % dict(yamlfile=yamlfile, encoding=encoding,
model=model and model.__name__ or '-'))
for d in yaml_records(file(yamlfile).read(), encoding=encoding, yaml_mark_subref=yaml_mark_subref):
if raw_data_format: output.write(raw_data_format % raw_data_pp.pformat(d))
if model:
i = model(**d)
if model_instance_format: output.write(model_instance_format % i)
# __ls(yamldata=file(yamlfile).read(), encoding=encoding, model=model, output=output
# yaml_mark_subref = yaml_mark_subref,
# yaml_data_header = '', # file header was output, skip data header
# raw_data_format = raw_data_format,
# raw_data_format_options = raw_data_format_options,
# model_instance_format = model_instance_format,
# )
return getattr(output, 'getvalue', lambda: None)() # output if stringio, or None
def test():
print "RescueTime class TEST"
rt = RescueTime(beginDay="20141015",endDay="20141019")
pp = PrettyPrinter()
print "confirm what raw data are"
pp.pprint(rt.jsonRawData)
print "getAllData"
rt.getAllData([["Twitter","Twitter for Android"],["Hulu"]],["2","-1"],["General Software Development"])
class TweetRawPrinter(TweetListener):
def __init__(self):
TweetListener.__init__(self,"TweetRawPrinter")
self.pp = PrettyPrinter(indent=4)
def processTweet(self,tweet):
print "Raw Tweet: "
self.pp.pprint(tweet.__dict__)
print '-' * 20
def main():
"""
Tester for the validate method.
"""
filepath = os.path.join(os.path.dirname(__file__), 'test_data.json')
validated_data = validate(filepath)
pp = PrettyPrinter(indent=4)
pp.pprint(validated_data)
def request_view(request):
"""
display the request metadata
:param request:
:return:
"""
pp = PrettyPrinter()
s = '<pre>request.META =\n' + pp.pformat(request.META) + '</pre>'
return HttpResponse(s)
def __call__(self):
'''
Check the portlets defined here and in some sublevels
'''
self.results = {}
self.update_results(self.context)
printer = PrettyPrinter(stream=StringIO())
printer.pprint(self.results)
return printer._stream.getvalue()
def pprint(tree):
"""
Utility function to pretty print the history tree of a piece.
Args:
tree (dict): History tree of a piece.
"""
p = PrettyPrinter(indent=2)
p.pprint(tree)
def handle(self, *args, **options):
award_map = {}
for r in Resident.objects.all():
award_map[ r.name ] = []
for q in Question.objects.filter(istwoans=False):
result = feud.top_k_for_question(q, 1)
if len(result['answers']):
award_map[ result['answers'][0]['one'] ].append( str(result['answers'][0]['number']) + ' - ' + q.qtext )
pp = PrettyPrinter(indent=2)
pp.pprint(award_map)
def handle_disconnected(self, event):
endtime= int(time.time())
filename = "rodney.{0}.log".format(endtime)
f = open(filename, "w")
from pprint import PrettyPrinter
pp = PrettyPrinter(depth=10, stream=f)
pp.pprint(self.logs)
print "My transcript has been written to {0}".format(filename)
"""Quit the main loop upon disconnection."""
return QUIT
"""
Copyright (c) 2019-present, Facebook, Inc.
All rights reserved.
This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree. An additional grant
of patent rights can be found in the PATENTS file in the same directory.
"""
import argparse
from pprint import PrettyPrinter
from magma.configuration import service_configs
LOG_LEVEL_KEY = 'log_level'
pp = PrettyPrinter(width=1)
def load_override_config(args):
cfg = service_configs.load_override_config(args.service)
pp.pprint(cfg)
def get_log_level(args):
cfg = service_configs.load_service_config(args.service)
if cfg is None or LOG_LEVEL_KEY not in cfg:
print('No log level set!')
return
print('Log level:', cfg[LOG_LEVEL_KEY])
from packetbeat import (BaseTest, FLOWS_REQUIRED_FIELDS)
from pprint import PrettyPrinter
pprint = lambda x: PrettyPrinter().pprint(x)
def check_fields(flow, fields):
for k, v in fields.iteritems():
assert flow[k] == v
class Test(BaseTest):
def test_mysql_flow(self):
self.render_config_template(
flows=True,
)
self.run_packetbeat(
pcap="mysql_long.pcap",
wait_stop=1,
debug_selectors=["*"])
objs = self.read_output(
types=["flow"],
required_fields=FLOWS_REQUIRED_FIELDS)
pprint(objs)
assert len(objs) == 1
check_fields(objs[0], {
'final': True,
__doc__ = """
Generates the data to be used.
However, it repeated itself. Hence the switch to uniprot_master_parser.UniprotMasterReader.
"""
from pprint import PrettyPrinter
pprint = PrettyPrinter().pprint
import os, json
from warnings import warn
from ._protein_gatherer import ProteinGatherer
from .uniprot_master_parser import UniprotMasterReader
from .split_gnomAD import gnomAD
from .PDB_blast import Blaster
from ..settings_handler import global_settings #the instance not the class.
import random
def announce(*msgs):
print('*' * 50)
print(*msgs)
print('*' * 50)
class ProteomeGathererOLD:
"""
Gets everything ready and parses!
>>> ProteomeGatherer(skip=True)
Will assume all is prepared.
"""
# A minimax tree which interfaces like gameState
# state.getNumAgents()
# state.isWin()
# state.isLose()
# state.generateSuccessor(agentIndex, action)
# state.getScore()
# used by multiAgents.scoreEvaluationFunction, which is the default
#
import testClasses
import json
from collections import defaultdict
from pprint import PrettyPrinter
pp = PrettyPrinter()
from game import Agent
from pacman import GameState
from ghostAgents import RandomGhost, DirectionalGhost
import random, math, traceback, sys, os
import layout, pacman
import autograder
# import grading
VERBOSE = False
class MultiagentTreeState(object):
def __init__(self, problem, state):
def tree(user):
""" Shows a tree of all repositories in bitbucket
"""
from pprint import PrettyPrinter
P = PrettyPrinter(indent=2)
P.pprint(bitbucket_repo_tree(user))
import json
import argparse
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=2)
parser = argparse.ArgumentParser()
parser.add_argument('-f',
type=str,
help='Result filename to summarize the result',
dest='filename')
parser.add_argument('-exp',
type=str,
help='Result filename to summarize the result',
dest='filename')
parser.add_argument(choices=['learn_crf', 'predict_crf', 'entity', 'crf_entity', \
'init', 'result', 'iter_crf'],
dest = 'prog')
args = parser.parse_args()
filename = args.filename
with open(filename, 'r') as fp:
data = json.load(fp)
if args.prog == 'crf_entity':
source_nums = []
char_f1s = []
char_macrof1s = []
entity_accuracies = []
def log(obj):
from pprint import PrettyPrinter
PrettyPrinter(indent=4).pprint(obj)
class TestPackageValidation(OPennTestCase):
pass
pp = PrettyPrinter(indent=2)
staging_dir = os.path.join(os.path.dirname(__file__), 'staging')
template_dir = os.path.join(os.path.dirname(__file__),
'data/package_validation')
staged_package = os.path.join(staging_dir, 'package_validation')
validation_config = {
'valid_names': ['*.tif', 'bibid.txt'],
'invalid_names': ['CaptureOne', 'Output', '*[()]*'],
'required_names': ['*.tif', 'bibid.txt']
}
def touch(self, fname):
with open(fname, 'a'):
os.utime(fname, None)
def setUp(self):
if not os.path.exists(TestPackageValidation.staging_dir):
os.mkdir(TestPackageValidation.staging_dir)
def tearDown(self):
if os.path.exists(TestPackageValidation.staging_dir):
shutil.rmtree(TestPackageValidation.staging_dir)
def stage_template(self):
shutil.copytree(TestPackageValidation.template_dir,
TestPackageValidation.staged_package)
def test_run(self):
# setup
self.stage_template()
validation = PackageValidation(
**TestPackageValidation.validation_config)
# run
errors = validation.validate(TestPackageValidation.staged_package)
self.assertTrue(len(errors) == 0)
def test_invalid_dir(self):
self.stage_template()
os.mkdir(os.path.join(self.staged_package, 'CaptureOne'))
validation = PackageValidation(
**TestPackageValidation.validation_config)
# run
errors = validation.validate(TestPackageValidation.staged_package)
self.assertTrue(len(errors) > 0)
self.assertIn(
'CaptureOne',
validation.check_invalid_names(
TestPackageValidation.staged_package))
def test_invalid_valid_file(self):
# test a file that passes the check_valid_names test '\.tif$', but fails the
# check_invalid_names test '[()]'
self.stage_template()
self.touch(os.path.join(self.staged_package, 'somefile(1).tif'))
validation = PackageValidation(**self.validation_config)
# run
errors = validation.validate(self.staged_package)
self.assertTrue(len(errors) > 0)
errors = validation.check_invalid_names(
TestPackageValidation.staged_package)
self.assertIn('somefile(1).tif', errors)
def test_missing_required(self):
self.stage_template()
os.remove(os.path.join(self.staged_package, 'bibid.txt'))
validation = PackageValidation(**self.validation_config)
# run
errors = validation.validate(self.staged_package)
self.assertTrue(len(errors) > 0)
errors = validation.check_required(
TestPackageValidation.staged_package)
self.assertIn('bibid.txt', errors)
def format(self, obj, context, maxlevels, level):
if isinstance(obj, SWRealReference):
return (str(obj), False, False)
else:
return PrettyPrinter.format(self, obj, context, maxlevels, level)
def setup(self):
self.auth_creds = {
'': {
'user': '******',
'pwd': 'qwe123',
'context': 'default',
'description': '{ "defaultAllow": "true" }'
},
'read_only': {
'user':
'******',
'pwd':
'qwe123',
'context':
'read_only',
'description':
'{'
'{ "cache": "cache_ro*", "permissions":["CACHE_READ"] }, '
'{ "cache": "cache_rw*", "permissions":["CACHE_READ", "CACHE_PUT", "CACHE_REMOVE"] }, '
'{ "task": "*", "permissions":["TASK_EXECUTE"] }, '
'{ "system":["ADMIN_CACHE", "CACHE_CREATE"] }, '
'"defaultAllow":"false"'
'}'
},
'utility': {
'user': '******',
'pwd': 'qwe123',
'context': 'utility',
'description': '{ "defaultAllow":"true" }'
},
}
pp = PrettyPrinter()
for auth_cred_name in self.auth_creds:
context_name = self.auth_creds[auth_cred_name]['context']
if context_name != 'default':
context = self.create_test_context(context_name)
else:
context = self.contexts['default']
context.add_context_variables(
ssl_enabled=is_enabled(self.config.get('ssl_enabled')),
authentication_enabled=True,
client_key_store_file_path='%s/client.jks' %
self.config['rt']['remote']['test_module_dir'],
server_key_store_file_path='%s/server.jks' %
self.config['rt']['remote']['test_module_dir'],
trust_store_file_path='%s/trust.jks' %
self.config['rt']['remote']['test_module_dir'],
zookeeper_enabled=is_enabled(
self.config.get('zookeeper_enabled')),
pitr_enabled=is_enabled(self.config.get('pitr_enabled')),
rolling_updates_enabled=is_enabled(
self.config.get('rolling_updates_enabled')),
auth_login=self.auth_creds[auth_cred_name]['user'],
auth_password=self.auth_creds[auth_cred_name]['pwd'],
)
self.cu.enable_authentication('utility', 'qwe123')
self.su.enable_authentication('utility', 'qwe123')
print_blue('Credentials: \n' + pp.pformat(self.auth_creds))
super().setup()
return json.loads(stdout)
def get_custom_distribution_metadata(product_name) -> List[CustomDistributionMeta]:
"""Get metadata for reconstructing custom distribution buckets in Glean metrics."""
# GleanPing.get_repos -> List[Tuple[name: str, app_id: str]]
glean = GleanPing(product_name)
probes = glean.get_probes()
custom = []
for probe in probes:
if probe.get_type() != "custom_distribution":
continue
meta = CustomDistributionMeta(
probe.get_name(),
probe.get("range_min"),
probe.get("range_max"),
probe.get("bucket_count"),
probe.get("histogram_type"),
)
custom.append(meta)
return custom
if __name__ == "__main__":
from pprint import PrettyPrinter
pp = PrettyPrinter()
pp.pprint(get_custom_distribution_metadata("fenix"))
item = end_key
elif what.lower() == 'value':
item = self[end_key]
elif what.lower() == 'item':
item = (end_key, self[end_key])
else:
raise ValueError(
what +
'. The \'what\' keyword argument must be one of \'key\', \'value\', or \'item\'.'
)
return item
if __name__ == '__main__':
from pprint import PrettyPrinter
pp = PrettyPrinter()
a = DeepOrderedDict([('a', 1)])
b = DeepOrderedDict([('b', 10)])
c = DeepOrderedDict([('c', 100), ('d', 200)])
a['b'] = b.copy()
a['b']['c'] = c.copy()
a['c'] = c.copy()
a['d'] = 300
print('original dict:')
pp.pprint(a)
a.move_to_end('d')
# a['c']['d'] moves nearer to the end than a['d'] because a['d'] is the
# sibling of a['c']
# where as a['d'] moves closer to the end than a['b']['c']['d'] becaues
# they dont satisfy the aformentioned relationship and a['d'] comes later
# in a depth-first search
'-s', '--size',
default='tiny',
help='tiny, main'
)
parser.add_argument(
'--dataset_root',
default=os.environ['THIS_REPO'],
)
parser.add_argument(
'-k', '--k',
default=10,
type=int,
help='the number of choices',
)
parser.add_argument(
"-nuc", '--not_unify_category',
action='store_false',
dest='unify_category',
)
parser.add_argument(
'--seed',
type=int,
default=12345,
)
args = parser.parse_args()
args = vars(args)
pp = PrettyPrinter(indent=4)
pp.pprint(args)
main(**args)
load_dotenv()
API_KEY = os.getenv('API_KEY')
# Settings for image endpoint
# Written with help from http://dataviztalk.blogspot.com/2016/01/serving-matplotlib-plot-that-follows.html
matplotlib.use('agg')
plt.style.use('ggplot')
my_loader = jinja2.ChoiceLoader([
app.jinja_loader,
jinja2.FileSystemLoader('data'),
])
app.jinja_loader = my_loader
pp = PrettyPrinter(indent=4)
################################################################################
## ROUTES
################################################################################
@app.route('/')
def home():
"""Displays the homepage with forms for current or historical data."""
context = {
'min_date': (datetime.now() - timedelta(days=5)),
'max_date': datetime.now()
}
return render_template('home.html', **context)
def test_builtin_prettyprinter():
# non regression test than ensures we can still use the builtin
# PrettyPrinter class for estimators (as done e.g. by joblib).
# Used to be a bug
PrettyPrinter().pprint(LogisticRegression())
config = {
"ip": "172.16.102.59",
"credentials": {
"userName": "******",
"password": ""
},
"enclosure_hostname": "172.178.209.32",
"enclosure_username": "******",
"enclosure_password": "",
"vcmUsername": "******",
"vcmPassword": "",
"enclosure_group_uri": None
}
pp = PrettyPrinter()
# Try load config from a file (if there is a config file)
print("Loading configuration.")
config = try_load_from_file(config)
# Obtain the master OneView client
print("Setting up OneView client.")
oneview_client = OneViewClient(config)
# Create the dict that VC Migration Manager requires to start the process
migrationInformation = MigratableVcDomains.make_migration_information(
config['enclosure_hostname'],
config['enclosure_username'],
config['enclosure_password'],
config['vcmUsername'],
def _repr(self, *args, **kwargs):
try:
return PrettyPrinter._repr(self, *args, **kwargs)
except Exception as e:
return "!! Cannot format: %s" % e
def solution_error_vs_row_dim():
'''
Increase `n` the input dimension of the problem and
measure the solution error in both:
(i) Euclidean norm (`mean_square_error`)
(ii) Prediction norm (`prediction_error`).
Error measurements are taken with respect to:
(i) the optimal solution x_opt
(ii) the ground truth
'''
print('Experimental setup:')
print(f'IHS sketch size {SKETCH_SIZE}')
print(f'Sketch and solve sketch size {CLASSICAL_SKETCH_SIZE}')
print(f'Number of rounds {ROUNDS}')
# Output dictionaries
MSE_OPT = {
sketches[i]: np.zeros(len(ROWDIMS), )
for i in range(len(sketches))
}
PRED_ERROR_OPT = {
sketches[i]: np.zeros(len(ROWDIMS), )
for i in range(len(sketches))
}
MSE_TRUTH = {
sketches[i]: np.zeros(len(ROWDIMS), )
for i in range(len(sketches))
}
PRED_ERROR_TRUTH = {
sketches[i]: np.zeros(len(ROWDIMS), )
for i in range(len(sketches))
}
MSE_OPT['Sketch & Solve'] = np.zeros(len(ROWDIMS), )
PRED_ERROR_OPT['Sketch & Solve'] = np.zeros(len(ROWDIMS), )
MSE_TRUTH['Sketch & Solve'] = np.zeros(len(ROWDIMS), )
PRED_ERROR_TRUTH['Sketch & Solve'] = np.zeros(len(ROWDIMS), )
MSE_TRUTH['Exact'] = np.zeros(len(ROWDIMS), )
PRED_ERROR_TRUTH['Exact'] = np.zeros(len(ROWDIMS), )
## Experiment
for n in ROWDIMS:
print(f'Testing {n} rows')
experiment_index = ROWDIMS.index(n)
_iters = ROUNDS[experiment_index]
ihs_sketch_size = SKETCH_SIZE
classic_sketch_size = CLASSICAL_SKETCH_SIZE[experiment_index]
for trial in range(NTRIALS):
print("TRIAL {}".format(trial))
X, y, x_true = gaussian_design_unconstrained(n, D, variance=1.0)
x_opt = np.linalg.lstsq(X, y)[0]
for sketch_method in METHODS:
print('*' * 80)
if sketch_method in sketches or sketch_method == 'Sketch & Solve':
if sketch_method == 'sjlt':
col_sparsity = 4
else:
col_sparsity = 1
if sketch_method == 'Sketch & Solve':
_sketch = rp(X, classic_sketch_size, 'countSketch',
col_sparsity)
SA, Sb = _sketch.sketch_data_targets(y)
x_ss = np.linalg.lstsq(SA, Sb)[0]
MSE_OPT[sketch_method][
experiment_index] += mean_square_error(
x_opt, x_ss)
PRED_ERROR_OPT[sketch_method][
experiment_index] += prediction_error(
X, x_opt, x_ss)
MSE_TRUTH[sketch_method][
experiment_index] += mean_square_error(
x_true, x_ss)
PRED_ERROR_TRUTH[sketch_method][
experiment_index] += prediction_error(
X, x_true, x_ss)
else:
print(f'{sketch_method} IHS')
my_ihs = ihs(X, y, sketch_method, ihs_sketch_size,
col_sparsity)
x_ihs, x_iters = my_ihs.ols_fit_new_sketch_track_errors(
_iters)
x_errors = x_opt[:, None] - x_iters
print(x_errors.shape)
MSE_OPT[sketch_method][
experiment_index] += mean_square_error(
x_opt, x_ihs)
PRED_ERROR_OPT[sketch_method][
experiment_index] += prediction_error(
X, x_opt, x_ihs)
MSE_TRUTH[sketch_method][
experiment_index] += mean_square_error(
x_true, x_ihs)
PRED_ERROR_TRUTH[sketch_method][
experiment_index] += prediction_error(
X, x_true, x_ihs)
else:
# solve exactly
#x_opt = np.linalg.lstsq(X,y)[0]
MSE_TRUTH["Exact"][experiment_index] += mean_square_error(
x_opt, x_true)
PRED_ERROR_TRUTH["Exact"][
experiment_index] += prediction_error(
X, x_opt, x_true)
for _dict in [MSE_OPT, PRED_ERROR_OPT, MSE_TRUTH, PRED_ERROR_TRUTH]:
for _key in _dict.keys():
_dict[_key] /= NTRIALS
pretty = PrettyPrinter(indent=4)
pretty.pprint(MSE_OPT)
pretty.pprint(PRED_ERROR_OPT)
pretty.pprint(MSE_TRUTH)
pretty.pprint(PRED_ERROR_TRUTH)
save_dir = '../../output/baselines/'
np.save(save_dir + 'ihs_ols_mse_OPT', MSE_OPT)
np.save(save_dir + 'ihs_ols_pred_error_OPT', PRED_ERROR_OPT)
np.save(save_dir + 'ihs_ols_mse_TRUTH', MSE_TRUTH)
np.save(save_dir + 'ihs_ols_pred_error_TRUTH', PRED_ERROR_TRUTH)
from csv import reader
from pprint import PrettyPrinter
pp = PrettyPrinter()
with open('satfaces.csv', newline='') as sat:
next(sat)
colors = set()
satread = reader(sat, delimiter=',')
for i in satread:
colors.add(i[3])
print("{} different colors".format(len(colors)))
pp.pprint(colors)
#! /usr/bin/env python
from sys import argv
from pprint import PrettyPrinter
from keyword import iskeyword
if __name__ == "__main__":
pp = PrettyPrinter(indent = 4, compact = True).pprint
for arg in argv[1:]:
if arg.isidentifier():
if not iskeyword(arg):
code = 'import {}'.format(arg)
env = {}
try:
exec(code, env)
print(repr(env[arg]))
print('the following symbols were added:')
pp(dir(env[arg]))
except ImportError:
print('import {} failed'.format(arg))
else:
print('\'{}\' is a python keyword'.format(arg))
else:
print('invalid identifier: {}'.format(repr(arg)))
#!/usr/bin/env python3
from pprint import PrettyPrinter
import slacker
import peewee
import settings
import models as m
token = settings.token
slack = slacker.Slacker(token)
pp = PrettyPrinter(indent=2).pprint
def assert_auth():
try:
return slack.auth.test().body
except slacker.Error as err:
print('Auth failed: ' + str(err))
def save_team_metadata(auth_resp):
del auth_resp['ok']
# fixme: warn if user use a different database to backup
with m.db.atomic():
for prop in auth_resp:
meta, _ = m.Information.get_or_create(key=prop, defaults={'value': auth_resp[prop]})
if prop == 'team_id' and meta.value != auth_resp['team_id']:
print(' Warning: Team metadata is inconsistent. You may be corrupting a existing database!')
exit()
def fetch_user_list():
def message():
if 'json' in params:
return dumps(mylights.input_state())
else:
pp = PrettyPrinter(indent=2)
return pp.pprint(mylights.input_state())
def print_spec(*args, **kwargs):
pp = PrettyPrinter(width=200)
pp.pprint(*args, **kwargs)
import os
from logging import getLogger
from logging.config import dictConfig
from pprint import PrettyPrinter
from config import LOG_DIR, LOGGING
pp = PrettyPrinter(indent=4, width=120, compact=True)
def configure_logger():
logger = getLogger(__name__)
try:
os.mkdir(LOG_DIR)
except OSError as e:
if e.errno == 17:
pass
else:
logger.exception('%s %s', e.strerror, e.filename)
dictConfig(LOGGING)
return logger
import easyargs
import logging
import json
from dataclasses import dataclass
from dataclasses_json import dataclass_json
from pprint import PrettyPrinter
from datetime import datetime, timedelta
from urllib.parse import urlparse
from tempfile import mkdtemp
from contextlib import contextmanager
from shutil import rmtree
from os import path
from base64 import b64encode
from time import sleep
pretty_printer = PrettyPrinter(indent=4)
@contextmanager
def defer_close(thing):
try:
yield thing
finally:
thing.close()
@contextmanager
def tmpdir():
try:
temp_dir = mkdtemp(suffix="images")
yield temp_dir
# Standard Library
import os
from lxml import etree
import email.mime.text
from pprint import PrettyPrinter
from cStringIO import StringIO
# Third Party
import suds
import six
# Local
import vars
from extensions import AttributeDict, StrOrEtree
PP = PrettyPrinter()
def create_file(path=None, mode=0777):
# type: (str, int) -> None
"""
Creates a file, recursively creating the directory path if required
:param str path: Full path to the file to create
:param int mode: Octal notation for the file/directories mode (Default 0777)
:returns: None
"""
if path is None:
err = 'No path defined for `create_file()`'
raise vars.DMSIKnownFatalException(err)
fp, fn = os.path.split(path)
if calls[j][0] == 'i':
param += tpl % (iprefix + calls[j], rge[0],
rge[1])
elif calls[j][0] == 's':
param += tpl % (calls[j][1:], rge[0],
rge[1])
else:
param += tpl % (prefix + calls[j], rge[0],
rge[1])
r.append(call % param)
r += ["}"]
txt += r + txtf
if tb_name.find('.') != -1:
for i, l in enumerate(txt):
txt[i] = re.sub('nt2::', 'boost::simd::', l)
h = Headers(os.path.join(self.bg.get_nt2_rel_tb_path(tb_name), 'bench',
mode),
name,
inner=txt,
guard_begin=[],
guard_end=[]).txt()
return h.split('\n')
if __name__ == "__main__":
print __doc__
from pprint import PrettyPrinter
bg = Base_gen("exponential", 'pipo', 'simd')
bbg = Bench_gen(bg)
PrettyPrinter().pprint(bbg.get_gen_result())
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
from dataclasses import dataclass
from pprint import PrettyPrinter
from typing import Dict, List, NamedTuple
custom_printer = PrettyPrinter(
indent=4,
width=100,
depth=2,
compact=True,
sort_dicts=False,
underscore_numbers=True,
)
INPUT_FILE = "input.txt"
MAX_GRID_SIZE = 10
class Point(NamedTuple):
i: int
j: int
@dataclass
class Octopus:
energy_level: int
has_flashed: bool
def __str__(self):
return PrettyPrinter(indent=4).pformat(self.__data)
def assignment_7():
print("*** ASSIGNMENT 7 ***")
samples = 100
fractions = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.925, 0.95,
0.99)
datasets = {
'monk1': {
'training': monk1,
'test': monk1test
},
'monk3': {
'training': monk3,
'test': monk3test
}
}
results = {}
for dataset_name, dataset_data in datasets.items():
dataset = dataset_data['training']
dataset_test = dataset_data['test']
results[dataset_name] = {}
for fraction in fractions:
errors = []
for _ in range(samples):
train, validation = partition(dataset, fraction)
tree = buildTree(train, attributes)
opt_tree, _ = optimum_prune(tree, validation)
errors.append(1.0 - check(opt_tree, dataset_test))
results[dataset_name][fraction] = {
'mean': np.mean(errors),
'median': np.median(errors),
'std': np.std(errors),
'max': max(errors),
'min': min(errors)
}
pp = PrettyPrinter(indent=4)
pp.pprint(results)
y_monk1 = [(stats['mean'], stats['std'])
for fraction, stats in results['monk1'].items()]
y_monk3 = [(stats['mean'], stats['std'])
for fraction, stats in results['monk3'].items()]
plt.figure()
plt.errorbar(fractions, [e[0] for e in y_monk1],
yerr=[e[1] for e in y_monk1],
fmt='or',
capsize=5,
label='MONK-1')
plt.errorbar(fractions, [e[0] for e in y_monk3],
yerr=[e[1] for e in y_monk3],
fmt='ob',
capsize=5,
label='MONK-3')
plt.xlabel('Pruning fraction size (relative size of training set)')
plt.ylabel('Classification error')
plt.legend()
plt.title(
'Error vs. fraction size (mean of {} samples, errorbars represent one standard deviation)'
.format(samples))
plt.show()
