def main(term=DEFAULT_SEARCH_TERM):
'''Given a search term, execute the search.
Also, print the results to console.
'''
# Make a search object with our given term
if DEBUG:
print('Searching eBay for %s' % term)
search = EbaySearch(term)
# Get our results
results = search.results
if DEBUG:
print(search.result_count.text)
printer = PrettyPrinter()
printer.pprint(results)
newlistings = 0
for listing in results:
if listing['New']:
newlistings += 1
if DEBUG:
print('There are %i new listings out of %i on this page.' %
(newlistings, len(results)))
def main():
printer = PrettyPrinter(depth=2)
config = Config()
system_table = SystemTable.from_json(config.system_file)
panddas = {}
for system_id, system_info in system_table.to_dict().items():
if system_info is None:
logs.LOG[system_id] = {}
logs.LOG[system_id]["path"] = system_info
logs.LOG[system_id]["started"] = False
logs.LOG[system_id]["result"] = None
continue
logs.LOG[system_id] = {}
logs.LOG[system_id]["path"] = system_info
logs.LOG[system_id]["started"] = True
printer.pprint(logs.LOG.dict)
pandda = PanDDA.from_system(
Path(system_info),
config.panddas_dir / system_id,
script_path=Path("/tmp") / f"pandda_{system_id}",
)
result = pandda.poll()
panddas[system_id] = result
logs.LOG[system_id]["result"] = result
printer.pprint(logs.LOG.dict)
to_json(
panddas,
config.panddas_dir / "results.json",
)
def __process_article(self, article, print_out=False):
'''
Process an individual article as necessary
article - individual article dictionary to look at
return the article if usable or None
'''
# debug items
if (print_out):
pp = PrettyPrinter(indent = 2)
pp.pprint(article)
# process into numbers
for item in self.__LABELS:
target = self.__values[item]
article_item = article[item]
# ignore bad articles
if (not article_item):
return None
# assign new index if it doesn't exist
if (not target.has_key(article_item)):
length = len(target)
target[article_item] = length
return article
def run_grid_search(pipeline, parameters) -> dict:
from pprint import PrettyPrinter
pp = PrettyPrinter()
print("parameters:\n ", end='')
pp.pprint(parameters)
grid_search = GridSearchCV(pipeline,
parameters,
n_jobs=-1,
verbose=1,
cv=num_cv_folds)
grid_search.fit(unzip(train_data, 0), unzip(train_data, 1))
prediction = grid_search.predict(unzip(train_data, 0))
print(
classification_report(unzip(train_data, 1),
prediction,
target_names=model_names))
best_parameters = grid_search.best_estimator_.get_params()
print("Best score : %0.3f" % grid_search.best_score_)
ret = {}
print("Best parameter:")
try:
for param_name in sorted(parameters.keys()):
print(" %s: %r" % (param_name, best_parameters[param_name]))
ret[param_name] = (best_parameters[param_name], )
except AttributeError:
for param_name in sorted(parameters[0].keys()):
print(" %s: %r" % (param_name, best_parameters[param_name]))
ret[param_name] = (best_parameters[param_name], )
print("")
return ret
def main():
top_level = check_output(TOP_LEVEL, shell=True,
universal_newlines=True).strip()
source_files = []
test_files = []
for extension in EXTENSIONS:
source_files += glob.glob(top_level + FOLDER_SRC + extension,
recursive=True)
test_files += glob.glob(top_level + FOLDER_TEST + extension,
recursive=True)
files = set(source_files) - set(test_files)
args_used = set()
args_docd = set()
for file in files:
with open(file, 'r', encoding='utf-8') as f:
content = f.read()
args_used |= set(re.findall(re.compile(REGEX_ARG), content))
args_docd |= set(re.findall(re.compile(REGEX_DOC), content))
args_used |= SET_FALSE_POSITIVE_UNKNOWNS
args_need_doc = args_used - args_docd
args_unknown = args_docd - args_used
pp = PrettyPrinter()
print("Args used : {}".format(len(args_used)))
print("Args documented : {}".format(len(args_docd)))
print("Args undocumented: {}".format(len(args_need_doc)))
pp.pprint(args_need_doc)
print("Args unknown : {}".format(len(args_unknown)))
pp.pprint(args_unknown)
def get_resource(self, arguments):
""" Gets the resource requested in the arguments """
user_id = arguments['--user_id']
object_id = arguments['--object_id']
start_date = arguments['--start_date']
end_date = arguments['--end_date']
detail = arguments['--detail']
misfit = Misfit(self.client_id, self.client_secret, self.access_token,
user_id)
if arguments['profile']:
result = misfit.profile(object_id)
elif arguments['device']:
result = misfit.device(object_id)
elif arguments['goal']:
result = misfit.goal(start_date, end_date, object_id)
elif arguments['summary']:
result = misfit.summary(start_date, end_date, detail, object_id)
elif arguments['session']:
result = misfit.session(start_date, end_date, object_id)
elif arguments['sleep']:
result = misfit.sleep(start_date, end_date, object_id)
pp = PrettyPrinter(indent=4)
if isinstance(result, list):
pp.pprint([res.data for res in result])
else:
pp.pprint(result.data)
def dump(self):
"""
A debug function to display the cache contents.
"""
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=4)
pp.pprint(self)
def test_config_load3():
""" test config many shards with many replicas """
print test_config_load3.__name__
test_config_file = BytesIO()
# non ordered
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs3')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs3')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs3')
test_config_file.seek(0)
# config file processing
config = configparser.ConfigParser()
config.read_file(test_config_file)
pp = PrettyPrinter()
all_settings = load_mongo_replicas_from_setting(config,
'mongo-oplog')
pp.pprint(all_settings)
assert(3 == len(all_settings.keys()))
assert(sorted(all_settings.keys()) == \
sorted(['mongo-oplog-shard1',
'mongo-oplog-shard2',
'mongo-oplog-shard3']))
def printKey(key, XmlMap):
if XmlMap.has_key(key):
PP = PrettyPrinter()
print "%s:" % key
PP.pprint(XmlMap[key])
else:
print "entry %s not found" % key
def scrape(self, soup_jobs: list) -> list:
"""
In goes a bunch of html files (in the form of beautiful soups),
out comes serial data, i.e. dictionaries of field name/value
pairs, where values are raw strings.
:param soup_jobs: Stamped(Stamp, BeautifulSoup)
:return: Stamped(Stamp, (job_details, addresses))
"""
assert soup_jobs is not None, 'Argument cannot be None.'
serial_jobs = list()
for i, soup_job in enumerate(soup_jobs):
self.stamp = soup_job.stamp
job_details, addresses = self._scrape_job(soup_job)
serial_job = Stamped(soup_job.stamp, (job_details, addresses))
serial_jobs.append(serial_job)
if DEBUG:
print('Scraped {n}/{N}: {date}-uuid-{uuid}.html'
.format(date=str(soup_job.stamp.date),
uuid=soup_job.stamp.uuid,
N=len(soup_jobs),
n=i+1))
pp = PrettyPrinter()
pp.pprint(job_details)
pp.pprint(addresses)
return serial_jobs
def dump(self):
"""
A debug function to display the cache contents.
"""
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=4)
pp.pprint(self)
def show_scores_from_results():
results_with_kinds_fdir = os.path.join(proj_path, 'experiments',
'web_commons_v2_results.tsv')
df = pd.read_csv(results_with_kinds_fdir, sep='\t')
scores = {1: {}, 3: {}, 5: {}, 10: {}}
for k in scores.keys():
for kind in commons.KINDS:
if commons.KINDS[kind] == []:
scores[k][kind] = compute_score_from_df(kind=kind,
df=df,
is_sub=False,
k=k)
else:
for sub in commons.KINDS[kind]:
scores[k][sub] = compute_score_from_df(kind=sub,
df=df,
is_sub=True,
k=k)
scores[k]["all"] = compute_score_from_df(kind="all",
df=df,
is_sub=None,
k=k)
del scores[k][commons.RANDOM]
del scores[k][commons.YEAR]
pp = PrettyPrinter(indent=2)
pp.pprint(scores)
return scores
def main():
pp = PrettyPrinter(indent=4)
slack = Slacker(SLACK_AUTH_TOKEN)
emails_logins = create_login_dictionary()
logins_passwords = create_password_dictionary()
# get a list of all member ids
response = slack.channels.info(slack.channels.get_channel_id('cp3402'))
members = response.body['channel']['members']
# get relevant details for just those members
users = get_member_details(slack, members)
# pp.pprint(users)
contacted = []
for user_id, email in users.items():
try:
login = emails_logins[email]
# print(user_id, "Your MySQL username and database on the ditwebtsv.jcu.edu.au server is {} and your password is {}".format(login, logins_passwords[login]))
message = "Your MySQL username and database on the ditwebtsv.jcu.edu.au server is {} and your password is {}".format(login, logins_passwords[login])
# print(message)
# Send Slack message
slack.chat.post_message(user_id, message, as_user=True)
contacted.append((email, login, user_id))
except KeyError:
print("Skipping", email)
pp.pprint(contacted)
def clusterization_k_analysis(data, cluster_list, seed, json_file, verbose):
output_data = {}
for attr_set in data.keys():
output_data[attr_set] = {}
output_data[attr_set]['n_clusters'] = []
output_data[attr_set]['inertia'] = []
for k in cluster_list:
print('Executing experiment %s with %d clusters...' %
(attr_set, k))
data_norm, min_norm, max_norm = normalizes(data[attr_set])
km = KMeans(n_clusters=k, random_state=seed, n_jobs=-1)
km.fit_predict(data_norm)
# Init data matrix with k lists
output_data[attr_set]['n_clusters'].append(k)
output_data[attr_set]['inertia'].append(km.inertia_)
if verbose:
print('\n\nOutput data summary:')
pp = PrettyPrinter(depth=3)
pp.pprint(output_data)
data_json = json.dumps(output_data, indent=4)
f = open(json_file, 'w')
f.writelines(data_json)
f.close()
print('\nOutput data successfully exported to file %s\n' % json_file)
return output_data
def main():
pp = PrettyPrinter(indent=4)
slack = Slacker(SLACK_AUTH_TOKEN)
emails_logins = create_login_dictionary()
logins_passwords = create_password_dictionary()
# get a list of all member ids
response = slack.channels.info(slack.channels.get_channel_id('cp3402'))
members = response.body['channel']['members']
# get relevant details for just those members
users = get_member_details(slack, members)
# pp.pprint(users)
contacted = []
for user_id, email in users.items():
try:
login = emails_logins[email]
# print(user_id, "Your MySQL username and database on the ditwebtsv.jcu.edu.au server is {} and your password is {}".format(login, logins_passwords[login]))
message = "Your MySQL username and database on the ditwebtsv.jcu.edu.au server is {} and your password is {}".format(
login, logins_passwords[login])
# print(message)
# Send Slack message
slack.chat.post_message(user_id, message, as_user=True)
contacted.append((email, login, user_id))
except KeyError:
print("Skipping", email)
pp.pprint(contacted)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset',
'--numpy-dataset',
dest='dataset',
help='Path to a previously serialized dataset',
required=True,
type=str,
metavar='PATH')
# parser.add_argument('--test-dataset', dest='test_dataset',
# help='Path to a previously serialized dataset (Test)',
# required=True, type=str, metavar='PATH')
parser.add_argument(
'--undersampling',
help='Whether to use undersampling to balance classes in the dataset',
action='store_true')
parser.add_argument('name', type=str, help='Classifier\'s name')
args = parser.parse_args()
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=4)
pp.pprint(vars(args))
print('\n')
return args
def test_config_load3():
""" test config many shards with many replicas """
print test_config_load3.__name__
test_config_file = BytesIO()
# non ordered
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs1')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs2')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard1-rs3')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard2-rs3')
append_to_file_mongo_setting(test_config_file, 'mongo-oplog-shard3-rs3')
test_config_file.seek(0)
# config file processing
config = configparser.ConfigParser()
config.read_file(test_config_file)
pp = PrettyPrinter()
all_settings = load_mongo_replicas_from_setting(config, 'mongo-oplog')
pp.pprint(all_settings)
assert (3 == len(all_settings.keys()))
assert(sorted(all_settings.keys()) == \
sorted(['mongo-oplog-shard1',
'mongo-oplog-shard2',
'mongo-oplog-shard3']))
def printKey(key, XmlMap):
if XmlMap.has_key(key):
PP = PrettyPrinter()
print "%s:" % key
PP.pprint(XmlMap[key])
else:
print "entry %s not found" % key
def exec_evaluator(ground_truth_dicts, detect_dicts, image_id_gt_dict,
total_iter, batch_size):
import sys
if "numpy" in sys.modules:
del sys.modules["numpy"]
if "np" in sys.modules:
del sys.modules["np"]
import numpy as np
def setUp():
np.round = round
from coco_detection_evaluator import CocoDetectionEvaluator
setUp()
evaluator = CocoDetectionEvaluator()
for step in range(total_iter):
# add single ground truth info detected
# add single image info detected
if step in list(detect_dicts.keys()):
evaluator.add_single_ground_truth_image_info(
image_id_gt_dict[step], ground_truth_dicts[step])
evaluator.add_single_detected_image_info(image_id_gt_dict[step],
detect_dicts[step])
if (step + 1) * batch_size >= COCO_NUM_VAL_IMAGES:
metrics = evaluator.evaluate()
if metrics:
pp = PrettyPrinter(indent=4)
self.logfile.info("Metrics:\n" + pp.pprint(metrics))
self.logfile.info("Detection Dicts:\n" + pp.pprint(detect_dicts))
def get_straddle_by_ticker(conn, driver, conf, op, tick, i, total_count):
if not isSaved(conn=conn, ticker=tick):
print("{} of {}, currently {}".format(str(i), str(total_count), tick))
# op.goto(ticker=tick, date=None)
# time.sleep(1)
# ticker_straddle = {'meta-data': [
# '10-day-vol' : get_vol(arr, days=10),
# '1-day-vol' : get_vol(arr, days=1)],
# 'data': arr}
try:
arr = get_straddles(driver=driver, ticker=tick, conf=conf)
save_count = 0
for straddle in arr:
if conf['verbose']:
pp = PrettyPrinter()
pp.pprint(straddle)
if save_straddle(conn, straddle):
save_count += 1
print("saved {} records".format(save_count))
print(dt.now().strftime("%Y-%m-%d %H:%M"))
op.dates = None
except Exception as e:
print("Unexpected error:", sys.exc_info()[0])
print(e)
def _return(self, req, template, content_type='text/html'):
""" Wrap the return so that things are processed by Stan
"""
if req.args.has_key('hdfdump'):
# FIXME: the administrator should probably be able to disable HDF
# dumps
from pprint import PrettyPrinter
outstream = StringIO()
pp = PrettyPrinter(stream=outstream)
pp.pprint(req.standata)
content_type = 'text/plain'
data = outstream.getvalue()
outstream.close()
else:
ct = content_type.split('/')[0]
data = self._render(req.standata, template)
req.send_response(200)
req.send_header('Cache-control', 'must-revalidate')
req.send_header('Expires', 'Fri, 01 Jan 1999 00:00:00 GMT')
req.send_header('Content-Type', content_type + ';charset=utf-8')
req.send_header('Content-Length', len(data))
req.end_headers()
if req.method != 'HEAD':
req.write(data)
pass
class WaterBeaconTestCase(APITestCase):
# REF: https://waterbeacon.docs.apiary.io/#reference/0/data/submit-data
fixtures = [
'./app/fixtures/test_data.json',
'./news/fixtures/locations.json',
'./news/fixtures/test_data.json',
]
def setUp(self):
self.pp = PrettyPrinter(indent=4)
self.print_results = True
def print_response(self, response, title):
# prety print json output
if self.print_results:
self.pp.pprint(response.data)
def test_get_locations(self):
# ./manage.py test api.v1.tests.WaterBeaconTestCase.test_get_locations --keepdb --settings=settings.dev
client = APIClient()
params = {'sources': 'locations,news,utilities',
'violation': 'true' }
response = client.get('/v1/data/', data = params)
self.assertEqual(response.status_code, 200)
self.print_response(response, "test_get_locations")
self.assertEqual(response.data['meta']['locations'], 86)
self.assertEqual(response.data['meta']['cities'], 104)
self.assertEqual(response.data['meta']['utilities'], 2918)
def dprint(object, stream=None, indent=1, width=80, depth=None):
"""
A small addition to pprint that converts any Django model objects to dictionaries so they print prettier.
h3. Example usage
>>> from toolbox.dprint import dprint
>>> from app.models import Dummy
>>> dprint(Dummy.objects.all().latest())
{'first_name': u'Ben',
'last_name': u'Welsh',
'city': u'Los Angeles',
'slug': u'ben-welsh',
"""
# Catch any singleton Django model object that might get passed in
if getattr(object, '__metaclass__', None):
if object.__metaclass__.__name__ == 'ModelBase':
# Convert it to a dictionary
object = object.__dict__
# Catch any Django QuerySets that might get passed in
elif isinstance(object, QuerySet):
# Convert it to a list of dictionaries
object = [i.__dict__ for i in object]
# Pass everything through pprint in the typical way
printer = PrettyPrinter(stream=stream,
indent=indent,
width=width,
depth=depth)
printer.pprint(object)
def _collect_data(main_process, filename, frequency_seconds, abort_event):
while True:
try:
if abort_event.is_set():
return
# here we MUST collect stats on the MAIN process, the one that
# instantiated the callback
logs = log_all_tree(main_process)
with open(filename, 'w') as f:
f.write(f'Time={datetime.datetime.now()}\n')
f.write(f'Logging from Process={os.getpid()}\n')
pp = PrettyPrinter(width=200, stream=f)
pp.pprint(logs)
time.sleep(frequency_seconds)
except KeyboardInterrupt:
abort_event.set()
print(
f'CollectDataThread={threading.get_ident()} Stopped (KeyboardInterrupt)!!'
)
return
except Exception as e:
# exception is intercepted and skip to next job
print(
f'CollectDataThread={threading.get_ident()} Exception in background worker thread_id={os.getpid()}, E={e}'
)
continue
def _return(self, req, template, content_type='text/html'):
""" Wrap the return so that things are processed by Stan
"""
if req.args.has_key('hdfdump'):
# FIXME: the administrator should probably be able to disable HDF
# dumps
from pprint import PrettyPrinter
outstream = StringIO()
pp = PrettyPrinter(stream=outstream)
pp.pprint(req.standata)
content_type = 'text/plain'
data = outstream.getvalue()
outstream.close()
else:
ct = content_type.split('/')[0]
data = self._render(req.standata, template)
req.send_response(200)
req.send_header('Cache-control', 'must-revalidate')
req.send_header('Expires', 'Fri, 01 Jan 1999 00:00:00 GMT')
req.send_header('Content-Type', content_type + ';charset=utf-8')
req.send_header('Content-Length', len(data))
req.end_headers()
if req.method != 'HEAD':
req.write(data)
pass
def test_reservoir_map(self):
# test various "workloads". Make sure no pipeline doesn't deadlock
np.random.seed(0)
for i in range(20):
print(f'---------- {i} -------------')
time_sleep_1 = np.random.uniform(0.001, 0.5)
time_sleep_2 = np.random.uniform(0.001, 0.5)
nb_jobs_at_once = int(np.random.uniform(1, 5))
nb_indices = int(np.random.uniform(10, 40))
nb_workers = int(np.random.uniform(1, 4))
nb_epochs = int(np.random.uniform(1, 7))
split = {
'indices': np.asarray(list(range(nb_indices))),
}
split = trw.train.SequenceArray(
split, sampler=trw.train.SamplerSequential())
split = split.async_reservoir(max_reservoir_samples=nb_indices,
function_to_run=partial(
load_data,
time_sleep=time_sleep_1),
max_jobs_at_once=nb_jobs_at_once)
split = split.map(partial(create_value, time_sleep=time_sleep_2),
nb_workers=nb_workers)
nb = 0
for epoch in range(nb_epochs):
print('Epoch=', epoch)
for b in split:
if nb == 1:
logs = log_all_tree()
pp = PrettyPrinter(width=300)
pp.pprint(logs)
nb += 1
def test_changed_files(self):
'''
Gets the Information from the changed file and from ldap and the gpg repo
Validates if a file is encrypted for the right users
'''
config.init(conf_path=CONF_PATH)
util_crypt.update_git_repo(config.GPG_REPO, path=KEY_PATH)
files = construct_gpg_information("master")
printer = PrettyPrinter(indent=2)
for file_info in files:
print(file_info['filename'])
print("+--- Encrypted for {}:".format(file_info['encrypted_for']))
users = extract_subkey_for_every_user(file_info['users'])
if users:
for user, data in users.items():
for key in data.keys():
if key in file_info['encrypted_for']:
users[user][key]['encrypted_for'] = True
else:
pass
printer.pprint(users)
for user, data in users.items():
if user == 'tobiass' or user == 'sebastiann' or user == 'jo':
self.assertFalse(check_encrypted_for_user(data))
else:
self.assertTrue(check_encrypted_for_user(data))
else:
pass
class Map:
def __init__(self, dryrun, verbose):
self.dryrun = dryrun
self.verbose = verbose
self.prettyp = PrettyPrinter(indent=4)
self.cache = Cache()
def p(self, msg):
if self.verbose:
print(msg)
def pp(self, dict):
if self.verbose:
self.prettyp.pprint(dict)
def _get_client(self,
service_name,
profile_name="default",
region_name="us-east-1"):
session = boto3.Session(profile_name=profile_name,
region_name=region_name)
return session.client(service_name)
def _make_cache_key(self, profile_name, region_name, topic):
key = f"{profile_name}_{region_name}_{topic}"
return key
def _tags(self, tags):
ro = {}
for tag in tags:
ro[tag['Key']] = tag['Value']
return ro
def run_grid_search(pipeline, parameters) -> dict:
from pprint import PrettyPrinter
pp = PrettyPrinter()
print("parameters:\n ", end='')
pp.pprint(parameters)
grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, cv=num_cv_folds)
grid_search.fit( unzip(train_data,0), unzip(train_data,1) )
prediction = grid_search.predict(unzip(train_data,0))
print(classification_report(unzip(train_data,1), prediction, target_names=model_names))
best_parameters = grid_search.best_estimator_.get_params()
print("Best score : %0.3f" % grid_search.best_score_)
ret = {}
print("Best parameter:")
try:
for param_name in sorted(parameters.keys()):
print(" %s: %r" % (param_name, best_parameters[param_name]))
ret[param_name] = (best_parameters[param_name],)
except AttributeError:
for param_name in sorted(parameters[0].keys()):
print(" %s: %r" % (param_name, best_parameters[param_name]))
ret[param_name] = (best_parameters[param_name],)
print("")
return ret
def main(args=None):
parser = ArgumentParser(usage="Usage: %(prog)s [options]", description="query datacatalog")
parser.add_argument("-H","--host",dest='host',help="hostname of influxdb instance")
parser.add_argument("-u","--user",dest="user",help="username")
parser.add_argument("-p","--password",dest="pw",help="password")
parser.add_argument("-P","--port",dest="port",type=int,default=8086,help="influxdb ingest port")
parser.add_argument("-n","--dbname", dest="dbname",help="name of DB to store data in.")
parser.add_argument("-d", "--dry", dest="dry", action="store_true", default=False, help="do not report results to grafana")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False, help="verbose mode")
opts = parser.parse_args(args)
json_bdy = []
for site in batchSites:
status_dict = {key: 0 for key in statii}
stats = JobInstance.objects.filter(site=site).item_frequencies("status")
status_dict.update(stats)
for stat, freq in status_dict.iteritems():
json_bdy.append(__makeEntry__(stat, site, freq))
print 'found %i measurements to add'%len(json_bdy)
pp = PrettyPrinter(indent=2)
if opts.verbose: pp.pprint(json_bdy)
if opts.dry:
return
if influxdb:
client = InfluxDBClient(opts.host,opts.port,opts.user,opts.pw,opts.dbname)
client.create_database(opts.dbname)
ret = client.write_points(json_bdy)
if not ret:
try:
raise Exception("Could not write points to DB")
except Exception:
print_exc()
sys_exit(int(ret))
def main():
pp = PrettyPrinter()
parser = ArgumentParser(
description='The max flow algorithm using the MWU method')
parser.add_argument('--eps',
type=float,
default=.1,
help='the epsilon parameter of the MWU method')
parser.add_argument('--file',
type=str,
help='Number of breeds to be selected',
default='network.json')
args = parser.parse_args()
if args.eps < 0 or args.eps > 0.5:
raise ValueError('Argument --eps must be within 0 and 1/2')
with open(args.file, 'r') as inp:
flow_net = json.load(inp)
flow, flow_val = max_flow(flow_net['net'],
flow_net['source'],
flow_net['sink'],
eps=args.eps)
print('(Approximately) optimal flow value: {0:.2f}'.format(flow_val))
print('Flow:')
pp.pprint(flow)
def main():
from boto3 import session, ec2
from pprint import PrettyPrinter
utc = UTC()
pp = PrettyPrinter(indent=4)
args = get_args()
delta_kill = datetime.now(utc) - timedelta(hours=args.terminate)
delta_warn = datetime.now(utc) - timedelta(hours=args.warn)
session = session.Session(region_name=args.region,
profile_name=args.profile)
ec2client = session.client('ec2')
response = ec2client.describe_instances()
warn_instances = []
for reservation in response['Reservations']:
for instance in reservation['Instances']:
launchtime = instance['LaunchTime']
if launchtime < delta_kill and args.yes:
print("Terminating instance", instance['InstanceId'])
# ec2client.terminate_instances(InstanceIds=[instance[u'InstanceId']])
elif launchtime < delta_kill and not args.yes:
print("Skipping instance", instance['InstanceId'])
elif launchtime < delta_warn:
warn_instances += instance
if warn_instances:
print("The following instances are more than ", args.warn, "hrs old.")
pp.pprint(warn_instances)
def view_spreadsheet():
pp = PrettyPrinter()
pp.pprint(ss.get_records())
print("\n")
def main():
for n, q in enumerate(queries_tested_ok):
print("\nQuery n %d " % n)
query_meta = q
tq = type(query_meta)
if tq is str:
query = query_meta
vars = None
oper = None
if len(q.strip()) < 2:
continue
elif tq is dict:
query = query_meta.get('query', None)
vars = query_meta.get('variables', None)
oper = query_meta.get('operationName', None)
if query is None or vars is None:
continue
else:
query = ""
vars = None
oper = None
continue
result = client.execute(query, variables=vars, operationName=oper)
pp = PrettyPrinter(indent=4)
pp.pprint(result)
def addInteraction(self, label, paramsAndValues):
self.info[label]=paramsAndValues
f=file(self.databaseFile, 'w')
pp=PrettyPrinter(stream=f)
pp.pprint(self.info)
# print >>f, self.info
f.close()
def main():
directory = "../../figures/ihs_timings/"
try:
if not os.path.exists(directory):
os.makedirs(directory)
except OSError:
print('Creating directory. ' + directory)
# n_trials = time_error_ihs_grid['num trials']
# for n,d in itertools.product(time_error_ihs_grid['rows'],time_error_ihs_grid['columns']):
sklearn_lasso_bound = 5.0
sampling_factors = [5, 10]
for data in datasets.keys():
if data != 'specular':
continue
print("Plotting for {}".format(data))
print("-" * 80)
file_name = 'ihs_time_' + data + '.npy'
exp_results = np.load(file_name)[()]
pretty = PrettyPrinter(indent=4)
pretty.pprint(exp_results)
make_time_plots(data, directory, exp_results)
pass
def qfile(doprint, interactive, qfile):
global m
# Reinitialize 'm' every time this command is run. This isn't normally
# needed for command line use, but is important for the test suite.
m = []
printer = PrettyPrinter(indent=4)
with open(qfile, 'rb') as fp:
while True:
try:
m.append(pickle.load(fp))
except EOFError:
break
if doprint:
print(_('[----- start pickle -----]'))
for i, obj in enumerate(m):
count = i + 1
print(_('<----- start object $count ----->'))
if isinstance(obj, (bytes, str)):
print(obj)
else:
printer.pprint(obj)
print(_('[----- end pickle -----]'))
count = len(m) # noqa: F841
banner = _("Number of objects found (see the variable 'm'): $count")
if interactive:
interact(banner=banner)
def dprint(object, stream=None, indent=1, width=80, depth=None):
"""
A small addition to pprint that converts any Django model objects to dictionaries so they print prettier.
h3. Example usage
>>> from toolbox.dprint import dprint
>>> from app.models import Dummy
>>> dprint(Dummy.objects.all().latest())
{'first_name': u'Ben',
'last_name': u'Welsh',
'city': u'Los Angeles',
'slug': u'ben-welsh',
"""
# Catch any singleton Django model object that might get passed in
if getattr(object, '__metaclass__', None):
if object.__metaclass__.__name__ == 'ModelBase':
# Convert it to a dictionary
object = object.__dict__
# Catch any Django QuerySets that might get passed in
elif isinstance(object, QuerySet):
# Convert it to a list of dictionaries
object = [i.__dict__ for i in object]
# Pass everything through pprint in the typical way
printer = PrettyPrinter(stream=stream, indent=indent, width=width, depth=depth)
printer.pprint(object)
def main():
import sys
import pprint
from pprint import PrettyPrinter
lc = Local_code()
lc.make_local_code()
area_code = lc.find_area_code('강서구')
# area_code = lc.find_area_code('연수구')
# area_code = lc.find_area_code('수원시 영통구')
# area_code = lc.find_area_code('고양시 일산서구')
# area_code = lc.find_area_code('오산시')
# area_code = lc.find_area_code('파주시')
result = make_restfull_query(area_code, '201810')
print(result.status)
# print(result.data)
# print_xml(result.data)
apt_ = xml_to_dict(result.data)
print(lineno())
my_apt_ = find_my_apt(apt_, '가양동')
pp = PrettyPrinter(indent=4)
pp.pprint(my_apt_)
def setInteraction(self, label, paramsAndValues):
self.info[label] = paramsAndValues
f = file(self.databaseFile, 'w')
pp = PrettyPrinter(stream=f)
pp.pprint(self.info)
# print >>f, self.info
f.close()
def dump(ctx):
bld = ctx.exec_dict['bld']
suite = bld.env.orch_suite
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=2)
pp.pprint(suite)
sys.exit(0)
def prettyPrint(obj): """ Prints the dictionary real nice like. Argument: dictionary 'obj' """ pp = PrettyPrinter(indent=1) pp.pprint(obj)
def get_resource(self, arguments):
""" Gets the resource requested in the arguments """
user_id = arguments['--user_id']
object_id = arguments['--object_id']
start_date = arguments['--start_date']
end_date = arguments['--end_date']
detail = arguments['--detail']
misfit = Misfit(self.client_id, self.client_secret, self.access_token,
user_id)
if arguments['profile']:
result = misfit.profile(object_id)
elif arguments['device']:
result = misfit.device(object_id)
elif arguments['goal']:
result = misfit.goal(start_date, end_date, object_id)
elif arguments['summary']:
result = misfit.summary(start_date, end_date, detail)
elif arguments['session']:
result = misfit.session(start_date, end_date, object_id)
elif arguments['sleep']:
result = misfit.sleep(start_date, end_date, object_id)
pp = PrettyPrinter(indent=4)
if isinstance(result, list):
pp.pprint([res.data for res in result])
else:
pp.pprint(result.data)
class Action(object):
def __init__(self, config, stream):
self.program = config.pop(0)
self.action = config.pop(0)
self.arguments = config
self.stream = stream
self.writer = csv.writer(self.stream, delimiter=' ')
self.pp = PrettyPrinter(stream=self.stream)
def start(self):
if self.action in ['display', 'zip']:
print("#" ,end='', file=self.stream)
print(*self.arguments, file=self.stream)
print("", file=self.stream)
def __call__(self, result):
getattr(self, self.action)(result, *self.arguments)
def report(self, result):
print(result, file=self.stream)
def inspect(self, result):
self.pp.pprint(result.hash())
def display(self, result, *cols):
self.writer.writerow(map(lambda x: x if x is not None else 'None',
[result.datum(col) for col in cols]))
result.files.clear()
def name(self, result):
print(result.name, file=self.stream)
def accept(self, result):
acceptance = open(os.path.join(result.path, 'acceptance.txt'), 'w')
acceptance.write("Accepted: OK")
acceptance.close()
def reject(self, result):
acceptance = open(os.path.join(result.path,'acceptance.txt'), 'w')
acceptance.write("Accepted: NO")
acceptance.close()
def delete(self, result):
shutil.rmtree(result.path, ignore_errors=True)
def cat(self, result, *files):
for pattern in files:
for fn in glob.iglob(os.path.join(result.path, pattern)):
with file(fn) as f:
content = f.read()
print(content, file=self.stream)
def zip(self, result, *cols):
print("#%s" % result.name, file=self.stream)
self.writer.writerows(
zip(*[result.datum(col) for col in cols])
)
print("\n", file=self.stream)
def pprint(tree):
"""
Utility function to pretty print the history tree of a piece
:param tree: History tree of a piece
:return: None
"""
p = PrettyPrinter(indent=2)
p.pprint(tree)
def status():
""" returns current status of the instance """
config = get_config()
pp = PrettyPrinter(indent=4)
pp.pprint(config)
if has_state():
state = load_state()
pp.pprint(state)
def test():
print "RescueTime class TEST"
rt = RescueTime(beginDay="20141015",endDay="20141019")
pp = PrettyPrinter()
print "confirm what raw data are"
pp.pprint(rt.jsonRawData)
print "getAllData"
rt.getAllData([["Twitter","Twitter for Android"],["Hulu"]],["2","-1"],["General Software Development"])
def main():
"""
Tester for the validate method.
"""
filepath = os.path.join(os.path.dirname(__file__), 'test_data.json')
validated_data = validate(filepath)
pp = PrettyPrinter(indent=4)
pp.pprint(validated_data)
def __call__(self):
'''
Check the portlets defined here and in some sublevels
'''
self.results = {}
self.update_results(self.context)
printer = PrettyPrinter(stream=StringIO())
printer.pprint(self.results)
return printer._stream.getvalue()
class TweetRawPrinter(TweetListener):
def __init__(self):
TweetListener.__init__(self,"TweetRawPrinter")
self.pp = PrettyPrinter(indent=4)
def processTweet(self,tweet):
print "Raw Tweet: "
self.pp.pprint(tweet.__dict__)
print '-' * 20
def handle(self, *args, **options):
award_map = {}
for r in Resident.objects.all():
award_map[ r.name ] = []
for q in Question.objects.filter(istwoans=False):
result = feud.top_k_for_question(q, 1)
if len(result['answers']):
award_map[ result['answers'][0]['one'] ].append( str(result['answers'][0]['number']) + ' - ' + q.qtext )
pp = PrettyPrinter(indent=2)
pp.pprint(award_map)
def handle_disconnected(self, event):
endtime= int(time.time())
filename = "rodney.{0}.log".format(endtime)
f = open(filename, "w")
from pprint import PrettyPrinter
pp = PrettyPrinter(depth=10, stream=f)
pp.pprint(self.logs)
print "My transcript has been written to {0}".format(filename)
"""Quit the main loop upon disconnection."""
return QUIT
def pprint(tree):
"""
Utility function to pretty print the history tree of a piece.
Args:
tree (dict): History tree of a piece.
"""
p = PrettyPrinter(indent=2)
p.pprint(tree)
def main():
# Initialize the parser
parser = build_parser()
args = parser.parse_args()
# Initialize the STOMP connection
stomp_connection = syncstomp_from_args(args, 'chazelle-admin')
s_stomp = SynchronousStomp(stomp_connection)
proxy = RpcProxy(args.destination, s_stomp)
# Reload if requested
if args.reload:
proxy.reload()
# Load configs
cfg = {}
cfg = {cfg_name: proxy.config(cfg_name) for cfg_name in CONFIGS}
# Pretty-print the configs
pp = PrettyPrinter(indent=2)
pruned = {}
for cfg_name, cfg_data in cfg.items():
if 'users' not in cfg_name:
pruned[cfg_name] = cfg_data
pp.pprint(pruned)
# Load proxies
print ''
u_engine_dst = cfg['queue']['u_server']
q_server_dst = cfg['queue']['q_server']
u_engine = RpcProxy(u_engine_dst, s_stomp)
print 'u_engine:'
explain(u_engine)
print 'q_server:'
q_server = RpcProxy(q_server_dst, s_stomp)
explain(q_server)
# Load teams
teams = list(u_engine.get_team_state_report().keys())
# Prepare admin commands
cmds = AdminCommands(u_engine, q_server)
# Drop to shell
from IPython.frontend.terminal.embed import InteractiveShellEmbed
ipshell = InteractiveShellEmbed()
local_ns = {'stomp_connection': stomp_connection,
's_stomp': s_stomp,
'cfg': cfg,
'u_engine': u_engine,
'q_server': q_server,
'teams': teams,
'cmds': cmds,
}
ipshell(local_ns=local_ns)
def dump(filename, start='start', formatter=None):
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=2)
if not formatter:
prefix ='/tmp/simple'
formatter = PkgFormatter(prefix=prefix, PREFIX=prefix)
data = load(filename, start=start, formatter=formatter)
print ('Starting from "%s"' % start)
pp.pprint(data)
def save (self, name):
f = open (name, 'w')
pp = PrettyPrinter (stream = f, indent = 2)
f.write ('nmoods = ')
pp.pprint (self.nmoods)
f.write ('\nchildren = ')
pp.pprint (self.children)
f.close ()
f = None
def main(cls):
"""
Parse command-line arguments, construct appropriate Marble subclass,
and either train / test the model.
"""
parser = OptionParser(usage="usage: prog <train|test> [options]")
parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False,
help="print status messages to stdout")
parser.add_option("-a", "--artists", dest="max_artists", type='int', default=sys.maxint,
help="number of artists to run")
parser.add_option("-c", "--conf", dest="conf", default="conf.json",
help="location of the mlp json config file, specifying model parameters")
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.print_help()
sys.exit(1)
else:
# train | test
mode = sys.argv[1]
# make sure it's either train | test
if mode != "train" and mode != "test":
parser.print_help()
sys.exit(1)
sys.stderr.write("mode = " + mode + "\n")
# open and parse the config file
with open(options.conf) as f:
conf = json.load(f)
# construct a pretty printer
pp = PrettyPrinter(stream=sys.stderr)
# log the parameters used
sys.stderr.write("Using parameters from " + options.conf + ":\n")
pp.pprint(conf)
# Verify that all paths are valid
verify_paths(conf["paths"],mode=mode)
# construct the marble
d = cls(conf,mode,verbose=options.verbose,max_artists=options.max_artists)
# test / train as appropriate
if mode == "train":
d.train()
else:
d.test()
def handle(self, *args, **options):
self.cursor = connections['default'].cursor()
results = {}
results['contracts_count'] = int(self.get_table_count('contracts_contract')[0])
results['grants_count'] = int(self.get_table_count('grants_grant')[0])
results['agg_sp_org_count'] = int(self.get_table_count('agg_spending_org')[0])
results['agg_sp_totals_count'] = int(self.get_table_count('agg_spending_totals')[0])
results['assoc_sp_contracts_count'] = int(self.get_table_count('assoc_spending_contracts')[0])
results['assoc_sp_grants_count'] = int(self.get_table_count('assoc_spending_grants')[0])
pp = PrettyPrinter()
pp.pprint(results)
def _display_experiment(exp):
pp = PrettyPrinter(indent=4)
pp.pprint("Experiment: " + exp._uri)
print_entity_info(exp, info_indent=6)
for s in exp.scans():
pps = PrettyPrinter(indent=6)
pps.pprint("Scan: " + s.id())
print_entity_info(s, info_indent=8)
for r in s.resources():
for f in r.files():
print("File: " + f._uri)
def handle(self, *args, **options):
self.cursor = connections['default'].cursor()
results = {}
results['lobbying_count'] = int(self.get_table_count('lobbying_lobbying')[0])
results['agency_count'] = int(self.get_table_count('lobbying_agency')[0])
results['lobbyist_count'] = int(self.get_table_count('lobbying_lobbyist')[0])
results['issue_count'] = int(self.get_table_count('lobbying_issue')[0])
results['bill_count'] = int(self.get_table_count('lobbying_bill')[0])
results['billtitle_count'] = int(self.get_table_count('lobbying_billtitle')[0])
pp = PrettyPrinter()
pp.pprint(results)
