def test_summary(mock_request):
# Given
url = 'http://www.google.com'
mock_response = Response()
mock_response.headers.get = MagicMock(return_value = 'html')
mock_response.url = url
mock_response.encoding = 'UTF-8'
mock_response.consumed = False
mock_response.raw = MagicMock()
mock_response.iter_content = lambda s: ['<html><head><title>Test Title</head><body></body></html>']
mock_request.return_value = mock_response
# When
summ = Summary(url)
# summ._html = '<html><head><title>Test Title</head><body></body></html>'
summ.extract()
# Then
# mock_response.raw.close.assert_called_with()
assert summ.title == 'Test Title'
def actionRunUnitTests():
"""
Run all specified unit tests.
"""
UnitTestRunner.init()
for target in Settings.targets:
for platform in Settings.targetPlatforms:
for cpu in Settings.targetCPUs:
for configuration in Settings.targetConfigurations:
if not Summary.checkIfActionFailed(ACTION_BUILD, target,
platform, cpu,
configuration):
Logger.printStartActionMessage(
'Running unit tests for ' + target + ' ' +
platform + ' ' + cpu + ' ' + configuration,
ColoredFormatter.YELLOW)
result = UnitTestRunner.run(target, platform, cpu,
configuration)
Summary.addSummary(ACTION_RUN_UNITTESTS, target,
platform, cpu, configuration,
result,
UnitTestRunner.executionTime)
if result != NO_ERROR:
Logger.printEndActionMessage(
'Failed to execute unit tests!')
else:
Logger.printEndActionMessage(
'Executed all unit tests')
def column_distributions(request):
username = request.GET.get("username")
dbname = request.GET.get("db", "intel")
tablename = request.GET.get("table", "readings")
where = request.GET.get("where", "")
try:
nbuckets = int(request.GET.get("nbuckets", 100))
except Exception as e:
print e
nbuckets = 100
full_tablename = "%s.%s" % (dbname, tablename)
summary = Summary(dbname, full_tablename, username, nbuckets=nbuckets, where=where)
print "where: %s" % where
try:
stats = summary()
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = []
for col, typ, col_stats in stats:
data.append({"col": col, "type": typ, "stats": col_stats})
context = {"data": data}
return context
def update_summary(self, batch: namedtuple):
"""
Add batch, then add or update summary in DB
:param batch:
:return:
"""
self.add_batch(batch)
summary = Summary(batch)
if summary.exists():
summary_data = summary.data()
summary_info = summary.misc_data()
batch_id = batch.batch_name
summary_line = (self.session
.query(PGSummary)
.filter_by(
batch_id=batch_id,
date=summary_info['date'],
time=summary_info['time']
)
.one_or_none())
if summary_line:
for key, val in summary_data._asdict().items():
setattr(summary_line, key, val)
summary_line.gate_homes = summary_info['gate_homes']
summary_line.lost_homing = summary_info['lost_homing']
else:
summary_line = PGSummary(
batch_id=batch_id, **summary_data._asdict(), **summary_info
)
self.session.add(summary_line)
self.session.flush()
return summary_line
def main(target_path: str = '', process: int = None):
if target_path == '':
_path = os.getcwd()
else:
_path = target_path
if not process:
# TODO: Set Proper Process Number for ProcessingPool!!!
num_pool = 8
else:
num_pool = process
t_total = TimeMonitor('\nTotal Time')
pool = multiprocessing.Pool(num_pool)
path_list = get_all_files(_path)
for each_path in path_list:
pool.apply_async(func=single_process, args=(each_path, ))
pool.close()
pool.join()
Summary.to_excel()
t_total.show()
print('------------------------------')
print('Finished!'.center(30))
print('------------------------------')
def column_distributions(request):
username = request.GET.get('username')
dbname = request.GET.get('db', 'intel')
tablename = request.GET.get('table', 'readings')
where = request.GET.get('where', '')
try:
nbuckets = int(request.GET.get('nbuckets', 100))
except Exception as e:
print e
nbuckets = 100
full_tablename = "%s.%s" % (dbname, tablename)
summary = Summary(dbname, full_tablename, username, nbuckets=nbuckets, where=where)
print 'where: %s' % where
try:
stats = summary()
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = []
for col, typ, col_stats in stats:
data.append({
'col': col,
'type': typ,
'stats': col_stats
})
context = { "data": data }
return context
def column_distributions(request):
username = request.GET.get('username')
dbname = request.GET.get('db', 'intel')
tablename = request.GET.get('table', 'readings')
where = request.GET.get('where', '')
try:
nbuckets = int(request.GET.get('nbuckets', 100))
except Exception as e:
print e
nbuckets = 100
full_tablename = "%s.%s" % (dbname, tablename)
summary = Summary(dbname,
full_tablename,
username,
nbuckets=nbuckets,
where=where)
print 'where: %s' % where
try:
stats = summary()
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = []
for col, typ, col_stats in stats:
data.append({'col': col, 'type': typ, 'stats': col_stats})
context = {"data": data}
return context
def summary(url): if utils.is_youtube(url): return summary_youtube(url) # all non-youtube summ = Summary(url) summ.extract() return (summ.title, summ.description, summ.image.url)
def column_distributions():
dbname = request.args.get('db', 'intel')
tablename = request.args.get('table', 'readings')
where = request.args.get('where', '')
try:
nbuckets = int(request.args.get('nbuckets', 100))
except Exception as e:
print e
nbuckets = 100
# #from monetdb import sql as msql
# #db = msql.connect(user='******', password='******', database=dbname)
summary = Summary(g.db, tablename, nbuckets=nbuckets, where=where)
print 'where: %s' % where
try:
stats = summary()
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = []
for col, typ, col_stats in stats:
data.append({
'col': col,
'type': typ,
'stats': col_stats
})
context = { "data": data }
return context
def actionCreateNuget():
CreateNuget.init()
for target in Settings.targets:
for platform in Settings.targetPlatforms:
Logger.printStartActionMessage('Create Nuget for ' + target)
if checkIfBuildWasSuccessful() == True:
result = CreateNuget.run(target, platform, Settings.targetCPUs,
Settings.targetConfigurations,
Settings.nugetFolderPath,
Settings.nugetVersionInfo)
else:
result = ERROR_BUILD_FAILED
Logger.printColorMessage(
'Create Nuget cannot run because build has failed',
ColoredFormatter.YELLOW)
Logger.printEndActionMessage(
'Create Nuget not run for ' + target + ' ' + platform,
ColoredFormatter.YELLOW)
CreateNuget.executionTime = 0
Summary.addNugetSummary(target, platform, result,
CreateNuget.executionTime)
if result != NO_ERROR:
Logger.printEndActionMessage(
'Failed to create NuGet package ' + target + ' ' +
platform, ColoredFormatter.RED)
#Terminate script execution if stopExecutionOnError is set to True in userdef
shouldEndOnError(result)
else:
Logger.printEndActionMessage('Create Nuget for ' + target +
' ' + platform)
def summarize(urls):
"""
Calls extract for each of the URLs,
Returns the list of Extracted instances as summaries,
the result of the process, and the speed.
"""
import time
from summary import Summary
fails = 0
err = lambda e: e.__class__.__name__
summaries = []
start = time.time()
for url in urls:
try:
print "-> %s" % url
summary = Summary(url)
summary.extract()
except KeyboardInterrupt:
break
except Exception, e:
fails += 1
summary = {
'titles': ["[%s]" % err(e)],
'urls': [url],
'descriptions': [str(e)],
'source': url,
}
print "[%s] (%s): %s" % (err(e), e, url)
summaries.append(summary)
end = time.time()
def shouldEndOnError(error):
"""
Terminates script execution if stopExecutionOnError is set to True in userdef
"""
if Settings.stopExecutionOnError:
System.stopExecution(error)
Summary.printSummary()
def column_distribution(request):
username = request.GET.get('username')
dbname = request.GET.get('db', 'intel')
tablename = request.GET.get('table', 'readings')
where = request.GET.get('where', '')
col = request.GET.get('col')
try:
nbuckets = int(request.GET.get('nbuckets', 100))
except Exception as e:
print e
nbuckets = 100
full_tablename = "%s.%s" % (dbname, tablename)
summary = Summary(dbname, full_tablename, username, nbuckets=nbuckets, where=where)
try:
typ = summary.get_type(col)
stats = summary.get_col_stats(col, typ)
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = {
'col': col,
'type': typ,
'stats': stats
}
context = { "data": data }
return context
def summary(url):
if utils.is_youtube(url):
return summary_youtube(url)
# all non-youtube
summ = Summary(url)
summ.extract()
return (summ.title, summ.description, summ.image.url)
def _build_summary(self, summary_step):
if self.with_summary is not True:
return
self.summary = Summary(self.sess, summary_step)
for s in self.for_summary_scalar:
self.summary.scalar(s)
for h in self.for_summary_hist:
self.summary.hist(h)
self.summary.merge()
def this_test_fails_test_summary_a16z_com():
# Given
url = 'http://a16z.com/2015/01/30/a16z-podcast-mobile-is-eating-the-world-and-apple-is-gobbling-fastest/'
# When
summ = Summary(url)
summ.extract()
# Then
assert summ.image.url == "http://i1.sndcdn.com/artworks-000105093900-p7g3yz-t500x500.jpg"
def get_schema(db_or_name, table):
summary = Summary(db_or_name, table)
try:
cols_and_types = summary.get_columns_and_types()
schema = dict(cols_and_types)
return schema
except Exception as e:
traceback.print_exc()
finally:
summary.close()
return {}
def test_summary_misc_data_bjkl(self):
test_segments = TestAid().test_segments_dict()
summary = Summary(test_segments['bjkl15'])
test_dict = {
'date': datetime(year=2016, month=7, day=15),
'time': time(hour=13, minute=17),
'lost_homing': 0,
'gate_homes': 0,
}
for key, value in test_dict.items():
self.assertEqual(value, summary.misc_data()[key])
def ignore_test_summary_gv_com():
# Given
url = 'http://www.gv.com/lib/the-product-design-sprint-a-five-day-recipe-for-startups'
# When
summ = Summary(url)
summ.extract()
# Then
assert summ.image.url == 'http://img.gv.com/wp-content/uploads/2012/10/sprint.jpg?fit=crop&crop=faces&w=500&h=500&fm=jpg&s=0452fc00eb1557bc421f8975be7850b7'
def test_summary_fcw_com():
# Given
url = 'http://fcw.com/articles/2015/01/30/health-it.aspx'
# When
summ = Summary(url)
summ.extract()
# Then
assert summ.image.url == "http://fcw.com/~/media/GIG/FCWNow/Topics/Health/stethoscope.png"
def this_fails_test_summary_forbes_com():
# Given
url = 'http://www.creativefreedom.co.uk/wp-content/uploads/2014/04/facebook-buys-oculus-virtual-reality-gaming-startup-for-2-billion'
# When
summ = Summary(url)
summ.extract()
# Then
assert summ.image.url == "http://blogs-images.forbes.com/briansolomon/files/2014/03/f0d9530bfbeede89ff78fb4a2dc156ae-e1395947625586.jpg"
def test_summary_github_com():
# Given
url = 'https://github.com/blog/1862-introducing-a-simpler-faster-github-for-mac'
# When
summ = Summary(url)
summ.extract()
# Then
#assert summ.image.url == "https://cloud.githubusercontent.com/assets/22635/3517580/2399aa10-06f1-11e4-8671-0923504c594a.png"
assert summ.image.url == "https://github.com/apple-touch-icon-144.png"
def update_pid(pid, bid, limit, prover, results, dohtml=True):
PidResults(pid,prover).update(results)
solved = [x for x in results if results[x].solved(limit)]
Solved("protos",bid,limit,prover).update(pid,solved)
summary = Summary("protos",bid,limit).update(pid,prover,results)
if dohtml: html.create_summary("protos",bid,limit,summary,usehash=True)
# additionally update data for lower available time limits
for lim in limits("protos",bid,prover):
if lim < limit:
solved = [x for x in results if results[x].solved(lim)]
Solved("protos",bid,lim,prover).update(pid,solved)
summary = Summary("protos",bid,lim).update(pid,prover,results)
if dohtml: html.create_summary("protos",bid,lim,summary,usehash=True)
def __init__(self,
level_filepath,
episodes=30000,
initial_epsilon=1.,
min_epsilon=0.1,
exploration_ratio=0.5,
max_steps=2000,
render_freq=500,
enable_render=True,
render_fps=20,
save_dir='checkpoints',
enable_save=True,
save_freq=500,
gamma=0.99,
batch_size=64,
min_replay_memory_size=1000,
replay_memory_size=100000,
target_update_freq=5,
seed=42):
self.set_random_seed(seed)
self.episodes = episodes
self.max_steps = max_steps
self.epsilon = initial_epsilon
self.min_epsilon = min_epsilon
self.exploration_ratio = exploration_ratio
self.render_freq = render_freq
self.enable_render = enable_render
self.render_fps = render_fps
self.save_dir = save_dir
self.enable_save = enable_save
self.save_freq = save_freq
if enable_save and not os.path.exists(save_dir):
os.makedirs(save_dir)
level_loader = LevelLoader(level_filepath)
self.agent = DQNAgent(level_loader.get_field_size(),
gamma=gamma,
batch_size=batch_size,
min_replay_memory_size=min_replay_memory_size,
replay_memory_size=replay_memory_size,
target_update_freq=target_update_freq)
self.env = Snake(level_loader)
self.summary = Summary()
self.current_episode = 0
self.max_average_length = 0
self.epsilon_decay = (initial_epsilon -
min_epsilon) / (exploration_ratio * episodes)
def actionBuild():
"""
Build all specified targets for all specified platforms.
"""
#Init builder logger
Builder.init()
for target in Settings.targets:
targetsToBuild, combineLibs, copyToOutput = Builder.getTargetGnPath(
target)
for platform in Settings.targetPlatforms:
for cpu in Settings.targetCPUs:
if System.checkIfCPUIsSupportedForPlatform(cpu, platform):
for configuration in Settings.targetConfigurations:
if not Summary.checkIfActionFailed(
ACTION_PREPARE, target, platform, cpu,
configuration):
Logger.printStartActionMessage(
'Build ' + target + ' ' + platform + ' ' +
cpu + ' ' + configuration,
ColoredFormatter.YELLOW)
result = Builder.run(target, targetsToBuild,
platform, cpu, configuration,
combineLibs, copyToOutput)
Summary.addSummary(ACTION_BUILD, target, platform,
cpu, configuration, result,
Builder.executionTime)
if result != NO_ERROR:
Logger.printEndActionMessage(
'Failed building ' + target + ' ' +
platform + ' ' + cpu + ' ' + configuration,
ColoredFormatter.RED)
#Terminate script execution if stopExecutionOnError is set to True in userdef
shouldEndOnError(result)
else:
Logger.printEndActionMessage('Build ' +
target + ' ' +
platform + ' ' +
cpu + ' ' +
configuration)
else:
Logger.printColorMessage(
'Build cannot run because preparation has failed for '
+ target + ' ' + platform + ' ' + cpu + ' ' +
configuration, ColoredFormatter.YELLOW)
Logger.printEndActionMessage(
'Build not run for ' + target + ' ' +
platform + ' ' + cpu + ' ' + configuration,
ColoredFormatter.YELLOW)
def parse_summary(row):
"""摘要处理"""
obj = Summary()
data = {}
for item in field_tags:
s_field, e_field = item
text = re.sub('\<.+?\>', '', row[s_field])
summary = obj.summary(text, phrases_num=3, sentence_num=12)
if len(summary) < 15:
continue
data[e_field] = summary
data['have_summary'] = 1 if len(data) else -1
return data
class TestRunner(object):
def __init__(self, run_test_method):
self.summary = Summary()
self.run_test = run_test_method
self.exception_suppressed = False
def run(self):
self.run_test(self)
return self.summary
def run_test(self):
raise Exception('Do not call this run_test method!')
def test(self, test_value):
self.summary.test(test_value)
def increment_pass(self):
self.summary.increment_pass()
def increment_fail(self):
self.summary.increment_fail()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type != None:
self.summary.append_error(
'Failed test with %s:%s\n%s\n' % (exc_type, exc_value, traceback))
return self.exception_suppressed
return True
class TestRunner(object):
def __init__(self, run_test_method):
self.summary = Summary()
self.run_test = run_test_method
self.exception_suppressed = False
def run(self):
self.run_test(self)
return self.summary
def run_test(self):
raise Exception('Do not call this run_test method!')
def test(self, test_value):
self.summary.test(test_value)
def increment_pass(self):
self.summary.increment_pass()
def increment_fail(self):
self.summary.increment_fail()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type != None:
self.summary.append_error('Failed test with %s:%s\n%s\n' %
(exc_type, exc_value, traceback))
return self.exception_suppressed
return True
def load_feats(self, mod=dict(kp_algo='FAST', des_algo='SIFT')):
su = Summary()
sin = su.info(self.root, self.name).iloc[0]
ns = sin.n_slides
key_root = "/{kp_algo}/{des_algo}/slide".format(**mod)
sfd = []
for si in range(1, ns + 1):
kk = key_root + "/kp/i{:03d}".format(si)
dk = key_root + "/desc/i{:03d}".format(si)
kps = self._unpickle_keypoints(self.load(kk))
des = self.load(dk)
des = None if des is None else des.values
sfd.append(dict(idx=si, kps=kps, des=des))
return sfd
def onGenerateLimitGraphs(self, event):
if self.currentFilePath is None:
return
try:
dev_limit = float(self.textEntryDEV.GetLineText(0))
rms_limit = float(self.textEntryRMS.GetLineText(0))
except ValueError:
#TODO create ALERT
return
results = ReadCSVWithLimit(self.currentFilePath, dev_limit, rms_limit)
self.summary = Summary(numHEADs=len(self.master),
dev=results[0],
rms=results[1])
self.summaryCanvas = FigureCanvas(self.summaryPanel, 1,
self.summary.PlotSummaryGraphs())
def load_feats(self,
mod=dict(kp_algo='FAST', des_algo='SIFT')):
su = Summary()
sin = su.info(self.root, self.name).iloc[0]
ns = sin.n_slides
key_root = "/{kp_algo}/{des_algo}/slide".format(**mod)
sfd = []
for si in range(1, ns+1):
kk = key_root + "/kp/i{:03d}".format(si)
dk = key_root + "/desc/i{:03d}".format(si)
kps = self._unpickle_keypoints(self.load(kk))
des = self.load(dk)
des = None if des is None else des.values
sfd.append(dict(idx=si, kps=kps, des=des))
return sfd
def extract():
url = request.args.get('url')
# g = Goose({'browser_user_agent': 'Mozilla'})
# print 'url', url
# article = g.extract(url=url)
# summarised_article = Summary({'title' : article.title , 'content' : article.cleaned_text})
#
article = Article(url)
article.download()
article.parse()
# print article.title
# print article.text
summarised_article = Summary({'title' : article.title , 'content' : article.text})
# summarised_article.print_result() if article.text else {}
return jsonify(**summarised_article.get_result())
def main():
opt = parse_opt()
use_cuda = torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
env = gym.make(game)
seed = 7122
env.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
agent = DQN(env, opt, device=device)
agent.network.apply(weights_init)
agent.sync_weight()
progress = trange(opt.episode, ascii=True)
summary = Summary()
last_rewards = 0
for episode in progress:
# Training
state = env.reset()
for s in range(opt.max_step):
# use epsilon-greedy in training
action = agent.egreedy_action(state)
next_state, reward, done, _ = env.step(action)
loss = agent.perceive(state, action, reward, next_state, done)
state = next_state
if done:
break
summary.add(episode, 'loss', loss)
# Testing
if opt.test_interval > 0 and (episode + 1) % opt.test_interval == 0:
rewards = 0
for t in trange(opt.test, ascii=True, leave=False):
state = env.reset()
for s in range(opt.max_step):
action = agent.action(state)
next_state, reward, done, _ = env.step(action)
state = next_state
rewards += reward
if done:
break
if opt.test > 0:
rewards /= opt.test
last_rewards = rewards
summary.add(episode, 'reward', rewards)
progress.set_description('Loss: {:.4f} | Reward: {:2}'.format(
loss, last_rewards))
if opt.log:
summary.write(opt.log)
def query_leaderboard_moves(period):
'''
return leaderboard entries for period from Moves storyline API
Moves: GET /user/storyline/daily (period)
'''
entries = []
users = store.get_all_users()
for user in users:
access_token = user['access_token']
first_date = user['first_date']
# validate user access_token
if validate_access_token(user, access_token) is False:
continue
# validate period for user
if utils.validate_period(period, first_date) is False:
continue
# Moves: GET /user/storyline/daily (day/week/month)
storyline = moves.user_storyline_daily(period, trackPoints={'false'}, access_token=access_token)
# sum all trips of the period (day/week/month) for each user
summary = Summary.fromstoryline(storyline, user, first_date)
entry = summary.format()
entries.append(entry)
return entries
def __init__(self, options, session):
self._options = options
self._session = session
np.random.seed(options.seed)
tf.set_random_seed(options.seed)
self.input_size = unet.input_size_needed(options.patch_size,
options.num_layers)
self.experiment_name = datetime.now().strftime("%Y-%m-%dT%Hh%Mm%Ss")
experiment_path = os.path.abspath(
os.path.join(options.save_path, self.experiment_name))
summary_path = os.path.join(options.logdir, self.experiment_name)
self._summary = Summary(options, session, summary_path)
self.build_graph()
def export_to_excel(self):
"""
Function which is called when user click export to excel button, export report to excel
"""
if self.loaded:
s = Summary(self.p)
t = s.export_to_excel()
if t == "":
tkinter.messagebox.showinfo("Blast", "Your didn't choose file")
else:
tkinter.messagebox.showinfo(
"Blast", "Your data has been saved in excel")
else:
tkinter.messagebox.showinfo("Blast",
"Data has not loaded, try again!")
def query_leaderboard_moves(period):
'''
return leaderboard entries for period from Moves storyline API
Moves: GET /user/storyline/daily (period)
'''
entries = []
users = store.get_all_users()
for user in users:
access_token = user['access_token']
first_date = user['first_date']
# validate user access_token
if validate_access_token(user, access_token) is False:
continue
# validate period for user
if utils.validate_period(period, first_date) is False:
continue
# Moves: GET /user/storyline/daily (day/week/month)
storyline = moves.user_storyline_daily(period,
trackPoints={'false'},
access_token=access_token)
# sum all trips of the period (day/week/month) for each user
summary = Summary.fromstoryline(storyline, user, first_date)
entry = summary.format()
entries.append(entry)
return entries
def train(gan, generator_noise_input, batch_handler, config):
summary_writer = Summary(config)
for batch_number, batch in enumerate(
batch_handler.get_next_batch(config["batch_size"])):
generator_output = gan.generate(generator_noise_input)
generator_loss, discriminator_loss = gan.train(generator_output, batch)
summary_writer.update_statistics(generator_loss, discriminator_loss,
batch_number)
if batch_number % config["report_batch_interval"] == 0:
summary_writer.print_statistics()
summary_writer.write_statistics()
if batch_number % config["save_batch_interval"] == 0:
gan.save()
summary_writer.write_images(generator_output, batch)
def RunSingle(self):
global sim_results
sim_results = []
StartSimulations(self.sim_configs, cache_config['memory'])
Summary(sim_results, self.title)
def __init__(self, options, session):
self._options = options
self._session = session
self.train_images_shape = None
np.random.seed(options.seed)
tf.set_random_seed(options.seed)
self.input_size = self._options.patch_size
self.experiment_name = datetime.now().strftime("%Y%m%d%H%M%S")
experiment_path = os.path.abspath(
os.path.join(options.save_path, self.experiment_name))
self.summary_path = os.path.join(
options.logdir, self.experiment_name + options.log_suffix)
self._summary = Summary(options, session)
self.build_graph()
def test_summary_data_bjkl(self):
test_dict = {
'inspected': 2,
'good': 1,
'good_percent': 50.0,
'fail_general': 1,
'fail_gen_percent': 50.0,
'fail_od': 0,
'fail_od_percent': 0.0,
'fail_backward': 0,
'fail_backward_percent': 0.0,
'n_a': 0,
'n_a_percent': 0.0
}
test_segments = TestAid().test_segments_dict()
summary = Summary(test_segments['bjkl15'])
for key, value in test_dict.items():
self.assertEqual(value, getattr(summary.data(), key))
def o_slides(self, gray=False, resize=None):
"""
Get slide images collection
"""
spm = self.slides_path(size='big')
su = Summary()
sin = su.info(self.root, self.name)
if resize is True:
resize = (sin.v_width, sin.v_height)
for si in range(1, sin.n_slides + 1):
sp = "{}/{:03d}.jpg".format(spm, si)
if gray:
img = cv2.imread(sp, cv2.COLOR_GRAY2BGR)
else:
img = cv2.imread(sp)
if resize is not None:
img = cv2.resize(img, resize)
yield (dict(img=img, idx=si))
def o_slides(self, gray=False, resize=None):
"""
Get slide images collection
"""
spm = self.slides_path(size='big')
su = Summary()
sin = su.info(self.root, self.name)
if resize is True:
resize = (sin.v_width, sin.v_height)
for si in range(1, sin.n_slides+1):
sp = "{}/{:03d}.jpg".format(spm, si)
if gray:
img = cv2.imread(sp, cv2.COLOR_GRAY2BGR)
else:
img = cv2.imread(sp)
if resize is not None:
img = cv2.resize(img, resize)
yield(dict(img=img, idx=si))
def actionCreateNuget():
CreateNuget.init()
for target in Settings.targets:
Logger.printStartActionMessage('Create Nuget for ' + target)
result = CreateNuget.run(target, Settings.targetPlatforms,
Settings.targetCPUs,
Settings.targetConfigurations,
Settings.nugetFolderPath,
Settings.nugetVersionInfo)
Summary.addNugetSummary(target, result, CreateNuget.executionTime)
if result != NO_ERROR:
Logger.printEndActionMessage(
'Failed to create NuGet package ' + target,
ColoredFormatter.RED)
#Terminate script execution if stopExecutionOnError is set to True in userdef
shouldEndOnError(result)
else:
Logger.printEndActionMessage('Create Nuget for ' + target)
def stream_terms(k):
tot_time = 0
with open(file) as f:
lines = f.read().splitlines()
for line in lines:
start_time = time.time()
parts = line.split("\t")
text = parts[0].lower()
lat = parts[1]
lon = parts[2]
cell_id = math.floor(float(lat)).__str__() + "/" + math.floor(float(lon)).__str__()
# cell = map_cells[cell_id]
terms = text.split(" ")
# pdb.set_trace()
for item in stop:
if item in terms:
terms = list(filter((item).__ne__, terms))
for item in terms:
if item == '':
continue
if len(item) == 1:
continue
if item in all_terms:
# print(item)
if item in map_terms.keys():
map_terms[item].update_summary(cell_id)
# map_terms[item].seen_cells.add(cell_id)
else:
temp_summary = Summary(k)
temp_summary.update_summary(cell_id)
map_terms.update({item : temp_summary})
# map_terms[item].seen_cells.add(cell_id)
# if item == 'paris':
# map_terms[item].update_replacements()
# print(item)
# print(time.time() - start_time)
tot_time += (time.time() - start_time)
print(tot_time/len(lines))
def load_link(self, session, url):
"""
Load link from URL.
Get it from database, or use `summary` to create it. The returned
`link.url` is most probably different from the `url` input param.
When `redo` mode perform resummarization and update current link.
"""
def check_url(url):
"""
Raise for existing or restricted URLs.
Param `url` is actually `summary.url` while extracting.
"""
logger.debug("Checking")
link = self.get_link(session, url)
if link: # existing
raise ExistingLinkException()
if self.redo: # redo
s = Summary(url)
logger.debug("Resummarizing")
s.extract()
logger.debug("Resummarized")
link = self.status.link # current
if link and \
link.url == s.url: # same url
link.load(s)
# link.updated = True
result = UPDATED_LINK
else: # new url
link = Link(s)
result = NEW_LINK
session.add(link)
else: # just do
link = self.get_link(session, url)
if link: # existing
result = EXISTING_LINK
else: # summarize
s = Summary(url)
try:
logger.debug("Summarizing")
s.extract(check_url=check_url)
logger.debug("Summarized")
except ExistingLinkException: # existing
link = self.get_link(session, s.url)
result = EXISTING_LINK
# except RestrictedLinkException:
# result = RESTRICTED_LINK
else: # new
link = Link(s)
session.add(link)
result = NEW_LINK
return link, result
def setUp(self):
# A simple test case set up on two symbols. The first four elements are
# valid test data, but trade_data[4] has a numeric symbol.
self.test = Summary("", "")
# Each list within the trade_data list has the form:
# [timeStamp, symbol, quantity, price]
self.test.trade_data = [['1', 'aaa', '1', '0'], ['2', 'aaa', '1', '1'],
['2', 'bbb', '2', '2'], ['5', 'aaa', '1', '3'],
['7', '123', '2', '3'], ['9', '', '1', '1'],
['10', 'ddd', 'large', '10'],
['11', 'ddd', '1.2', '2']]
def column_distribution(request):
username = request.GET.get('username')
dbname = request.GET.get('db', 'intel')
tablename = request.GET.get('table', 'readings')
where = request.GET.get('where', '')
col = request.GET.get('col')
try:
nbuckets = int(request.GET.get('nbuckets', 100))
except Exception as e:
print e
nbuckets = 100
full_tablename = "%s.%s" % (dbname, tablename)
summary = Summary(dbname,
full_tablename,
username,
nbuckets=nbuckets,
where=where)
try:
typ = summary.get_type(col)
stats = summary.get_col_stats(col, typ)
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = {'col': col, 'type': typ, 'stats': stats}
context = {"data": data}
return context
def summarize(ucr):
# Wraps calls to Summary
infiles = []
newfiles = getInfiles(False)
infiles.append(getMergedFile())
infiles.append(getMergedFile(True))
infiles.append(getMergedFile(imputed=True))
if ucr:
infiles.append(newfiles["ucr"])
infiles.append(newfiles["case"])
for i in infiles:
print(("\n\tSummarizing {}...").format(os.path.split(i)[1]))
Summary(i)
def console_summary():
"""Summarizes a csv report."""
args = summary_parser.parse_args()
conf = config.load(args.config_file)
logger.setLevel(LOG_LEVELS.get(args.log_level.lower(), 'info'))
clientmanager = create_client_manager(**conf.get('auth_kwargs', {}))
Summary(
domain_client=clientmanager.get_domain(),
input_file=args.csv_file,
project_id_column=args.project_id_field,
cost_column=args.cost_field,
group_by=args.group_by
).output()
def load_link(self, session, url):
"""
Load link from URL.
Get it from database, or use `summary` to create it. The returned
`link.url` is most probably different from the `url` input param.
When `redo` mode perform resummarization and update current link.
"""
def check_url(url):
"""
Raise for existing or restricted URLs.
Param `url` is actually `summary.url` while extracting.
"""
logger.debug("Checking")
link = self.get_link(session, url)
if link: # existing
raise ExistingLinkException()
if self.redo: # redo
s = Summary(url)
logger.debug("Resummarizing")
s.extract()
logger.debug("Resummarized")
link = self.status.link # current
if link and \
link.url == s.url: # same url
link.load(s)
# link.updated = True
result = UPDATED_LINK
else: # new url
link = Link(s)
result = NEW_LINK
session.add(link)
else: # just do
link = self.get_link(session, url)
if link: # existing
result = EXISTING_LINK
else: # summarize
s = Summary(url)
try:
logger.debug("Summarizing")
s.extract(check_url=check_url)
logger.debug("Summarized")
except ExistingLinkException: # existing
link = self.get_link(session, s.url)
result = EXISTING_LINK
# except RestrictedLinkException:
# result = RESTRICTED_LINK
else: # new
link = Link(s)
session.add(link)
result = NEW_LINK
return link, result
def fit(self, X, Y, valid_X = None, valid_Y = None):
self.sess.run(tf.initialize_all_variables())
summary_train = Summary('train_loss')
summary_valid = Summary('valid_loss')
summary_valid_acc = Summary('valid_acc')
summary_train.start()
summary_valid.start()
summary_valid_acc.start()
for iter in range(NUM_ITER):
total_loss = 0.0
for i in range(0,len(X), BATCH_SIZE):
tail = min(i+BATCH_SIZE, len(X))
batch_xs = X[i:tail,:]
batch_ys = Y[i:tail,:]
_, loss = self.sess.run([self.train_step, self.cross_entropy], feed_dict={self.x: batch_xs, self.y: batch_ys})
total_loss += loss*(tail-i+1)
total_loss /= float(len(X))
print('Avg training loss on {}th epoch: {}'.format(iter, total_loss))
summary_train.log(total_loss)
if valid_X is not None and valid_Y is not None and iter % 100 ==0:
valid_loss, valid_acc = self.eva(valid_X, valid_Y)
summary_valid.log(valid_loss)
summary_valid_acc.log(valid_acc)
def test_canonical_site_urls():
config.NONCANONIC_SITES = ["c2.com", "www.ecommercebytes.com", "www.lukew.com", "cyberdust.com", 'forums.station.sony.com']
config.USEFUL_QUERY_KEYS = ["page", "a", "h", "z"]
s = Summary()
# a sort query arguments, first by key, then by value
# b percent encode paths and query arguments. non-ASCII characters are
# c percent-encoded using UTF-8 (RFC-3986)
# d normalize all spaces (in query arguments) '+' (plus symbol)
# e normalize percent encodings case (%2f -> %2F)
# f remove query arguments with blank values (unless site in NONCANONIC_SITES)
# g remove fragments (unless #!)
# h remove username/password at front of domain
# i remove port if 80, keep if not
# j change domain to lowercase
# k remove query arguments (unless site in USEFUL_QUERY_KEYS)
url = 'http://www.gmo-toku.jp/item/70228568/【YONEX】+パワークッションSC4メン+（SHB-SC4M+）'
clean_url = s._clean_url(url)
assert clean_url == 'http://www.gmo-toku.jp/item/70228568/%E3%80%90YONEX%E3%80%91+%E3%83%91%E3%83%AF%E3%83%BC%E3%82%AF%E3%83%83%E3%82%B7%E3%83%A7%E3%83%B3SC4%E3%83%A1%E3%83%B3+%EF%BC%88SHB-SC4M+%EF%BC%89'
url = "http://www.mavs.com/videos/run-dmc-cant-be-stopped/"
clean_url = s._clean_url(url)
assert clean_url == "http://www.mavs.com/videos/run-dmc-cant-be-stopped"
# 1 - h g i
url = 'https://*****:*****@www.lunatech.com:8080/file.jpg;p=1?a=3422#!page1/98292'
clean_url = s._clean_url(url)
assert clean_url == "https://www.lunatech.com:8080/file.jpg;p=1?a=3422#!page1/98292"
# 2 - h g i
url = 'https://*****:*****@www.lunatech.com:8080/file.jpg;p=1?q=2#third'
clean_url = s._clean_url(url)
assert clean_url == "https://www.lunatech.com:8080/file.jpg;p=1"
# 3 - i a b
url = 'http://google.com:80/cgi?z=!@£$%^*()-+&a=!@£$%^ *()-+'
clean_url = s._clean_url(url)
assert clean_url == "http://google.com/cgi?a=%21%40%C2%A3%24%25%5E+%2A%28%29-+&z=%21%40%C2%A3%24%25%5E%2A%28%29-+"
# 4 - i a
url = "http://google.com:80/cgi?z=1&h=8&a=1"
clean_url = s._clean_url(url)
assert clean_url == "http://google.com/cgi?a=1&h=8&z=1"
# 5 - j f
url = "http://CYBERDUST.com:80/addme?urbandrones&page1"
clean_url = s._clean_url(url)
assert clean_url == 'http://cyberdust.com/addme?page1&urbandrones'
# 6 - a f
url = "http://www.lukew.com/ff/entry.asp?1696&893489&3333"
clean_url = s._clean_url(url)
assert clean_url == 'http://www.lukew.com/ff/entry.asp?1696&3333&893489'
# 7 - f j
url = "http://www.ecommercebytes.COM/C/blog/blog.pl?/pl/2014/12/1417625708.htm"
clean_url = s._clean_url(url)
assert clean_url == 'http://www.ecommercebytes.com/C/blog/blog.pl?%2Fpl%2F2014%2F12%2F1417625708.htm'
# 8 - f
url = "https://forums.station.sony.com/ps2/index.php?threads/welcome-to-the-planetside-2-closed-beta.212516/"
clean_url = s._clean_url(url)
assert clean_url == 'https://forums.station.sony.com/ps2/index.php?threads%2Fwelcome-to-the-planetside-2-closed-beta.212516%2F'
# 9 - a f
url = "http://c2.com/cgi/wiki?LispMacro&p=1&BParam#DROPME"
clean_url = s._clean_url(url)
assert clean_url == "http://c2.com/cgi/wiki?BParam&LispMacro&p=1"
# 11 - a f
url = "http://c2.com/cgi/wiki?LispMacro&p=1&BParam&zzz#!KEEPME"
clean_url = s._clean_url(url)
assert clean_url == "http://c2.com/cgi/wiki?BParam&LispMacro&p=1&zzz#!KEEPME"
# 12 - f j k
url = "http://I_AM_NORMAL_URL.com/cgi?xxx&yyy=1"
clean_url = s._clean_url(url)
assert clean_url == "http://i_am_normal_url.com/cgi"
# 13 - i k
url = "http://google.com:80/cgi?xxx&page=1"
clean_url = s._clean_url(url)
assert clean_url == "http://google.com/cgi?page=1"
# 14
url = "http://www.google.com/cgi?page=1 and 3"
clean_url = s._clean_url(url)
assert clean_url == "http://www.google.com/cgi?page=1+and+3"
def merge_summaries(summaries):
merged_summary = Summary()
for summary in summaries:
merged_summary.merge(summary)
return merged_summary
logging.info('skipping as instances are live on here')
continue
if 'fips' in cleanup and len(cleanup['fips']) > 0:
logging.info('skipping as there are floating ips live on here')
continue
resources_selected_str = pp.pformat(cleanup)
logging.debug(resources_selected_str)
if args.dryrun:
cleanup_resources(cloud, cleanup, dry_run=True)
if args.run_cleanup:
try:
cleanup_resources(cloud, cleanup, dry_run=False)
except shade.exc.OpenStackCloudException as e:
logging.error(
'We had a problem trying to clean up [{}]'
.format(substr))
logging.error(e)
if not args.old_instances and not args.unused and not args.floatingips:
substring = args.substring or ''
resources.select_resources(substring)
cleanup = resources.get_selection()
do_cleanup(cloud, cleanup, pp, args)
Summary.print_summary()
if not args.run_cleanup:
logging.info("Nothing cleaned up. To cleanup resources, "
"please use --cleanup")
epsilons = [0.1, 0.2, 0.4, 1.0, 2.0]
#epsilons = [0.1, 0.2]
degree = 2
dimension = 5
data_size = "1M"
jar_name = "PrivBayes.jar"
source_file = "%sdata_%ddim_%s-coarse.csv" % (resource_path, dimension, data_size)
domain_file = "%sdata_%ddim_%s-coarse.domain.csv" % (resource_path, dimension, data_size)
for ep in epsilons:
test_prefix = "%dD%s%sE%dK" % (dimension, data_size, convert_epislon(ep), degree)
arguments = " %s %s %f %d %s" % (source_file, domain_file, ep, degree, convert_epislon(ep))
command = "java -Xmx10g -jar %s%s %s" % (jar_path, jar_name, arguments)
result_file = "%s%s_SyntheticData.csv" % (result_path, test_prefix)
acc_check = Summary(dimension, data_size, ep, source_file, result_file)
for round in range(test_round):
# run the java command
print "Command: %s, Round %d" % (command, round+1)
run_command(command)
# after privbayes, do the accuracy check
acc_check.statistic_accuracy()
acc_check.query_accuracy()
acc_check.print_summaries("/privbayes_test/test_round_1")
def __init__(self, run_test_method):
self.summary = Summary()
self.run_test = run_test_method
self.exception_suppressed = False
def run(self, rhoNo=1, phiNo=0, tfNo=1):
dataset = Dataset(rhoNo, phiNo, tfNo)
texDimSize = 128
renderer = Renderer(self.eye, self.screen)
rho = dataset.rho
phi = dataset.phi
tf = dataset.tf
phiPlane = SplinePlane(phi, self.splineInterval, self.intersectTolerance)
boundingBox = phiPlane.createBoundingBox()
plotter = Plotter(self.splineInterval)
refSplinePlotter = plotter.refSplineModelPlotter
directSplinePlotter = plotter.directSplineModelPlotter
voxelPlotter = plotter.voxelModelPlotter
paramPlotter = plotter.paramPlotter
refSplinePlotter.plotGrid(phi.evaluate, 10, 10)
directSplinePlotter.plotGrid(phi.evaluate, 10, 10)
paramPlotter.plotGrid(10, 10)
paramPlotter.plotScalarField(rho, tf)
refSplinePlotter.plotBoundingBox(boundingBox)
directSplinePlotter.plotBoundingBox(boundingBox)
voxelPlotter.plotBoundingBox(boundingBox)
# Creating models
refSplineModel = SplineModel(tf, phiPlane, rho, self.refTolerance)
directSplineModel = SplineModel(tf, phiPlane, rho)
samplingScalars = refSplineModel.generateScalarMatrix(boundingBox, texDimSize, texDimSize,
self.voxelizationTolerance)
#voxelPlotter.plotScalars(samplingScalars, boundingBox)
scalarTexture = Texture2D(samplingScalars)
plotter.plotScalarTexture(scalarTexture)
voxelModel = VoxelModel(tf, scalarTexture, boundingBox)
choice = 0
if choice == 0:
model = voxelModel
modelType = ModelType.VOXEL
elif choice == 1:
model = BoundaryAccurateModel(tf, directSplineModel, voxelModel)
modelType = ModelType.BOUNDARYACCURATE
elif choice == 2:
voxelWidth = boundingBox.getHeight() / float(texDimSize)
criterion = GeometricCriterion(self.screen.pixelWidth, voxelWidth)
model = HybridModel(tf, directSplineModel, voxelModel, criterion)
modelType = ModelType.HYBRID
else:
lodTextures = [scalarTexture]
size = texDimSize / 2
while size >= 2:
scalars = refSplineModel.generateScalarMatrix(boundingBox, size, size, self.voxelizationTolerance)
lodTextures.append(Texture2D(scalars))
size /= 2
model = VoxelLodModel(tf, lodTextures, boundingBox, self.screen.pixelWidth)
modelType = ModelType.VOXEL
# Rendering
refRenderData = RenderData(ModelType.REFERENCE, self.viewRayDeltaRef)
refRenderData.renderResult = renderer.render(refSplineModel, self.viewRayDeltaRef, refSplinePlotter)
directRenderData = RenderData(ModelType.DIRECT, self.viewRayDelta)
directRenderData.renderResult = renderer.render(directSplineModel, self.viewRayDelta, directSplinePlotter)
renderData = RenderData(modelType, self.viewRayDelta, texSize=texDimSize)
renderData.renderResult = renderer.render(model, self.viewRayDelta, voxelPlotter)
# Plotting
refPixelColors = refRenderData.renderResult.colors
directPixelColors = directRenderData.renderResult.colors
pixelColors = renderData.renderResult.colors
plotter.pixelReferencePlot.plotPixelColors(refPixelColors)
plotter.pixelDirectPlot.plotPixelColors(directPixelColors)
plotter.pixelVoxelizedPlot.plotPixelColors(pixelColors)
directDiffs = colordiff.compare(refPixelColors, directPixelColors)
diffs = colordiff.compare(refPixelColors, pixelColors)
plotter.pixelDirectDiffPlot.plotPixelColorDiffs(directDiffs)
plotter.pixelVoxelizedDiffPlot.plotPixelColorDiffs(diffs)
plotter.draw()
# Printing
directSummary = Summary(directRenderData, directDiffs)
directSummary.printData()
print ""
summary = Summary(renderData, diffs)
summary.printData()
def build_library(repository=None, branch=None, namespace=None, push=False,
debug=False, prefill=True, registry=None, targetlist=None,
repos_folder=None, logger=None):
''' Entrypoint method build_library.
repository: Repository containing a library/ folder. Can be a
local path or git repository
branch: If repository is a git repository, checkout this branch
(default: DEFAULT_BRANCH)
namespace: Created repositories will use the following namespace.
(default: no namespace)
push: If set to true, push images to the repository
debug: Enables debug logging if set to True
prefill: Retrieve images from public repository before building.
Serves to prefill the builder cache.
registry: URL to the private registry where results should be
pushed. (only if push=True)
targetlist: String indicating which library files are targeted by
this build. Entries should be comma-separated. Default
is all files.
repos_folder: Fixed location where cloned repositories should be
stored. Default is None, meaning folders are temporary
and cleaned up after the build finishes.
logger: Logger instance to use. Default is None, in which case
build_library will create its own logger.
'''
dst_folder = None
summary = Summary()
if logger is None:
logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level='INFO')
if repository is None:
repository = DEFAULT_REPOSITORY
if branch is None:
branch = DEFAULT_BRANCH
if debug:
logger.setLevel('DEBUG')
if targetlist is not None:
targetlist = targetlist.split(',')
if not repository.startswith(('https://', 'git://')):
logger.info('Repository provided assumed to be a local path')
dst_folder = repository
try:
client.version()
except Exception as e:
logger.error('Could not reach the docker daemon. Please make sure it '
'is running.')
logger.warning('Also make sure you have access to the docker UNIX '
'socket (use sudo)')
return
if not dst_folder:
logger.info('Cloning docker repo from {0}, branch: {1}'.format(
repository, branch))
try:
rep, dst_folder = git.clone_branch(repository, branch)
except Exception as e:
logger.exception(e)
logger.error('Source repository could not be fetched. Check '
'that the address is correct and the branch exists.')
return
try:
dirlist = os.listdir(os.path.join(dst_folder, 'library'))
except OSError as e:
logger.error('The path provided ({0}) could not be found or didn\'t'
'contain a library/ folder.'.format(dst_folder))
return
for buildfile in dirlist:
if buildfile == 'MAINTAINERS':
continue
if (targetlist and buildfile not in targetlist):
continue
f = open(os.path.join(dst_folder, 'library', buildfile))
linecnt = 0
for line in f:
linecnt += 1
if not line or line.strip() == '':
continue
elif line.lstrip().startswith('#'): # # It's a comment!
continue
logger.debug('{0} ---> {1}'.format(buildfile, line))
try:
tag, url, ref, dfile = parse_line(line, logger)
if prefill:
logger.debug('Pulling {0} from official repository (cache '
'fill)'.format(buildfile))
try:
client.pull(buildfile)
except:
# Image is not on official repository, ignore prefill
pass
img, commit = build_repo(url, ref, buildfile, dfile, tag,
namespace, push, registry,
repos_folder, logger)
summary.add_success(buildfile, (linecnt, line), img, commit)
except Exception as e:
logger.exception(e)
summary.add_exception(buildfile, (linecnt, line), e)
f.close()
cleanup(dst_folder, dst_folder != repository, repos_folder is None)
summary.print_summary(logger)
return summary
