def load_link(self, session, url):
"""
Load link from URL.
Get it from database, or use `summary` to create it. The returned
`link.url` is most probably different from the `url` input param.
When `redo` mode perform resummarization and update current link.
"""
def check_url(url):
"""
Raise for existing or restricted URLs.
Param `url` is actually `summary.url` while extracting.
"""
logger.debug("Checking")
link = self.get_link(session, url)
if link: # existing
raise ExistingLinkException()
if self.redo: # redo
s = Summary(url)
logger.debug("Resummarizing")
s.extract()
logger.debug("Resummarized")
link = self.status.link # current
if link and \
link.url == s.url: # same url
link.load(s)
# link.updated = True
result = UPDATED_LINK
else: # new url
link = Link(s)
result = NEW_LINK
session.add(link)
else: # just do
link = self.get_link(session, url)
if link: # existing
result = EXISTING_LINK
else: # summarize
s = Summary(url)
try:
logger.debug("Summarizing")
s.extract(check_url=check_url)
logger.debug("Summarized")
except ExistingLinkException: # existing
link = self.get_link(session, s.url)
result = EXISTING_LINK
# except RestrictedLinkException:
# result = RESTRICTED_LINK
else: # new
link = Link(s)
session.add(link)
result = NEW_LINK
return link, result
def update_pid(pid, bid, limit, prover, results, dohtml=True):
PidResults(pid,prover).update(results)
solved = [x for x in results if results[x].solved(limit)]
Solved("protos",bid,limit,prover).update(pid,solved)
summary = Summary("protos",bid,limit).update(pid,prover,results)
if dohtml: html.create_summary("protos",bid,limit,summary,usehash=True)
# additionally update data for lower available time limits
for lim in limits("protos",bid,prover):
if lim < limit:
solved = [x for x in results if results[x].solved(lim)]
Solved("protos",bid,lim,prover).update(pid,solved)
summary = Summary("protos",bid,lim).update(pid,prover,results)
if dohtml: html.create_summary("protos",bid,lim,summary,usehash=True)
def column_distribution(request):
username = request.GET.get('username')
dbname = request.GET.get('db', 'intel')
tablename = request.GET.get('table', 'readings')
where = request.GET.get('where', '')
col = request.GET.get('col')
try:
nbuckets = int(request.GET.get('nbuckets', 100))
except Exception as e:
print e
nbuckets = 100
full_tablename = "%s.%s" % (dbname, tablename)
summary = Summary(dbname,
full_tablename,
username,
nbuckets=nbuckets,
where=where)
try:
typ = summary.get_type(col)
stats = summary.get_col_stats(col, typ)
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = {'col': col, 'type': typ, 'stats': stats}
context = {"data": data}
return context
def column_distributions(request):
username = request.GET.get('username')
dbname = request.GET.get('db', 'intel')
tablename = request.GET.get('table', 'readings')
where = request.GET.get('where', '')
try:
nbuckets = int(request.GET.get('nbuckets', 100))
except Exception as e:
print e
nbuckets = 100
full_tablename = "%s.%s" % (dbname, tablename)
summary = Summary(dbname,
full_tablename,
username,
nbuckets=nbuckets,
where=where)
print 'where: %s' % where
try:
stats = summary()
except Exception as e:
traceback.print_exc()
finally:
summary.close()
data = []
for col, typ, col_stats in stats:
data.append({'col': col, 'type': typ, 'stats': col_stats})
context = {"data": data}
return context
def summary(url):
if utils.is_youtube(url):
return summary_youtube(url)
# all non-youtube
summ = Summary(url)
summ.extract()
return (summ.title, summ.description, summ.image.url)
def RunSingle(self):
global sim_results
sim_results = []
StartSimulations(self.sim_configs, cache_config['memory'])
Summary(sim_results, self.title)
def main():
opt = parse_opt()
use_cuda = torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
env = gym.make(game)
seed = 7122
env.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
agent = DQN(env, opt, device=device)
agent.network.apply(weights_init)
agent.sync_weight()
progress = trange(opt.episode, ascii=True)
summary = Summary()
last_rewards = 0
for episode in progress:
# Training
state = env.reset()
for s in range(opt.max_step):
# use epsilon-greedy in training
action = agent.egreedy_action(state)
next_state, reward, done, _ = env.step(action)
loss = agent.perceive(state, action, reward, next_state, done)
state = next_state
if done:
break
summary.add(episode, 'loss', loss)
# Testing
if opt.test_interval > 0 and (episode + 1) % opt.test_interval == 0:
rewards = 0
for t in trange(opt.test, ascii=True, leave=False):
state = env.reset()
for s in range(opt.max_step):
action = agent.action(state)
next_state, reward, done, _ = env.step(action)
state = next_state
rewards += reward
if done:
break
if opt.test > 0:
rewards /= opt.test
last_rewards = rewards
summary.add(episode, 'reward', rewards)
progress.set_description('Loss: {:.4f} | Reward: {:2}'.format(
loss, last_rewards))
if opt.log:
summary.write(opt.log)
def _build_summary(self, summary_step):
if self.with_summary is not True:
return
self.summary = Summary(self.sess, summary_step)
for s in self.for_summary_scalar:
self.summary.scalar(s)
for h in self.for_summary_hist:
self.summary.hist(h)
self.summary.merge()
def setUp(self):
# A simple test case set up on two symbols. The first four elements are
# valid test data, but trade_data[4] has a numeric symbol.
self.test = Summary("", "")
# Each list within the trade_data list has the form:
# [timeStamp, symbol, quantity, price]
self.test.trade_data = [['1', 'aaa', '1', '0'], ['2', 'aaa', '1', '1'],
['2', 'bbb', '2', '2'], ['5', 'aaa', '1', '3'],
['7', '123', '2', '3'], ['9', '', '1', '1'],
['10', 'ddd', 'large', '10'],
['11', 'ddd', '1.2', '2']]
def summarize(ucr):
# Wraps calls to Summary
infiles = []
newfiles = getInfiles(False)
infiles.append(getMergedFile())
infiles.append(getMergedFile(True))
infiles.append(getMergedFile(imputed=True))
if ucr:
infiles.append(newfiles["ucr"])
infiles.append(newfiles["case"])
for i in infiles:
print(("\n\tSummarizing {}...").format(os.path.split(i)[1]))
Summary(i)
def console_summary():
"""Summarizes a csv report."""
args = summary_parser.parse_args()
conf = config.load(args.config_file)
logger.setLevel(LOG_LEVELS.get(args.log_level.lower(), 'info'))
clientmanager = create_client_manager(**conf.get('auth_kwargs', {}))
Summary(
domain_client=clientmanager.get_domain(),
input_file=args.csv_file,
project_id_column=args.project_id_field,
cost_column=args.cost_field,
group_by=args.group_by
).output()
def __init__(self,
level_filepath,
episodes=30000,
initial_epsilon=1.,
min_epsilon=0.1,
exploration_ratio=0.5,
max_steps=2000,
render_freq=500,
enable_render=True,
render_fps=20,
save_dir='checkpoints',
enable_save=True,
save_freq=500,
gamma=0.99,
batch_size=64,
min_replay_memory_size=1000,
replay_memory_size=100000,
target_update_freq=5,
seed=42):
self.set_random_seed(seed)
self.episodes = episodes
self.max_steps = max_steps
self.epsilon = initial_epsilon
self.min_epsilon = min_epsilon
self.exploration_ratio = exploration_ratio
self.render_freq = render_freq
self.enable_render = enable_render
self.render_fps = render_fps
self.save_dir = save_dir
self.enable_save = enable_save
self.save_freq = save_freq
if enable_save and not os.path.exists(save_dir):
os.makedirs(save_dir)
level_loader = LevelLoader(level_filepath)
self.agent = DQNAgent(level_loader.get_field_size(),
gamma=gamma,
batch_size=batch_size,
min_replay_memory_size=min_replay_memory_size,
replay_memory_size=replay_memory_size,
target_update_freq=target_update_freq)
self.env = Snake(level_loader)
self.summary = Summary()
self.current_episode = 0
self.max_average_length = 0
self.epsilon_decay = (initial_epsilon -
min_epsilon) / (exploration_ratio * episodes)
def dump(input_file):
with open(input_file, 'rb') as fd:
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
boxes = []
while fd.tell() < eof:
box = Box.parse_stream(fd)
boxes.append(box)
#print(box)
summary = Summary(input_file, boxes)
print(summary.data)
def load_feats(self, mod=dict(kp_algo='FAST', des_algo='SIFT')):
su = Summary()
sin = su.info(self.root, self.name).iloc[0]
ns = sin.n_slides
key_root = "/{kp_algo}/{des_algo}/slide".format(**mod)
sfd = []
for si in range(1, ns + 1):
kk = key_root + "/kp/i{:03d}".format(si)
dk = key_root + "/desc/i{:03d}".format(si)
kps = self._unpickle_keypoints(self.load(kk))
des = self.load(dk)
des = None if des is None else des.values
sfd.append(dict(idx=si, kps=kps, des=des))
return sfd
def parse_summary(row):
"""摘要处理"""
obj = Summary()
data = {}
for item in field_tags:
s_field, e_field = item
text = re.sub('\<.+?\>', '', row[s_field])
summary = obj.summary(text, phrases_num=3, sentence_num=12)
if len(summary) < 15:
continue
data[e_field] = summary
data['have_summary'] = 1 if len(data) else -1
return data
def onGenerateLimitGraphs(self, event):
if self.currentFilePath is None:
return
try:
dev_limit = float(self.textEntryDEV.GetLineText(0))
rms_limit = float(self.textEntryRMS.GetLineText(0))
except ValueError:
#TODO create ALERT
return
results = ReadCSVWithLimit(self.currentFilePath, dev_limit, rms_limit)
self.summary = Summary(numHEADs=len(self.master),
dev=results[0],
rms=results[1])
self.summaryCanvas = FigureCanvas(self.summaryPanel, 1,
self.summary.PlotSummaryGraphs())
def extract():
url = request.args.get('url')
# g = Goose({'browser_user_agent': 'Mozilla'})
# print 'url', url
# article = g.extract(url=url)
# summarised_article = Summary({'title' : article.title , 'content' : article.cleaned_text})
#
article = Article(url)
article.download()
article.parse()
# print article.title
# print article.text
summarised_article = Summary({'title' : article.title , 'content' : article.text})
# summarised_article.print_result() if article.text else {}
return jsonify(**summarised_article.get_result())
def export_to_excel(self):
"""
Function which is called when user click export to excel button, export report to excel
"""
if self.loaded:
s = Summary(self.p)
t = s.export_to_excel()
if t == "":
tkinter.messagebox.showinfo("Blast", "Your didn't choose file")
else:
tkinter.messagebox.showinfo(
"Blast", "Your data has been saved in excel")
else:
tkinter.messagebox.showinfo("Blast",
"Data has not loaded, try again!")
def train(gan, generator_noise_input, batch_handler, config):
summary_writer = Summary(config)
for batch_number, batch in enumerate(
batch_handler.get_next_batch(config["batch_size"])):
generator_output = gan.generate(generator_noise_input)
generator_loss, discriminator_loss = gan.train(generator_output, batch)
summary_writer.update_statistics(generator_loss, discriminator_loss,
batch_number)
if batch_number % config["report_batch_interval"] == 0:
summary_writer.print_statistics()
summary_writer.write_statistics()
if batch_number % config["save_batch_interval"] == 0:
gan.save()
summary_writer.write_images(generator_output, batch)
def __init__(self, options, session):
self._options = options
self._session = session
np.random.seed(options.seed)
tf.set_random_seed(options.seed)
self.input_size = unet.input_size_needed(options.patch_size,
options.num_layers)
self.experiment_name = datetime.now().strftime("%Y-%m-%dT%Hh%Mm%Ss")
experiment_path = os.path.abspath(
os.path.join(options.save_path, self.experiment_name))
summary_path = os.path.join(options.logdir, self.experiment_name)
self._summary = Summary(options, session, summary_path)
self.build_graph()
def RunMultiple(self):
total_results = []
global sim_results
sim_results = []
for i in range(0, cache_config['mult_sims']):
# Generate new memory
cache_config['memory'] = RandomMem(
size=cache_config['mem_size'],
max_address=cache_config['mem_range'],
filename='gen_mem' + str(i + 1) + '.csv',
pattern_type=cache_config['mem_pattern'],
save_mem=True)
print('Beginning simulation batch', i + 1, 'of',
cache_config['mult_sims'],
'using gen_mem' + str(i + 1) + '.txt')
StartSimulations(self.sim_configs, cache_config['memory'])
# Add results together
if total_results == []:
total_results = sim_results.copy()
else:
for index in range(0, len(total_results)):
total_results[index]['history'] += sim_results[index][
'history']
total_results[index]['hits'] += sim_results[index]['hits']
total_results[index]['misses'] += sim_results[index][
'misses']
for sim in total_results:
name = '{:25s}'.format(sim['cache'].config_name)
hits = '{:6s}'.format(str(sim['hits']))
misses = '{:6s}'.format(str(sim['misses']))
logging.info(name + ' - Hits:' + hits + '- Misses:' + misses)
logging.info('')
sim_results = []
Summary(total_results, self.title)
def __init__(self, options, session):
self._options = options
self._session = session
self.train_images_shape = None
np.random.seed(options.seed)
tf.set_random_seed(options.seed)
self.input_size = self._options.patch_size
self.experiment_name = datetime.now().strftime("%Y%m%d%H%M%S")
experiment_path = os.path.abspath(
os.path.join(options.save_path, self.experiment_name))
self.summary_path = os.path.join(
options.logdir, self.experiment_name + options.log_suffix)
self._summary = Summary(options, session)
self.build_graph()
def summarize(self, url, **kwargs):
params = self.kwargs2params(url, kwargs)
response = requests.get(self.endpoint, params=params)
response.close()
smmry_dict = response.json()
if smmry_dict.get('sm_api_error'):
raise SmmryAPIException("%s: %s" % (smmry_dict['sm_api_error'], smmry_dict['sm_api_message']))
if params.get('SM_WITH_BREAK'):
smmry_dict['sm_api_content'] = smmry_dict['sm_api_content'].replace('[BREAK]', params['SM_BREAK_WITH'])
smmry_dict['sm_api_content'] = smmry_dict['sm_api_content'].strip()
return Summary(smmry_dict, params, response)
def o_slides(self, gray=False, resize=None):
"""
Get slide images collection
"""
spm = self.slides_path(size='big')
su = Summary()
sin = su.info(self.root, self.name)
if resize is True:
resize = (sin.v_width, sin.v_height)
for si in range(1, sin.n_slides + 1):
sp = "{}/{:03d}.jpg".format(spm, si)
if gray:
img = cv2.imread(sp, cv2.COLOR_GRAY2BGR)
else:
img = cv2.imread(sp)
if resize is not None:
img = cv2.resize(img, resize)
yield (dict(img=img, idx=si))
def single_process(path):
"""
Operation for every TXT file
:param path: Path of target TXT file
"""
try:
print(f'Processing: {path}')
info = InfoPrint(path)
data, data_effect, data_avg, data_median, data_sum, amplitude_99, info.line, mu_avg, info.t_convert = \
data_convert(path)
info.t_plot = data_plot(path, data, data_effect, data_avg)
info.t_excel = excel_write(path, data, data_effect)
Summary(path, data_effect, data_avg, data_median, data_sum, amplitude_99, mu_avg)
info.show()
except Exception as e:
print(f'Fail: {path}\n\t{e}')
def stream_terms(k):
tot_time = 0
with open(file) as f:
lines = f.read().splitlines()
for line in lines:
start_time = time.time()
parts = line.split("\t")
text = parts[0].lower()
lat = parts[1]
lon = parts[2]
cell_id = math.floor(float(lat)).__str__() + "/" + math.floor(float(lon)).__str__()
# cell = map_cells[cell_id]
terms = text.split(" ")
# pdb.set_trace()
for item in stop:
if item in terms:
terms = list(filter((item).__ne__, terms))
for item in terms:
if item == '':
continue
if len(item) == 1:
continue
if item in all_terms:
# print(item)
if item in map_terms.keys():
map_terms[item].update_summary(cell_id)
# map_terms[item].seen_cells.add(cell_id)
else:
temp_summary = Summary(k)
temp_summary.update_summary(cell_id)
map_terms.update({item : temp_summary})
# map_terms[item].seen_cells.add(cell_id)
# if item == 'paris':
# map_terms[item].update_replacements()
# print(item)
# print(time.time() - start_time)
tot_time += (time.time() - start_time)
print(tot_time/len(lines))
def update_sid(sid, bid, limit, prover, results, dohtml=True):
SidResults(sid,prover).update(results)
solved = [x for x in results if results[x].solved(limit)]
Solved("scheds",bid,limit,prover).update(sid,solved)
summary = Summary("scheds",bid,limit).update(sid,prover,results)
if dohtml: html.create_summary("scheds",bid,limit,summary)
def main():
#webclient = WebClient()
#webclient.save_as_csv('https://www.worldometers.info/coronavirus/', './data/corona.csv')
summary = Summary('./data/corona.csv', './data/muslims.csv')
summary.show()
def build_library(repository=None,
branch=None,
namespace=None,
push=False,
debug=False,
prefill=True,
registry=None,
targetlist=None,
repos_folder=None,
logger=None):
''' Entrypoint method build_library.
repository: Repository containing a library/ folder. Can be a
local path or git repository
branch: If repository is a git repository, checkout this branch
(default: DEFAULT_BRANCH)
namespace: Created repositories will use the following namespace.
(default: no namespace)
push: If set to true, push images to the repository
debug: Enables debug logging if set to True
prefill: Retrieve images from public repository before building.
Serves to prefill the builder cache.
registry: URL to the private registry where results should be
pushed. (only if push=True)
targetlist: String indicating which library files are targeted by
this build. Entries should be comma-separated. Default
is all files.
repos_folder: Fixed location where cloned repositories should be
stored. Default is None, meaning folders are temporary
and cleaned up after the build finishes.
logger: Logger instance to use. Default is None, in which case
build_library will create its own logger.
'''
dst_folder = None
summary = Summary()
if logger is None:
logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level='INFO')
if repository is None:
repository = DEFAULT_REPOSITORY
if branch is None:
branch = DEFAULT_BRANCH
if debug:
logger.setLevel('DEBUG')
if targetlist is not None:
targetlist = targetlist.split(',')
if not repository.startswith(('https://', 'git://')):
logger.info('Repository provided assumed to be a local path')
dst_folder = repository
try:
client.version()
except Exception as e:
logger.error('Could not reach the docker daemon. Please make sure it '
'is running.')
logger.warning('Also make sure you have access to the docker UNIX '
'socket (use sudo)')
return
if not dst_folder:
logger.info('Cloning docker repo from {0}, branch: {1}'.format(
repository, branch))
try:
rep, dst_folder = git.clone_branch(repository, branch)
except Exception as e:
logger.exception(e)
logger.error('Source repository could not be fetched. Check '
'that the address is correct and the branch exists.')
return
try:
dirlist = os.listdir(os.path.join(dst_folder, 'library'))
except OSError as e:
logger.error('The path provided ({0}) could not be found or didn\'t'
'contain a library/ folder.'.format(dst_folder))
return
for buildfile in dirlist:
if buildfile == 'MAINTAINERS':
continue
if (targetlist and buildfile not in targetlist):
continue
f = open(os.path.join(dst_folder, 'library', buildfile))
linecnt = 0
for line in f:
linecnt += 1
if not line or line.strip() == '':
continue
elif line.lstrip().startswith('#'): # # It's a comment!
continue
logger.debug('{0} ---> {1}'.format(buildfile, line))
try:
tag, url, ref, dfile = parse_line(line, logger)
if prefill:
logger.debug('Pulling {0} from official repository (cache '
'fill)'.format(buildfile))
try:
client.pull(buildfile)
except:
# Image is not on official repository, ignore prefill
pass
img, commit = build_repo(url, ref, buildfile, dfile, tag,
namespace, push, registry,
repos_folder, logger)
summary.add_success(buildfile, (linecnt, line), img, commit)
except Exception as e:
logger.exception(e)
summary.add_exception(buildfile, (linecnt, line), e)
f.close()
cleanup(dst_folder, dst_folder != repository, repos_folder is None)
summary.print_summary(logger)
return summary
def __init__(self, run_test_method):
self.summary = Summary()
self.run_test = run_test_method
self.exception_suppressed = False
