def load_link(self, session, url): """ Load link from URL. Get it from database, or use `summary` to create it. The returned `link.url` is most probably different from the `url` input param. When `redo` mode perform resummarization and update current link. """ def check_url(url): """ Raise for existing or restricted URLs. Param `url` is actually `summary.url` while extracting. """ logger.debug("Checking") link = self.get_link(session, url) if link: # existing raise ExistingLinkException() if self.redo: # redo s = Summary(url) logger.debug("Resummarizing") s.extract() logger.debug("Resummarized") link = self.status.link # current if link and \ link.url == s.url: # same url link.load(s) # link.updated = True result = UPDATED_LINK else: # new url link = Link(s) result = NEW_LINK session.add(link) else: # just do link = self.get_link(session, url) if link: # existing result = EXISTING_LINK else: # summarize s = Summary(url) try: logger.debug("Summarizing") s.extract(check_url=check_url) logger.debug("Summarized") except ExistingLinkException: # existing link = self.get_link(session, s.url) result = EXISTING_LINK # except RestrictedLinkException: # result = RESTRICTED_LINK else: # new link = Link(s) session.add(link) result = NEW_LINK return link, result
def update_pid(pid, bid, limit, prover, results, dohtml=True): PidResults(pid,prover).update(results) solved = [x for x in results if results[x].solved(limit)] Solved("protos",bid,limit,prover).update(pid,solved) summary = Summary("protos",bid,limit).update(pid,prover,results) if dohtml: html.create_summary("protos",bid,limit,summary,usehash=True) # additionally update data for lower available time limits for lim in limits("protos",bid,prover): if lim < limit: solved = [x for x in results if results[x].solved(lim)] Solved("protos",bid,lim,prover).update(pid,solved) summary = Summary("protos",bid,lim).update(pid,prover,results) if dohtml: html.create_summary("protos",bid,lim,summary,usehash=True)
def column_distribution(request): username = request.GET.get('username') dbname = request.GET.get('db', 'intel') tablename = request.GET.get('table', 'readings') where = request.GET.get('where', '') col = request.GET.get('col') try: nbuckets = int(request.GET.get('nbuckets', 100)) except Exception as e: print e nbuckets = 100 full_tablename = "%s.%s" % (dbname, tablename) summary = Summary(dbname, full_tablename, username, nbuckets=nbuckets, where=where) try: typ = summary.get_type(col) stats = summary.get_col_stats(col, typ) except Exception as e: traceback.print_exc() finally: summary.close() data = {'col': col, 'type': typ, 'stats': stats} context = {"data": data} return context
def column_distributions(request): username = request.GET.get('username') dbname = request.GET.get('db', 'intel') tablename = request.GET.get('table', 'readings') where = request.GET.get('where', '') try: nbuckets = int(request.GET.get('nbuckets', 100)) except Exception as e: print e nbuckets = 100 full_tablename = "%s.%s" % (dbname, tablename) summary = Summary(dbname, full_tablename, username, nbuckets=nbuckets, where=where) print 'where: %s' % where try: stats = summary() except Exception as e: traceback.print_exc() finally: summary.close() data = [] for col, typ, col_stats in stats: data.append({'col': col, 'type': typ, 'stats': col_stats}) context = {"data": data} return context
def summary(url): if utils.is_youtube(url): return summary_youtube(url) # all non-youtube summ = Summary(url) summ.extract() return (summ.title, summ.description, summ.image.url)
def RunSingle(self): global sim_results sim_results = [] StartSimulations(self.sim_configs, cache_config['memory']) Summary(sim_results, self.title)
def main(): opt = parse_opt() use_cuda = torch.cuda.is_available() device = torch.device('cuda' if use_cuda else 'cpu') env = gym.make(game) seed = 7122 env.seed(seed) random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) agent = DQN(env, opt, device=device) agent.network.apply(weights_init) agent.sync_weight() progress = trange(opt.episode, ascii=True) summary = Summary() last_rewards = 0 for episode in progress: # Training state = env.reset() for s in range(opt.max_step): # use epsilon-greedy in training action = agent.egreedy_action(state) next_state, reward, done, _ = env.step(action) loss = agent.perceive(state, action, reward, next_state, done) state = next_state if done: break summary.add(episode, 'loss', loss) # Testing if opt.test_interval > 0 and (episode + 1) % opt.test_interval == 0: rewards = 0 for t in trange(opt.test, ascii=True, leave=False): state = env.reset() for s in range(opt.max_step): action = agent.action(state) next_state, reward, done, _ = env.step(action) state = next_state rewards += reward if done: break if opt.test > 0: rewards /= opt.test last_rewards = rewards summary.add(episode, 'reward', rewards) progress.set_description('Loss: {:.4f} | Reward: {:2}'.format( loss, last_rewards)) if opt.log: summary.write(opt.log)
def _build_summary(self, summary_step): if self.with_summary is not True: return self.summary = Summary(self.sess, summary_step) for s in self.for_summary_scalar: self.summary.scalar(s) for h in self.for_summary_hist: self.summary.hist(h) self.summary.merge()
def setUp(self): # A simple test case set up on two symbols. The first four elements are # valid test data, but trade_data[4] has a numeric symbol. self.test = Summary("", "") # Each list within the trade_data list has the form: # [timeStamp, symbol, quantity, price] self.test.trade_data = [['1', 'aaa', '1', '0'], ['2', 'aaa', '1', '1'], ['2', 'bbb', '2', '2'], ['5', 'aaa', '1', '3'], ['7', '123', '2', '3'], ['9', '', '1', '1'], ['10', 'ddd', 'large', '10'], ['11', 'ddd', '1.2', '2']]
def summarize(ucr): # Wraps calls to Summary infiles = [] newfiles = getInfiles(False) infiles.append(getMergedFile()) infiles.append(getMergedFile(True)) infiles.append(getMergedFile(imputed=True)) if ucr: infiles.append(newfiles["ucr"]) infiles.append(newfiles["case"]) for i in infiles: print(("\n\tSummarizing {}...").format(os.path.split(i)[1])) Summary(i)
def console_summary(): """Summarizes a csv report.""" args = summary_parser.parse_args() conf = config.load(args.config_file) logger.setLevel(LOG_LEVELS.get(args.log_level.lower(), 'info')) clientmanager = create_client_manager(**conf.get('auth_kwargs', {})) Summary( domain_client=clientmanager.get_domain(), input_file=args.csv_file, project_id_column=args.project_id_field, cost_column=args.cost_field, group_by=args.group_by ).output()
def __init__(self, level_filepath, episodes=30000, initial_epsilon=1., min_epsilon=0.1, exploration_ratio=0.5, max_steps=2000, render_freq=500, enable_render=True, render_fps=20, save_dir='checkpoints', enable_save=True, save_freq=500, gamma=0.99, batch_size=64, min_replay_memory_size=1000, replay_memory_size=100000, target_update_freq=5, seed=42): self.set_random_seed(seed) self.episodes = episodes self.max_steps = max_steps self.epsilon = initial_epsilon self.min_epsilon = min_epsilon self.exploration_ratio = exploration_ratio self.render_freq = render_freq self.enable_render = enable_render self.render_fps = render_fps self.save_dir = save_dir self.enable_save = enable_save self.save_freq = save_freq if enable_save and not os.path.exists(save_dir): os.makedirs(save_dir) level_loader = LevelLoader(level_filepath) self.agent = DQNAgent(level_loader.get_field_size(), gamma=gamma, batch_size=batch_size, min_replay_memory_size=min_replay_memory_size, replay_memory_size=replay_memory_size, target_update_freq=target_update_freq) self.env = Snake(level_loader) self.summary = Summary() self.current_episode = 0 self.max_average_length = 0 self.epsilon_decay = (initial_epsilon - min_epsilon) / (exploration_ratio * episodes)
def dump(input_file): with open(input_file, 'rb') as fd: fd.seek(0, io.SEEK_END) eof = fd.tell() fd.seek(0) boxes = [] while fd.tell() < eof: box = Box.parse_stream(fd) boxes.append(box) #print(box) summary = Summary(input_file, boxes) print(summary.data)
def load_feats(self, mod=dict(kp_algo='FAST', des_algo='SIFT')): su = Summary() sin = su.info(self.root, self.name).iloc[0] ns = sin.n_slides key_root = "/{kp_algo}/{des_algo}/slide".format(**mod) sfd = [] for si in range(1, ns + 1): kk = key_root + "/kp/i{:03d}".format(si) dk = key_root + "/desc/i{:03d}".format(si) kps = self._unpickle_keypoints(self.load(kk)) des = self.load(dk) des = None if des is None else des.values sfd.append(dict(idx=si, kps=kps, des=des)) return sfd
def parse_summary(row): """摘要处理""" obj = Summary() data = {} for item in field_tags: s_field, e_field = item text = re.sub('\<.+?\>', '', row[s_field]) summary = obj.summary(text, phrases_num=3, sentence_num=12) if len(summary) < 15: continue data[e_field] = summary data['have_summary'] = 1 if len(data) else -1 return data
def onGenerateLimitGraphs(self, event): if self.currentFilePath is None: return try: dev_limit = float(self.textEntryDEV.GetLineText(0)) rms_limit = float(self.textEntryRMS.GetLineText(0)) except ValueError: #TODO create ALERT return results = ReadCSVWithLimit(self.currentFilePath, dev_limit, rms_limit) self.summary = Summary(numHEADs=len(self.master), dev=results[0], rms=results[1]) self.summaryCanvas = FigureCanvas(self.summaryPanel, 1, self.summary.PlotSummaryGraphs())
def extract(): url = request.args.get('url') # g = Goose({'browser_user_agent': 'Mozilla'}) # print 'url', url # article = g.extract(url=url) # summarised_article = Summary({'title' : article.title , 'content' : article.cleaned_text}) # article = Article(url) article.download() article.parse() # print article.title # print article.text summarised_article = Summary({'title' : article.title , 'content' : article.text}) # summarised_article.print_result() if article.text else {} return jsonify(**summarised_article.get_result())
def export_to_excel(self): """ Function which is called when user click export to excel button, export report to excel """ if self.loaded: s = Summary(self.p) t = s.export_to_excel() if t == "": tkinter.messagebox.showinfo("Blast", "Your didn't choose file") else: tkinter.messagebox.showinfo( "Blast", "Your data has been saved in excel") else: tkinter.messagebox.showinfo("Blast", "Data has not loaded, try again!")
def train(gan, generator_noise_input, batch_handler, config): summary_writer = Summary(config) for batch_number, batch in enumerate( batch_handler.get_next_batch(config["batch_size"])): generator_output = gan.generate(generator_noise_input) generator_loss, discriminator_loss = gan.train(generator_output, batch) summary_writer.update_statistics(generator_loss, discriminator_loss, batch_number) if batch_number % config["report_batch_interval"] == 0: summary_writer.print_statistics() summary_writer.write_statistics() if batch_number % config["save_batch_interval"] == 0: gan.save() summary_writer.write_images(generator_output, batch)
def __init__(self, options, session): self._options = options self._session = session np.random.seed(options.seed) tf.set_random_seed(options.seed) self.input_size = unet.input_size_needed(options.patch_size, options.num_layers) self.experiment_name = datetime.now().strftime("%Y-%m-%dT%Hh%Mm%Ss") experiment_path = os.path.abspath( os.path.join(options.save_path, self.experiment_name)) summary_path = os.path.join(options.logdir, self.experiment_name) self._summary = Summary(options, session, summary_path) self.build_graph()
def RunMultiple(self): total_results = [] global sim_results sim_results = [] for i in range(0, cache_config['mult_sims']): # Generate new memory cache_config['memory'] = RandomMem( size=cache_config['mem_size'], max_address=cache_config['mem_range'], filename='gen_mem' + str(i + 1) + '.csv', pattern_type=cache_config['mem_pattern'], save_mem=True) print('Beginning simulation batch', i + 1, 'of', cache_config['mult_sims'], 'using gen_mem' + str(i + 1) + '.txt') StartSimulations(self.sim_configs, cache_config['memory']) # Add results together if total_results == []: total_results = sim_results.copy() else: for index in range(0, len(total_results)): total_results[index]['history'] += sim_results[index][ 'history'] total_results[index]['hits'] += sim_results[index]['hits'] total_results[index]['misses'] += sim_results[index][ 'misses'] for sim in total_results: name = '{:25s}'.format(sim['cache'].config_name) hits = '{:6s}'.format(str(sim['hits'])) misses = '{:6s}'.format(str(sim['misses'])) logging.info(name + ' - Hits:' + hits + '- Misses:' + misses) logging.info('') sim_results = [] Summary(total_results, self.title)
def __init__(self, options, session): self._options = options self._session = session self.train_images_shape = None np.random.seed(options.seed) tf.set_random_seed(options.seed) self.input_size = self._options.patch_size self.experiment_name = datetime.now().strftime("%Y%m%d%H%M%S") experiment_path = os.path.abspath( os.path.join(options.save_path, self.experiment_name)) self.summary_path = os.path.join( options.logdir, self.experiment_name + options.log_suffix) self._summary = Summary(options, session) self.build_graph()
def summarize(self, url, **kwargs): params = self.kwargs2params(url, kwargs) response = requests.get(self.endpoint, params=params) response.close() smmry_dict = response.json() if smmry_dict.get('sm_api_error'): raise SmmryAPIException("%s: %s" % (smmry_dict['sm_api_error'], smmry_dict['sm_api_message'])) if params.get('SM_WITH_BREAK'): smmry_dict['sm_api_content'] = smmry_dict['sm_api_content'].replace('[BREAK]', params['SM_BREAK_WITH']) smmry_dict['sm_api_content'] = smmry_dict['sm_api_content'].strip() return Summary(smmry_dict, params, response)
def o_slides(self, gray=False, resize=None): """ Get slide images collection """ spm = self.slides_path(size='big') su = Summary() sin = su.info(self.root, self.name) if resize is True: resize = (sin.v_width, sin.v_height) for si in range(1, sin.n_slides + 1): sp = "{}/{:03d}.jpg".format(spm, si) if gray: img = cv2.imread(sp, cv2.COLOR_GRAY2BGR) else: img = cv2.imread(sp) if resize is not None: img = cv2.resize(img, resize) yield (dict(img=img, idx=si))
def single_process(path): """ Operation for every TXT file :param path: Path of target TXT file """ try: print(f'Processing: {path}') info = InfoPrint(path) data, data_effect, data_avg, data_median, data_sum, amplitude_99, info.line, mu_avg, info.t_convert = \ data_convert(path) info.t_plot = data_plot(path, data, data_effect, data_avg) info.t_excel = excel_write(path, data, data_effect) Summary(path, data_effect, data_avg, data_median, data_sum, amplitude_99, mu_avg) info.show() except Exception as e: print(f'Fail: {path}\n\t{e}')
def stream_terms(k): tot_time = 0 with open(file) as f: lines = f.read().splitlines() for line in lines: start_time = time.time() parts = line.split("\t") text = parts[0].lower() lat = parts[1] lon = parts[2] cell_id = math.floor(float(lat)).__str__() + "/" + math.floor(float(lon)).__str__() # cell = map_cells[cell_id] terms = text.split(" ") # pdb.set_trace() for item in stop: if item in terms: terms = list(filter((item).__ne__, terms)) for item in terms: if item == '': continue if len(item) == 1: continue if item in all_terms: # print(item) if item in map_terms.keys(): map_terms[item].update_summary(cell_id) # map_terms[item].seen_cells.add(cell_id) else: temp_summary = Summary(k) temp_summary.update_summary(cell_id) map_terms.update({item : temp_summary}) # map_terms[item].seen_cells.add(cell_id) # if item == 'paris': # map_terms[item].update_replacements() # print(item) # print(time.time() - start_time) tot_time += (time.time() - start_time) print(tot_time/len(lines))
def update_sid(sid, bid, limit, prover, results, dohtml=True): SidResults(sid,prover).update(results) solved = [x for x in results if results[x].solved(limit)] Solved("scheds",bid,limit,prover).update(sid,solved) summary = Summary("scheds",bid,limit).update(sid,prover,results) if dohtml: html.create_summary("scheds",bid,limit,summary)
def main(): #webclient = WebClient() #webclient.save_as_csv('https://www.worldometers.info/coronavirus/', './data/corona.csv') summary = Summary('./data/corona.csv', './data/muslims.csv') summary.show()
def build_library(repository=None, branch=None, namespace=None, push=False, debug=False, prefill=True, registry=None, targetlist=None, repos_folder=None, logger=None): ''' Entrypoint method build_library. repository: Repository containing a library/ folder. Can be a local path or git repository branch: If repository is a git repository, checkout this branch (default: DEFAULT_BRANCH) namespace: Created repositories will use the following namespace. (default: no namespace) push: If set to true, push images to the repository debug: Enables debug logging if set to True prefill: Retrieve images from public repository before building. Serves to prefill the builder cache. registry: URL to the private registry where results should be pushed. (only if push=True) targetlist: String indicating which library files are targeted by this build. Entries should be comma-separated. Default is all files. repos_folder: Fixed location where cloned repositories should be stored. Default is None, meaning folders are temporary and cleaned up after the build finishes. logger: Logger instance to use. Default is None, in which case build_library will create its own logger. ''' dst_folder = None summary = Summary() if logger is None: logger = logging.getLogger(__name__) logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level='INFO') if repository is None: repository = DEFAULT_REPOSITORY if branch is None: branch = DEFAULT_BRANCH if debug: logger.setLevel('DEBUG') if targetlist is not None: targetlist = targetlist.split(',') if not repository.startswith(('https://', 'git://')): logger.info('Repository provided assumed to be a local path') dst_folder = repository try: client.version() except Exception as e: logger.error('Could not reach the docker daemon. Please make sure it ' 'is running.') logger.warning('Also make sure you have access to the docker UNIX ' 'socket (use sudo)') return if not dst_folder: logger.info('Cloning docker repo from {0}, branch: {1}'.format( repository, branch)) try: rep, dst_folder = git.clone_branch(repository, branch) except Exception as e: logger.exception(e) logger.error('Source repository could not be fetched. Check ' 'that the address is correct and the branch exists.') return try: dirlist = os.listdir(os.path.join(dst_folder, 'library')) except OSError as e: logger.error('The path provided ({0}) could not be found or didn\'t' 'contain a library/ folder.'.format(dst_folder)) return for buildfile in dirlist: if buildfile == 'MAINTAINERS': continue if (targetlist and buildfile not in targetlist): continue f = open(os.path.join(dst_folder, 'library', buildfile)) linecnt = 0 for line in f: linecnt += 1 if not line or line.strip() == '': continue elif line.lstrip().startswith('#'): # # It's a comment! continue logger.debug('{0} ---> {1}'.format(buildfile, line)) try: tag, url, ref, dfile = parse_line(line, logger) if prefill: logger.debug('Pulling {0} from official repository (cache ' 'fill)'.format(buildfile)) try: client.pull(buildfile) except: # Image is not on official repository, ignore prefill pass img, commit = build_repo(url, ref, buildfile, dfile, tag, namespace, push, registry, repos_folder, logger) summary.add_success(buildfile, (linecnt, line), img, commit) except Exception as e: logger.exception(e) summary.add_exception(buildfile, (linecnt, line), e) f.close() cleanup(dst_folder, dst_folder != repository, repos_folder is None) summary.print_summary(logger) return summary
def __init__(self, run_test_method): self.summary = Summary() self.run_test = run_test_method self.exception_suppressed = False