def __get_text(self, chapter_start_url, book_name, topic): ''' 小说正文爬取并写入磁盘 :param chapter_start_url: :param book_name: :param topic: :return: ''' # 文章正文部分是js 用selenium模拟浏览器获取html页面 driver = dr() driver.get(chapter_start_url) # 网络时整个页面没有加载外全会导致抓取内容错误 # 直接调用sheep对整个线程不安全 故采用 ▲t1-▲t2的方式拖延程序 start_time = int(time.time()) while True: end_time = int(time.time()) if end_time - start_time >= 1: break soup = BeautifulSoup(driver.page_source, "html5lib") ps = soup.select("#chapter-content p") text_list = [ "{0}\n".format(p.string.encode("utf-8")) for p in ps if p.string ] driver.quit() io = IO() io.write(text_list, book_name, topic) logging.info( "--------------------{0}{1} ..下载完成--------------------".format( book_name.encode("utf-8"), topic))
def pcd2img(name, load_path, save_path, k, factor_path=None): """Convert point clouds to depth images. """ mask = IO.get(os.path.join(load_path, "%s/instance_segment.png" % name)) mask_bg = np.array(mask[:, :, 2] == 0, dtype=np.float32) depth_in = IO.get(os.path.join(load_path, "%s/depth.exr" % name)) depth_out = depth_in * mask_bg mask_vals = np.unique(mask[:, :, 2])[1:] for i in range(len(mask_vals)): pcd_i = IO.get( os.path.join(load_path, "%s/depth2pcd_pred_%d.pcd" % (name, i))) if factor_path is not None: with open( os.path.join(factor_path, "%s/scale_factor_%d.json" % (name, i))) as f: factors = json.loads(f.read()) if factors['normalize']: pcd_i *= float(factors['normalize_factor']) if factors['centering']: pcd_i += np.array(factors['center_position']) depth_i = project_to_image(k, pcd_i) mask_i = np.array(mask[:, :, 2] == mask_vals[i], dtype=np.float32) depth_out += depth_i * mask_i # write predicted depth image in exr depth_out = np.expand_dims(depth_out, -1).repeat(3, axis=2) pyexr.write(os.path.join(save_path, "%s/depth_pred.exr" % name), depth_out)
def main(config): Misc.set_seed() print(config) data = IO.load_matrix_from_file(config.data) train, test = IO.split_matrix(data, config.ratio) config.num_user, config.num_item = data.shape model = PerisJointModel(config) model.fit(train) m = model.evaluate(train, test) return m
def rms(path, gt_name, pred_name): """ Compute the average root-mean-squared error of predicted depth images. """ errors = [] for subdir, dirs, files in os.walk(path): for dirname in sorted(dirs): depth_gt = IO.get(os.path.join(path, '%s/%s' % (dirname, gt_name))) depth_pred = IO.get(os.path.join(path, '%s/%s' % (dirname, pred_name))) diff = np.where(depth_pred > 0, depth_pred - depth_gt, 0) errors.append(np.sqrt(np.mean(diff**2))) return np.array(errors).mean(), np.array(errors).sum()
def main(): # Part one t0 = time.time() # Get dependency formula formula = IO.readFile("../data/input.txt") part_one = compute(formula, 'ORE', 1) time_part_one = round((time.time() - t0) * 1e3) print("Solution to part one: %s (time taken %s[ms])" % (part_one, time_part_one)) t0 = time.time() formula = IO.readFile("../data/input.txt") part_two = compute_fuel(formula) time_part_two = round((time.time() - t0) * 1e3) print("Solution to part two: %s (time taken %s[ms])" % (part_two, time_part_two))
def __call__(self, k): self._k = k # Starting entries : all tid with pid set to -1 starting_entries = [(i, -1) for i in range(len(self._dataset._db))] starting_pattern = [] # Void pattern # Start the main process with void pattern self._main_recursive(starting_pattern, starting_entries) # Wait all threads for thread in self._threads: thread.join() # Output the results on the stdout IO.to_stdout(self._dataset, self._results) return self._results
def __init__(self, file, input=[], verbose=True, reset=True): """ Read file and give input to the intcode computer.""" self.memory = IO.read_file(file) self.x, self.output = None, None self.input = input if isinstance(input, list) else [input] self.verbose = verbose self.reset = reset self.idle = False self.n = len(self.memory) self.i, self.base = 0, 0
def main(): x = IO.read_image('../data/input.txt') t0 = time.time() part_one, spot_index = compute_vision(x) time_part_one = round((time.time() - t0) * 1e3) print("Solution to part one: %s (time taken %s[ms])" % (part_one, time_part_one)) t0 = time.time() part_two = blast(x, spot_index, n_asteriods=200) time_part_two = round((time.time() - t0) * 1e3) print("Solution to part two: %s (time taken %s[ms])" % (part_two, time_part_two))
def __init__(self, number_of_epochs=1000, content_weight=1e3, style_weight=1e-2, model_name="mobilenetv2_coco_voctrainaug", enable_gpu=False, verbose=False): self.number_of_epochs = number_of_epochs self.content_weight = content_weight self.style_weight = style_weight self.content_layers = ["block5_conv2"] self.style_layers = ["block1_conv1", "block2_conv1", "block3_conv1", "block4_conv1", "block5_conv1"] self.model_name = model_name self.verbose = verbose self.enable_gpu = enable_gpu self._set_device() tf.compat.v1.enable_eager_execution() self.path = IO() if(self.verbose == True): print ("Eager Execution: {}".format(tf.executing_eagerly()))
def img2pcd(name, load_path, save_path, inv_k, normalize=False, plot=False): """Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds. """ mask = IO.get(os.path.join(load_path, "%s-mask.png" % name)) # Separate multiple objects into multiple point clouds mask_copy = np.array(mask * 255, dtype=np.uint8) if plot: cv2.imshow("%s mask" % name, mask_copy) cv2.waitKey(0) contours, hierarchy = cv2.findContours(mask_copy, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) obj_idx = 0 for i in range(len(contours)): if cv2.contourArea(contours[i]) > 100: mask_i = np.zeros_like(mask) cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=-1) if plot: cv2.imshow("%s idx:%d" % (name, i), mask_i) cv2.waitKey(0) mask_pcd = deproject(mask_i / 255, inv_k).sum(axis=1) > 0 opaque_depth = IO.get(os.path.join(load_path, "%s-opaque-depth-img.exr" % name)) opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd] IO.put(os.path.join(save_path, "opaque/%s-%d.pcd" % (name, obj_idx)), opaque_pcd) transp_depth = IO.get(os.path.join(load_path, "%s-transparent-depth-img.exr" % name)) transp_pcd = deproject(transp_depth, inv_k)[mask_pcd] IO.put(os.path.join(save_path, "transparent/%s-%d.pcd" % (name, obj_idx)), transp_pcd) if normalize: # normalize pcd to be within [-1, 1] for gridding max_val = np.nanmax((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd)))) if max_val > 1: print(max_val) opaque_pcd /= max_val * 1.01 transp_pcd /= max_val * 1.01 with open(os.path.join(save_path, "norm_factors/%s-%d.json" % (name, obj_idx)), 'w') as outfile: json.dump(max_val * 1.01, outfile) obj_idx += 1 # print(mask_pcd.sum()) # print(opaque_pcd.shape) # print(transp_pcd.shape) else: if plot: mask_i = np.zeros_like(mask) cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=5) cv2.imshow("%s idx:%d" % (name, i), mask_i) cv2.waitKey(0)
def main(): dir_path = dirname(realpath(__file__)) file_location = join(dir_path, "../data/test_input0.txt") # Part one t0 = time.time() # Get dependency formula formula = IO.readFile(file_location) part_one = compute(formula, "ORE", 1) time_part_one = round((time.time() - t0) * 1e3) print( "Solution to part one: %s (time taken %s[ms])" % (part_one, time_part_one) ) t0 = time.time() formula = IO.readFile(file_location) part_two = compute_fuel(formula) time_part_two = round((time.time() - t0) * 1e3) print( "Solution to part two: %s (time taken %s[ms])" % (part_two, time_part_two) )
def main(): dir_path = dirname(realpath(__file__)) file_location = join(dir_path, "../data/input.txt") x = IO.read_image(file_location) t0 = time.time() part_one, spot_index = compute_vision(x) time_part_one = round((time.time() - t0) * 1e3) print("Solution to part one: %s (time taken %s[ms])" % (part_one, time_part_one)) t0 = time.time() part_two = blast(x, spot_index, n_asteriods=200) time_part_two = round((time.time() - t0) * 1e3) print("Solution to part two: %s (time taken %s[ms])" % (part_two, time_part_two))
def collect_probes(glob_wildcardORLst=None, pathLst=[], out_path=''): """ Imports either from probe list or from glob wildcard file or list :param glob_wildcardORLst: :param pathLst: :param out_path: :return: """ if not (glob_wildcardORLst is None): if type(glob_wildcardORLst) is str: glob_wildcardORLst = [glob_wildcardORLst] for glob_wildcard in glob_wildcardORLst: print('glob_wildcard', glob_wildcard) pathLst.extend(glob.glob(glob_wildcard)) print("collect_probes: [n=%i]" % (len(pathLst))) LL_Lst = [] probeLst = [] seqLst = [] s = 0 for csv_path in pathLst: plst = IO.simple_inCSV(csv_path) LL = plst.pop(0) seq_idx = LL.index('seq') if LL_Lst: assert len(LL_Lst[-1]) == len(LL) LL_Lst.append(LL) probe_lst = [] for line in plst: if len(line) < len(LL): break seq = line[seq_idx] if not (seq in seqLst): probe_lst.append(line) seqLst.append(seq) s += 1 source = "%s [n=%i]" % (os.path.basename(csv_path), len(probe_lst)) for p in probe_lst: p.append(source) probeLst.append(p) LL.append('source') print('collect_probes: %i probes %i probes deduped' % (s, len(probeLst))) probe_df = pd.DataFrame.from_records(probeLst, columns=LL) probe_df.to_csv(out_path) return pd.read_csv(out_path, header=0)
particles_num = 22 elif task_mode == '5_sin': from CPG_core.butterfly_osc.butterfly_oscillator_5_sin import oscillator_nw particles_num = 29 else: raise print('task mode does not exist') else: raise print("env :{} task does not implemented.".format(env_name)) results_dir = os.path.join(results_path, 'results') monitor_dir = os.path.join(results_dir, 'monitor') results = IO(os.path.join(results_path, 'results.pkl')).read_pickle() #plot # gen = len(results) # fitness =[] # for g in range (1, gen+1): # fitness.append(results['gen{}'.format(g)]['best_fitness']) # # fitness =np.array(fitness) # gen_x = np.linspace(1,30,30) # plt.plot(gen_x, fitness) # plt.show() # # if not os.path.isdir(results_dir): # os.makedirs(results_dir) # create path # if not os.path.isdir(monitor_dir):
def __init__(self, file_location, image_width, image_height): """Constructor: read image and specify dimensinons.""" self.img = IO.read_image(file_location, image_width, image_height) self.wide, self.tall = image_width, image_height
positive_part = p * self.positive_part negative_part = n * self.negative_part wracc = positive_part - negative_part # Get at least one occurence in the dataset if p == 0: positive_part = -negative_part / n # Return the upper bound and wracc score if return_supports: # Returned supports are needed for CloSpan algo return round(positive_part, 5), round(wracc, 5), p, n return round(positive_part, 5), round(wracc, 5) if __name__ == "__main__": import cProfile, pstats args = IO.from_stdin() ds = Dataset(args.negative_filepath, args.positive_filepath) algo = WraccPrefixSpan(ds) if args.cprofile: profiler = cProfile.Profile() profiler.enable() results = algo(args.k) profiler.disable() stats = pstats.Stats(profiler).sort_stats("cumtime") stats.print_stats() else: results = algo(args.k)
"filenames":["theTargeFile1","targetfile2"], "words":[ "escribir","todas","las palabras a buscar" ] }""") # argument parsing args = parser.parse_args() parsed_args = json.loads(args.json) filenames = parsed_args["filenames"] words = parsed_args["words"] # data retrieving files_contents = [(IO.read_function(filename), filename) for filename in filenames] def show(result): console.message(result, console.bold) def persist(filename, word, ocurrences): db.upsert( { 'date': currentDate, 'ocurrence': { 'filename': filename, 'ocurrences': ocurrences, 'word': word } }, (query.filename == filename) & (query.word == word))
opaque = o3d.io.read_point_cloud("./%s/%s/depth2pcd_GT_%d.pcd" % (dset, name, obj_idx)) print(np.array(opaque.points).shape) o3d.visualization.draw_geometries([opaque]) opaque = o3d.io.read_point_cloud("./%s/%s/Ground_Truth_recenter_%d.pcd" % (dset, name, obj_idx)) print(np.array(opaque.points).shape) o3d.visualization.draw_geometries([opaque]) pred_i = o3d.io.read_point_cloud("./%s/%s/depth2pcd_pred_%d.pcd" % (dset, name, obj_idx)) print(np.array(pred_i.points).shape) o3d.visualization.draw_geometries([pred_i]) depth_raw = IO.get("./%s/%s/depth.exr" % (dset, name)) plt.imshow(depth_raw) plt.show() pt_raw = deproject(depth_raw, INV_K) pcd = o3d.geometry.PointCloud() pcd.points = o3d.utility.Vector3dVector(pt_raw) print(np.array(pcd.points).shape) o3d.visualization.draw_geometries([pcd]) depth_gt = IO.get("./%s/%s/detph_GroundTruth.exr" % (dset, name)) plt.imshow(depth_gt) plt.show() pt_gt = deproject(depth_gt, INV_K) pcd = o3d.geometry.PointCloud() pcd.points = o3d.utility.Vector3dVector(pt_gt) print(np.array(pcd.points).shape)
def __init__(self, model_name, verbose): self.model_name = model_name self.verbose = verbose self.path = IO()
parser.add_argument('--MUTPB', type=float, default=0.1) parser.add_argument('--gain_max', type=float, default=2.0) parser.add_argument('--speed_max', type=float, default=2.0) args = parser.parse_args() env_name = args.env_name env = gym.make(env_name) log_name = 'PSO4_open' # Set the logging variables # This also creates a new log file # Create log files log_dir = configure_log_dir(env_name, txt=log_name, No_time=False) logging_output(log_dir) logger = LoggerCsv(log_dir, csvname='log_results') results_IO = IO(os.path.join(log_dir, 'results.pkl')) args_IO = IO(os.path.join(log_dir, 'args.pkl')).to_pickle(args) def parmeter_generate(pmin, pmax): parm_list = [random.uniform(pmin, pmax) for _ in range(27)] return parm_list def generate(size, pmin, pmax, smin, smax): part = creator.Particle(parmeter_generate(pmin, pmax)) part.speed = [random.uniform(smin, smax) for _ in range(size)] part.smin = smin part.smax = smax return part
def img2pcd(name, load_path, save_path, inv_k, centering=False, normalize=False, skip=False, plot=False): """Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds. """ mask = IO.get(os.path.join(load_path, "%s/instance_segment.png" % name)) # Separate multiple objects into multiple point clouds mask_vals = np.unique(mask[:, :, 2])[ 1:] # each object is indicated by a distinct value in RED channel maxdis = [] skip_count = 0 for i in range(len(mask_vals)): mask_i = np.array(mask[:, :, 2] == mask_vals[i], dtype=np.float32) if plot: cv2.imshow("%s idx:%d" % (name, i), mask_i) cv2.waitKey(0) mask_pcd = deproject(mask_i, inv_k).sum(axis=1) > 0 # opaque_depth = IO.get(os.path.join(load_path, "%s/detph_GroundTruth.exr" % name)) # opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd] opaque_pcd = IO.get( os.path.join(load_path, "%s/Ground_Truth_%d.pcd" % (name, i))) transp_depth = IO.get(os.path.join(load_path, "%s/depth.exr" % name)) transp_pcd = deproject(transp_depth, inv_k)[mask_pcd] center = transp_pcd.mean(axis=0) if centering: opaque_pcd -= center transp_pcd -= center maxdis.append( np.max((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd))))) if normalize and not skip: opaque_pcd /= maxdis[-1] * 1.01 transp_pcd /= maxdis[-1] * 1.01 if maxdis[-1] >= 1 and skip: skip_count += 1 continue if not os.path.exists(os.path.join(save_path, name)): os.mkdir(os.path.join(save_path, name)) if maxdis[-1] == 0: print((name, i)) # save centering and scaling factors factors = { 'centering': centering, 'center_position': center.tolist(), 'normalize': normalize, 'normalize_factor': maxdis[-1] * 1.01, } with open( os.path.join(save_path, "%s/scale_factor_%d.json" % (name, i)), 'w') as outfile: json.dump(factors, outfile) # IO.put(os.path.join(save_path, "%s/depth2pcd_GT_%d.pcd" % (name, i)), opaque_pcd) IO.put( os.path.join(save_path, "%s/Ground_Truth_recenter_%d.pcd" % (name, i)), opaque_pcd) IO.put(os.path.join(save_path, "%s/depth2pcd_%d.pcd" % (name, i)), transp_pcd) # print(mask_pcd.sum()) # print(opaque_pcd.shape) # print(transp_pcd.shape) return np.max(maxdis), skip_count