def __get_text(self, chapter_start_url, book_name, topic):
'''
小说正文爬取并写入磁盘
:param chapter_start_url:
:param book_name:
:param topic:
:return:
'''
# 文章正文部分是js 用selenium模拟浏览器获取html页面
driver = dr()
driver.get(chapter_start_url)
# 网络时整个页面没有加载外全会导致抓取内容错误
# 直接调用sheep对整个线程不安全 故采用 ▲t1-▲t2的方式拖延程序
start_time = int(time.time())
while True:
end_time = int(time.time())
if end_time - start_time >= 1:
break
soup = BeautifulSoup(driver.page_source, "html5lib")
ps = soup.select("#chapter-content p")
text_list = [
"{0}\n".format(p.string.encode("utf-8")) for p in ps if p.string
]
driver.quit()
io = IO()
io.write(text_list, book_name, topic)
logging.info(
"--------------------{0}{1} ..下载完成--------------------".format(
book_name.encode("utf-8"), topic))
def pcd2img(name, load_path, save_path, k, factor_path=None):
"""Convert point clouds to depth images.
"""
mask = IO.get(os.path.join(load_path, "%s/instance_segment.png" % name))
mask_bg = np.array(mask[:, :, 2] == 0, dtype=np.float32)
depth_in = IO.get(os.path.join(load_path, "%s/depth.exr" % name))
depth_out = depth_in * mask_bg
mask_vals = np.unique(mask[:, :, 2])[1:]
for i in range(len(mask_vals)):
pcd_i = IO.get(
os.path.join(load_path, "%s/depth2pcd_pred_%d.pcd" % (name, i)))
if factor_path is not None:
with open(
os.path.join(factor_path,
"%s/scale_factor_%d.json" % (name, i))) as f:
factors = json.loads(f.read())
if factors['normalize']:
pcd_i *= float(factors['normalize_factor'])
if factors['centering']:
pcd_i += np.array(factors['center_position'])
depth_i = project_to_image(k, pcd_i)
mask_i = np.array(mask[:, :, 2] == mask_vals[i], dtype=np.float32)
depth_out += depth_i * mask_i
# write predicted depth image in exr
depth_out = np.expand_dims(depth_out, -1).repeat(3, axis=2)
pyexr.write(os.path.join(save_path, "%s/depth_pred.exr" % name), depth_out)
def main(config):
Misc.set_seed()
print(config)
data = IO.load_matrix_from_file(config.data)
train, test = IO.split_matrix(data, config.ratio)
config.num_user, config.num_item = data.shape
model = PerisJointModel(config)
model.fit(train)
m = model.evaluate(train, test)
return m
def rms(path, gt_name, pred_name):
""" Compute the average root-mean-squared error of predicted depth images.
"""
errors = []
for subdir, dirs, files in os.walk(path):
for dirname in sorted(dirs):
depth_gt = IO.get(os.path.join(path, '%s/%s' % (dirname, gt_name)))
depth_pred = IO.get(os.path.join(path, '%s/%s' % (dirname, pred_name)))
diff = np.where(depth_pred > 0, depth_pred - depth_gt, 0)
errors.append(np.sqrt(np.mean(diff**2)))
return np.array(errors).mean(), np.array(errors).sum()
def main():
# Part one
t0 = time.time()
# Get dependency formula
formula = IO.readFile("../data/input.txt")
part_one = compute(formula, 'ORE', 1)
time_part_one = round((time.time() - t0) * 1e3)
print("Solution to part one: %s (time taken %s[ms])" %
(part_one, time_part_one))
t0 = time.time()
formula = IO.readFile("../data/input.txt")
part_two = compute_fuel(formula)
time_part_two = round((time.time() - t0) * 1e3)
print("Solution to part two: %s (time taken %s[ms])" %
(part_two, time_part_two))
def __call__(self, k):
self._k = k
# Starting entries : all tid with pid set to -1
starting_entries = [(i, -1) for i in range(len(self._dataset._db))]
starting_pattern = [] # Void pattern
# Start the main process with void pattern
self._main_recursive(starting_pattern, starting_entries)
# Wait all threads
for thread in self._threads:
thread.join()
# Output the results on the stdout
IO.to_stdout(self._dataset, self._results)
return self._results
def __init__(self, file, input=[], verbose=True, reset=True):
""" Read file and give input to the intcode computer."""
self.memory = IO.read_file(file)
self.x, self.output = None, None
self.input = input if isinstance(input, list) else [input]
self.verbose = verbose
self.reset = reset
self.idle = False
self.n = len(self.memory)
self.i, self.base = 0, 0
def main():
x = IO.read_image('../data/input.txt')
t0 = time.time()
part_one, spot_index = compute_vision(x)
time_part_one = round((time.time() - t0) * 1e3)
print("Solution to part one: %s (time taken %s[ms])" %
(part_one, time_part_one))
t0 = time.time()
part_two = blast(x, spot_index, n_asteriods=200)
time_part_two = round((time.time() - t0) * 1e3)
print("Solution to part two: %s (time taken %s[ms])" %
(part_two, time_part_two))
def __init__(self, number_of_epochs=1000, content_weight=1e3, style_weight=1e-2, model_name="mobilenetv2_coco_voctrainaug", enable_gpu=False, verbose=False):
self.number_of_epochs = number_of_epochs
self.content_weight = content_weight
self.style_weight = style_weight
self.content_layers = ["block5_conv2"]
self.style_layers = ["block1_conv1", "block2_conv1", "block3_conv1", "block4_conv1", "block5_conv1"]
self.model_name = model_name
self.verbose = verbose
self.enable_gpu = enable_gpu
self._set_device()
tf.compat.v1.enable_eager_execution()
self.path = IO()
if(self.verbose == True):
print ("Eager Execution: {}".format(tf.executing_eagerly()))
def img2pcd(name, load_path, save_path, inv_k, normalize=False, plot=False):
"""Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds.
"""
mask = IO.get(os.path.join(load_path, "%s-mask.png" % name))
# Separate multiple objects into multiple point clouds
mask_copy = np.array(mask * 255, dtype=np.uint8)
if plot:
cv2.imshow("%s mask" % name, mask_copy)
cv2.waitKey(0)
contours, hierarchy = cv2.findContours(mask_copy, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
obj_idx = 0
for i in range(len(contours)):
if cv2.contourArea(contours[i]) > 100:
mask_i = np.zeros_like(mask)
cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=-1)
if plot:
cv2.imshow("%s idx:%d" % (name, i), mask_i)
cv2.waitKey(0)
mask_pcd = deproject(mask_i / 255, inv_k).sum(axis=1) > 0
opaque_depth = IO.get(os.path.join(load_path, "%s-opaque-depth-img.exr" % name))
opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd]
IO.put(os.path.join(save_path, "opaque/%s-%d.pcd" % (name, obj_idx)), opaque_pcd)
transp_depth = IO.get(os.path.join(load_path, "%s-transparent-depth-img.exr" % name))
transp_pcd = deproject(transp_depth, inv_k)[mask_pcd]
IO.put(os.path.join(save_path, "transparent/%s-%d.pcd" % (name, obj_idx)), transp_pcd)
if normalize: # normalize pcd to be within [-1, 1] for gridding
max_val = np.nanmax((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd))))
if max_val > 1:
print(max_val)
opaque_pcd /= max_val * 1.01
transp_pcd /= max_val * 1.01
with open(os.path.join(save_path, "norm_factors/%s-%d.json" % (name, obj_idx)), 'w') as outfile:
json.dump(max_val * 1.01, outfile)
obj_idx += 1
# print(mask_pcd.sum())
# print(opaque_pcd.shape)
# print(transp_pcd.shape)
else:
if plot:
mask_i = np.zeros_like(mask)
cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=5)
cv2.imshow("%s idx:%d" % (name, i), mask_i)
cv2.waitKey(0)
def main():
dir_path = dirname(realpath(__file__))
file_location = join(dir_path, "../data/test_input0.txt")
# Part one
t0 = time.time()
# Get dependency formula
formula = IO.readFile(file_location)
part_one = compute(formula, "ORE", 1)
time_part_one = round((time.time() - t0) * 1e3)
print(
"Solution to part one: %s (time taken %s[ms])"
% (part_one, time_part_one)
)
t0 = time.time()
formula = IO.readFile(file_location)
part_two = compute_fuel(formula)
time_part_two = round((time.time() - t0) * 1e3)
print(
"Solution to part two: %s (time taken %s[ms])"
% (part_two, time_part_two)
)
def main():
dir_path = dirname(realpath(__file__))
file_location = join(dir_path, "../data/input.txt")
x = IO.read_image(file_location)
t0 = time.time()
part_one, spot_index = compute_vision(x)
time_part_one = round((time.time() - t0) * 1e3)
print("Solution to part one: %s (time taken %s[ms])" %
(part_one, time_part_one))
t0 = time.time()
part_two = blast(x, spot_index, n_asteriods=200)
time_part_two = round((time.time() - t0) * 1e3)
print("Solution to part two: %s (time taken %s[ms])" %
(part_two, time_part_two))
def collect_probes(glob_wildcardORLst=None, pathLst=[], out_path=''):
"""
Imports either from probe list or from glob wildcard file or list
:param glob_wildcardORLst:
:param pathLst:
:param out_path:
:return:
"""
if not (glob_wildcardORLst is None):
if type(glob_wildcardORLst) is str:
glob_wildcardORLst = [glob_wildcardORLst]
for glob_wildcard in glob_wildcardORLst:
print('glob_wildcard', glob_wildcard)
pathLst.extend(glob.glob(glob_wildcard))
print("collect_probes: [n=%i]" % (len(pathLst)))
LL_Lst = []
probeLst = []
seqLst = []
s = 0
for csv_path in pathLst:
plst = IO.simple_inCSV(csv_path)
LL = plst.pop(0)
seq_idx = LL.index('seq')
if LL_Lst:
assert len(LL_Lst[-1]) == len(LL)
LL_Lst.append(LL)
probe_lst = []
for line in plst:
if len(line) < len(LL): break
seq = line[seq_idx]
if not (seq in seqLst):
probe_lst.append(line)
seqLst.append(seq)
s += 1
source = "%s [n=%i]" % (os.path.basename(csv_path), len(probe_lst))
for p in probe_lst:
p.append(source)
probeLst.append(p)
LL.append('source')
print('collect_probes: %i probes %i probes deduped' % (s, len(probeLst)))
probe_df = pd.DataFrame.from_records(probeLst, columns=LL)
probe_df.to_csv(out_path)
return pd.read_csv(out_path, header=0)
particles_num = 22
elif task_mode == '5_sin':
from CPG_core.butterfly_osc.butterfly_oscillator_5_sin import oscillator_nw
particles_num = 29
else:
raise print('task mode does not exist')
else:
raise print("env :{} task does not implemented.".format(env_name))
results_dir = os.path.join(results_path, 'results')
monitor_dir = os.path.join(results_dir, 'monitor')
results = IO(os.path.join(results_path, 'results.pkl')).read_pickle()
#plot
# gen = len(results)
# fitness =[]
# for g in range (1, gen+1):
# fitness.append(results['gen{}'.format(g)]['best_fitness'])
#
# fitness =np.array(fitness)
# gen_x = np.linspace(1,30,30)
# plt.plot(gen_x, fitness)
# plt.show()
#
# if not os.path.isdir(results_dir):
# os.makedirs(results_dir) # create path
# if not os.path.isdir(monitor_dir):
def __init__(self, file_location, image_width, image_height):
"""Constructor: read image and specify dimensinons."""
self.img = IO.read_image(file_location, image_width, image_height)
self.wide, self.tall = image_width, image_height
positive_part = p * self.positive_part
negative_part = n * self.negative_part
wracc = positive_part - negative_part
# Get at least one occurence in the dataset
if p == 0:
positive_part = -negative_part / n
# Return the upper bound and wracc score
if return_supports:
# Returned supports are needed for CloSpan algo
return round(positive_part, 5), round(wracc, 5), p, n
return round(positive_part, 5), round(wracc, 5)
if __name__ == "__main__":
import cProfile, pstats
args = IO.from_stdin()
ds = Dataset(args.negative_filepath, args.positive_filepath)
algo = WraccPrefixSpan(ds)
if args.cprofile:
profiler = cProfile.Profile()
profiler.enable()
results = algo(args.k)
profiler.disable()
stats = pstats.Stats(profiler).sort_stats("cumtime")
stats.print_stats()
else:
results = algo(args.k)
"filenames":["theTargeFile1","targetfile2"],
"words":[
"escribir","todas","las palabras a buscar"
]
}""")
# argument parsing
args = parser.parse_args()
parsed_args = json.loads(args.json)
filenames = parsed_args["filenames"]
words = parsed_args["words"]
# data retrieving
files_contents = [(IO.read_function(filename), filename)
for filename in filenames]
def show(result):
console.message(result, console.bold)
def persist(filename, word, ocurrences):
db.upsert(
{
'date': currentDate,
'ocurrence': {
'filename': filename,
'ocurrences': ocurrences,
'word': word
}
}, (query.filename == filename) & (query.word == word))
opaque = o3d.io.read_point_cloud("./%s/%s/depth2pcd_GT_%d.pcd" %
(dset, name, obj_idx))
print(np.array(opaque.points).shape)
o3d.visualization.draw_geometries([opaque])
opaque = o3d.io.read_point_cloud("./%s/%s/Ground_Truth_recenter_%d.pcd" %
(dset, name, obj_idx))
print(np.array(opaque.points).shape)
o3d.visualization.draw_geometries([opaque])
pred_i = o3d.io.read_point_cloud("./%s/%s/depth2pcd_pred_%d.pcd" %
(dset, name, obj_idx))
print(np.array(pred_i.points).shape)
o3d.visualization.draw_geometries([pred_i])
depth_raw = IO.get("./%s/%s/depth.exr" % (dset, name))
plt.imshow(depth_raw)
plt.show()
pt_raw = deproject(depth_raw, INV_K)
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(pt_raw)
print(np.array(pcd.points).shape)
o3d.visualization.draw_geometries([pcd])
depth_gt = IO.get("./%s/%s/detph_GroundTruth.exr" % (dset, name))
plt.imshow(depth_gt)
plt.show()
pt_gt = deproject(depth_gt, INV_K)
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(pt_gt)
print(np.array(pcd.points).shape)
def __init__(self, model_name, verbose):
self.model_name = model_name
self.verbose = verbose
self.path = IO()
parser.add_argument('--MUTPB', type=float, default=0.1)
parser.add_argument('--gain_max', type=float, default=2.0)
parser.add_argument('--speed_max', type=float, default=2.0)
args = parser.parse_args()
env_name = args.env_name
env = gym.make(env_name)
log_name = 'PSO4_open'
# Set the logging variables
# This also creates a new log file
# Create log files
log_dir = configure_log_dir(env_name, txt=log_name, No_time=False)
logging_output(log_dir)
logger = LoggerCsv(log_dir, csvname='log_results')
results_IO = IO(os.path.join(log_dir, 'results.pkl'))
args_IO = IO(os.path.join(log_dir, 'args.pkl')).to_pickle(args)
def parmeter_generate(pmin, pmax):
parm_list = [random.uniform(pmin, pmax) for _ in range(27)]
return parm_list
def generate(size, pmin, pmax, smin, smax):
part = creator.Particle(parmeter_generate(pmin, pmax))
part.speed = [random.uniform(smin, smax) for _ in range(size)]
part.smin = smin
part.smax = smax
return part
def img2pcd(name,
load_path,
save_path,
inv_k,
centering=False,
normalize=False,
skip=False,
plot=False):
"""Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds.
"""
mask = IO.get(os.path.join(load_path, "%s/instance_segment.png" % name))
# Separate multiple objects into multiple point clouds
mask_vals = np.unique(mask[:, :, 2])[
1:] # each object is indicated by a distinct value in RED channel
maxdis = []
skip_count = 0
for i in range(len(mask_vals)):
mask_i = np.array(mask[:, :, 2] == mask_vals[i], dtype=np.float32)
if plot:
cv2.imshow("%s idx:%d" % (name, i), mask_i)
cv2.waitKey(0)
mask_pcd = deproject(mask_i, inv_k).sum(axis=1) > 0
# opaque_depth = IO.get(os.path.join(load_path, "%s/detph_GroundTruth.exr" % name))
# opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd]
opaque_pcd = IO.get(
os.path.join(load_path, "%s/Ground_Truth_%d.pcd" % (name, i)))
transp_depth = IO.get(os.path.join(load_path, "%s/depth.exr" % name))
transp_pcd = deproject(transp_depth, inv_k)[mask_pcd]
center = transp_pcd.mean(axis=0)
if centering:
opaque_pcd -= center
transp_pcd -= center
maxdis.append(
np.max((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd)))))
if normalize and not skip:
opaque_pcd /= maxdis[-1] * 1.01
transp_pcd /= maxdis[-1] * 1.01
if maxdis[-1] >= 1 and skip:
skip_count += 1
continue
if not os.path.exists(os.path.join(save_path, name)):
os.mkdir(os.path.join(save_path, name))
if maxdis[-1] == 0:
print((name, i))
# save centering and scaling factors
factors = {
'centering': centering,
'center_position': center.tolist(),
'normalize': normalize,
'normalize_factor': maxdis[-1] * 1.01,
}
with open(
os.path.join(save_path, "%s/scale_factor_%d.json" % (name, i)),
'w') as outfile:
json.dump(factors, outfile)
# IO.put(os.path.join(save_path, "%s/depth2pcd_GT_%d.pcd" % (name, i)), opaque_pcd)
IO.put(
os.path.join(save_path,
"%s/Ground_Truth_recenter_%d.pcd" % (name, i)),
opaque_pcd)
IO.put(os.path.join(save_path, "%s/depth2pcd_%d.pcd" % (name, i)),
transp_pcd)
# print(mask_pcd.sum())
# print(opaque_pcd.shape)
# print(transp_pcd.shape)
return np.max(maxdis), skip_count