def test_do_remove_folder_with_contents(self):
# does nothing if folder does not exist
_, remove_count = util.do_remove_folder_with_contents(
self._utilTestFolder + '/xxxxx', dry_run='DRY_RUN')
self.assertEqual(remove_count, 0)
# set up TestData folder
localFolder = self._utilTestFolder + "/Remove Folder"
util.ensure_directory(localFolder)
self.assertTrue(os.path.isdir(localFolder))
projc = localFolder + "/Proj C"
util.ensure_directory(projc)
self.assertTrue(os.path.isdir(projc))
bryson = os.path.join(projc, 'Bryson')
util.ensure_directory(bryson)
emily = os.path.join(projc, 'Emily', 'Nine')
util.ensure_directory(emily)
corlan = os.path.join(projc, 'Corlan', 'Three')
util.ensure_directory(corlan)
util.make_small_textfile(corlan, 'red')
util.make_small_textfile(corlan, 'green')
util.make_small_textfile(emily, 'blue')
util.make_small_textfile(emily, 'pink')
# verify some local folder contents
self.assertTrue(os.path.isdir(bryson))
self.assertTrue(os.path.isdir(corlan))
self.assertTrue(os.path.exists(os.path.join(emily, 'blue')))
# attempt to remove folder does not remove regular file
util.do_remove_folder_with_contents(os.path.join(emily, 'blue'))
self.assertTrue(os.path.exists(os.path.join(emily, 'blue')))
# removing interior folder removes that folder
util.do_remove_folder_with_contents(corlan)
self.assertFalse(os.path.exists(corlan))
self.assertTrue(os.path.exists(emily))
# top folder corlan should still be present; remove now
self.assertTrue(os.path.isdir(os.path.join(projc, 'Corlan')))
util.do_remove_folder_with_contents(os.path.join(projc, 'Corlan'))
self.assertFalse(os.path.isdir(os.path.join(projc, 'Corlan')))
# full folder corlan should still be present; remove now
self.assertTrue(os.path.isdir(os.path.join(projc, 'Emily')))
util.do_remove_folder_with_contents(os.path.join(projc, 'Emily'))
self.assertFalse(os.path.isdir(os.path.join(projc, 'Emily')))
# remove this test's local folder
util.do_remove_folder_with_contents(localFolder)
self.assertFalse(os.path.isdir(localFolder))
def save_screenshot(self):
ensure_directory('screenshots')
FILENAME_FORMAT = "screenshots/{:04d}.png"
index = 0
while os.path.isfile(FILENAME_FORMAT.format(index)):
index += 1
filename = FILENAME_FORMAT.format(index)
image = self.get_image(flip_red_blue=True)
cv2.imwrite(filename, image)
print("Screenshot saved to " + filename + ".")
def test_ensure_directory(self):
# ensure test folder
localFolder = self._utilTestFolder + "/EnsureDirectory"
util.ensure_directory(localFolder)
# ok if directory already present
util.ensure_directory(localFolder)
# does nothing if regular file already present at path
# add a regular file inside test folder
textfile = os.path.join(localFolder, 'textfile')
util.make_small_textfile(localFolder, 'textfile')
self.assertTrue(os.path.exists(textfile))
# attempt to overwrite regular file with directory
util.ensure_directory(textfile)
self.assertTrue(os.path.exists(textfile))
self.assertFalse(os.path.isdir(textfile))
# can add another folder inside
util.ensure_directory(localFolder + '/textfile2')
self.assertTrue(os.path.exists(localFolder + '/textfile2'))
# clean up
os.rmdir(localFolder + '/textfile2')
os.remove(textfile)
os.rmdir(localFolder)
self.assertFalse(os.path.isdir(localFolder))
def render_image_for_index(sdf_net,
latent_codes,
index,
crop=False,
resolution=800):
ensure_directory('screenshots')
FILENAME = 'screenshots/raymarching-examples/image-{:d}-{:d}.png'
filename = FILENAME.format(index, resolution)
if os.path.isfile(filename):
return Image.open(filename)
img = render_image(sdf_net,
latent_codes[index],
resolution=resolution,
crop=crop)
img.save(filename)
return img
def process_model_files():
ensure_directory(DIRECTORY_SDF)
files = list(get_model_files())
worker_count = os.cpu_count() // 2
print("Using {:d} processes.".format(worker_count))
pool = Pool(worker_count)
progress = tqdm(total=len(files))
def on_complete(*_):
progress.update()
for filename in files:
pool.apply_async(process_model_file,
args=(filename, ),
callback=on_complete)
pool.close()
pool.join()
def create_image_sequence():
ensure_directory('images')
frame_index = 0
viewer = MeshRenderer(size=1080, start_thread=False)
progress_bar = tqdm(total=SAMPLE_COUNT * TRANSITION_FRAMES)
for sample_index in range(SAMPLE_COUNT):
for step in range(TRANSITION_FRAMES):
code = torch.tensor(spline(float(sample_index) + step / TRANSITION_FRAMES), dtype=torch.float32, device=device)
if ROTATE_MODEL:
viewer.rotation = (147 + frame_index / (SAMPLE_COUNT * TRANSITION_FRAMES) * 360 * 6, 40)
viewer.set_mesh(sdf_net.get_mesh(code, voxel_resolution=128, sphere_only=False, level=SURFACE_LEVEL))
image = viewer.get_image(flip_red_blue=True)
cv2.imwrite("images/frame-{:05d}.png".format(frame_index), image)
frame_index += 1
progress_bar.update()
print("\n\nUse this command to create a video:\n")
print('ffmpeg -framerate 30 -i images/frame-%05d.png -c:v libx264 -profile:v high -crf 19 -pix_fmt yuv420p video.mp4')
def test_do_remove_fs_item(self):
# set up TestData folder
folder = self._utilTestFolder + "/RemoveItem"
self.assertEqual(
folder,
"/Users/carolclark/Library/CCDev/TestData/util_py/RemoveItem")
util.ensure_directory(folder)
# dry-run remove top folder: check values returned, actual file status
self.assertTrue(os.path.isdir(folder))
output, remove_count = util.do_remove_fs_item(folder, 'DRY_RUN')
self.assertEqual(output, 'remove directory {}\n'.format(folder))
self.assertEqual(remove_count, 1)
self.assertTrue(os.path.isdir(folder))
# remove top folder without dry-run: check values returned, actual file status
output, remove_count = util.do_remove_fs_item(folder)
self.assertEqual(output, 'remove directory {}\n'.format(folder))
self.assertEqual(remove_count, 1)
self.assertFalse(os.path.isdir(folder))
def process_model_file(filename):
voxels_filename = get_voxel_filename(filename)
sdf_cloud_filename = get_sdf_cloud_filename(filename)
if is_bad_mesh(filename):
return
if not (CREATE_VOXELS and not os.path.isfile(voxels_filename)
or CREATE_SDF_CLOUDS and not os.path.isfile(sdf_cloud_filename)):
return
mesh = trimesh.load(filename)
if ROTATION is not None:
mesh.apply_transform(ROTATION)
mesh = scale_to_unit_sphere(mesh)
surface_point_cloud = get_surface_point_cloud(mesh)
if CREATE_SDF_CLOUDS:
try:
points, sdf = surface_point_cloud.sample_sdf_near_surface(
number_of_points=SDF_CLOUD_SAMPLE_SIZE,
sign_method='depth',
min_size=0.015)
combined = np.concatenate((points, sdf[:, np.newaxis]), axis=1)
ensure_directory(os.path.dirname(sdf_cloud_filename))
np.save(sdf_cloud_filename, combined)
except BadMeshException:
tqdm.write("Skipping bad mesh. ({:s})".format(filename))
mark_bad_mesh(filename)
return
if CREATE_VOXELS:
try:
voxels = surface_point_cloud.get_voxels(
voxel_resolution=VOXEL_RESOLUTION, use_depth_buffer=True)
ensure_directory(os.path.dirname(voxels_filename))
np.save(voxels_filename, voxels)
except BadMeshException:
tqdm.write("Skipping bad mesh. ({:s})".format(filename))
mark_bad_mesh(filename)
return
def create_objects():
from util import ensure_directory
from rendering.raymarching import render_image
from rendering.math import get_rotation_matrix
import os
ensure_directory('generated_objects/')
image_filename = 'generated_objects/chair-{:03d}.png'
mesh_filename = 'generated_objects/chair-{:03d}.stl'
index = 0
while True:
if os.path.exists(image_filename.format(index)) or os.path.exists(mesh_filename.format(index)):
index += 1
continue
latent_code = standard_normal_distribution.sample((LATENT_CODE_SIZE,)).to(device)
image = render_image(sdf_net, latent_code, resolution=128, sdf_offset=-SURFACE_LEVEL, ssaa=2, radius=1.4, color=(0.7, 0.7, 0.7))
image.save(image_filename.format(index))
mesh = sdf_net.get_mesh(latent_code, voxel_resolution=256, sphere_only=False, level=SURFACE_LEVEL)
mesh.apply_transform(get_rotation_matrix(90, 'x'))
mesh.apply_translation((0, 0, -np.min(mesh.vertices[:, 2])))
mesh.export(mesh_filename.format(index))
print("Created mesh for index {:d}".format(index))
index += 1
def process_model_files():
for res in VOXEL_RESOLUTIONS:
ensure_directory(DIRECTORY_VOXELS.format(res))
if CREATE_UNIFORM_AND_SURFACE:
ensure_directory(DIRECTORY_UNIFORM)
ensure_directory(DIRECTORY_SURFACE)
if CREATE_SDF_CLOUDS:
ensure_directory(DIRECTORY_SDF_CLOUD)
ensure_directory(DIRECTORY_BAD_MESHES)
files = list(get_model_files())
worker_count = os.cpu_count() // 2
print("Using {:d} processes.".format(worker_count))
pool = Pool(worker_count)
progress = tqdm(total=len(files))
def on_complete(*_):
progress.update()
for filename in files:
pool.apply_async(process_model_file, args=(filename,), callback=on_complete)
pool.close()
pool.join()
TRANSITION_FRAMES, :] = np.linspace(colors[indices[i]],
colors[indices[i + 1]],
num=TRANSITION_FRAMES)
embedded_spline = scipy.interpolate.CubicSpline(
np.arange(SAMPLE_COUNT + 1),
latent_codes_embedded[indices, :],
axis=0,
bc_type='periodic')
frame_latent_codes_embedded = embedded_spline(progress)
frame_latent_codes_embedded[0, :] = frame_latent_codes_embedded[-1, :]
width, height = 40, 40
PLOT_FILE_NAME = 'tsne.png'
ensure_directory('images')
margin = 2
range_x = (latent_codes_embedded[:, 0].min() - margin,
latent_codes_embedded[:, 0].max() + margin)
range_y = (latent_codes_embedded[:, 1].min() - margin,
latent_codes_embedded[:, 1].max() + margin)
plt.ioff()
def create_plot(index, resolution=1080, filename=PLOT_FILE_NAME, dpi=100):
frame_color = frame_colors[index, :]
frame_color = (frame_color[0], frame_color[1], frame_color[2], 1.0)
size_inches = resolution / dpi
def generate_sample_tests(destination):
# set up folder for sample test files
util.ensure_directory(destination)
test_equality_pass = SampleTestFile("test_equality_pass", [
SampleTestClass("TestEquality_Pass", [
SampleTestMethod(
"test_equality_pass",
["self.assertTrue (1 == 1)", "self.assertTrue (2 == 2)"])
])
])
test_equality_pass.write_test_file(destination)
test_equality_fail = SampleTestFile("test_equality_fail", [
SampleTestClass("TestEquality_Fail", [
SampleTestMethod(
"test_equality_fail",
["self.assertTrue (1 == 1)", "self.assertTrue (2 == 1)"])
])
])
test_equality_fail.write_test_file(destination)
test_equality_error = SampleTestFile("test_equality_error", [
SampleTestClass("TestEquality_Error", [
SampleTestMethod(
"test_equality_error",
["self.assertTrueX (1 == 1)", "self.assertTrue (2 == 1)"])
])
])
test_equality_error.write_test_file(destination)
test_equality_PF = SampleTestFile("test_equality_PF", [
SampleTestClass("TestEquality_PF", [
SampleTestMethod("test_equality_pass",
["self.assertTrue (1 == 1)"]),
SampleTestMethod("test_equality_fail",
["self.assertTrue (2 == 1)"])
])
])
test_equality_PF.write_test_file(destination)
test_equality_FP = SampleTestFile("test_equality_FP", [
SampleTestClass("TestEquality_FP", [
SampleTestMethod("test_equality_fail",
["self.assertTrue (2 == 1)"]),
SampleTestMethod("test_equality_pass",
["self.assertTrue (1 == 1)"])
])
])
test_equality_FP.write_test_file(destination)
test_equality_FF = SampleTestFile("test_equality_FF", [
SampleTestClass("TestEquality_FF", [
SampleTestMethod("test_equality_fail",
["self.assertTrue (1 == 2)"]),
SampleTestMethod("test_equality_fail1",
["self.assertTrue (2 == 1)"])
])
])
test_equality_FF.write_test_file(destination)
test_equality_PP = SampleTestFile("test_equality_PP", [
SampleTestClass("TestEquality_PP", [
SampleTestMethod("test_equality_pass",
["self.assertTrue (1 == 1)"]),
SampleTestMethod("test_equality_pass1",
["self.assertTrue (1 == 1)"])
])
])
test_equality_PP.write_test_file(destination)
test_equality_FPE = SampleTestFile("test_equality_FPE", [
SampleTestClass("TestEquality_FP", [
SampleTestMethod("test_equality_fail",
["self.assertTrue (2 == 1)"]),
SampleTestMethod("test_equality_pass",
["self.assertTrue (1 == 1)"])
]),
SampleTestClass("TestEquality_Error", [
SampleTestMethod(
"test_equality_error",
["self.assertTrueX (1 == 1)", "self.assertTrue (2 == 1)"])
])
])
test_equality_FPE.write_test_file(destination)
def mark_bad_mesh(model_filename):
filename = get_bad_mesh_filename(model_filename)
ensure_directory(os.path.dirname(filename))
open(filename, 'w').close()
def setUpModule():
# set up folder for testUtil files
util.ensure_directory(utilTestFolder())
import requests import patterns import oauth import util # Parameters for this script report_dirname = '/Users/kkolmar/common/iorj/report2020/' data_start_date = '2019-12-01' data_end_date = '2020-12-01' interesting_issues = ['2019-14-4', '2020-15-1', '2020-15-2', '2020-15-3'] dirpath = report_dirname archive_dir = os.path.join(dirpath, 'archive') issues_dir = os.path.join(archive_dir, 'issues') util.ensure_directory(issues_dir) # Формирование файла rawdownloads.txt: # Отчеты - Стандартные отчеты - Содержание - Загрузки файлов # страница загрузки - хост - iorj.hse.ru; загрузка файла - хост - iorj.hse.ru # группировка по страница загрузки - путь raw_downloads_path = os.path.join(dirpath, 'rawdownloads.txt') # Данные приложения IORJReport: # ID: 12f41230e56a42a78d25afefbfdf93b5 # Пароль: 98b2e0efb06a4cb08baedafeb82d93e1 # Callback URL: https://oauth.yandex.ru/verification_code # Получение токена: # https://oauth.yandex.ru/authorize?response_type=token&client_id=<идентификатор приложения> # https://oauth.yandex.ru/authorize?response_type=token&client_id=12f41230e56a42a78d25afefbfdf93b5
