def convert_to_lmdb(self, lmdb_converter_path):
"""
Converts octree files to lmdb files
Args:
lmdb_converter_path (str): Path to octree to lmdb converter executable.
"""
octree_train_db_path = os.path.join(self.dataset_directory,
Dataset.OCTREE_TRAIN_DB_FOLDER)
octree_test_db_path = os.path.join(self.dataset_directory,
Dataset.OCTREE_TEST_DB_FOLDER)
self.prepare_octree_file_list()
if (find_files(octree_train_db_path, "*.mdb", find_first_instance=True)
or find_files(
octree_test_db_path, "*.mdb", find_first_instance=True)):
raise AttributeError(
"*.mdb file found in dataset path ({0}) please remove them".
format(self.dataset_directory))
subprocess.check_call([
lmdb_converter_path, self.dataset_directory,
self.octree_train_list_path, octree_train_db_path
])
subprocess.check_call([
lmdb_converter_path, self.dataset_directory,
self.octree_test_list_path, octree_test_db_path
])
def main():
parse_args()
if not os.path.isdir('src') or not os.path.isdir('utils'):
print('CWD is not the root of the repository.')
return 1
sys.stdout.write('Fixing Lua tabs ')
for filename in find_files('.', ['.lua']):
sys.stdout.write('.')
sys.stdout.flush()
lines = read_text_file(filename).strip().split('\n')
new_lines = []
for line in lines:
m = LEADING_TABS.match(line)
if m is not None:
line = line[m.start():m.end()].expandtabs(
SPACES_PER_TAB) + line[m.end():]
new_lines.append(line.rstrip() + '\n')
write_text_file(filename, ''.join(new_lines))
call(['git', 'add', '--renormalize', filename])
print(' done.')
sys.stdout.write('\nFormatting C++ ')
for filename in find_files('./src', ['.cc', '.h']):
if 'third_party' in filename:
continue
sys.stdout.write('.')
sys.stdout.flush()
call(['clang-format', '-i', filename])
call(['git', 'add', '--renormalize', filename])
print(' done.')
sys.stdout.write('\nFormatting Python utils ')
for filename in find_files('./utils', ['.py']):
sys.stdout.write('.')
sys.stdout.flush()
call(['pyformat', '-i', filename])
call(['git', 'add', '--renormalize', filename])
print(' done.')
print('Formatting finished.')
return 0
def main():
parse_args()
if not os.path.isdir('src') or not os.path.isdir('utils'):
print('CWD is not the root of the repository.')
return 1
sys.stdout.write('Fixing Lua tabs ')
for filename in find_files('.', ['.lua']):
sys.stdout.write('.')
sys.stdout.flush()
lines = read_text_file(filename).strip().split('\n')
new_lines = []
for line in lines:
m = LEADING_TABS.match(line)
if m is not None:
line = line[m.start():m.end()].expandtabs(
SPACES_PER_TAB) + line[m.end():]
new_lines.append(line.rstrip() + '\n')
write_text_file(filename, ''.join(new_lines))
print(' done.')
sys.stdout.write('\nFormatting C++ ')
for filename in find_files('./src', ['.cc', '.h']):
if 'third_party' in filename:
continue
sys.stdout.write('.')
sys.stdout.flush()
call(['clang-format', '-i', filename])
print(' done.')
sys.stdout.write('\nFormatting Python utils ')
for filename in find_files('./utils', ['.py']):
sys.stdout.write('.')
sys.stdout.flush()
call(['pyformat', '-i', filename])
print(' done.')
print 'Formatting finished.'
return 0
def prepare_octree_file_list(self):
"""
Creates file list in dataset directory. File list is composed of all octree files and
their corresponding classification category.
The file list should be of the form:
<file_name.octree> <category_number>
"""
model_directory_path = os.path.join(self.dataset_directory,
self.dataset_filename_stem)
class_list = os.listdir(model_directory_path)
class_map = self._generate_class_map(class_list)
test_list = find_files(self.dataset_directory, '*.octree')
train_list = find_files(self.dataset_directory, '*.octree')
test_list_generator = self.class_file_map_generator(
test_list, class_map)
train_list_generator = self.class_file_map_generator(
train_list, class_map)
write_out_iterable(self.octree_train_list_path, train_list_generator)
write_out_iterable(self.octree_test_list_path, test_list_generator)
def main(ext):
# arg_list is a list of tuples, each tuple is one call to mol_to_sdf
#arg_list = [ (i["path"], i["filename"][:-4]) for i in find_files("/".join(os.path.abspath("").split("/")[:-1]) + "/datasets/dss_tox/DSSToxAll_20151019/ToxAll/", "mol$") ]
arg_list = [(i["path"], i["filename"][:-(len(ext) + 1)])
for i in find_files(
"/".join(os.path.abspath("").split("/")[:-1]) +
"/datasets/activity_cliffs/", ext)]
mp = Pool(number_of_processes)
#mp.starmap(mol_to_sdf, arg_list, chunksize=1)
#mp.starmap(del_mol, arg_list, chunksize=1)
#mp.starmap(del_xyz, arg_list, chunksize=1)
mp.starmap(mol2_to_sdf, arg_list, chunksize=1)
#mp.starmap(del_sdf, arg_list, chunksize=1)
mp.close()
mp.join()
def main():
"""Updates the copyright year in all source files to the given year."""
if len(sys.argv) != 2:
print('Usage: update_copyright.py <year>')
return 1
try:
year = sys.argv[1]
sys.stdout.write('Updating copyright year to: ' + year + ' ')
src_path = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../src'))
# Fix copyright headers in C++ files
for filename in find_files(src_path, ['.h', '.cc']):
sys.stdout.write('.')
sys.stdout.flush()
lines = read_text_file(filename).strip().split('\n')
new_lines = []
regex = re.compile(
'(.*Copyright \(C\) \d\d\d\d)(.*)( by the Widelands Development Team.*)')
for line in lines:
match = regex.match(line)
if match:
if match.group(1).endswith(year):
line = match.group(1) + match.group(3)
else:
line = match.group(1) + '-' + year + match.group(3)
new_lines.append(line.rstrip() + '\n')
write_text_file(filename, ''.join(new_lines))
# Now update the Buildinfo
filename = os.path.join(src_path, 'build_info.h')
lines = read_text_file(filename).strip().split('\n')
new_lines = []
regex = re.compile(
'(.*constexpr uint16_t kWidelandsCopyrightEnd = )(\d\d\d\d)(;)')
for line in lines:
match = regex.match(line)
if match:
line = match.group(1) + year + match.group(3)
new_lines.append(line.rstrip() + '\n')
write_text_file(filename, ''.join(new_lines))
print(' done.')
except Exception:
print('Something went wrong:')
traceback.print_exc()
return 1
def clean_dataset(input_folder):
"""
Fixes headers of all OFF files in a given folder.
Args:
input_folder (str): Folder to search for off files
"""
executor = Pool()
file_list = file_utils.find_files(input_folder, '*.[Oo][Ff][Ff]')
files_cleaned_list = executor.map(clean_off_file, file_list)
num_files_cleaned = 0
for file_cleaned in files_cleaned_list:
if file_cleaned:
num_files_cleaned += 1
print("{0} out of {1} files cleaned".format(num_files_cleaned,
len(files_cleaned_list)))
def get_initial_action(self, conversion_arguments, starting_action):
"""
Determines initial action to take with dataset when automatically preparing dataset.
Args:
conversion_arguments (ConversionArguments): Arguments specified to convert to octree
starting_action (DatasetActions): starting action to start when preparing dataset.
Returns:
DatasetActions: Action to perform when preparing the dataset.
Raises:
AttributeError: Any error in the conversion arguments or starting action which
would break the conversion process.
"""
try:
next_action, cached_conversion_arguments = self.get_cached_data()
except IOError:
next_action = DatasetActions.Retrieve
cached_conversion_arguments = conversion_arguments
if starting_action:
if starting_action <= next_action:
next_action = starting_action
else:
raise AttributeError(
"starting action {0} is ahead of cached action {1}")
if cached_conversion_arguments != conversion_arguments:
if next_action <= DatasetActions.CreateOctree:
if (find_files(self.dataset_directory,
"*.octree",
find_first_instance=True)):
raise AttributeError(
Dataset._get_conversion_error_str(
cached_conversion_arguments,
conversion_arguments) +
"remove octree file or use same conversion arguments")
else:
raise AttributeError(
Dataset._get_conversion_error_str(
cached_conversion_arguments, conversion_arguments) +
"specify earlier action or use same conversion arguments")
return next_action
def main():
"""Updates the copyright year in all source files to the given year."""
if len(sys.argv) != 2:
print('Usage: update_copyright.py <year>')
return 1
try:
year = sys.argv[1]
sys.stdout.write('Updating copyright year to: ' + year + ' ')
src_path = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../src'))
# Fix copyright headers in C++ files
for filename in find_files(src_path, ['.h', '.cc']):
sys.stdout.write('.')
sys.stdout.flush()
lines = read_text_file(filename).strip().split('\n')
new_lines = []
regex = re.compile(
'(.*Copyright \(C\) \d\d\d\d)(.*)( by the Widelands Development Team.*)')
for line in lines:
match = regex.match(line)
if match:
line = match.group(1) + '-' + year + match.group(3)
new_lines.append(line.rstrip() + '\n')
write_text_file(filename, ''.join(new_lines))
# Now update the Buildinfo
filename = os.path.join(src_path, 'build_info.h')
lines = read_text_file(filename).strip().split('\n')
new_lines = []
regex = re.compile(
'(.*constexpr uint16_t kWidelandsCopyrightEnd = )(\d\d\d\d)(;)')
for line in lines:
match = regex.match(line)
if match:
line = match.group(1) + year + match.group(3)
new_lines.append(line.rstrip() + '\n')
write_text_file(filename, ''.join(new_lines))
print(' done.')
except Exception:
print('Something went wrong:')
traceback.print_exc()
return 1
def convert_to_octree(self, octree_converter_path, conversion_arguments):
"""
Converts points files to octrees
Args:
octree_converter_path (str): Path to points to octree converter executable
conversion_arguments (ConversionArguments): Arguments specified to convert to octree
"""
#check for filelist
points_list_path = os.path.join(self.dataset_directory,
Dataset.POINTS_LIST_FILE)
file_list = line_separator_generator(find_files(
self.dataset_directory, '*.points'),
use_os_sep=False)
write_out_iterable(points_list_path, file_list)
subprocess.check_call([
octree_converter_path, points_list_path,
str(conversion_arguments.depth),
str(conversion_arguments.full_layer),
str(conversion_arguments.displacement),
str(conversion_arguments.augmentation),
str(int(conversion_arguments.for_segmentation))
])
def create_files(in_dir):
print("Beginning Creation of txt Files")
for in_file in find_files(in_dir, ".json"):
with open(os.path.join(in_file["path"], in_file["filename"]), 'r') as raw_in:
data = json.load(raw_in)
for s in data:
m_id = s["material_id"]
path = in_dir + m_id
if not os.path.exists(path):
os.mkdir(path)
if in_file["filename"] == "surfaces.json":
pre = "surfaces_"
else:
pre = "details_"
final_path = os.path.join(path, pre + m_id + ".txt")
f = open(final_path, 'w')
json.dump(s, f)
f.close()
print("Creation of txt Files Complete")
def main():
"""Updates the copyright year in all source files to the given year."""
if len(sys.argv) != 2:
print('Usage: update_copyright.py <year>')
return 1
try:
year = sys.argv[1]
sys.stdout.write('Updating copyright year to: ' + year + ' ')
base_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..'))
src_path = os.path.join(base_path, 'src')
# Fix copyright headers in C++ files
regex_header = re.compile(
'(.*Copyright \(C\) \d\d\d\d)(.*)( by the Widelands Development Team.*)'
)
def repl_header(match):
if match.group(1).endswith(year):
return match.group(1) + match.group(3)
else:
return match.group(1) + '-' + year + match.group(3)
for filename in find_files(src_path, ['.h', '.cc']):
update_file(filename, regex_header, repl_header)
# Now update the Buildinfo
filename = os.path.join(src_path, 'build_info.h')
regex = re.compile(
'(.*constexpr uint16_t kWidelandsCopyrightEnd = )(\d\d\d\d)(;)')
def replace(match):
return match.group(1) + year + match.group(3)
update_file(filename, regex, replace)
# Now update special files
filename = os.path.join(base_path, 'data/txts/LICENSE.lua')
regex = re.compile(
'(.*\"Copyright 2002 - %1% by the Widelands Development Team\.\"\):bformat\()(\d\d\d\d)(\).*)'
)
update_file(filename, regex, replace)
filename = os.path.join(base_path,
'utils/win32/innosetup/Widelands.iss')
regex = re.compile(
'(#define Copyright \"Widelands Development Team 2001\-)(\d\d\d\d)(\")'
)
update_file(filename, regex, replace)
filename = os.path.join(base_path, 'utils/win32/widelands.rc.cmake')
update_file(filename, regex_header, repl_header)
filename = os.path.join(base_path, 'debian/copyright')
regex = re.compile(
'(Copyright: 2002\-)(\d\d\d\d)( by the Widelands Development Team)'
)
update_file(filename, regex, replace)
print(' done.')
print('You might need to update some copyright years in %s manually.' %
filename)
except Exception:
print('Something went wrong:')
traceback.print_exc()
return 1
errors = 0
for line in known_clean_lines:
e = check_line('selftest false-positives', 1, line)
errors += e
if e > 0:
print('SELFTEST ERROR: false-positive in "{}"'.format(line))
for line in known_issues:
e = check_line('selftest false-negatives', 1, line, False)
if e == 0:
errors += 1
print('SELFTEST ERROR: false-negative in "{}"'.format(line))
if errors > 0:
print('\nThere were selftest errors, please fix!')
sys.exit(1)
# Actual check
for filename in find_files(
os.path.abspath(os.path.join(os.path.dirname(__file__), '..')),
['.lua', '.rst', '.rst.org']):
for lidx, line in enumerate(read_text_file(filename).strip().split('\n')):
errors += check_line(
re.compile(r""".*widelands.(.*)""").search(filename).group(1),
lidx, line)
if errors > 0:
print('\nThere were {} error(s), please fix!'.format(errors))
sys.exit(1)
def main():
args = parse_args()
format_cplusplus = args['c++'] or not (args['lua'] or args['python'])
format_lua = args['lua'] or not (args['c++'] or args['python'])
format_python = args['python'] or not (args['c++'] or args['lua'])
if not os.path.isdir('src') or not os.path.isdir('utils'):
print('CWD is not the root of the repository.')
return 1
if format_cplusplus:
directory = args['dir']
if not directory:
directory = './src'
sys.stdout.write('\nFormatting C++ in directory: ' + directory + ' ')
for filename in find_files(directory, ['.cc', '.h']):
if 'third_party' in filename:
continue
sys.stdout.write('.')
sys.stdout.flush()
call(['clang-format', '-i', filename])
call(['git', 'add', '--renormalize', filename])
print(' done.')
if format_lua:
directories = set()
if args['dir']:
directories.add(args['dir'])
else:
directories = {'./data', './test'}
for directory in directories:
sys.stdout.write('\nFixing Lua tabs in directory: ' + directory +
' ')
for filename in find_files(directory, ['.lua']):
sys.stdout.write('.')
sys.stdout.flush()
lines = read_text_file(filename).strip().split('\n')
new_lines = []
for line in lines:
m = LEADING_TABS.match(line)
if m is not None:
line = line[m.start():m.end()].expandtabs(
SPACES_PER_TAB) + line[m.end():]
new_lines.append(line.rstrip() + '\n')
write_text_file(filename, ''.join(new_lines))
call(['git', 'add', '--renormalize', filename])
print(' done.')
if format_python:
directories = set()
if args['dir']:
directories.add(args['dir'])
else:
directories = {'./utils', './cmake/codecheck'}
for directory in directories:
sys.stdout.write('\nFormatting Python scripts in directory: ' +
directory + ' ')
for filename in find_files(directory, ['.py']):
sys.stdout.write('.')
sys.stdout.flush()
call(['pyformat', '-i', filename])
call(['git', 'add', '--renormalize', filename])
print(' done.')
print('Formatting finished.')
return 0
