def increase_index_and_move(src_folder, dst_folder, file, extension, src_index, dst_index, max_index):
# Helper function to format the full source and destination path
path = lambda f, i: os_path_join(f, extension.format(file, i))
# If destination file's index is lesser than
# the maximum number of backups allowed
if src_index <= max_index:
src = path(src_folder, src_index)
dst = path(dst_folder, dst_index)
# If the destination file exists
if os_path_isfile(dst):
# Call this function recursivly
increase_index_and_move(
src_folder=dst_folder,
dst_folder=dst_folder,
file=file,
extension=extension,
src_index=dst_index,
dst_index=dst_index + 1,
max_index=max_index,
)
cleanup = ""
# If destination file's index is equal or
# greater than the maximum number of backups allowed
else:
src = path(src_folder, max_index - 1)
dst = path(dst_folder, max_index)
cleanup = path(src_folder, src_index)
# Move source file to destination
try:
shutil_move(src, dst)
return cleanup
# If source does not found
except FileNotFoundError:
return ""
def move_file(obj,Config): dst = obj2dst(obj,Config) debuglog(u'moving %s \n=====> %s'%(obj['src'],dst)) shutil_move(obj['src'],dst) os.chmod(dst, 0644) obj['dst']=dst return obj
def move(self, destination): # pragma: no cover
"""
Move the globally given file path to the given destination.
:param str destination: The destination of the file.
"""
if self.exists(self.path):
shutil_move(self.path, destination)
def edit_folder(self, path: str, name: str) -> Tuple[str, int]:
new_folder_path = os.path.dirname(os.path.join(path, name))
if path == new_folder_path:
return (
f"{path} was not renamed because the name didn't change",
0,
)
try:
shutil_move(path, new_folder_path)
except OSError:
return f"Could not move {path}", 1
return f"The folder {path} was successfully renamed to {new_folder_path}", 0
def evaluate_one_model_keras(model_dirpath):
# rename _trained as _evaluating
new_folder_name = model_dirpath.replace('_trained', '_evaluating')
shutil_move(model_dirpath, new_folder_name)
model_name = os_path_basename(new_folder_name)
copied_scan_battery_dirname = os_path_join(
new_folder_name, os_path_basename(SCAN_BATTERIES_DIRNAME))
copy_anything(SCAN_BATTERIES_DIRNAME, copied_scan_battery_dirname)
time_start = time_time()
# with Pool() as pool:
# list(pool.imap_unordered(process_single_target, target_dirnames))
for scan_battery_dirname in glob_glob(
os_path_join(SCAN_BATTERIES_DIRNAME, '*')):
process_single_scan_battery_keras(new_folder_name,
scan_battery_dirname)
print('{}: it took {:.2f} to evaluate model {} for all scan batteries'.
format(SCRIPT_FNAME,
time_time() - time_start, model_name))
shutil_move(new_folder_name,
new_folder_name.replace('_evaluating', '_evaluated'))
def fetch_input_file(self, file):
if not os.path.isdir(self.input_cache_dir):
os.mkdir(self.input_cache_dir)
#url = 'https://www.metanetx.org/cgi-bin/mnxget/mnxref/'
url = 'ftp://ftp.vital-it.ch/databases/metanetx/MNXref/3.2/'
# 3xCommon + rpReader
if file in [
'reac_xref.tsv', 'chem_xref.tsv', 'chem_prop.tsv',
'comp_xref.tsv'
]:
urllib_request_urlretrieve(url + file,
self.input_cache_dir + '/' + file)
#TODO: need to add this file to the git or another location
if file in ['rr_compounds.tsv', 'rxn_recipes.tsv']:
urllib_request_urlretrieve(
'https://retrorules.org/dl/this/is/not/a/secret/path/rr02',
self.input_cache_dir + '/rr02_more_data.tar.gz')
tar = tarfile_open(self.input_cache_dir + '/rr02_more_data.tar.gz',
'r:gz')
tar.extractall(self.input_cache_dir)
tar.close()
shutil_move(self.input_cache_dir + '/rr02_more_data/compounds.tsv',
self.input_cache_dir + '/rr_compounds.tsv')
shutil_move(
self.input_cache_dir + '/rr02_more_data/rxn_recipes.tsv',
self.input_cache_dir)
os.remove(self.input_cache_dir + 'rr02_more_data.tar.gz')
shutil_rmtree(self.input_cache_dir + 'rr02_more_data')
if file == 'rules_rall.tsv':
urllib_request_urlretrieve(
'https://retrorules.org/dl/preparsed/rr02/rp3/hs',
self.input_cache_dir + '/retrorules_rr02_rp3_hs.tar.gz')
tar = tarfile_open(
self.input_cache_dir + '/retrorules_rr02_rp3_hs.tar.gz',
'r:gz')
tar.extractall(self.input_cache_dir)
tar.close()
shutil_move(
self.input_cache_dir +
'/retrorules_rr02_rp3_hs/retrorules_rr02_flat_all.tsv',
self.input_cache_dir + '/rules_rall.tsv')
os.remove(self.input_cache_dir + '/retrorules_rr02_rp3_hs.tar.gz')
shutil_rmtree(self.input_cache_dir + '/retrorules_rr02_rp3_hs')
+ str(bpy.data.objects["Camera"].rotation_axis_angle[1])
+ " "
+ str(bpy.data.objects["Camera"].rotation_axis_angle[2])
+ " "
+ str(bpy.data.objects["Camera"].rotation_axis_angle[3])
+ "\n"
)
f.close()
process_sp = Popen(
("python process_sp.py -- " + modelname + " " + intermediate_folder + " " + result_folder).split()
)
process_sp.communicate()
shutil_move(
intermediate_folder + "/tmp_c1/Image0001.png",
intermediate_folder + "/" + obj_name + "_cd/" + obj_name + "_" + str(i).zfill(2) + "_c.png",
)
shutil_move(
intermediate_folder + "/tmp_d1/Image0001.png",
intermediate_folder + "/" + obj_name + "_cd/" + obj_name + "_" + str(i).zfill(2) + "_d.png",
)
shutil_move(
intermediate_folder + "/tmp_n1/Image0001.png",
intermediate_folder + "/" + obj_name + "_cd/" + obj_name + "_" + str(i).zfill(2) + "_n.png",
)
with open(result_folder + "/" + obj_name + ".off", "r") as file:
data = file.readlines()
f = open(result_folder + "/" + obj_name + ".off", "w")
def move(self, source, target):
source = self._resolve_path(source)
target = self._resolve_path(target)
# If target is a folder, move inside it
return shutil_move(source, target)
def move(self, source, target):
source = self._resolve_path(source)
target = self._resolve_path(target)
# If target is a folder, move inside it
return shutil_move(source, target)
def train_one_model_keras(model_dirpath):
new_model_folder_name = model_dirpath.replace('_created', '_training')
shutil_move(model_dirpath, new_model_folder_name)
for frequency in FREQUENCIES_TO_TRAIN_SEPARATELY:
train_one_model_per_frequency(model_dirpath, frequency)
def process_files(
targets, user_media=None, test_run=False, id_key=None, **config
):
""" Processes targets, relocating them as needed
"""
notify = Notify()
# Begin processing files
detection_count = 0
success_count = 0
for file_path in dir_crawl(
targets, config.get("recurse", False), config.get("extension_mask")
):
notify.heading("Detected File")
blacklist = config.get("blacklist", ())
if any(match(b, file_stem(file_path)) for b in blacklist):
notify.info("%s (blacklisted)" % file_path)
continue
else:
print(file_stem(file_path))
# Print metadata fields
meta = meta_parse(file_path, user_media)
if config["verbose"] is True:
for field, value in meta.items():
notify.verbose("%s: %s" % (field, value), True)
# Print search results
detection_count += 1
notify.heading("Query Results")
results = provider_search(meta, id_key, **config)
i = 1
hits = []
while i < int(config.get("max_hits", 15)):
try:
hit = next(results)
print(" [%s] %s" % (i, hit))
hits.append(hit)
i += 1
except (StopIteration, MapiNotFoundException):
break
# Skip hit if no hits
if not hits:
notify.info("None found! Skipping.", True)
continue
# Select first if batch
if config.get("batch") is True:
meta.update(hits[0])
# Prompt user for input
else:
print(" [RETURN] for default, [s]kip, [q]uit")
abort = skip = None
while True:
selection = notify.prompt("Your Choice")
# Catch default selection
if not selection:
meta.update(hits[0])
break
# Catch skip hit (just break w/o changes)
elif selection in ["s", "S", "skip", "SKIP"]:
skip = True
break
# Quit (abort and exit)
elif selection in ["q", "Q", "quit", "QUIT"]:
abort = True
break
# Catch result choice within presented range
elif selection.isdigit() and 0 < int(selection) < len(hits) + 1:
meta.update(hits[int(selection) - 1])
break
# Re-prompt if user input is invalid wrt to presented options
else:
print("\nInvalid selection, please try again.")
# User requested to skip file...
if skip is True:
notify.info("Skipping rename, as per user request.", True)
continue
# User requested to exit...
elif abort is True:
notify.info("\nAborting, as per user request.")
return
# Create file path
notify.heading("Processing File")
media = meta["media"]
template = config.get("%s_template" % media)
dest_path = meta.format(template)
if config.get("%s_destination" % media):
dest_dir = meta.format(config.get("%s_destination" % media, ""))
dest_path = "%s/%s" % (dest_dir, dest_path)
dest_path = filename_sanitize(dest_path)
dest_path = filename_replace(dest_path, config.get("replacements"))
if config.get("scene") is True:
dest_path = filename_scenify(dest_path)
dest_path = realpath(dest_path)
# Attempt to process file
try:
if not test_run:
# TODO: create parent paths
shutil_move(str(file_path), str(dest_path))
notify.info("Relocating file to '%s'" % dest_path, True)
except IOError as e:
notify.error(" Failed!", True)
if config.get("verbose") is True:
notify.verbose(e)
else:
notify.success("Success!", True)
success_count += 1
# Summarize session outcome
if not detection_count:
print("")
notify.alert('No media files found. "mnamer --help" for usage.')
return
if success_count == 0:
outcome_colour = "red"
elif success_count < detection_count:
outcome_colour = "yellow"
else:
outcome_colour = "green"
cprint(
"\nSuccessfully processed %s out of %s files"
% (success_count, detection_count),
outcome_colour,
)
