def create_zshrc(content, zshrc):
if exists(zshrc):
if not read_zshrc_omz(zshrc):
shutil_copyfile(zshrc, f"{zshrc}-{datetime.today().isoformat()}")
snakypy_file_create(content, zshrc, force=True)
return True
elif not exists(zshrc):
snakypy_file_create(content, zshrc)
return True
return
def run(self):
dump_logger = getLogger('dumpscraper')
# Let's invoke the getscore runner and tell him to work on training data
dump_logger.info("Calculating dump score...")
running = getscore.DumpScraperGetscore(self.settings, self.parentArgs)
running.run()
# First of all let's feed the classifier with the training data
training = scipy_genfromtxt(self.settings['data_dir'] + "/" + "training/features.csv", delimiter=",", skip_header=1, usecols=(0, 1, 2))
target = scipy_genfromtxt(self.settings['data_dir'] + "/" + "training/features.csv", delimiter=",", skip_header=1, usecols=(-2))
clf = sklearn.neighbors.KNeighborsClassifier(10, weights='uniform')
clf.fit(training, target)
trash_count = hash_count = plain_count = 0
cleared = []
with open(self.settings['data_dir'] + "/" + 'features.csv', 'rb') as csvfile:
reader = csv_reader(csvfile)
for line in reader:
if line[0] == 'Trash score':
continue
features = np_array(line[0:3])
features = features.reshape(1, -1)
label = clf.predict(features)
if label == 0:
folder = 'trash'
trash_count += 1
elif label == 1:
folder = 'hash'
hash_count += 1
elif label == 2:
folder = 'plain'
plain_count += 1
target_file = self.settings['data_dir'] + "/" + 'organized/' + folder + "/" + line[-1]
target_dir = path.dirname(target_file)
# If asked for a clean run, let's delete the entire folder before copying any file
if self.parentArgs.clean and target_dir not in cleared and path.exists(target_dir):
cleared.append(target_dir)
shutil_rmtree(target_dir)
if not path.exists(target_dir):
makedirs(target_dir)
shutil_copyfile(self.settings['data_dir'] + "/" + 'raw/' + line[-1], target_file)
dump_logger.info("Trash files: " + str(trash_count))
dump_logger.info("Hash files: " + str(hash_count))
dump_logger.info("Plain files: " + str(plain_count))
dump_logger.info("Operation completed")
def index_subcommand(args):
_ = include_rematch_dependencies_path()
utils.required_programs({'bowtie2-build': ['--version', '>=', '2.2.9']})
if args.reference is not None:
args.reference = [
os.path.abspath(reference.name) for reference in args.reference
]
else:
args.reference, _ = get_fasta_config(args.org)
print('\n'
'Settings that will be used:\n'
' reference: {reference}\n'
'\n'.format(reference=args.reference))
for reference in args.reference:
reference_file_outdir = os.path.join(args.outdir,
os.path.basename(reference))
new_reference_file, header_gene_list, seq_reference_dict = \
run_rematch.clean_headers_reference_file(reference_file=reference, outdir=args.outdir)
if new_reference_file == reference:
# Copy fasta file to outdir
shutil_copyfile(reference,
reference_file_outdir,
follow_symlinks=True)
else:
utils.Bcolors_print(
'WARNING: Sequences headers were renamed for {}'.format(
reference), 'WARNING')
# Rename files
os.rename(new_reference_file, reference_file_outdir)
pickle_file = os.path.join(args.outdir, reference_file_outdir + '.pkl')
utils.saveVariableToPickle((header_gene_list, seq_reference_dict),
pickle_file)
# Create Bowtie2 index
run_successfully = run_rematch.run_bowtie_build(
reference_file=reference_file_outdir,
outdir=args.outdir,
threads=args.threads)
if not run_successfully:
sys.exit(
'Something went wrong while creating Bowtie2 index for {}'.
format(reference))
exit(0)
def main(self):
checking_init(self.HOME)
if not read_zshrc_omz(self.zsh_rc):
title = f"Do you want to uninstall {package.info['name']}?"
options = ["Yes", "No"]
reply = pick(title, options, colorful=True, index=True)
if reply is None or reply[0] == 1:
printer("Whew! Thanks! :)", foreground=FG.GREEN)
exit(0)
rm_init_file_package(self.init_file)
rm_source_zshrc(self.zsh_rc)
else:
title = f"What did you want to uninstall?"
options = [
f"{package.info['name']}",
f"{package.info['name']} and Oh My ZSH",
"Cancel",
]
reply = pick(title, options, colorful=True, index=True)
if reply is None or reply[0] == 2:
printer("Whew! Thanks! :)", foreground=FG.GREEN)
exit(0)
with contextlib_suppress(Exception):
os_remove(self.theme_file)
rm_init_file_package(self.init_file)
change_theme_in_zshrc(self.zsh_rc, "robbyrussell")
if reply[0] == 1:
shutil_rmtree(self.omz_root, ignore_errors=True)
with contextlib_suppress(Exception):
shutil_copyfile(
self.zsh_rc,
f"{self.zsh_rc}-D{datetime.today().isoformat()}")
with contextlib_suppress(Exception):
os_remove(self.zsh_rc)
snakypy_file_create("", f"{self.HOME}/.zshrc", force=True)
reload_zsh()
printer("Uninstall process finished.", foreground=FG.FINISH)
def move_file(origin_name, destination_name):
"""
Moves or duplicates a file into a destination directory. Deletes
a file in the destination if replace_files is True. Otherwise, avoids
the operation in case the filename already exists.
:param origin_name: File's original name
:param destination_name: Name of the file in its new directory
:return: None
"""
# Take care of removing or lowercase transformation if desired
destination_name = replace_lower(destination_name) if removing[0] \
or lowercase else destination_name
# Create the pathname of the file in its new directory
final_pathname = os.path.join(destination, destination_name)
# If files should be replaced, get rid of any file
# that already has the same name in the destination folder
if replace_files and os.path.exists(final_pathname):
# Log information about the file being removed
log_info("Removing {} from {} as it will be overwritten".format(
destination_name, destination))
os.remove(final_pathname)
# If the files should not be deleted from the original folder,
# use shutil_copyfile to move the file; don't try if file exists
if duplicate and not os.path.exists(final_pathname):
shutil_copyfile(os.path.join(origin, origin_name), final_pathname)
# Else, if the files are to be moved and the filename doesn't
# already exist in the destination folder, move with os' rename
elif not os.path.exists(final_pathname):
os.rename(os.path.join(origin, origin_name), final_pathname)
# Log the new file movement
log_info("File {} in {} moved with name {} to {}".format(origin_name,
origin,
destination_name,
destination))
def preload_repo(
tupdater3_exe: str,
stage2_exe: str,
repodir_s: pathlib.Path
) -> ServerRepoCtx:
with git.Repo.init(str(repodir_s)) as repo:
rc = ServerRepoCtx(repodir_s, repo)
# BEG populate with dummy data
for ff in [("a.txt", b"aaa"), ("d/b.txt", b"bbb")]:
with _file_open_mkdirp(str(rc.repodir.joinpath(ff[0]))) as f:
f.write(ff[1])
rc.repo.index.add([ff[0]])
# END populate with dummy data
# BEG populate with executables
for gg in [("updater.exe", tupdater3_exe), ("stage2.exe", stage2_exe)]:
shutil_copyfile(gg[1], str(rc.repodir.joinpath(gg[0])))
rc.repo.index.add([gg[0]])
# END populate with executables
commit = rc.repo.index.commit("ccc")
ref_master = git.Reference(rc.repo, "refs/heads/master")
ref_master.set_object(commit)
def copyfile(src, dst, follow_symlinks=True):
"""
Copies a source file to a destination file.
Equivalent to "shutil.copyfile".
Source and destination can also be binary opened file-like objects.
Args:
src (path-like object or file-like object): Source file.
dst (path-like object or file-like object): Destination file.
follow_symlinks (bool): Follow symlinks.
Not supported on cloud storage objects.
Raises:
IOError: Destination directory not found.
"""
# Handles path-like objects and checks if storage
src, src_is_storage = format_and_is_storage(src)
dst, dst_is_storage = format_and_is_storage(dst)
# Local files: Redirects to "shutil.copyfile"
if not src_is_storage and not dst_is_storage:
return shutil_copyfile(src, dst, follow_symlinks=follow_symlinks)
# Checks destination
try:
if not hasattr(dst, 'read') and not isdir(dirname(dst)):
raise FileNotFoundError("No such file or directory: '%s'" % dst)
except PermissionError:
# Unable to check target directory due to missing read access, but
# do not raise to allow to write if possible
pass
# Performs copy
_copy(src, dst, src_is_storage, dst_is_storage)
def copyfile(src, dst, *, follow_symlinks=True):
"""
Copies a source file to a destination file.
Equivalent to "shutil.copyfile".
Source and destination can also be binary opened file-like objects.
.. versionadded:: 1.2.0
Args:
src (path-like object or file-like object): Source file.
dst (path-like object or file-like object): Destination file.
follow_symlinks (bool): Follow symlinks.
Raises:
FileNotFoundError: Destination directory not found.
"""
src, src_is_storage = format_and_is_storage(src, True)
dst, dst_is_storage = format_and_is_storage(dst, True)
if not src_is_storage and not dst_is_storage:
return shutil_copyfile(src, dst, follow_symlinks=follow_symlinks)
_copy(src, dst, src_is_storage, dst_is_storage, follow_symlinks)
def _gettrainingdata(self):
files = [
path.join(w_path, filename)
for w_path, dirs, files in walk(self.settings['data_dir'] + "/" +
'raw') for filename in files
if not filename.endswith(".csv")
]
while 1:
rfile = random_choice(files)
trash = len(
listdir(self.settings['data_dir'] + "/" + 'training/trash'))
plain = len(
listdir(self.settings['data_dir'] + "/" + 'training/plain'))
hashes = len(
listdir(self.settings['data_dir'] + "/" + 'training/hash'))
# Clear the screen before displaying the text
system('cls' if name == 'nt' else 'clear')
# Let's the terminal size, so I can fill it with the file text
cols, rows = get_terminal_size()
print colorama.Fore.YELLOW + rfile
print("")
with open(rfile) as tfile:
i = 0
for line in tfile:
i += 1
if i >= (rows - 6):
break
if len(line) <= cols:
print line.strip('\n\r')
else:
print line[0:cols].strip('\n\r')
print("")
print colorama.Fore.YELLOW + "Trash: " + str(
trash) + " Plain: " + str(plain) + " Hash: " + str(hashes)
input_descr = colorama.Fore.MAGENTA + "[o]"
input_descr += colorama.Fore.CYAN + "open "
input_descr += colorama.Fore.MAGENTA + "[t]"
input_descr += colorama.Fore.CYAN + "rash "
input_descr += colorama.Fore.MAGENTA + "[p]"
input_descr += colorama.Fore.CYAN + "lain "
input_descr += colorama.Fore.MAGENTA + "[h]"
input_descr += colorama.Fore.CYAN + "ash "
input_descr += colorama.Fore.MAGENTA + "[s]"
input_descr += colorama.Fore.CYAN + "kip "
input_descr += colorama.Fore.MAGENTA + "[q]"
input_descr += colorama.Fore.CYAN + "uit=> "
sys_stdout.write(input_descr)
sys_stdout.flush()
answer = getch()
while answer == '':
pass
# Opening a file with the default application AND being cross platform is a PITA...
if answer == 'o':
current_os = platform_system()
if current_os == 'Windows':
from os import startfile as os_startfile
os_startfile(rfile)
elif current_os == 'Linux':
subprocess_call(["xdg-open", rfile])
elif current_os == 'Darwin':
system("open " + rfile)
# Let's start the loop again to read the new key
answer = getch()
while answer == '':
pass
if answer == 't':
shutil_copyfile(
rfile, self.settings['data_dir'] + "/" +
'training/trash/' + path.basename(rfile))
elif answer == 'p':
shutil_copyfile(
rfile, self.settings['data_dir'] + "/" +
'training/plain/' + path.basename(rfile))
elif answer == 'h':
shutil_copyfile(
rfile, self.settings['data_dir'] + "/" + 'training/hash/' +
path.basename(rfile))
elif answer == 's':
print("")
continue
elif answer == 'q':
print("")
print(colorama.Fore.GREEN + "Training complete")
break
else:
print("")
continue
def _gettrainingdata(self):
files = [path.join(w_path, filename)
for w_path, dirs, files in walk(self.settings['data_dir'] + "/" + 'raw')
for filename in files
if not filename.endswith(".csv")]
while 1:
rfile = random_choice(files)
trash = len(listdir(self.settings['data_dir'] + "/" + 'training/trash'))
plain = len(listdir(self.settings['data_dir'] + "/" + 'training/plain'))
hashes = len(listdir(self.settings['data_dir'] + "/" + 'training/hash'))
# Clear the screen before displaying the text
system('cls' if name == 'nt' else 'clear')
# Let's the terminal size, so I can fill it with the file text
cols, rows = get_terminal_size()
print colorama.Fore.YELLOW + rfile
print("")
with open(rfile) as tfile:
i = 0
for line in tfile:
i += 1
if i >= (rows - 6):
break
if len(line) <= cols:
print line.strip('\n\r')
else:
print line[0:cols].strip('\n\r')
print("")
print colorama.Fore.YELLOW + "Trash: " + str(trash) + " Plain: " + str(plain) + " Hash: " + str(hashes)
input_descr = colorama.Fore.MAGENTA + "[o]"
input_descr += colorama.Fore.CYAN + "open "
input_descr += colorama.Fore.MAGENTA + "[t]"
input_descr += colorama.Fore.CYAN + "rash "
input_descr += colorama.Fore.MAGENTA + "[p]"
input_descr += colorama.Fore.CYAN + "lain "
input_descr += colorama.Fore.MAGENTA + "[h]"
input_descr += colorama.Fore.CYAN + "ash "
input_descr += colorama.Fore.MAGENTA + "[s]"
input_descr += colorama.Fore.CYAN + "kip "
input_descr += colorama.Fore.MAGENTA + "[q]"
input_descr += colorama.Fore.CYAN + "uit=> "
sys_stdout.write(input_descr)
sys_stdout.flush()
answer = getch()
while answer == '':
pass
# Opening a file with the default application AND being cross platform is a PITA...
if answer == 'o':
current_os = platform_system()
if current_os == 'Windows':
from os import startfile as os_startfile
os_startfile(rfile)
elif current_os == 'Linux':
subprocess_call(["xdg-open", rfile])
elif current_os == 'Darwin':
system("open " + rfile)
# Let's start the loop again to read the new key
answer = getch()
while answer == '':
pass
if answer == 't':
shutil_copyfile(rfile, self.settings['data_dir'] + "/" + 'training/trash/' + path.basename(rfile))
elif answer == 'p':
shutil_copyfile(rfile, self.settings['data_dir'] + "/" + 'training/plain/' + path.basename(rfile))
elif answer == 'h':
shutil_copyfile(rfile, self.settings['data_dir'] + "/" + 'training/hash/' + path.basename(rfile))
elif answer == 's':
print("")
continue
elif answer == 'q':
print("")
print(colorama.Fore.GREEN + "Training complete")
break
else:
print("")
continue