def _is_compress_filetype(self, inpath):
"""private method that performs magic number and size check on file to determine whether to compress the file"""
# check for common file type suffixes in order to avoid the need for file reads to check magic number for binary vs. text file
if self._is_common_binary(inpath):
return False
elif self._is_common_text(inpath):
return True
else:
# files > 10kB get checked for compression (arbitrary decision to skip compression on small files)
the_file_size = file_size(inpath)
if the_file_size > 10240:
if the_file_size > 512000: # seems to be a break point at ~ 500kb where file compression offset by additional file read, so limit tests to files > 500kB
try:
system_command = "file --mime-type -b " + quote(inpath)
response = muterun(system_command)
if response.stdout[0:5] == "text/": # check for a text file mime type
return True # appropriate size, appropriate file mime type
else:
return False # appropriate size, inappropriate file mime type
except Exception:
return False
else:
return True # if file size is < 500kB, skip the additional file read and just go with compression
else:
return False # below minimum size to consider compression, do not compress
def _is_compress_filetype(self, inpath):
"""private method that performs magic number and size check on file to determine whether to compress the file"""
# check for common file type suffixes in order to avoid the need for file reads to check magic number for binary vs. text file
if self._is_common_binary(inpath):
return False
elif self._is_common_text(inpath):
return True
else:
# files > 10kB get checked for compression (arbitrary decision to skip compression on small files)
the_file_size = file_size(inpath)
if the_file_size > 10240:
if the_file_size > 512000: # seems to be a break point at ~ 500kb where file compression offset by additional file read, so limit tests to files > 500kB
try:
system_command = "file --mime-type -b " + quote(inpath)
response = muterun(system_command)
if response.stdout[0:5] == "text/": # check for a text file mime type
return True # appropriate size, appropriate file mime type
else:
return False # appropriate size, inappropriate file mime type
except Exception:
return False
else:
return True # if file size is < 500kB, skip the additional file read and just go with compression
else:
return False # below minimum size to consider compression, do not compress
def download_Tiktok_signature():
try:
e = muterun("npm i tiktok-signature") # install tiktok-signature
if e.exitcode == 1:
print("Not able to install tiktok-signature. Please make sure Node.js is installed properly and restart your script running environment.")
rewrite_browserJS()
except():
print("Not able to install tiktok-signature. Please make sure Node.js is installed properly and restart your script running environment.")
def controllProcess(name, var, port, files):
#TODO terminate this process
nodeProcess = Process(target=startNode, args=(name,port), daemon=True)
nodeProcess.start()
getProcess = Process(target=getJSON, args=(files, var, port), daemon=True)
getProcess.start()
while(var.value == 0):
pass
getProcess.join()
# Close the node server without promting the keyboard interrupt
muterun("TASKKILL /F /IM node.exe /T")
return
def encrypt_file(self,
inpath,
force_nocompress=False,
force_compress=False,
armored=False,
checksum=False):
"""public method for single file encryption with optional compression, ASCII armored formatting, and file hash digest generation"""
if armored:
if force_compress:
command_stub = self.command_maxcompress_armored
elif force_nocompress:
command_stub = self.command_nocompress_armored
else:
if self._is_compress_filetype(inpath):
command_stub = self.command_default_armored
else:
command_stub = self.command_nocompress_armored
else:
if force_compress:
command_stub = self.command_maxcompress
elif force_nocompress:
command_stub = self.command_nocompress
else:
if self._is_compress_filetype(inpath):
command_stub = self.command_default
else:
command_stub = self.command_nocompress
encrypted_outpath = self._create_outfilepath(inpath)
system_command = command_stub + encrypted_outpath + " --passphrase " + quote(
self.passphrase) + " --symmetric " + quote(inpath)
try:
response = muterun(system_command)
# check returned status code
if response.exitcode == 0:
stdout(encrypted_outpath + " was generated from " + inpath)
if checksum: # add a SHA256 hash digest of the encrypted file - requested by user --hash flag in command
from crypto.library import hash
encrypted_file_hash = hash.generate_hash(encrypted_outpath)
if len(encrypted_file_hash) == 64:
stdout("SHA256 hash digest for " + encrypted_outpath +
" :")
stdout(encrypted_file_hash)
else:
stdout(
"Unable to generate a SHA256 hash digest for the file "
+ encrypted_outpath)
else:
stderr(response.stderr, 0)
stderr("Encryption failed")
sys.exit(1)
except Exception as e:
stderr(
"There was a problem with the execution of gpg. Encryption failed. Error: ["
+ str(e) + "]")
sys.exit(1)
def execute_babel_node(self, expression_or_path, arguments):
try:
if len(arguments) > 0:
js_command = 'babel-node ' + self.babel_arguments + expression_or_path + ' ' + arguments
else:
js_command = 'babel-node ' + self.babel_arguments + expression_or_path
return muterun(js_command) # return result of execute_babel_node() of node.js file
except Exception as e:
raise e
def test_execute_command(self):
"""Test execution of a helpy command"""
state = StateObject()
if (state.py_major == 2 and state.py_minor > 6) or state.py3:
response = muterun('helpy sys')
self.assertEqual(response.exitcode, 0)
self.assertTrue(len(response.stdout) > 0)
self.assertEqual(response.stderr, b'')
elif state.py_major == 2 and state.py_minor == 6:
res = subprocess.check_call(['helpy', 'sys'])
self.assertEqual(res, 0)
def test_execute_command(self):
"""Test execution of a helpy command"""
state = StateObject()
if (state.py_major == 2 and state.py_minor > 6) or state.py3:
response = muterun('helpy sys')
self.assertEqual(response.exitcode, 0)
self.assertTrue(len(response.stdout) > 0)
self.assertEqual(response.stderr, b'')
elif state.py_major == 2 and state.py_minor == 6:
res = subprocess.check_call(['helpy', 'sys'])
self.assertEqual(res, 0)
def test_make_template_file(self):
try:
os.chdir(self.test_dir)
response = muterun('doxx make template')
if not response.exitcode == 0:
stderr("Error with 'doxx make template' command. Error: " + response.stderr, exit=1)
else:
self.assertTrue(file_exists(self.template_file)) # confirm that the template file was made
os.remove(self.template_file) # remove the test file after the test
os.chdir(self.cwd)
except Exception as e:
os.chdir(self.cwd)
raise e
def encrypt_file(self, inpath, force_nocompress=False, force_compress=False, armored=False, checksum=False):
"""public method for single file encryption with optional compression, ASCII armored formatting, and file hash digest generation"""
if armored:
if force_compress:
command_stub = self.command_maxcompress_armored
elif force_nocompress:
command_stub = self.command_nocompress_armored
else:
if self._is_compress_filetype(inpath):
command_stub = self.command_default_armored
else:
command_stub = self.command_nocompress_armored
else:
if force_compress:
command_stub = self.command_maxcompress
elif force_nocompress:
command_stub = self.command_nocompress
else:
if self._is_compress_filetype(inpath):
command_stub = self.command_default
else:
command_stub = self.command_nocompress
encrypted_outpath = self._create_outfilepath(inpath)
system_command = command_stub + encrypted_outpath + " --passphrase " + quote(self.passphrase) + " --symmetric " + quote(inpath)
try:
response = muterun(system_command)
# check returned status code
if response.exitcode == 0:
stdout(encrypted_outpath + " was generated from " + inpath)
if checksum: # add a SHA256 hash digest of the encrypted file - requested by user --hash flag in command
from crypto.library import hash
encrypted_file_hash = hash.generate_hash(encrypted_outpath)
if len(encrypted_file_hash) == 64:
stdout("SHA256 hash digest for " + encrypted_outpath + " :")
stdout(encrypted_file_hash)
else:
stdout("Unable to generate a SHA256 hash digest for the file " + encrypted_outpath)
else:
stderr(response.stderr, 0)
stderr("Encryption failed")
sys.exit(1)
except Exception as e:
stderr("There was a problem with the execution of gpg. Encryption failed. Error: [" + str(e) + "]")
sys.exit(1)
def test_make_project(self):
try:
os.chdir(self.test_dir)
response = muterun('doxx make project')
if not response.exitcode == 0:
stderr("Error with 'doxx make project' command. Error: " + response.stderr, exit=1)
else:
self.assertTrue(file_exists(self.pkey_file_path)) # confirm that the pkey.yaml file was made
self.assertTrue(file_exists(self.projyaml_file_path)) # confirm that the project.yaml file was made
self.assertTrue(file_exists(self.template_file_path)) # confirm that the template file was made
self.assertTrue(dir_exists(self.templates_dir)) # confirm that the 'templates' directory was made
os.remove(self.pkey_file_path) # remove the test key file after the test
os.remove(self.projyaml_file_path)
os.remove(self.template_file_path) # remove the test template file
shutil.rmtree(self.templates_dir) # remove the templates directory
os.chdir(self.cwd)
except Exception as e:
os.chdir(self.cwd)
raise e
def test_muterun_bad_command_return_type(self):
self.assertEqual(type(NakedObject()), type(muterun(self.bad_command))) # returns NakedObject on error
from Naked.toolshed.shell import muterun
import os
curr_dir = os.path.dirname(os.path.realpath(__file__))
print 'Starting zombies..'
cmd = 'node ' + curr_dir + '/poll_terms.js'
result1 = muterun(cmd)
dumps = result1.stdout.split(',')
print 'Starting filedump..'
filo = open(curr_dir + '/dump_links.txt', 'w')
for line in dumps:
if(line != ''):
filo.write(line + '\n')
filo.close()
print 'Cleaning up..'
def test_muterun_bad_command_stdout(self):
out = muterun(self.bad_command)
self.assertEqual(b"", out.stdout) # std out is empty string on failure
def start_selenium():
muterun(location("./node_modules/protractor/bin/webdriver-manager start"))
def test_muterun_good_command_exitcode(self):
out = muterun(self.good_command)
self.assertEqual(0, out.exitcode) # exit code = 0 = success
def test_muterun_good_command_stderr(self):
out = muterun(self.good_command)
self.assertEqual(
b"", out.stderr) # stderr empty string when successful command
def test_muterun_bad_command_stderr(self):
out = muterun(self.bad_command)
self.assertTrue(b'bogusapp: command not found' in out.stderr) # has std err message on failure
def test_muterun_missing_option_exitcode(self):
out = muterun(self.missing_option)
self.assertEqual(1, out.exitcode) # returns 1 on missing option to ls
def test_muterun_bad_command_exitcode(self):
out = muterun(self.bad_command)
self.assertEqual(127, out.exitcode) # returns 127 on absent executable
def test_muterun_bad_command_stdout(self):
out = muterun(self.bad_command)
self.assertEqual(b"", out.stdout) # std out is empty string on failure
def test_muterun_good_command_stderr(self):
out = muterun(self.good_command)
self.assertEqual(b"", out.stderr) # stderr empty string when successful command
def test_muterun_good_command_stdout(self):
out = muterun(self.good_command)
self.assertEqual(b"test command\n", out.stdout) # stdout string is correct
def test_muterun_good_command_exitcode(self):
out = muterun(self.good_command)
self.assertEqual(0, out.exitcode) # exit code = 0 = success
def test_muterun_good_command_return_type(self):
self.assertEqual(type(NakedObject()), type(muterun(
self.good_command))) # returns NakedObject on success
def test_muterun_missing_option_stderr(self):
out = muterun(self.missing_option)
self.assertTrue(len(out.stderr) > 0) # there is a stderr message present
def test_muterun_bad_command_return_type(self):
self.assertEqual(type(NakedObject()), type(muterun(
self.bad_command))) # returns NakedObject on error
term = after.split('/')[0]
tail = '/'.join(after.split('/')[1:])
for doc_type in pages:
urls.append(lead + 'diseases-conditions/' + doc_type + '/' + tail)
#file_arr.append(term)
terms[term] = ['', [[] for i in xrange(len(pages))]]
if(count > max_range):
break
filo.close()
for url in urls:
curr_dir = os.getcwd()
cmd = 'node ' + curr_dir + '/test_parse.js ' + str(i)
result1 = muterun(cmd)
piece = result1.stdout
print piece.split('\n')[0]
'''
for i in range(0, max_range, increm):
curr_dir = os.getcwd()
cmd = 'node ' + curr_dir + '/test_parse.js ' + str(i)
result1 = muterun(cmd)
piece = result1.stdout
for pices in xrange(len(splices)):
contents = []
piece = splices[pices]
def test_muterun_good_command_stdout(self):
out = muterun(self.good_command)
self.assertEqual(b"test command\n",
out.stdout) # stdout string is correct
def test_muterun_missing_option_stdout(self):
out = muterun(self.missing_option)
self.assertEqual(b"", out.stdout) # std out is empty string on failure
def test_muterun_bad_command_exitcode(self):
out = muterun(self.bad_command)
self.assertEqual(127, out.exitcode) # returns 127 on absent executable
def main():
import os
import sys
from time import sleep
import getpass
import tarfile
from Naked.commandline import Command
from Naked.toolshed.shell import execute, muterun
from Naked.toolshed.system import dir_exists, file_exists, list_all_files, make_path, stdout, stderr, is_dir
from shellescape import quote
# ------------------------------------------------------------------------------------------
# [ Instantiate command line object ]
# used for all subsequent conditional logic in the CLI application
# ------------------------------------------------------------------------------------------
c = Command(sys.argv[0], sys.argv[1:])
# ------------------------------------------------------------------------------------------
# [ VALIDATION LOGIC ] - early validation of appropriate command syntax
# Test that user entered at least one argument to the executable, print usage if not
# ------------------------------------------------------------------------------------------
if not c.command_suite_validates():
from Crypto.settings import usage as crypto_usage
print(crypto_usage)
sys.exit(1)
# ------------------------------------------------------------------------------------------
# [ HELP, VERSION, USAGE LOGIC ]
# Naked framework provides default help, usage, and version commands for all applications
# --> settings for user messages are assigned in the lib/Crypto/settings.py file
# ------------------------------------------------------------------------------------------
if c.help(): # User requested Crypto help information
from Crypto.settings import help as crypto_help
print(crypto_help)
sys.exit(0)
elif c.usage(): # User requested Crypto usage information
from Crypto.settings import usage as crypto_usage
print(crypto_usage)
sys.exit(0)
elif c.version(): # User requested Crypto version information
from Crypto.settings import app_name, major_version, minor_version, patch_version
version_display_string = app_name + ' ' + major_version + '.' + minor_version + '.' + patch_version
print(version_display_string)
sys.exit(0)
# ------------------------------------------------------------------------------------------
# [ APPLICATION LOGIC ]
#
# ------------------------------------------------------------------------------------------
elif c.argc > 1:
# code for multi-file processing and commands that include options
use_standard_output = False # print to stdout flag
use_file_overwrite = False # overwrite existing file
untar_archives = True # untar decrypted tar archives, true by default
# set user option flags
if c.option('--stdout') or c.option('-s'):
use_standard_output = True
if c.option('--overwrite') or c.option('-o'):
use_file_overwrite = True
if c.option('--nountar'):
untar_archives = False
directory_list = [
] # directory paths included in the user entered paths from the command line
file_list = [
] # file paths included in the user entered paths from the command line (and inside directories entered)
for argument in c.argv:
if file_exists(
argument
): # user included a file, add it to the file_list for decryption
if argument.endswith('.crypt'):
file_list.append(
argument
) # add .crypt files to the list of files for decryption
elif argument.endswith('.gpg'):
file_list.append(argument)
elif argument.endswith('.asc'):
file_list.append(argument)
elif argument.endswith('.pgp'):
file_list.append(argument)
else:
# cannot identify as an encrypted file, give it a shot anyways but warn user
file_list.append(argument)
stdout(
"Could not confirm that '" + argument +
"' is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed..."
)
elif dir_exists(
argument
): # user included a directory, add it to the directory_list
directory_list.append(argument)
else:
if argument[0] == "-":
pass # if it is an option, do nothing
else:
stderr(
"'" + argument +
"' does not appear to be an existing file or directory. Aborting decryption attempt for this request."
)
# unroll the contained directory files into the file_list IF they are encrypted file types
if len(directory_list) > 0:
for directory in directory_list:
directory_file_list = list_all_files(directory)
for contained_file in directory_file_list:
if contained_file.endswith('.crypt'):
file_list.append(
make_path(directory, contained_file)
) # include the file with a filepath 'directory path/contained_file path'
elif contained_file.endswith('.gpg'):
file_list.append(make_path(directory, contained_file))
elif contained_file.endswith('asc'):
file_list.append(make_path(directory, contained_file))
elif contained_file.endswith('.pgp'):
file_list.append(make_path(directory, contained_file))
# confirm that there are files for decryption, if not abort
if len(file_list) == 0:
stderr("Could not identify files for decryption")
sys.exit(1)
# get passphrase used to symmetrically decrypt the file
passphrase = getpass.getpass("Please enter your passphrase: ")
if len(passphrase) == 0: # confirm that user entered a passphrase
stderr(
"You did not enter a passphrase. Please repeat your command and try again."
)
sys.exit(1)
passphrase_confirm = getpass.getpass(
"Please enter your passphrase again: ")
if passphrase == passphrase_confirm:
# begin decryption of each requested file. the directory path was already added to the file path above
for encrypted_file in file_list:
# create the decrypted file name
decrypted_filename = ""
if encrypted_file.endswith('.crypt'):
decrypted_filename = encrypted_file[0:-6]
elif encrypted_file.endswith(
'.gpg') or encrypted_file.endswith(
'.asc') or encrypted_file.endswith('.pgp'):
decrypted_filename = encrypted_file[0:-4]
else:
decrypted_filename = encrypted_file + '.decrypt' # if it was a file without a known encrypted file type, add the .decrypt suffix
# determine whether file overwrite will take place with the decrypted file
skip_file = False # flag that indicates this file should not be encrypted
created_tmp_files = False
if not use_standard_output: # if not writing a file, no need to check for overwrite
if file_exists(decrypted_filename):
if use_file_overwrite: # rename the existing file to temp file which will be erased or replaced (on decryption failures) below
tmp_filename = decrypted_filename + '.tmp'
os.rename(decrypted_filename, tmp_filename)
created_tmp_files = True
else:
stdout(
"The file path '" + decrypted_filename +
"' already exists. This file was not decrypted."
)
skip_file = True
# begin decryption
if not skip_file:
if use_standard_output: # using --quiet flag to suppress stdout messages from gpg, just want the file data in stdout stream
system_command = "gpg --batch --quiet --passphrase " + quote(
passphrase) + " -d " + quote(encrypted_file)
successful_execution = execute(
system_command
) # use naked execute function to directly push to stdout, rather than return stdout
if not successful_execution:
stderr(
"Unable to decrypt file '" + encrypted_file +
"'", 0)
if created_tmp_files: # restore the moved tmp file to original if decrypt failed
tmp_filename = decrypted_filename + '.tmp'
if file_exists(tmp_filename):
os.rename(tmp_filename, decrypted_filename)
else: # decryption successful but we are in stdout flag so do not include any other output from decrypto
pass
else:
system_command = "gpg --batch -o " + quote(
decrypted_filename) + " --passphrase " + quote(
passphrase) + " -d " + quote(encrypted_file)
response = muterun(system_command)
if response.exitcode == 0:
stdout("'" + encrypted_file + "' decrypted to '" +
decrypted_filename + "'")
else: # failed decryption
if created_tmp_files: # restore the moved tmp file to original if decrypt failed
tmp_filename = decrypted_filename + '.tmp'
if file_exists(tmp_filename):
os.rename(tmp_filename, decrypted_filename)
# report the error
stderr(response.stderr)
stderr("Decryption failed for " + encrypted_file)
# cleanup: remove the tmp file
if created_tmp_files:
tmp_filename = decrypted_filename + '.tmp'
if file_exists(tmp_filename):
os.remove(tmp_filename)
# untar/extract any detected archive file(s)
if untar_archives is True:
if decrypted_filename.endswith(
'.tar') and tarfile.is_tarfile(decrypted_filename):
untar_path_tuple = os.path.split(decrypted_filename)
untar_path = untar_path_tuple[0]
if use_file_overwrite:
with tarfile.open(decrypted_filename) as tar:
if len(untar_path) > 0:
tar.extractall(
path=untar_path
) # use dir path from the decrypted_filename if not CWD
stdout(
"'" + decrypted_filename +
"' unpacked in the directory path '" +
untar_path + "'")
else:
tar.extractall() # else use CWD
stdout(
"'" + decrypted_filename +
"' unpacked in the current working directory"
)
else:
with tarfile.TarFile(decrypted_filename,
'r',
errorlevel=1) as tar:
for tarinfo in tar:
t_file = tarinfo.name
if len(untar_path) > 0:
t_file_path = os.path.join(
untar_path, t_file)
else:
t_file_path = t_file
if not os.path.exists(t_file_path):
try:
if len(untar_path) > 0:
tar.extract(
t_file, path=untar_path
) # write to the appropriate dir
else:
tar.extract(
t_file) # write to CWD
except IOError as e:
stderr(
"Failed to unpack the file '" +
t_file_path + "' [" + str(e) +
"]")
elif is_dir(t_file_path):
pass # do nothing if it exists and is a directory, no need to warn
else: # it is a file and it already exists, provide user error message
stderr(
"Failed to unpack the file '" +
t_file_path +
"'. File already exists. Use the --overwrite flag to replace existing files."
)
# remove the decrypted tar archive file
os.remove(decrypted_filename)
# overwrite the entered passphrases after file decryption is complete for all files
passphrase = ""
passphrase_confirm = ""
# add a short pause to hinder brute force pexpect style password attacks with decrypto
sleep(0.2) # 200ms pause
else: # passphrases did not match
passphrase = ""
passphrase_confirm = ""
stderr(
"The passphrases did not match. Please enter your command again."
)
sys.exit(1)
elif c.argc == 1:
# simple single file or directory processing with default settings
path = c.arg0
if file_exists(path): # SINGLE FILE
check_existing_file = False # check for a file with the name of new decrypted filename in the directory
if path.endswith('.crypt'):
decrypted_filename = path[0:-6] # remove the .crypt suffix
check_existing_file = True
elif path.endswith('.gpg') or path.endswith(
'.pgp') or path.endswith('.asc'):
decrypted_filename = path[0:-4]
check_existing_file = True
else:
decrypted_filename = path + ".decrypt" # if there is not a standard file type, then add a .decrypt suffix to the decrypted file name
stdout(
"Could not confirm that the requested file is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed..."
)
# confirm that the decrypted path does not already exist, if so abort with warning message to user
if check_existing_file is True:
if file_exists(decrypted_filename):
stderr(
"Your file will be decrypted to '" +
decrypted_filename +
"' and this file path already exists. Please move the file or use the --overwrite option with your command if you intend to replace the current file."
)
sys.exit(1)
# get passphrase used to symmetrically decrypt the file
passphrase = getpass.getpass("Please enter your passphrase: ")
if len(passphrase) == 0: # confirm that user entered a passphrase
stderr(
"You did not enter a passphrase. Please repeat your command and try again."
)
sys.exit(1)
passphrase_confirm = getpass.getpass(
"Please enter your passphrase again: ")
# confirm that the passphrases match
if passphrase == passphrase_confirm:
system_command = "gpg --batch -o " + quote(
decrypted_filename) + " --passphrase " + quote(
passphrase) + " -d " + quote(path)
response = muterun(system_command)
if response.exitcode == 0:
# unpack tar archive generated from the decryption, if present
if decrypted_filename.endswith(
'.tar') and tarfile.is_tarfile(decrypted_filename):
untar_path_tuple = os.path.split(decrypted_filename)
untar_path = untar_path_tuple[0]
with tarfile.TarFile(decrypted_filename,
'r',
errorlevel=1) as tar:
for tarinfo in tar:
t_file = tarinfo.name
if len(untar_path) > 0:
t_file_path = os.path.join(
untar_path, t_file)
else:
t_file_path = t_file
if not os.path.exists(t_file_path):
try:
if len(untar_path) > 0:
tar.extract(
t_file, path=untar_path
) # write to the appropriate dir
else:
tar.extract(t_file) # write to CWD
except IOError as e:
stderr("Failed to unpack the file '" +
t_file_path + "' [" + str(e) +
"]")
elif is_dir(t_file_path):
pass # do nothing if it exists and is a directory, no need to warn
else: # it is a file and it already exists, provide user error message
stderr(
"Failed to unpack the file '" +
t_file_path +
"'. File already exists. Use the --overwrite flag to replace existing files."
)
# remove the decrypted tar archive
os.remove(decrypted_filename)
stdout("Decryption complete")
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
sys.exit(0)
else:
stderr(response.stderr)
stderr("Decryption failed")
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
# add a short pause to hinder brute force pexpect style password attacks with decrypto
sleep(0.2) # 200ms pause
sys.exit(1)
else:
stderr(
"The passphrases did not match. Please enter your command again."
)
sys.exit(1)
elif dir_exists(path): # SINGLE DIRECTORY
dirty_directory_file_list = list_all_files(path)
directory_file_list = [
x for x in dirty_directory_file_list
if (x.endswith('.crypt') or x.endswith('.gpg')
or x.endswith('.pgp') or x.endswith('.asc'))
]
# if there are no encrypted files found, warn and abort
if len(directory_file_list) == 0:
stderr("There are no encrypted files in the directory")
sys.exit(1)
# prompt for the passphrase
passphrase = getpass.getpass("Please enter your passphrase: ")
if len(passphrase) == 0: # confirm that user entered a passphrase
stderr(
"You did not enter a passphrase. Please repeat your command and try again."
)
sys.exit(1)
passphrase_confirm = getpass.getpass(
"Please enter your passphrase again: ")
if passphrase == passphrase_confirm:
# decrypt all of the encypted files in the directory
for filepath in directory_file_list:
absolute_filepath = make_path(
path, filepath
) # combine the directory path and file name into absolute path
# remove file suffix from the decrypted file path that writes to disk
if absolute_filepath.endswith('.crypt'):
decrypted_filepath = absolute_filepath[
0:-6] # remove the .crypt suffix
elif absolute_filepath.endswith(
'.gpg') or absolute_filepath.endswith(
'.pgp') or absolute_filepath.endswith('.asc'):
decrypted_filepath = absolute_filepath[0:-4]
# confirm that the file does not already exist
if file_exists(decrypted_filepath):
stdout(
"The file path '" + decrypted_filepath +
"' already exists. This file was not decrypted.")
else:
system_command = "gpg --batch -o " + quote(
decrypted_filepath) + " --passphrase " + quote(
passphrase) + " -d " + quote(absolute_filepath)
response = muterun(system_command)
if response.exitcode == 0:
stdout("'" + absolute_filepath +
"' decrypted to '" + decrypted_filepath +
"'")
else:
stderr(response.stderr)
stderr("Decryption failed for " +
absolute_filepath)
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
# add a short pause to hinder brute force pexpect style password attacks with decrypto
sleep(0.2) # 200ms pause
else:
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
stderr(
"The passphrases did not match. Please enter your command again."
)
sys.exit(1)
else:
# error message, not a file or directory. user entry error
stderr(
"The path that you entered does not appear to be an existing file or directory. Please try again."
)
sys.exit(1)
# ------------------------------------------------------------------------------------------
# [ DEFAULT MESSAGE FOR MATCH FAILURE ]
# Message to provide to the user when all above conditional logic fails to meet a true condition
# ------------------------------------------------------------------------------------------
else:
print("Could not complete your request. Please try again.")
sys.exit(1)
def test_muterun_bad_command_stderr(self):
out = muterun(self.bad_command)
self.assertTrue(b'bogusapp: command not found'
in out.stderr) # has std err message on failure
def test_muterun_missing_option_exitcode(self):
out = muterun(self.missing_option)
self.assertEqual(1, out.exitcode) # returns 1 on missing option to ls
def start_selenium():
muterun(location("./node_modules/protractor/bin/webdriver-manager start"))
def test_muterun_missing_option_stderr(self):
out = muterun(self.missing_option)
self.assertTrue(
len(out.stderr) > 0) # there is a stderr message present
filo = open('dump_links.txt')
for line in filo:
count += 1
term = line.split('\n')[0].split('diseases-conditions/')[1].split('/')[0]
file_arr.append(term)
terms[term] = ['', [[] for i in xrange(len(pages))]]
if(count > max_range):
break
filo.close()
curr_dir = os.path.dirname(os.path.realpath(__file__))
cmd = 'node ' + curr_dir + '/test_parse.js '
for i in range(0, max_range, increm):
cmd_tmp = cmd + str(i)
result1 = muterun(cmd_tmp)
splices = result1.stdout.split('##################################################################\n')[1:]
for pices in xrange(len(splices)):
contents = []
piece = splices[pices]
url = piece.split('\n')[0]
print url
term = url.split('diseases-conditions/')[1].split('/')[0]
f = url.split('/')
doc_type = f[len(f)-2]
print term, doc_type
soup = BeautifulSoup(piece)
def test_muterun_missing_option_stdout(self):
out = muterun(self.missing_option)
self.assertEqual(b"", out.stdout) # std out is empty string on failure
def main():
import os
import sys
from time import sleep
import getpass
import tarfile
from Naked.commandline import Command
from Naked.toolshed.shell import execute, muterun
from Naked.toolshed.system import dir_exists, file_exists, list_all_files, make_path, stdout, stderr, is_dir
#------------------------------------------------------------------------------------------
# [ Instantiate command line object ]
# used for all subsequent conditional logic in the CLI application
#------------------------------------------------------------------------------------------
c = Command(sys.argv[0], sys.argv[1:])
#------------------------------------------------------------------------------------------
# [ VALIDATION LOGIC ] - early validation of appropriate command syntax
# Test that user entered at least one argument to the executable, print usage if not
#------------------------------------------------------------------------------------------
if not c.command_suite_validates():
from crypto.settings import usage as crypto_usage
print(crypto_usage)
sys.exit(1)
#------------------------------------------------------------------------------------------
# [ HELP, VERSION, USAGE LOGIC ]
# Naked framework provides default help, usage, and version commands for all applications
# --> settings for user messages are assigned in the lib/crypto/settings.py file
#------------------------------------------------------------------------------------------
if c.help(): # User requested crypto help information
from crypto.settings import help as crypto_help
print(crypto_help)
sys.exit(0)
elif c.usage(): # User requested crypto usage information
from crypto.settings import usage as crypto_usage
print(crypto_usage)
sys.exit(0)
elif c.version(): # User requested crypto version information
from crypto.settings import app_name, major_version, minor_version, patch_version
version_display_string = app_name + ' ' + major_version + '.' + minor_version + '.' + patch_version
print(version_display_string)
sys.exit(0)
#------------------------------------------------------------------------------------------
# [ APPLICATION LOGIC ]
#
#------------------------------------------------------------------------------------------
elif c.argc > 1:
# code for multi-file processing and commands that include options
use_standard_output = False # print to stdout flag
use_file_overwrite = False # overwrite existing file
untar_archives = True # untar decrypted tar archives, true by default
# set user option flags
if c.option('--stdout') or c.option('-s'):
use_standard_output = True
if c.option('--overwrite') or c.option('-o'):
use_file_overwrite = True
if c.option('--nountar'):
untar_archives = False
directory_list = [] # directory paths included in the user entered paths from the command line
file_list = [] # file paths included in the user entered paths from the command line (and inside directories entered)
for argument in c.argv:
if file_exists(argument): # user included a file, add it to the file_list for decryption
if argument.endswith('.crypt'):
file_list.append(argument) # add .crypt files to the list of files for decryption
elif argument.endswith('.gpg'):
file_list.append(argument)
elif argument.endswith('.asc'):
file_list.append(argument)
elif argument.endswith('.pgp'):
file_list.append(argument)
else:
# cannot identify as an encrypted file, give it a shot anyways but warn user
file_list.append(argument)
stdout("Could not confirm that '" + argument + "' is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed...")
elif dir_exists(argument): # user included a directory, add it to the directory_list
directory_list.append(argument)
else:
if argument[0] == "-":
pass # if it is an option, do nothing
else:
stderr("'" + argument + "' does not appear to be an existing file or directory. Aborting decryption attempt for this request.")
# unroll the contained directory files into the file_list IF they are encrypted file types
if len(directory_list) > 0:
for directory in directory_list:
directory_file_list = list_all_files(directory)
for contained_file in directory_file_list:
if contained_file.endswith('.crypt'):
file_list.append(make_path(directory, contained_file)) # include the file with a filepath 'directory path/contained_file path'
elif contained_file.endswith('.gpg'):
file_list.append(make_path(directory, contained_file))
elif contained_file.endswith('asc'):
file_list.append(make_path(directory, contained_file))
elif contained_file.endswith('.pgp'):
file_list.append(make_path(directory, contained_file))
# confirm that there are files for decryption, if not abort
if len(file_list) == 0:
stderr("Could not identify files for decryption")
sys.exit(1)
# get passphrase used to symmetrically decrypt the file
passphrase = getpass.getpass("Please enter your passphrase: ")
if len(passphrase) == 0: # confirm that user entered a passphrase
stderr("You did not enter a passphrase. Please repeat your command and try again.")
sys.exit(1)
passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")
if passphrase == passphrase_confirm:
# begin decryption of each requested file. the directory path was already added to the file path above
for encrypted_file in file_list:
# create the decrypted file name
decrypted_filename = ""
if encrypted_file.endswith('.crypt'):
decrypted_filename = encrypted_file[0:-6]
elif encrypted_file.endswith('.gpg') or encrypted_file.endswith('.asc') or encrypted_file.endswith('.pgp'):
decrypted_filename = encrypted_file[0:-4]
else:
decrypted_filename = encrypted_file + '.decrypt' # if it was a file without a known encrypted file type, add the .decrypt suffix
# determine whether file overwrite will take place with the decrypted file
skip_file = False # flag that indicates this file should not be encrypted
created_tmp_files = False
if not use_standard_output: # if not writing a file, no need to check for overwrite
if file_exists(decrypted_filename):
if use_file_overwrite: # rename the existing file to temp file which will be erased or replaced (on decryption failures) below
tmp_filename = decrypted_filename + '.tmp'
os.rename(decrypted_filename, tmp_filename)
created_tmp_files = True
else:
stdout("The file path '" + decrypted_filename + "' already exists. This file was not decrypted.")
skip_file = True
# begin decryption
if not skip_file:
if use_standard_output: # using --quiet flag to suppress stdout messages from gpg, just want the file data in stdout stream
system_command = "gpg --batch --quiet --passphrase '" + passphrase + "' -d " + encrypted_file
successful_execution = execute(system_command) # use naked execute function to directly push to stdout, rather than return stdout
if not successful_execution:
stderr("Unable to decrypt file '" + encrypted_file + "'", 0)
if created_tmp_files: # restore the moved tmp file to original if decrypt failed
tmp_filename = decrypted_filename + '.tmp'
if file_exists(tmp_filename):
os.rename(tmp_filename, decrypted_filename)
else: # decryption successful but we are in stdout flag so do not include any other output from decrypto
pass
else:
system_command = "gpg --batch -o " + decrypted_filename + " --passphrase '" + passphrase + "' -d " + encrypted_file
response = muterun(system_command)
if response.exitcode == 0:
stdout("'" + encrypted_file + "' decrypted to '" + decrypted_filename + "'")
else: # failed decryption
if created_tmp_files: # restore the moved tmp file to original if decrypt failed
tmp_filename = decrypted_filename + '.tmp'
if file_exists(tmp_filename):
os.rename(tmp_filename, decrypted_filename)
# report the error
stderr(response.stderr)
stderr("Decryption failed for " + encrypted_file)
# cleanup: remove the tmp file
if created_tmp_files:
tmp_filename = decrypted_filename + '.tmp'
if file_exists(tmp_filename):
os.remove(tmp_filename)
# untar/extract any detected archive file(s)
if untar_archives is True:
if decrypted_filename.endswith('.tar') and tarfile.is_tarfile(decrypted_filename):
untar_path_tuple = os.path.split(decrypted_filename)
untar_path = untar_path_tuple[0]
if use_file_overwrite:
with tarfile.open(decrypted_filename) as tar:
if len(untar_path) > 0:
tar.extractall(path=untar_path) # use dir path from the decrypted_filename if not CWD
stdout("'" + decrypted_filename + "' unpacked in the directory path '" + untar_path + "'")
else:
tar.extractall() # else use CWD
stdout("'" + decrypted_filename + "' unpacked in the current working directory")
else:
with tarfile.TarFile(decrypted_filename, 'r', errorlevel=1) as tar:
for tarinfo in tar:
t_file = tarinfo.name
if len(untar_path) > 0:
t_file_path = os.path.join(untar_path, t_file)
else:
t_file_path = t_file
if not os.path.exists(t_file_path):
try:
if len(untar_path) > 0:
tar.extract(t_file, path=untar_path) # write to the appropriate dir
else:
tar.extract(t_file) # write to CWD
except IOError as e:
stderr(
"Failed to unpack the file '" + t_file_path + "' [" + str(
e) + "]")
elif is_dir(t_file_path):
pass # do nothing if it exists and is a directory, no need to warn
else: # it is a file and it already exists, provide user error message
stderr(
"Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files.")
# remove the decrypted tar archive file
os.remove(decrypted_filename)
# overwrite the entered passphrases after file decryption is complete for all files
passphrase = ""
passphrase_confirm = ""
# add a short pause to hinder brute force pexpect style password attacks with decrypto
sleep(0.2) # 200ms pause
else: # passphrases did not match
passphrase = ""
passphrase_confirm = ""
stderr("The passphrases did not match. Please enter your command again.")
sys.exit(1)
elif c.argc == 1:
# simple single file or directory processing with default settings
path = c.arg0
if file_exists(path): # SINGLE FILE
check_existing_file = False # check for a file with the name of new decrypted filename in the directory
if path.endswith('.crypt'):
decrypted_filename = path[0:-6] # remove the .crypt suffix
check_existing_file = True
elif path.endswith('.gpg') or path.endswith('.pgp') or path.endswith('.asc'):
decrypted_filename = path[0:-4]
check_existing_file = True
else:
decrypted_filename = path + ".decrypt" # if there is not a standard file type, then add a .decrypt suffix to the decrypted file name
stdout("Could not confirm that the requested file is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed...")
# confirm that the decrypted path does not already exist, if so abort with warning message to user
if check_existing_file is True:
if file_exists(decrypted_filename):
stderr("Your file will be decrypted to '" + decrypted_filename + "' and this file path already exists. Please move the file or use the --overwrite option with your command if you intend to replace the current file.")
sys.exit(1)
# get passphrase used to symmetrically decrypt the file
passphrase = getpass.getpass("Please enter your passphrase: ")
if len(passphrase) == 0: # confirm that user entered a passphrase
stderr("You did not enter a passphrase. Please repeat your command and try again.")
sys.exit(1)
passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")
# confirm that the passphrases match
if passphrase == passphrase_confirm:
system_command = "gpg --batch -o " + decrypted_filename + " --passphrase '" + passphrase + "' -d " + path
response = muterun(system_command)
if response.exitcode == 0:
# unpack tar archive generated from the decryption, if present
if decrypted_filename.endswith('.tar') and tarfile.is_tarfile(decrypted_filename):
untar_path_tuple = os.path.split(decrypted_filename)
untar_path = untar_path_tuple[0]
with tarfile.TarFile(decrypted_filename, 'r', errorlevel=1) as tar:
for tarinfo in tar:
t_file = tarinfo.name
if len(untar_path) > 0:
t_file_path = os.path.join(untar_path, t_file)
else:
t_file_path = t_file
if not os.path.exists(t_file_path):
try:
if len(untar_path) > 0:
tar.extract(t_file, path=untar_path) # write to the appropriate dir
else:
tar.extract(t_file) # write to CWD
except IOError as e:
stderr("Failed to unpack the file '" + t_file_path + "' [" + str(e) + "]")
elif is_dir(t_file_path):
pass # do nothing if it exists and is a directory, no need to warn
else: # it is a file and it already exists, provide user error message
stderr("Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files.")
# remove the decrypted tar archive
os.remove(decrypted_filename)
stdout("Decryption complete")
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
sys.exit(0)
else:
stderr(response.stderr)
stderr("Decryption failed")
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
# add a short pause to hinder brute force pexpect style password attacks with decrypto
sleep(0.2) # 200ms pause
sys.exit(1)
else:
stderr("The passphrases did not match. Please enter your command again.")
sys.exit(1)
elif dir_exists(path): # SINGLE DIRECTORY
dirty_directory_file_list = list_all_files(path)
directory_file_list = [x for x in dirty_directory_file_list if (x.endswith('.crypt') or x.endswith('.gpg') or x.endswith('.pgp') or x.endswith('.asc'))]
# if there are no encrypted files found, warn and abort
if len(directory_file_list) == 0:
stderr("There are no encrypted files in the directory")
sys.exit(1)
# prompt for the passphrase
passphrase = getpass.getpass("Please enter your passphrase: ")
if len(passphrase) == 0: # confirm that user entered a passphrase
stderr("You did not enter a passphrase. Please repeat your command and try again.")
sys.exit(1)
passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")
if passphrase == passphrase_confirm:
# decrypt all of the encypted files in the directory
for filepath in directory_file_list:
absolute_filepath = make_path(path, filepath) # combine the directory path and file name into absolute path
# remove file suffix from the decrypted file path that writes to disk
if absolute_filepath.endswith('.crypt'):
decrypted_filepath = absolute_filepath[0:-6] # remove the .crypt suffix
elif absolute_filepath.endswith('.gpg') or absolute_filepath.endswith('.pgp') or absolute_filepath.endswith('.asc'):
decrypted_filepath = absolute_filepath[0:-4]
# confirm that the file does not already exist
if file_exists(decrypted_filepath):
stdout("The file path '" + decrypted_filepath + "' already exists. This file was not decrypted.")
else:
system_command = "gpg --batch -o " + decrypted_filepath + " --passphrase '" + passphrase + "' -d " + absolute_filepath
response = muterun(system_command)
if response.exitcode == 0:
stdout("'" + absolute_filepath + "' decrypted to '" + decrypted_filepath + "'")
else:
stderr(response.stderr)
stderr("Decryption failed for " + absolute_filepath)
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
# add a short pause to hinder brute force pexpect style password attacks with decrypto
sleep(0.2) # 200ms pause
else:
# overwrite user entered passphrases
passphrase = ""
passphrase_confirm = ""
stderr("The passphrases did not match. Please enter your command again.")
sys.exit(1)
else:
# error message, not a file or directory. user entry error
stderr("The path that you entered does not appear to be an existing file or directory. Please try again.")
sys.exit(1)
# ------------------------------------------------------------------------------------------
# [ DEFAULT MESSAGE FOR MATCH FAILURE ]
# Message to provide to the user when all above conditional logic fails to meet a true condition
# ------------------------------------------------------------------------------------------
else:
print("Could not complete your request. Please try again.")
sys.exit(1)
def test_muterun_good_command_return_type(self):
self.assertEqual(type(NakedObject()), type(muterun(self.good_command))) # returns NakedObject on success
filo = open('dump_links.txt')
for line in filo:
count += 1
term = line.split('\n')[0].split('diseases-conditions/')[1].split('/')[0]
file_arr.append(term)
terms[term] = ['', [[] for i in xrange(len(pages))]]
if (count > max_range):
break
filo.close()
curr_dir = os.path.dirname(os.path.realpath(__file__))
cmd = 'node ' + curr_dir + '/test_parse.js '
for i in range(0, max_range, increm):
cmd_tmp = cmd + str(i)
result1 = muterun(cmd_tmp)
splices = result1.stdout.split(
'##################################################################\n'
)[1:]
for pices in xrange(len(splices)):
contents = []
piece = splices[pices]
url = piece.split('\n')[0]
print url
term = url.split('diseases-conditions/')[1].split('/')[0]
f = url.split('/')
doc_type = f[len(f) - 2]
print term, doc_type
def startNode(name, port=8081, script="server"):
'Starts Node.js server with given port'
commands = ["cd {:s}".format(name), "SET PORT={:d}".format(port), "node {:s}.js".format(script)]
muterun(" && ".join(commands))
