def _is_compress_filetype(self, inpath): """private method that performs magic number and size check on file to determine whether to compress the file""" # check for common file type suffixes in order to avoid the need for file reads to check magic number for binary vs. text file if self._is_common_binary(inpath): return False elif self._is_common_text(inpath): return True else: # files > 10kB get checked for compression (arbitrary decision to skip compression on small files) the_file_size = file_size(inpath) if the_file_size > 10240: if the_file_size > 512000: # seems to be a break point at ~ 500kb where file compression offset by additional file read, so limit tests to files > 500kB try: system_command = "file --mime-type -b " + quote(inpath) response = muterun(system_command) if response.stdout[0:5] == "text/": # check for a text file mime type return True # appropriate size, appropriate file mime type else: return False # appropriate size, inappropriate file mime type except Exception: return False else: return True # if file size is < 500kB, skip the additional file read and just go with compression else: return False # below minimum size to consider compression, do not compress
def download_Tiktok_signature(): try: e = muterun("npm i tiktok-signature") # install tiktok-signature if e.exitcode == 1: print("Not able to install tiktok-signature. Please make sure Node.js is installed properly and restart your script running environment.") rewrite_browserJS() except(): print("Not able to install tiktok-signature. Please make sure Node.js is installed properly and restart your script running environment.")
def controllProcess(name, var, port, files): #TODO terminate this process nodeProcess = Process(target=startNode, args=(name,port), daemon=True) nodeProcess.start() getProcess = Process(target=getJSON, args=(files, var, port), daemon=True) getProcess.start() while(var.value == 0): pass getProcess.join() # Close the node server without promting the keyboard interrupt muterun("TASKKILL /F /IM node.exe /T") return
def encrypt_file(self, inpath, force_nocompress=False, force_compress=False, armored=False, checksum=False): """public method for single file encryption with optional compression, ASCII armored formatting, and file hash digest generation""" if armored: if force_compress: command_stub = self.command_maxcompress_armored elif force_nocompress: command_stub = self.command_nocompress_armored else: if self._is_compress_filetype(inpath): command_stub = self.command_default_armored else: command_stub = self.command_nocompress_armored else: if force_compress: command_stub = self.command_maxcompress elif force_nocompress: command_stub = self.command_nocompress else: if self._is_compress_filetype(inpath): command_stub = self.command_default else: command_stub = self.command_nocompress encrypted_outpath = self._create_outfilepath(inpath) system_command = command_stub + encrypted_outpath + " --passphrase " + quote( self.passphrase) + " --symmetric " + quote(inpath) try: response = muterun(system_command) # check returned status code if response.exitcode == 0: stdout(encrypted_outpath + " was generated from " + inpath) if checksum: # add a SHA256 hash digest of the encrypted file - requested by user --hash flag in command from crypto.library import hash encrypted_file_hash = hash.generate_hash(encrypted_outpath) if len(encrypted_file_hash) == 64: stdout("SHA256 hash digest for " + encrypted_outpath + " :") stdout(encrypted_file_hash) else: stdout( "Unable to generate a SHA256 hash digest for the file " + encrypted_outpath) else: stderr(response.stderr, 0) stderr("Encryption failed") sys.exit(1) except Exception as e: stderr( "There was a problem with the execution of gpg. Encryption failed. Error: [" + str(e) + "]") sys.exit(1)
def execute_babel_node(self, expression_or_path, arguments): try: if len(arguments) > 0: js_command = 'babel-node ' + self.babel_arguments + expression_or_path + ' ' + arguments else: js_command = 'babel-node ' + self.babel_arguments + expression_or_path return muterun(js_command) # return result of execute_babel_node() of node.js file except Exception as e: raise e
def test_execute_command(self): """Test execution of a helpy command""" state = StateObject() if (state.py_major == 2 and state.py_minor > 6) or state.py3: response = muterun('helpy sys') self.assertEqual(response.exitcode, 0) self.assertTrue(len(response.stdout) > 0) self.assertEqual(response.stderr, b'') elif state.py_major == 2 and state.py_minor == 6: res = subprocess.check_call(['helpy', 'sys']) self.assertEqual(res, 0)
def test_make_template_file(self): try: os.chdir(self.test_dir) response = muterun('doxx make template') if not response.exitcode == 0: stderr("Error with 'doxx make template' command. Error: " + response.stderr, exit=1) else: self.assertTrue(file_exists(self.template_file)) # confirm that the template file was made os.remove(self.template_file) # remove the test file after the test os.chdir(self.cwd) except Exception as e: os.chdir(self.cwd) raise e
def encrypt_file(self, inpath, force_nocompress=False, force_compress=False, armored=False, checksum=False): """public method for single file encryption with optional compression, ASCII armored formatting, and file hash digest generation""" if armored: if force_compress: command_stub = self.command_maxcompress_armored elif force_nocompress: command_stub = self.command_nocompress_armored else: if self._is_compress_filetype(inpath): command_stub = self.command_default_armored else: command_stub = self.command_nocompress_armored else: if force_compress: command_stub = self.command_maxcompress elif force_nocompress: command_stub = self.command_nocompress else: if self._is_compress_filetype(inpath): command_stub = self.command_default else: command_stub = self.command_nocompress encrypted_outpath = self._create_outfilepath(inpath) system_command = command_stub + encrypted_outpath + " --passphrase " + quote(self.passphrase) + " --symmetric " + quote(inpath) try: response = muterun(system_command) # check returned status code if response.exitcode == 0: stdout(encrypted_outpath + " was generated from " + inpath) if checksum: # add a SHA256 hash digest of the encrypted file - requested by user --hash flag in command from crypto.library import hash encrypted_file_hash = hash.generate_hash(encrypted_outpath) if len(encrypted_file_hash) == 64: stdout("SHA256 hash digest for " + encrypted_outpath + " :") stdout(encrypted_file_hash) else: stdout("Unable to generate a SHA256 hash digest for the file " + encrypted_outpath) else: stderr(response.stderr, 0) stderr("Encryption failed") sys.exit(1) except Exception as e: stderr("There was a problem with the execution of gpg. Encryption failed. Error: [" + str(e) + "]") sys.exit(1)
def test_make_project(self): try: os.chdir(self.test_dir) response = muterun('doxx make project') if not response.exitcode == 0: stderr("Error with 'doxx make project' command. Error: " + response.stderr, exit=1) else: self.assertTrue(file_exists(self.pkey_file_path)) # confirm that the pkey.yaml file was made self.assertTrue(file_exists(self.projyaml_file_path)) # confirm that the project.yaml file was made self.assertTrue(file_exists(self.template_file_path)) # confirm that the template file was made self.assertTrue(dir_exists(self.templates_dir)) # confirm that the 'templates' directory was made os.remove(self.pkey_file_path) # remove the test key file after the test os.remove(self.projyaml_file_path) os.remove(self.template_file_path) # remove the test template file shutil.rmtree(self.templates_dir) # remove the templates directory os.chdir(self.cwd) except Exception as e: os.chdir(self.cwd) raise e
def test_muterun_bad_command_return_type(self): self.assertEqual(type(NakedObject()), type(muterun(self.bad_command))) # returns NakedObject on error
from Naked.toolshed.shell import muterun import os curr_dir = os.path.dirname(os.path.realpath(__file__)) print 'Starting zombies..' cmd = 'node ' + curr_dir + '/poll_terms.js' result1 = muterun(cmd) dumps = result1.stdout.split(',') print 'Starting filedump..' filo = open(curr_dir + '/dump_links.txt', 'w') for line in dumps: if(line != ''): filo.write(line + '\n') filo.close() print 'Cleaning up..'
def test_muterun_bad_command_stdout(self): out = muterun(self.bad_command) self.assertEqual(b"", out.stdout) # std out is empty string on failure
def start_selenium(): muterun(location("./node_modules/protractor/bin/webdriver-manager start"))
def test_muterun_good_command_exitcode(self): out = muterun(self.good_command) self.assertEqual(0, out.exitcode) # exit code = 0 = success
def test_muterun_good_command_stderr(self): out = muterun(self.good_command) self.assertEqual( b"", out.stderr) # stderr empty string when successful command
def test_muterun_bad_command_stderr(self): out = muterun(self.bad_command) self.assertTrue(b'bogusapp: command not found' in out.stderr) # has std err message on failure
def test_muterun_missing_option_exitcode(self): out = muterun(self.missing_option) self.assertEqual(1, out.exitcode) # returns 1 on missing option to ls
def test_muterun_bad_command_exitcode(self): out = muterun(self.bad_command) self.assertEqual(127, out.exitcode) # returns 127 on absent executable
def test_muterun_good_command_stderr(self): out = muterun(self.good_command) self.assertEqual(b"", out.stderr) # stderr empty string when successful command
def test_muterun_good_command_stdout(self): out = muterun(self.good_command) self.assertEqual(b"test command\n", out.stdout) # stdout string is correct
def test_muterun_good_command_return_type(self): self.assertEqual(type(NakedObject()), type(muterun( self.good_command))) # returns NakedObject on success
def test_muterun_missing_option_stderr(self): out = muterun(self.missing_option) self.assertTrue(len(out.stderr) > 0) # there is a stderr message present
def test_muterun_bad_command_return_type(self): self.assertEqual(type(NakedObject()), type(muterun( self.bad_command))) # returns NakedObject on error
term = after.split('/')[0] tail = '/'.join(after.split('/')[1:]) for doc_type in pages: urls.append(lead + 'diseases-conditions/' + doc_type + '/' + tail) #file_arr.append(term) terms[term] = ['', [[] for i in xrange(len(pages))]] if(count > max_range): break filo.close() for url in urls: curr_dir = os.getcwd() cmd = 'node ' + curr_dir + '/test_parse.js ' + str(i) result1 = muterun(cmd) piece = result1.stdout print piece.split('\n')[0] ''' for i in range(0, max_range, increm): curr_dir = os.getcwd() cmd = 'node ' + curr_dir + '/test_parse.js ' + str(i) result1 = muterun(cmd) piece = result1.stdout for pices in xrange(len(splices)): contents = [] piece = splices[pices]
def test_muterun_missing_option_stdout(self): out = muterun(self.missing_option) self.assertEqual(b"", out.stdout) # std out is empty string on failure
def main(): import os import sys from time import sleep import getpass import tarfile from Naked.commandline import Command from Naked.toolshed.shell import execute, muterun from Naked.toolshed.system import dir_exists, file_exists, list_all_files, make_path, stdout, stderr, is_dir from shellescape import quote # ------------------------------------------------------------------------------------------ # [ Instantiate command line object ] # used for all subsequent conditional logic in the CLI application # ------------------------------------------------------------------------------------------ c = Command(sys.argv[0], sys.argv[1:]) # ------------------------------------------------------------------------------------------ # [ VALIDATION LOGIC ] - early validation of appropriate command syntax # Test that user entered at least one argument to the executable, print usage if not # ------------------------------------------------------------------------------------------ if not c.command_suite_validates(): from Crypto.settings import usage as crypto_usage print(crypto_usage) sys.exit(1) # ------------------------------------------------------------------------------------------ # [ HELP, VERSION, USAGE LOGIC ] # Naked framework provides default help, usage, and version commands for all applications # --> settings for user messages are assigned in the lib/Crypto/settings.py file # ------------------------------------------------------------------------------------------ if c.help(): # User requested Crypto help information from Crypto.settings import help as crypto_help print(crypto_help) sys.exit(0) elif c.usage(): # User requested Crypto usage information from Crypto.settings import usage as crypto_usage print(crypto_usage) sys.exit(0) elif c.version(): # User requested Crypto version information from Crypto.settings import app_name, major_version, minor_version, patch_version version_display_string = app_name + ' ' + major_version + '.' + minor_version + '.' + patch_version print(version_display_string) sys.exit(0) # ------------------------------------------------------------------------------------------ # [ APPLICATION LOGIC ] # # ------------------------------------------------------------------------------------------ elif c.argc > 1: # code for multi-file processing and commands that include options use_standard_output = False # print to stdout flag use_file_overwrite = False # overwrite existing file untar_archives = True # untar decrypted tar archives, true by default # set user option flags if c.option('--stdout') or c.option('-s'): use_standard_output = True if c.option('--overwrite') or c.option('-o'): use_file_overwrite = True if c.option('--nountar'): untar_archives = False directory_list = [ ] # directory paths included in the user entered paths from the command line file_list = [ ] # file paths included in the user entered paths from the command line (and inside directories entered) for argument in c.argv: if file_exists( argument ): # user included a file, add it to the file_list for decryption if argument.endswith('.crypt'): file_list.append( argument ) # add .crypt files to the list of files for decryption elif argument.endswith('.gpg'): file_list.append(argument) elif argument.endswith('.asc'): file_list.append(argument) elif argument.endswith('.pgp'): file_list.append(argument) else: # cannot identify as an encrypted file, give it a shot anyways but warn user file_list.append(argument) stdout( "Could not confirm that '" + argument + "' is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed..." ) elif dir_exists( argument ): # user included a directory, add it to the directory_list directory_list.append(argument) else: if argument[0] == "-": pass # if it is an option, do nothing else: stderr( "'" + argument + "' does not appear to be an existing file or directory. Aborting decryption attempt for this request." ) # unroll the contained directory files into the file_list IF they are encrypted file types if len(directory_list) > 0: for directory in directory_list: directory_file_list = list_all_files(directory) for contained_file in directory_file_list: if contained_file.endswith('.crypt'): file_list.append( make_path(directory, contained_file) ) # include the file with a filepath 'directory path/contained_file path' elif contained_file.endswith('.gpg'): file_list.append(make_path(directory, contained_file)) elif contained_file.endswith('asc'): file_list.append(make_path(directory, contained_file)) elif contained_file.endswith('.pgp'): file_list.append(make_path(directory, contained_file)) # confirm that there are files for decryption, if not abort if len(file_list) == 0: stderr("Could not identify files for decryption") sys.exit(1) # get passphrase used to symmetrically decrypt the file passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr( "You did not enter a passphrase. Please repeat your command and try again." ) sys.exit(1) passphrase_confirm = getpass.getpass( "Please enter your passphrase again: ") if passphrase == passphrase_confirm: # begin decryption of each requested file. the directory path was already added to the file path above for encrypted_file in file_list: # create the decrypted file name decrypted_filename = "" if encrypted_file.endswith('.crypt'): decrypted_filename = encrypted_file[0:-6] elif encrypted_file.endswith( '.gpg') or encrypted_file.endswith( '.asc') or encrypted_file.endswith('.pgp'): decrypted_filename = encrypted_file[0:-4] else: decrypted_filename = encrypted_file + '.decrypt' # if it was a file without a known encrypted file type, add the .decrypt suffix # determine whether file overwrite will take place with the decrypted file skip_file = False # flag that indicates this file should not be encrypted created_tmp_files = False if not use_standard_output: # if not writing a file, no need to check for overwrite if file_exists(decrypted_filename): if use_file_overwrite: # rename the existing file to temp file which will be erased or replaced (on decryption failures) below tmp_filename = decrypted_filename + '.tmp' os.rename(decrypted_filename, tmp_filename) created_tmp_files = True else: stdout( "The file path '" + decrypted_filename + "' already exists. This file was not decrypted." ) skip_file = True # begin decryption if not skip_file: if use_standard_output: # using --quiet flag to suppress stdout messages from gpg, just want the file data in stdout stream system_command = "gpg --batch --quiet --passphrase " + quote( passphrase) + " -d " + quote(encrypted_file) successful_execution = execute( system_command ) # use naked execute function to directly push to stdout, rather than return stdout if not successful_execution: stderr( "Unable to decrypt file '" + encrypted_file + "'", 0) if created_tmp_files: # restore the moved tmp file to original if decrypt failed tmp_filename = decrypted_filename + '.tmp' if file_exists(tmp_filename): os.rename(tmp_filename, decrypted_filename) else: # decryption successful but we are in stdout flag so do not include any other output from decrypto pass else: system_command = "gpg --batch -o " + quote( decrypted_filename) + " --passphrase " + quote( passphrase) + " -d " + quote(encrypted_file) response = muterun(system_command) if response.exitcode == 0: stdout("'" + encrypted_file + "' decrypted to '" + decrypted_filename + "'") else: # failed decryption if created_tmp_files: # restore the moved tmp file to original if decrypt failed tmp_filename = decrypted_filename + '.tmp' if file_exists(tmp_filename): os.rename(tmp_filename, decrypted_filename) # report the error stderr(response.stderr) stderr("Decryption failed for " + encrypted_file) # cleanup: remove the tmp file if created_tmp_files: tmp_filename = decrypted_filename + '.tmp' if file_exists(tmp_filename): os.remove(tmp_filename) # untar/extract any detected archive file(s) if untar_archives is True: if decrypted_filename.endswith( '.tar') and tarfile.is_tarfile(decrypted_filename): untar_path_tuple = os.path.split(decrypted_filename) untar_path = untar_path_tuple[0] if use_file_overwrite: with tarfile.open(decrypted_filename) as tar: if len(untar_path) > 0: tar.extractall( path=untar_path ) # use dir path from the decrypted_filename if not CWD stdout( "'" + decrypted_filename + "' unpacked in the directory path '" + untar_path + "'") else: tar.extractall() # else use CWD stdout( "'" + decrypted_filename + "' unpacked in the current working directory" ) else: with tarfile.TarFile(decrypted_filename, 'r', errorlevel=1) as tar: for tarinfo in tar: t_file = tarinfo.name if len(untar_path) > 0: t_file_path = os.path.join( untar_path, t_file) else: t_file_path = t_file if not os.path.exists(t_file_path): try: if len(untar_path) > 0: tar.extract( t_file, path=untar_path ) # write to the appropriate dir else: tar.extract( t_file) # write to CWD except IOError as e: stderr( "Failed to unpack the file '" + t_file_path + "' [" + str(e) + "]") elif is_dir(t_file_path): pass # do nothing if it exists and is a directory, no need to warn else: # it is a file and it already exists, provide user error message stderr( "Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files." ) # remove the decrypted tar archive file os.remove(decrypted_filename) # overwrite the entered passphrases after file decryption is complete for all files passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause else: # passphrases did not match passphrase = "" passphrase_confirm = "" stderr( "The passphrases did not match. Please enter your command again." ) sys.exit(1) elif c.argc == 1: # simple single file or directory processing with default settings path = c.arg0 if file_exists(path): # SINGLE FILE check_existing_file = False # check for a file with the name of new decrypted filename in the directory if path.endswith('.crypt'): decrypted_filename = path[0:-6] # remove the .crypt suffix check_existing_file = True elif path.endswith('.gpg') or path.endswith( '.pgp') or path.endswith('.asc'): decrypted_filename = path[0:-4] check_existing_file = True else: decrypted_filename = path + ".decrypt" # if there is not a standard file type, then add a .decrypt suffix to the decrypted file name stdout( "Could not confirm that the requested file is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed..." ) # confirm that the decrypted path does not already exist, if so abort with warning message to user if check_existing_file is True: if file_exists(decrypted_filename): stderr( "Your file will be decrypted to '" + decrypted_filename + "' and this file path already exists. Please move the file or use the --overwrite option with your command if you intend to replace the current file." ) sys.exit(1) # get passphrase used to symmetrically decrypt the file passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr( "You did not enter a passphrase. Please repeat your command and try again." ) sys.exit(1) passphrase_confirm = getpass.getpass( "Please enter your passphrase again: ") # confirm that the passphrases match if passphrase == passphrase_confirm: system_command = "gpg --batch -o " + quote( decrypted_filename) + " --passphrase " + quote( passphrase) + " -d " + quote(path) response = muterun(system_command) if response.exitcode == 0: # unpack tar archive generated from the decryption, if present if decrypted_filename.endswith( '.tar') and tarfile.is_tarfile(decrypted_filename): untar_path_tuple = os.path.split(decrypted_filename) untar_path = untar_path_tuple[0] with tarfile.TarFile(decrypted_filename, 'r', errorlevel=1) as tar: for tarinfo in tar: t_file = tarinfo.name if len(untar_path) > 0: t_file_path = os.path.join( untar_path, t_file) else: t_file_path = t_file if not os.path.exists(t_file_path): try: if len(untar_path) > 0: tar.extract( t_file, path=untar_path ) # write to the appropriate dir else: tar.extract(t_file) # write to CWD except IOError as e: stderr("Failed to unpack the file '" + t_file_path + "' [" + str(e) + "]") elif is_dir(t_file_path): pass # do nothing if it exists and is a directory, no need to warn else: # it is a file and it already exists, provide user error message stderr( "Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files." ) # remove the decrypted tar archive os.remove(decrypted_filename) stdout("Decryption complete") # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" sys.exit(0) else: stderr(response.stderr) stderr("Decryption failed") # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause sys.exit(1) else: stderr( "The passphrases did not match. Please enter your command again." ) sys.exit(1) elif dir_exists(path): # SINGLE DIRECTORY dirty_directory_file_list = list_all_files(path) directory_file_list = [ x for x in dirty_directory_file_list if (x.endswith('.crypt') or x.endswith('.gpg') or x.endswith('.pgp') or x.endswith('.asc')) ] # if there are no encrypted files found, warn and abort if len(directory_file_list) == 0: stderr("There are no encrypted files in the directory") sys.exit(1) # prompt for the passphrase passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr( "You did not enter a passphrase. Please repeat your command and try again." ) sys.exit(1) passphrase_confirm = getpass.getpass( "Please enter your passphrase again: ") if passphrase == passphrase_confirm: # decrypt all of the encypted files in the directory for filepath in directory_file_list: absolute_filepath = make_path( path, filepath ) # combine the directory path and file name into absolute path # remove file suffix from the decrypted file path that writes to disk if absolute_filepath.endswith('.crypt'): decrypted_filepath = absolute_filepath[ 0:-6] # remove the .crypt suffix elif absolute_filepath.endswith( '.gpg') or absolute_filepath.endswith( '.pgp') or absolute_filepath.endswith('.asc'): decrypted_filepath = absolute_filepath[0:-4] # confirm that the file does not already exist if file_exists(decrypted_filepath): stdout( "The file path '" + decrypted_filepath + "' already exists. This file was not decrypted.") else: system_command = "gpg --batch -o " + quote( decrypted_filepath) + " --passphrase " + quote( passphrase) + " -d " + quote(absolute_filepath) response = muterun(system_command) if response.exitcode == 0: stdout("'" + absolute_filepath + "' decrypted to '" + decrypted_filepath + "'") else: stderr(response.stderr) stderr("Decryption failed for " + absolute_filepath) # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause else: # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" stderr( "The passphrases did not match. Please enter your command again." ) sys.exit(1) else: # error message, not a file or directory. user entry error stderr( "The path that you entered does not appear to be an existing file or directory. Please try again." ) sys.exit(1) # ------------------------------------------------------------------------------------------ # [ DEFAULT MESSAGE FOR MATCH FAILURE ] # Message to provide to the user when all above conditional logic fails to meet a true condition # ------------------------------------------------------------------------------------------ else: print("Could not complete your request. Please try again.") sys.exit(1)
def test_muterun_missing_option_stderr(self): out = muterun(self.missing_option) self.assertTrue( len(out.stderr) > 0) # there is a stderr message present
filo = open('dump_links.txt') for line in filo: count += 1 term = line.split('\n')[0].split('diseases-conditions/')[1].split('/')[0] file_arr.append(term) terms[term] = ['', [[] for i in xrange(len(pages))]] if(count > max_range): break filo.close() curr_dir = os.path.dirname(os.path.realpath(__file__)) cmd = 'node ' + curr_dir + '/test_parse.js ' for i in range(0, max_range, increm): cmd_tmp = cmd + str(i) result1 = muterun(cmd_tmp) splices = result1.stdout.split('##################################################################\n')[1:] for pices in xrange(len(splices)): contents = [] piece = splices[pices] url = piece.split('\n')[0] print url term = url.split('diseases-conditions/')[1].split('/')[0] f = url.split('/') doc_type = f[len(f)-2] print term, doc_type soup = BeautifulSoup(piece)
def main(): import os import sys from time import sleep import getpass import tarfile from Naked.commandline import Command from Naked.toolshed.shell import execute, muterun from Naked.toolshed.system import dir_exists, file_exists, list_all_files, make_path, stdout, stderr, is_dir #------------------------------------------------------------------------------------------ # [ Instantiate command line object ] # used for all subsequent conditional logic in the CLI application #------------------------------------------------------------------------------------------ c = Command(sys.argv[0], sys.argv[1:]) #------------------------------------------------------------------------------------------ # [ VALIDATION LOGIC ] - early validation of appropriate command syntax # Test that user entered at least one argument to the executable, print usage if not #------------------------------------------------------------------------------------------ if not c.command_suite_validates(): from crypto.settings import usage as crypto_usage print(crypto_usage) sys.exit(1) #------------------------------------------------------------------------------------------ # [ HELP, VERSION, USAGE LOGIC ] # Naked framework provides default help, usage, and version commands for all applications # --> settings for user messages are assigned in the lib/crypto/settings.py file #------------------------------------------------------------------------------------------ if c.help(): # User requested crypto help information from crypto.settings import help as crypto_help print(crypto_help) sys.exit(0) elif c.usage(): # User requested crypto usage information from crypto.settings import usage as crypto_usage print(crypto_usage) sys.exit(0) elif c.version(): # User requested crypto version information from crypto.settings import app_name, major_version, minor_version, patch_version version_display_string = app_name + ' ' + major_version + '.' + minor_version + '.' + patch_version print(version_display_string) sys.exit(0) #------------------------------------------------------------------------------------------ # [ APPLICATION LOGIC ] # #------------------------------------------------------------------------------------------ elif c.argc > 1: # code for multi-file processing and commands that include options use_standard_output = False # print to stdout flag use_file_overwrite = False # overwrite existing file untar_archives = True # untar decrypted tar archives, true by default # set user option flags if c.option('--stdout') or c.option('-s'): use_standard_output = True if c.option('--overwrite') or c.option('-o'): use_file_overwrite = True if c.option('--nountar'): untar_archives = False directory_list = [] # directory paths included in the user entered paths from the command line file_list = [] # file paths included in the user entered paths from the command line (and inside directories entered) for argument in c.argv: if file_exists(argument): # user included a file, add it to the file_list for decryption if argument.endswith('.crypt'): file_list.append(argument) # add .crypt files to the list of files for decryption elif argument.endswith('.gpg'): file_list.append(argument) elif argument.endswith('.asc'): file_list.append(argument) elif argument.endswith('.pgp'): file_list.append(argument) else: # cannot identify as an encrypted file, give it a shot anyways but warn user file_list.append(argument) stdout("Could not confirm that '" + argument + "' is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed...") elif dir_exists(argument): # user included a directory, add it to the directory_list directory_list.append(argument) else: if argument[0] == "-": pass # if it is an option, do nothing else: stderr("'" + argument + "' does not appear to be an existing file or directory. Aborting decryption attempt for this request.") # unroll the contained directory files into the file_list IF they are encrypted file types if len(directory_list) > 0: for directory in directory_list: directory_file_list = list_all_files(directory) for contained_file in directory_file_list: if contained_file.endswith('.crypt'): file_list.append(make_path(directory, contained_file)) # include the file with a filepath 'directory path/contained_file path' elif contained_file.endswith('.gpg'): file_list.append(make_path(directory, contained_file)) elif contained_file.endswith('asc'): file_list.append(make_path(directory, contained_file)) elif contained_file.endswith('.pgp'): file_list.append(make_path(directory, contained_file)) # confirm that there are files for decryption, if not abort if len(file_list) == 0: stderr("Could not identify files for decryption") sys.exit(1) # get passphrase used to symmetrically decrypt the file passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr("You did not enter a passphrase. Please repeat your command and try again.") sys.exit(1) passphrase_confirm = getpass.getpass("Please enter your passphrase again: ") if passphrase == passphrase_confirm: # begin decryption of each requested file. the directory path was already added to the file path above for encrypted_file in file_list: # create the decrypted file name decrypted_filename = "" if encrypted_file.endswith('.crypt'): decrypted_filename = encrypted_file[0:-6] elif encrypted_file.endswith('.gpg') or encrypted_file.endswith('.asc') or encrypted_file.endswith('.pgp'): decrypted_filename = encrypted_file[0:-4] else: decrypted_filename = encrypted_file + '.decrypt' # if it was a file without a known encrypted file type, add the .decrypt suffix # determine whether file overwrite will take place with the decrypted file skip_file = False # flag that indicates this file should not be encrypted created_tmp_files = False if not use_standard_output: # if not writing a file, no need to check for overwrite if file_exists(decrypted_filename): if use_file_overwrite: # rename the existing file to temp file which will be erased or replaced (on decryption failures) below tmp_filename = decrypted_filename + '.tmp' os.rename(decrypted_filename, tmp_filename) created_tmp_files = True else: stdout("The file path '" + decrypted_filename + "' already exists. This file was not decrypted.") skip_file = True # begin decryption if not skip_file: if use_standard_output: # using --quiet flag to suppress stdout messages from gpg, just want the file data in stdout stream system_command = "gpg --batch --quiet --passphrase '" + passphrase + "' -d " + encrypted_file successful_execution = execute(system_command) # use naked execute function to directly push to stdout, rather than return stdout if not successful_execution: stderr("Unable to decrypt file '" + encrypted_file + "'", 0) if created_tmp_files: # restore the moved tmp file to original if decrypt failed tmp_filename = decrypted_filename + '.tmp' if file_exists(tmp_filename): os.rename(tmp_filename, decrypted_filename) else: # decryption successful but we are in stdout flag so do not include any other output from decrypto pass else: system_command = "gpg --batch -o " + decrypted_filename + " --passphrase '" + passphrase + "' -d " + encrypted_file response = muterun(system_command) if response.exitcode == 0: stdout("'" + encrypted_file + "' decrypted to '" + decrypted_filename + "'") else: # failed decryption if created_tmp_files: # restore the moved tmp file to original if decrypt failed tmp_filename = decrypted_filename + '.tmp' if file_exists(tmp_filename): os.rename(tmp_filename, decrypted_filename) # report the error stderr(response.stderr) stderr("Decryption failed for " + encrypted_file) # cleanup: remove the tmp file if created_tmp_files: tmp_filename = decrypted_filename + '.tmp' if file_exists(tmp_filename): os.remove(tmp_filename) # untar/extract any detected archive file(s) if untar_archives is True: if decrypted_filename.endswith('.tar') and tarfile.is_tarfile(decrypted_filename): untar_path_tuple = os.path.split(decrypted_filename) untar_path = untar_path_tuple[0] if use_file_overwrite: with tarfile.open(decrypted_filename) as tar: if len(untar_path) > 0: tar.extractall(path=untar_path) # use dir path from the decrypted_filename if not CWD stdout("'" + decrypted_filename + "' unpacked in the directory path '" + untar_path + "'") else: tar.extractall() # else use CWD stdout("'" + decrypted_filename + "' unpacked in the current working directory") else: with tarfile.TarFile(decrypted_filename, 'r', errorlevel=1) as tar: for tarinfo in tar: t_file = tarinfo.name if len(untar_path) > 0: t_file_path = os.path.join(untar_path, t_file) else: t_file_path = t_file if not os.path.exists(t_file_path): try: if len(untar_path) > 0: tar.extract(t_file, path=untar_path) # write to the appropriate dir else: tar.extract(t_file) # write to CWD except IOError as e: stderr( "Failed to unpack the file '" + t_file_path + "' [" + str( e) + "]") elif is_dir(t_file_path): pass # do nothing if it exists and is a directory, no need to warn else: # it is a file and it already exists, provide user error message stderr( "Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files.") # remove the decrypted tar archive file os.remove(decrypted_filename) # overwrite the entered passphrases after file decryption is complete for all files passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause else: # passphrases did not match passphrase = "" passphrase_confirm = "" stderr("The passphrases did not match. Please enter your command again.") sys.exit(1) elif c.argc == 1: # simple single file or directory processing with default settings path = c.arg0 if file_exists(path): # SINGLE FILE check_existing_file = False # check for a file with the name of new decrypted filename in the directory if path.endswith('.crypt'): decrypted_filename = path[0:-6] # remove the .crypt suffix check_existing_file = True elif path.endswith('.gpg') or path.endswith('.pgp') or path.endswith('.asc'): decrypted_filename = path[0:-4] check_existing_file = True else: decrypted_filename = path + ".decrypt" # if there is not a standard file type, then add a .decrypt suffix to the decrypted file name stdout("Could not confirm that the requested file is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed...") # confirm that the decrypted path does not already exist, if so abort with warning message to user if check_existing_file is True: if file_exists(decrypted_filename): stderr("Your file will be decrypted to '" + decrypted_filename + "' and this file path already exists. Please move the file or use the --overwrite option with your command if you intend to replace the current file.") sys.exit(1) # get passphrase used to symmetrically decrypt the file passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr("You did not enter a passphrase. Please repeat your command and try again.") sys.exit(1) passphrase_confirm = getpass.getpass("Please enter your passphrase again: ") # confirm that the passphrases match if passphrase == passphrase_confirm: system_command = "gpg --batch -o " + decrypted_filename + " --passphrase '" + passphrase + "' -d " + path response = muterun(system_command) if response.exitcode == 0: # unpack tar archive generated from the decryption, if present if decrypted_filename.endswith('.tar') and tarfile.is_tarfile(decrypted_filename): untar_path_tuple = os.path.split(decrypted_filename) untar_path = untar_path_tuple[0] with tarfile.TarFile(decrypted_filename, 'r', errorlevel=1) as tar: for tarinfo in tar: t_file = tarinfo.name if len(untar_path) > 0: t_file_path = os.path.join(untar_path, t_file) else: t_file_path = t_file if not os.path.exists(t_file_path): try: if len(untar_path) > 0: tar.extract(t_file, path=untar_path) # write to the appropriate dir else: tar.extract(t_file) # write to CWD except IOError as e: stderr("Failed to unpack the file '" + t_file_path + "' [" + str(e) + "]") elif is_dir(t_file_path): pass # do nothing if it exists and is a directory, no need to warn else: # it is a file and it already exists, provide user error message stderr("Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files.") # remove the decrypted tar archive os.remove(decrypted_filename) stdout("Decryption complete") # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" sys.exit(0) else: stderr(response.stderr) stderr("Decryption failed") # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause sys.exit(1) else: stderr("The passphrases did not match. Please enter your command again.") sys.exit(1) elif dir_exists(path): # SINGLE DIRECTORY dirty_directory_file_list = list_all_files(path) directory_file_list = [x for x in dirty_directory_file_list if (x.endswith('.crypt') or x.endswith('.gpg') or x.endswith('.pgp') or x.endswith('.asc'))] # if there are no encrypted files found, warn and abort if len(directory_file_list) == 0: stderr("There are no encrypted files in the directory") sys.exit(1) # prompt for the passphrase passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr("You did not enter a passphrase. Please repeat your command and try again.") sys.exit(1) passphrase_confirm = getpass.getpass("Please enter your passphrase again: ") if passphrase == passphrase_confirm: # decrypt all of the encypted files in the directory for filepath in directory_file_list: absolute_filepath = make_path(path, filepath) # combine the directory path and file name into absolute path # remove file suffix from the decrypted file path that writes to disk if absolute_filepath.endswith('.crypt'): decrypted_filepath = absolute_filepath[0:-6] # remove the .crypt suffix elif absolute_filepath.endswith('.gpg') or absolute_filepath.endswith('.pgp') or absolute_filepath.endswith('.asc'): decrypted_filepath = absolute_filepath[0:-4] # confirm that the file does not already exist if file_exists(decrypted_filepath): stdout("The file path '" + decrypted_filepath + "' already exists. This file was not decrypted.") else: system_command = "gpg --batch -o " + decrypted_filepath + " --passphrase '" + passphrase + "' -d " + absolute_filepath response = muterun(system_command) if response.exitcode == 0: stdout("'" + absolute_filepath + "' decrypted to '" + decrypted_filepath + "'") else: stderr(response.stderr) stderr("Decryption failed for " + absolute_filepath) # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause else: # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" stderr("The passphrases did not match. Please enter your command again.") sys.exit(1) else: # error message, not a file or directory. user entry error stderr("The path that you entered does not appear to be an existing file or directory. Please try again.") sys.exit(1) # ------------------------------------------------------------------------------------------ # [ DEFAULT MESSAGE FOR MATCH FAILURE ] # Message to provide to the user when all above conditional logic fails to meet a true condition # ------------------------------------------------------------------------------------------ else: print("Could not complete your request. Please try again.") sys.exit(1)
def test_muterun_good_command_return_type(self): self.assertEqual(type(NakedObject()), type(muterun(self.good_command))) # returns NakedObject on success
filo = open('dump_links.txt') for line in filo: count += 1 term = line.split('\n')[0].split('diseases-conditions/')[1].split('/')[0] file_arr.append(term) terms[term] = ['', [[] for i in xrange(len(pages))]] if (count > max_range): break filo.close() curr_dir = os.path.dirname(os.path.realpath(__file__)) cmd = 'node ' + curr_dir + '/test_parse.js ' for i in range(0, max_range, increm): cmd_tmp = cmd + str(i) result1 = muterun(cmd_tmp) splices = result1.stdout.split( '##################################################################\n' )[1:] for pices in xrange(len(splices)): contents = [] piece = splices[pices] url = piece.split('\n')[0] print url term = url.split('diseases-conditions/')[1].split('/')[0] f = url.split('/') doc_type = f[len(f) - 2] print term, doc_type
def startNode(name, port=8081, script="server"): 'Starts Node.js server with given port' commands = ["cd {:s}".format(name), "SET PORT={:d}".format(port), "node {:s}.js".format(script)] muterun(" && ".join(commands))