def setUp(self): self.cwd = os.getcwd() self.remote_packagerepo_testdir = "build-tests/remote-package-repo/build" self.package_repo_keypath = "build-tests/remote-package-repo/key.yaml" self.package_repo_missing_keypath = "build-tests/remote-package-repo/key_missingpackage.yaml" # remove all contents of the build directory and build directory path from prior test run if dir_exists(self.remote_packagerepo_testdir): shutil.rmtree(self.remote_packagerepo_testdir) self.assertFalse(dir_exists(self.remote_packagerepo_testdir)) # create the build directory path with empty directory os.makedirs(self.remote_packagerepo_testdir) self.assertTrue(dir_exists(self.remote_packagerepo_testdir)) # move the key with good file path into the build directory fr_key = FileReader('build-tests/remote-package-repo/key.yaml') key_data = fr_key.read() fw_key = FileWriter('build-tests/remote-package-repo/build/key.yaml') fw_key.write(key_data) # confirm that the build files are present self.assertTrue(file_exists('build-tests/remote-package-repo/build/key.yaml'))
def _write_text_file(self, file_path, file_text): try: fw = FileWriter(file_path) fw.write(file_text) return True except Exception: return False
def setUp(self): self.cwd = os.getcwd() self.local_project_testdir = "build-tests/remote-project/build" self.local_proj_zip_key = "zip_key.yaml" # executed from the same directory # remove all contents of the build directory and build directory path from prior test run if dir_exists('build-tests/remote-project/build'): shutil.rmtree('build-tests/remote-project/build') self.assertFalse(dir_exists('build-tests/remote-project/build')) # create the build directory path os.makedirs('build-tests/remote-project/build') self.assertTrue(dir_exists('build-tests/remote-project/build')) # move the key into the build directory fr_key = FileReader('build-tests/remote-project/zip_key.yaml') zip_key_data = fr_key.read() fw_key = FileWriter('build-tests/remote-project/build/zip_key.yaml') fw_key.write(zip_key_data) # confirm that the build files are present self.assertTrue(file_exists('build-tests/remote-project/build/zip_key.yaml')) # get the expected text for outfile write assertions self.fourohfour_text = FileReader('standards/404.html').read() self.indexhtml_text = FileReader('standards/index.html').read() self.jquery_text = FileReader('standards/jquery.js').read() self.normalize_text = FileReader('standards/normalize-min.css').read()
def make_template(self, outpath): try: fw = FileWriter(outpath) fw.write(template_stub) if file_exists(outpath): stdout("[+] doxx: The template stub '" + outpath + "' is now available in the current directory.") except Exception as e: stderr("[!] doxx: Unable to write the template stub to disk. Error: " + str(e), exit=1)
def test_http_get_text_exists_request_overwrite(self): """Test HTTP GET request with text file write does overwrite existing file when requested to do so""" filepath = os.path.join('testfiles', 'testdir', 'test.txt') http = HTTP('https://raw.github.com/chrissimpkins/six-four/master/LICENSE') fw = FileWriter(filepath) fw.write("test") http.get_txt_write_file(filepath, overwrite_existing=True) self.assertEqual(FileReader(filepath).read().strip(), self.http_string.strip())
def test_http_post_text_file_present_request_overwrite(self): """Test HTTP request text file write does occur when file present and request overwrite""" filepath = os.path.join('testfiles', 'testdir', 'post.txt') if not file_exists(filepath): fw = FileWriter(filepath) fw.write('test') http = HTTP('http://httpbin.org/gzip') response = http.post_bin_write_file(filepath, overwrite_existing=True) self.assertEqual(True, response)
def test_http_get_text_exists_request_overwrite(self): """Test HTTP GET request with text file write does overwrite existing file when requested to do so""" filepath = os.path.join('testfiles', 'testdir', 'test.txt') http = HTTP( 'https://raw.github.com/chrissimpkins/six-four/master/LICENSE') fw = FileWriter(filepath) fw.write("test") http.get_txt_write_file(filepath, overwrite_existing=True) self.assertEqual( FileReader(filepath).read().strip(), self.http_string.strip())
def test_http_post_binary_file_present_request_overwrite(self): """Test HTTP POST request binary file write when file does exist and request for overwrite""" filepath = os.path.join('testfiles', 'testdir', 'post.gz') if not file_exists(filepath): fw = FileWriter(filepath) fw.write('test') http = HTTP('http://httpbin.org/gzip') response = http.post_bin_write_file(filepath, overwrite_existing=True) self.assertEqual(True, response) self.assertEqual(True, file_exists(filepath))
def test_http_get_binary_file_exists_request_overwrite(self): """Test HTTP GET request and write binary file executes the write when the file exists and overwrite requested""" filepath = os.path.join('testfiles', 'testdir', 'test.tar.gz') fw = FileWriter(filepath) fw.write("test") if not file_exists(filepath): raise RuntimeError("Missing test file for the unit test") http = HTTP("https://github.com/chrissimpkins/six-four/tarball/master") http.get_bin_write_file(filepath, overwrite_existing=True) self.assertTrue(file_exists(filepath))
def pull_text_file(url, text_file_name): """pulls a remote text file and writes to disk""" # pull the binary file data http = HTTP(url) try: if http.get_status_ok(): text_data = http.res.text # write text data to disk try: fw = FileWriter(text_file_name) fw.write(text_data) except Exception as e: stderr("[!] doxx: File write failed for '" + text_file_name + "'. Error: " + str(e), exit=1) else: fail_status_code = http.res.status_code stderr("[!] doxx: Unable to pull '" + url + "' (HTTP status code " + str(fail_status_code) + ")", exit=1) except Exception as e: stderr("[!] doxx: Unable to pull '" + url + "'. Error: " + str(e), exit=1)
def run_pullkey(package_name): normalized_package_name = package_name.lower().strip() package = OfficialPackage() key_file_url = package.get_package_key_url(normalized_package_name) try: stdout("[*] doxx: Pulling the remote key file...") http = HTTP(key_file_url) if http.get_status_ok(): key_file_text = http.res.text fr = FileWriter('key.yaml') try: fr.write(key_file_text) except Exception as e: stderr("[!] doxx: Unable to write the 'key.yaml' file to disk. Error: " + str(e), exit=1) stdout("[*] doxx: Key file pull complete") elif http.res.status_code == 404: stderr("[!] doxx: Unable to pull the key file because the requested package could not be found. (HTTP status code: 404)", exit=1) else: stderr("[!] doxx: Unable to pull the key file. (HTTP status code: " + str(http.res.status_code) + ")", exit=1) except Exception as e: stderr("[!] doxx: Unable to pull the key file. Error: " + str(e), exit=1)
def replace_test_files(self): """replace the erased test files in the test directories""" # replace the files in the with-only-templates directory # make the directory if not dir_exists(make_path(self.onlytemplates_dir, 'templates')): os.makedirs(make_path(self.onlytemplates_dir, 'templates')) with_only_templates_files = ['key.yaml', 'pkey.yaml', 'project.yaml', 'templates/404.doxt', 'templates/index.doxt', 'templates/normalize.doxt'] for the_file in with_only_templates_files: fw = FileWriter(make_path(self.onlytemplates_dir, the_file)) fw.write("unimportant text") # replace the files in the with-templates-other-files directory # make the directory if not present if not dir_exists(make_path(self.otherfiles_dir, 'templates')): os.makedirs(make_path(self.otherfiles_dir, 'templates')) with_other_files = ['key.yaml', 'pkey.yaml', 'project.yaml', 'templates/404.doxt', 'templates/index.doxt', 'templates/normalize.doxt', 'templates/dontdeleteme.txt'] for the_file in with_other_files: fw = FileWriter(make_path(self.otherfiles_dir, the_file)) fw.write("unimportant text")
def decompress_gzip(gz_filename): """decompress gzip compressed file""" # decompress the gzip'd file f = gzip.open(gz_filename, 'rb') file_content = f.read() f.close() # monkeypatch for Python 3 byte string read issue if is_py3(): file_content = str(file_content) # get the base file name for the decompressed file write filename_split = gz_filename.split('.') if len(filename_split) == 2: basename = filename_split[0] elif len(filename_split) > 2: basename = filename_split[0] + '.' + filename_split[1] # concatenate first two parts of file name (e.g. example + '.' + 'tar' ) else: basename = gz_filename # write the file locally fw = FileWriter(basename) fw.write(file_content)
def make_project(self): try: # project.yaml file write fw_projyaml = FileWriter('project.yaml') fw_projyaml.write(project_yaml_stub) # pkey.yaml file write fw_pkey = FileWriter('pkey.yaml') fw_pkey.write(key_stub) # templates directory write if not dir_exists('templates'): os.mkdir('templates') # template.doxt file in templates directory template_stub_path = os.path.join('templates', 'stub.doxt') fw_template = FileWriter(template_stub_path) fw_template.write(template_stub) # confirm for user if file_exists('project.yaml'): stdout("[+] doxx: 'project.yaml' ... check") else: stderr("[!] doxx: There was an error writing the 'project.yaml' key file to your project directory") if file_exists('pkey.yaml'): stdout("[+] doxx: 'pkey.yaml' ... check") else: stderr("[!] doxx: There was an error writing the 'pkey.yaml' key file to your project directory") if file_exists(template_stub_path): stdout("[+] doxx: '" + template_stub_path + "' ... check") else: stderr("[!] doxx: There was an error writing the '" + template_stub_path + "' template file to your project directory") except Exception as e: stderr("[!] doxx: Unable to write project files to disk. Error: " + str(e), exit=1)
def single_template_run(self, template_path): """Render replacements using a single template file as defined in a doxx key file (public method)""" #---------------------------------------------------------------------------- # NOTE : changes in this method require the same changes to multi_process_run #---------------------------------------------------------------------------- ## Load the data # remote templates if len(template_path) > 6 and (template_path[0:7] == "http://" or template_path[0:8] == "https://"): template = RemoteDoxxTemplate(template_path) try: result = template.load_data() if result[0] == False: # if the method responds False, then HTTP data load did not work stderr(result[1], exit=1) # write out the returned error message in result[1] position of the tuple # halt execution if unsuccessful except Exception as e: stderr("[!] doxx: Unable to load the remote template file '" + template_path + "'. Error message: " + str(e), exit=1) # local templates elif file_exists(template_path): template = DoxxTemplate(template_path) try: template.load_data() except Exception as e: stderr("[!] doxx: Unable to read the local template file '" + template_path + "'. Error message: " + str(e), exit=1) else: stderr("[!] doxx: Unable to find the requested template file " + template_path, exit=1) # print error message and halt execution of application ## Split the data try: template.split_data() except Exception as e: stderr("[!] doxx: Unable to parse the template data. Please verify the template syntax and try again. Error message: " + str(e), exit=1) ## Parse data for errors error_parse_result = template.parse_template_for_errors() if error_parse_result[0] == True: # if there was a parsing error stderr(error_parse_result[1], exit=1) # print the returned error message to stderr and exit application ## Then parse the template text and load instance attributes for the text replacement with Ink below try: template.parse_template_text() except Exception as e: stderr("[!] doxx: An error occurred while parsing the template file. Error message: " + str(e), exit=1) # determine whether this is a verbatim template file (no replacements) or the key file did not include replacement keys if template.verbatim is True or self.no_key_replacements is True: # write template.text out verbatim try: # if the requested destination directory path does not exist, make it outfile_dir_path = make_path(os.path.dirname(self.key_path), os.path.dirname(template.outfile)) if not outfile_dir_path == '' and not dir_exists(outfile_dir_path): make_dirs(outfile_dir_path) # write the file outfile_path = make_path(os.path.dirname(self.key_path), template.outfile) fw = FileWriter(outfile_path) fw.write(template.text) stdout("[+] doxx: '" + outfile_path + "' build... check") except Exception as e: stderr("[!] doxx: There was a file write error. Error message: " + str(e), exit=1) else: # template meta data is in template.meta_data # template text is in template.text # perform the text replacements: try: ink_template = InkTemplate(template.text) ink_renderer = InkRenderer(ink_template, self.key_data) rendered_text = ink_renderer.render() except Exception as e: stderr("[!] doxx: An error occurred during the text replacement attempt. Error message: " + str(e), exit=1) # if the requested destination directory path does not exist, make it outfile_dir_path = make_path(os.path.dirname(self.key_path), os.path.dirname(template.outfile)) if not outfile_dir_path == '' and not dir_exists(outfile_dir_path): make_dirs(outfile_dir_path) # write rendered file to disk try: outfile_path = make_path(os.path.dirname(self.key_path), template.outfile) fw = FileWriter(outfile_path) fw.write(rendered_text) stdout("[+] doxx: -- " + outfile_path + " ... check") except Exception as e: stderr("[!] doxx: There was an error with the rendered file write. Error message: " + str(e), exit=1)
def multi_process_run(self, template_path, iolock, outputlock): """Render replacements over multiple template files as defined in doxx key file using multiple processes (public method)""" #------------------------------------------------------------------------------- # NOTE : changes in this method require the same changes to single_template_run #------------------------------------------------------------------------------- ## Load the data if len(template_path) > 6 and (template_path[0:7] == "http://" or template_path[0:8] == "https://"): template = RemoteDoxxTemplate(template_path) try: result = template.load_data() # load remote template file through HTTP or HTTPS protocol if result[0] == False: # if the method responds False, then HTTP data load was not successful outputlock.acquire() stderr(result[1], exit=0) # write out the returned error message in result[1] position of the tuple outputlock.release() sys.exit(1) # release the lock before raising SystemExit except Exception as e: outputlock.acquire() stderr("[!] doxx: Unable to load the remote template file '" + template_path + "'. Error message: " + str(e), exit=0) outputlock.release() sys.exit(1) # release the lock before raising SystemExit elif file_exists(template_path): template = DoxxTemplate(template_path) iolock.acquire() # acquire the IO lock for template file read try: template.load_data() # load local data except Exception as e: outputlock.acquire() stderr("[!] doxx: Unable to read the local template file '" + template_path + "'. Error message: " + str(e), exit=0) outputlock.release() sys.exit(1) # release the lock before raising SystemExit iolock.release() # release the IO lock for template file read else: outputlock.acquire() # acquire the stderr lock stdout("[!] doxx: Unable to find the requested template file " + template_path) # print error message in standard output, multi-file run so do not end execution of application outputlock.release() # release the stderr lock ## Split the data try: template.split_data() # split the data sections except Exception as e: outputlock.acquire() stderr("[!] doxx: Unable to parse the template data. Please verify the template syntax and try again. Error message: " + str(e), exit=0) outputlock.release() sys.exit(1) # release the lock before raising SystemExit ## Parse the data for errors error_parse_result = template.parse_template_for_errors() if error_parse_result[0] == True: # if there was a parsing error outputlock.acquire() stderr(error_parse_result[1], exit=0) # print the returned error message to stderr and exit application outputlock.release() sys.exit(1) # release the lock before raising SystemExit ## Parse the template text try: template.parse_template_text() except Exception as e: outputlock.acquire() stderr("[!] doxx: An error occurred during the attempt to parse the template file. Error message: " + str(e), exit=0) outputlock.release() sys.exit(1) # release the lock before raising SystemExit # determine whether this is a verbatim template file (no replacements) or the key file did not include replacement keys if template.verbatim is True or self.no_key_replacements is True: # write template.text out verbatim try: # if the requested destination directory path does not exist, make it outfile_dir_path = make_path(os.path.dirname(self.key_path), os.path.dirname(template.outfile)) try: iolock.acquire() if not outfile_dir_path == '' and not dir_exists(outfile_dir_path): make_dirs(outfile_dir_path) iolock.release() except Exception as e: iolock.release() # release the lock then re-raise the exception raise e # write the file outfile_path = make_path(os.path.dirname(self.key_path), template.outfile) # then write the file out verbatim try: iolock.acquire() fw = FileWriter(outfile_path) fw.write(template.text) iolock.release() except Exception as e: iolock.release() # catch and release the iolock before re-raising the exception raise e outputlock.acquire() stdout("[+] doxx: -- " + outfile_path + " ... check") outputlock.release() except Exception as e: outputlock.acquire() stderr("[!] doxx: There was a file write error with '" + template_path + "'. Error message: " + str(e), exit=0) outputlock.release() sys.exit(1) # release the lock before raising SystemExit else: # template meta data is in template.meta_data # template text is in template.text # perform the text replacements: try: ink_template = InkTemplate(template.text) ink_renderer = InkRenderer(ink_template, self.key_data) rendered_text = ink_renderer.render() except Exception as e: outputlock.acquire() stderr("[!] doxx: An error occurred during the text replacement attempt. Error message: " + str(e), exit=0) outputlock.release() sys.exit(1) # release the lock before raising SystemExit # if the requested destination directory path does not exist, make it outfile_dir_path = make_path(os.path.dirname(self.key_path), os.path.dirname(template.outfile)) try: iolock.acquire() if not outfile_dir_path == '' and not dir_exists(outfile_dir_path): make_dirs(outfile_dir_path) iolock.release() except Exception as e: outputlock.acquire() stderr("[!] doxx: Unable to create directory path '" + outfile_dir_path + "' for your file write. Error: " + str(e), exit=0) outputlock.release() iolock.release() sys.exit(1) # release the iolock before raising SystemExit outfile_path = make_path(os.path.dirname(self.key_path), template.outfile) try: iolock.acquire() fw = FileWriter(outfile_path) fw.write(rendered_text) iolock.release() except Exception as e: outputlock.acquire() stderr("[!] doxx: Unable to write the file '" + outfile_path + "'. Error: " + str(e), exit=0) outputlock.release() iolock.release() sys.exit(1) # release the iolock before raising SystemExit outputlock.acquire() stdout("[+] doxx: -- " + outfile_path + " ... check") outputlock.release()