def test_safe_path_posix_style_chinese_char(self): if py2: test = paths.safe_path('/includes/webform.compon\xd2\xaants.inc/') else: test = paths.safe_path(b'/includes/webform.compon\xd2\xaants.inc/') expected = 'includes/webform.componS_nts.inc' assert expected == test
def test_safe_path(self): # tuples of test data, expected results tests = [ # mixed slashes (r'C:\Documents and Settings\Boki\Desktop\head\patches\drupal6/drupal.js', 'c_/documents_and_settings/boki/desktop/head/patches/drupal6/drupal.js' ), # mixed slashes and spaces (r'C:\Documents and Settings\Boki\Desktop\head\patches\parallel uploads/drupal.js', 'c_/documents_and_settings/boki/desktop/head/patches/parallel_uploads/drupal.js' ), # windows style (r'C:\Documents and Settings\Administrator\Desktop\siftDemoV4_old\defs.h', 'c_/documents_and_settings/administrator/desktop/siftdemov4_old/defs.h' ), # windows style, mixed slashes, no spaces (r'C:\Documents and Settings\Boki\Desktop\head\patches\imagefield/imagefield.css', 'c_/documents_and_settings/boki/desktop/head/patches/imagefield/imagefield.css' ), # windows style, spaces (r'C:\Documents and Settings\Boki\Desktop\head\patches\js delete\imagefield.css', 'c_/documents_and_settings/boki/desktop/head/patches/js_delete/imagefield.css' ), # windows style, posix slashes (r'C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java', 'c_/documents_and_settings/alex_burgel/workspace/hibernate3.2/test/org/hibernate/test/alltests.java' ), # windows style, relative (r'includes\webform.components.inc', 'includes/webform.components.inc'), # windows style, absolute, trailing slash ('\\includes\\webform.components.inc\\', 'includes/webform.components.inc'), # posix style, relative (r'includes/webform.components.inc', 'includes/webform.components.inc'), # posix style, absolute, trailing slash (r'/includes/webform.components.inc/', 'includes/webform.components.inc'), # posix style, french char ('/includes/webform.compon\xc3nts.inc/', 'includes/webform.compon_nts.inc'), # posix style, chinese char ('/includes/webform.compon\xd2\xaants.inc/', 'includes/webform.compon__nts.inc'), # windows style, dots ('\\includes\\..\\webform.components.inc\\', 'webform.components.inc'), # windows style, many dots ('.\\includes\\.\\..\\..\\..\webform.components.inc\\.', 'dotdot/dotdot/webform.components.inc'), # posix style, dots (r'includes/../webform.components.inc', 'webform.components.inc'), # posix style, many dots (r'./includes/./../../../../webform.components.inc/.', 'dotdot/dotdot/dotdot/webform.components.inc'), ] for tst, expected in tests: assert expected == paths.safe_path(tst)
def test_safe_path(self): # tuples of test data, expected results tests = [ # mixed slashes ( r"C:\Documents and Settings\Boki\Desktop\head\patches\drupal6/drupal.js", "c_/documents_and_settings/boki/desktop/head/patches/drupal6/drupal.js", ), # mixed slashes and spaces ( r"C:\Documents and Settings\Boki\Desktop\head\patches\parallel uploads/drupal.js", "c_/documents_and_settings/boki/desktop/head/patches/parallel_uploads/drupal.js", ), # windows style ( r"C:\Documents and Settings\Administrator\Desktop\siftDemoV4_old\defs.h", "c_/documents_and_settings/administrator/desktop/siftdemov4_old/defs.h", ), # windows style, mixed slashes, no spaces ( r"C:\Documents and Settings\Boki\Desktop\head\patches\imagefield/imagefield.css", "c_/documents_and_settings/boki/desktop/head/patches/imagefield/imagefield.css", ), # windows style, spaces ( r"C:\Documents and Settings\Boki\Desktop\head\patches\js delete\imagefield.css", "c_/documents_and_settings/boki/desktop/head/patches/js_delete/imagefield.css", ), # windows style, posix slashes ( r"C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java", "c_/documents_and_settings/alex_burgel/workspace/hibernate3.2/test/org/hibernate/test/alltests.java", ), # windows style, relative (r"includes\webform.components.inc", "includes/webform.components.inc"), # windows style, absolute, trailing slash ("\\includes\\webform.components.inc\\", "includes/webform.components.inc"), # posix style, relative (r"includes/webform.components.inc", "includes/webform.components.inc"), # posix style, absolute, trailing slash (r"/includes/webform.components.inc/", "includes/webform.components.inc"), # posix style, french char ("/includes/webform.compon\xc3nts.inc/", "includes/webform.compon_nts.inc"), # posix style, chinese char ("/includes/webform.compon\xd2\xaants.inc/", "includes/webform.compon__nts.inc"), # windows style, dots ("\\includes\\..\\webform.components.inc\\", "webform.components.inc"), # windows style, many dots (".\\includes\\.\\..\\..\\..\webform.components.inc\\.", "dotdot/dotdot/webform.components.inc"), # posix style, dots (r"includes/../webform.components.inc", "webform.components.inc"), # posix style, many dots (r"./includes/./../../../../webform.components.inc/.", "dotdot/dotdot/dotdot/webform.components.inc"), ] for tst, expected in tests: assert expected == paths.safe_path(tst)
def extract(location, target_dir): """ Extract each patch of a patch file at `location` as files in a target_dir directory tree mimicking the directory in which the patches would be applied with the patch command. This treats a patch file as if it were an archive containing one file for each patch applied to a file to be patched. Return a list of warning messages. Raise Exceptionon errors. """ for source, target, text in patch_info(location): # prefer the target path for writing the patch text to a subfile # unless target is /dev/null (a deletion) if '/dev/null' in target: patch_subfile_path = source else: patch_subfile_path = target # make the path safe to use as a subfile path # ensure this a good and clean posix relative path patch_subfile_path = paths.safe_path(patch_subfile_path) # create directories parent_dir = posixpath.dirname(patch_subfile_path) parent_target_dir = os.path.join(target_dir, parent_dir) fileutils.create_dir(parent_target_dir) # find a unique name using a simple counter base_subfile_path = os.path.join(target_dir, patch_subfile_path) counter = 0 fp = base_subfile_path while os.path.exists(fp + extractcode.EXTRACT_SUFFIX): fp = base_subfile_path + '_%d' % counter counter += 1 base_subfile_path = fp # write the location proper, with a suffix extension to avoid # recursive extraction if py2: mode = 'wb' eol = b'\n' if py3: mode = 'w' eol = u'\n' subfile_path = base_subfile_path + extractcode.EXTRACT_SUFFIX with open(subfile_path, mode) as subfile: subfile.write(eol.join(text)) return []
def extract(location, target_dir): """ Extract each patch of a patch file at `location` as files in a target_dir directory tree mimicking the directory in which the patches would be applied with the patch command. This treats a patch file as if it were an archive containing one file for each patch applied to a file to be patched. Return a list of warning messages. Raise Exceptionon errors. """ for source, target, text in patch_info(location): # prefer the target path for writing the patch text to a subfile # unless target is /dev/null (a deletion) if '/dev/null' in target: patch_subfile_path = source else: patch_subfile_path = target # make the path safe to use as a subfile path # ensure this a good and clean posix relative path patch_subfile_path = paths.safe_path(patch_subfile_path) # create directories parent_dir = posixpath.dirname(patch_subfile_path) parent_target_dir = os.path.join(target_dir, parent_dir) fileutils.create_dir(parent_target_dir) # find a unique name using a simple counter base_subfile_path = os.path.join(target_dir, patch_subfile_path) counter = 0 fp = base_subfile_path while os.path.exists(fp + extractcode.EXTRACT_SUFFIX): fp = base_subfile_path + '_%d' % counter counter += 1 base_subfile_path = fp # write the location proper, with a suffix extension to avoid # recursive extraction subfile_path = base_subfile_path + extractcode.EXTRACT_SUFFIX with open(subfile_path, 'wb') as subfile: subfile.write(u'\n'.join(text)) return []
def test_safe_path_mixed_slashes_and_spaces(self): test = paths.safe_path( 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js' ) expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/parallel_uploads/drupal.js' assert expected == test
def test_safe_path_windows_style_absolute_trailing_slash(self): test = paths.safe_path('\\includes\\webform.components.inc\\') expected = 'includes/webform.components.inc' assert expected == test
def test_safe_path_posix_style_french_char(self): test = paths.safe_path('/includes/webform.compon\xc3nts.inc/') expected = 'includes/webform.componAnts.inc' assert expected == test
def extract_file_by_file(location, target_dir, arch_type='*', skip_symlinks=True): """ Extract all files using a one-by-one process from a 7zip-supported archive file at location in the `target_dir` directory. Return a list of warning messages if any or an empty list. Raise exception on errors. `arch_type` is the type of 7zip archive passed to the -t 7zip option. Can be None. """ abs_location = os.path.abspath(os.path.expanduser(location)) abs_target_dir = os.path.abspath(os.path.expanduser(target_dir)) entries, errors_msgs = list_entries(location, arch_type) entries = list(entries) # Determine if we need a one-by-one approach: technically the aproach is to # check if we have files that are in the same dir and have the same name # when the case is ignored. We take a simpler approach: we check if all # paths are unique when we ignore the case: for that we only check that the # length of two paths sets are the same: one set as-is and the other # lowercased. paths_as_is = set(e.path for e in entries) paths_no_case = set(p.lower() for p in paths_as_is) need_by_file = len(paths_as_is) != len(paths_no_case) if not need_by_file: # use regular extract return extract_all_files_at_once(location=location, target_dir=target_dir, arch_type=arch_type) # now we are extracting one file at a time. this is a tad painful because we # are dealing with a full command execution at each time. errors = {} warnings = {} tmp_dir = fileutils.get_temp_dir(prefix='extractcode-extract-') for i, entry in enumerate(entries): if not entry.is_file: continue tmp_extract_dir = os.path.join(tmp_dir, str(i)) fileutils.create_dir(tmp_extract_dir) ex_args = build_7z_extract_command( location=location, target_dir=tmp_extract_dir, single_entry=entry, arch_type=arch_type, ) rc, stdout, stderr = command.execute2(**ex_args) error = get_7z_errors(stdout, stderr) if error or rc != 0: error = error or UNKNOWN_ERROR if TRACE: logger.debug( 'extract: failure: {rc}\n' 'stderr: {stderr}\nstdout: {stdout}'.format(**locals())) errors[entry.path] = error continue # these are all for a single file path warns = get_7z_warnings(stdout) or {} wmsg = '\n'.join(warns.values()) if wmsg: if entry.path in warnings: warnings[entry.path] += '\n' + wmsg else: warnings[entry.path] = wmsg # finally move that extracted file to its target location, possibly renamed source_file_name = fileutils.file_name(entry.path) source_file_loc = os.path.join(tmp_extract_dir, source_file_name) if not os.path.exists(source_file_loc): if entry.path in errors: errors[entry.path] += '\nNo file name extracted.' else: errors[entry.path] = 'No file name extracted.' continue safe_path = paths.safe_path(entry.path, posix=True) target_file_loc = os.path.join(target_dir, safe_path) target_file_dir = os.path.dirname(target_file_loc) fileutils.create_dir(target_file_dir) unique_target_file_loc = extractcode.new_name(target_file_loc, is_dir=False) if TRACE: logger.debug( 'extract: unique_target_file_loc: from {} to {}'.format( target_file_loc, unique_target_file_loc)) if os.path.isfile(source_file_loc): fileutils.copyfile(source_file_loc, unique_target_file_loc) else: fileutils.copytree(source_file_loc, unique_target_file_loc) extractcode.remove_backslashes_and_dotdots(abs_target_dir) if errors: raise ExtractErrorFailedToExtract(errors) return convert_warnings_to_list(warnings)
def test_safe_path_windows_style_posix_slashes(self): test = paths.safe_path( 'C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java' ) expected = 'C/Documents_and_Settings/Alex_Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java' assert expected == test
def test_safe_path_posix_style_relative(self): test = paths.safe_path('includes/webform.components.inc') expected = 'includes/webform.components.inc' assert expected == test
def test_safe_path_mixed_slashes(self): test = paths.safe_path( 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\drupal6/drupal.js' ) expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js' assert test == expected
def test_safe_path_windows_style_mixed_slashes_no_spaces(self): test = paths.safe_path( 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\imagefield/imagefield.css' ) expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/imagefield/imagefield.css' assert test == expected
def test_safe_path_posix_style_chinese_char(self): test = paths.safe_path('/includes/webform.compon\xd2\xaants.inc/') expected = 'includes/webform.componS_nts.inc' assert expected == test
def test_safe_path_windows_style_many_dots(self): test = paths.safe_path('.\\includes\\.\\..\\..\\..\\webform.components.inc\\.') expected = 'dotdot/dotdot/webform.components.inc' assert expected == test
def test_safe_path_posix_style_absolute_trailing_slash(self): test = paths.safe_path('/includes/webform.components.inc/') expected = 'includes/webform.components.inc' assert expected == test
def test_safe_path_windows_style(self): test = paths.safe_path( 'C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h' ) expected = 'C/Documents_and_Settings/Administrator/Desktop/siftDemoV4_old/defs.h' assert expected == test
def test_safe_path_windows_style_relative(self): test = paths.safe_path('includes\\webform.components.inc') expected = 'includes/webform.components.inc' assert test == expected
def test_safe_path_windows_style_spaces(self): test = paths.safe_path( 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css' ) expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/js_delete/imagefield.css' assert expected == test
def test_safe_path_windows_style_dots(self): test = paths.safe_path('\\includes\\..\\webform.components.inc\\') expected = 'webform.components.inc' assert test == expected
def test_safe_path_posix_style_dots(self): test = paths.safe_path('includes/../webform.components.inc') expected = 'webform.components.inc' assert test == expected
def test_safe_path_mixed_slashes_and_spaces(self): test = paths.safe_path('C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js') expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/parallel_uploads/drupal.js' assert expected == test
def test_safe_path_windows_style_spaces(self): test = paths.safe_path('C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css') expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/js_delete/imagefield.css' assert expected == test
def test_safe_path_windows_style_many_dots(self): test = paths.safe_path( '.\\includes\\.\\..\\..\\..\\webform.components.inc\\.') expected = 'dotdot/dotdot/webform.components.inc' assert expected == test
def test_safe_path_windows_style(self): test = paths.safe_path('C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h') expected = 'C/Documents_and_Settings/Administrator/Desktop/siftDemoV4_old/defs.h' assert expected == test
def test_safe_path_posix_style_many_dots(self): test = paths.safe_path( './includes/./../../../../webform.components.inc/.') expected = 'dotdot/dotdot/dotdot/webform.components.inc' assert expected == test
def test_safe_path_posix_style_many_dots(self): test = paths.safe_path('./includes/./../../../../webform.components.inc/.') expected = 'dotdot/dotdot/dotdot/webform.components.inc' assert expected == test
def test_safe_path_windows_style_posix_slashes(self): test = paths.safe_path('C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java') expected = 'C/Documents_and_Settings/Alex_Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java' assert expected == test