def relocate(self, target_dir, new_key=None):
"""
Return a copy of this license object relocated to a new `src_dir`.
The data and license text files are persisted in the new `src_dir`.
"""
if not target_dir or target_dir == self.src_dir:
raise ValueError(
'Cannot relocate a License to empty directory or same directory.'
)
if new_key:
key = new_key
else:
key = self.key
newl = License(key, target_dir)
# copy attributes
excluded_attrs = (
'key',
'src_dir',
'data_file',
'text_file',
)
attrs = [a for a in self.__slots__ if a not in excluded_attrs]
for name in attrs:
setattr(newl, name, getattr(self, name))
# save it all to files
if self.text:
fileutils.copyfile(self.text_file, newl.text_file)
newl.dump()
return newl
def get_test_loc(self, test_path, copy=False, debug=False):
"""
Given a `test_path` relative to the self.test_data_dir directory, return the
location to a test file or directory for this path. Copy to a temp
test location if `copy` is True.
"""
if debug:
import inspect
caller = inspect.stack()[1][3]
print('\nself.get_test_loc,%(caller)s,"%(test_path)s"' % locals())
test_loc = get_test_loc(test_path, self.test_data_dir, debug=debug)
if copy:
base_name = os.path.basename(test_loc)
if filetype.is_file(test_loc):
# target must be an existing dir
target_dir = self.get_temp_dir()
fileutils.copyfile(test_loc, target_dir)
test_loc = os.path.join(target_dir, base_name)
else:
# target must be a NON existing dir
target_dir = os.path.join(self.get_temp_dir(), base_name)
fileutils.copytree(test_loc, target_dir)
# cleanup of VCS that could be left over from checkouts
self.remove_vcs(target_dir)
test_loc = target_dir
return test_loc
def get_test_loc(self, test_path, copy=False, debug=False):
"""
Given a `test_path` relative to the self.test_data_dir directory, return the
location to a test file or directory for this path. Copy to a temp
test location if `copy` is True.
"""
if debug:
import inspect
caller = inspect.stack()[1][3]
print('\nself.get_test_loc,%(caller)s,"%(test_path)s"' % locals())
test_loc = get_test_loc(test_path, self.test_data_dir, debug=debug)
if copy:
base_name = os.path.basename(test_loc)
if filetype.is_file(test_loc):
# target must be an existing dir
target_dir = self.get_temp_dir()
fileutils.copyfile(test_loc, target_dir)
test_loc = os.path.join(target_dir, base_name)
else:
# target must be a NON existing dir
target_dir = os.path.join(self.get_temp_dir(), base_name)
fileutils.copytree(test_loc, target_dir)
# cleanup of VCS that could be left over from checkouts
self.remove_vcs(target_dir)
test_loc = target_dir
return test_loc
def relocate(self, target_dir, new_key=None):
"""
Return f copy of this license object relocated to f new `src_dir`.
The data and license text files are persisted in the new `src_dir`.
"""
if not target_dir or target_dir == self.src_dir:
raise ValueError(
'Cannot relocate {} License to empty directory or same directory.'.format(self.key))
if new_key:
key = new_key
else:
key = self.key
newl = License(key, target_dir)
# copy fields
excluded_fields = ('key', 'src_dir', 'data_file', 'text_file',)
all_fields = attr.fields(self.__class__)
attrs = [f.name for f in all_fields if f.name not in excluded_fields]
for name in attrs:
setattr(newl, name, getattr(self, name))
# save it all to files
if self.text:
copyfile(self.text_file, newl.text_file)
newl.dump()
return newl
def relocate(self, target_dir, new_key=None):
"""
Return a copy of this license object relocated to a new `src_dir`.
The data and license text files are persisted in the new `src_dir`.
"""
if not target_dir or target_dir == self.src_dir:
raise ValueError(
'Cannot relocate a License to empty directory or same directory.')
if new_key:
key = new_key
else:
key = self.key
newl = License(key, target_dir)
# copy attributes
excluded_attrs = ('key', 'src_dir', 'data_file', 'text_file',)
attrs = [a for a in self.__slots__ if a not in excluded_attrs]
for name in attrs:
setattr(newl, name, getattr(self, name))
# save it all to files
if self.text:
fileutils.copyfile(self.text_file, newl.text_file)
newl.dump()
return newl
def test_parse_debian_files_list_with_arch(self):
test_file = self.get_test_loc('debian/files-md5sums/mokutil.md5sums',
copy=True)
from commoncode import fileutils
from pathlib import Path
src = Path(test_file)
test_dir = src.parent
test_file = str(test_dir / "mockutil:amd64.md5sum")
fileutils.copyfile(str(src), test_file)
results = debian.parse_debian_files_list(
location=test_file,
datasource_id=debian.DebianInstalledMd5sumFilelistHandler.
datasource_id,
package_type=debian.DebianInstalledMd5sumFilelistHandler.
default_package_type,
)
expected_loc = self.get_test_loc(
'debian/files-md5sums/mokutil-amd64.md5sums.expected.json',
must_exist=False)
self.check_packages_data(results,
expected_loc,
must_exist=False,
regen=REGEN_TEST_FIXTURES)
def test_copyfile_keeps_modified_date(self):
test_file = self.get_test_loc('fileutils/exec/subtxt/a.txt', copy=True)
dest = self.get_temp_file()
expected = 1289918700
os.utime(test_file, (expected, expected))
fileutils.copyfile(test_file, dest)
result = os.stat(dest).st_mtime
assert expected == result
def relocate(self, src_dir):
"""
Return a copy of this license object relocated to a new `src_dir`.
Also copy the LICENSE file.
"""
newl = copy(self)
newl.src_dir = src_dir
newl.set_file_paths()
if self.text:
fileutils.copyfile(self.text_file, newl.text_file)
return newl
def setup_vscode():
"""
Add base settings for .vscode
"""
from scancode_config import scancode_root_dir
from commoncode.fileutils import create_dir
from commoncode.fileutils import copyfile
settings = os.path.join(scancode_root_dir, 'etc', 'vscode',
'settings.json')
if os.path.exists(settings):
vscode = os.path.join(scancode_root_dir, '.vscode')
create_dir(vscode)
copyfile(settings, vscode)
def test_copyfile_does_not_keep_permissions(self):
src_file = self.get_temp_file()
dest = self.get_temp_dir()
with open(src_file, 'wb') as f:
f.write('')
try:
make_non_readable(src_file)
if on_posix:
assert not filetype.is_readable(src_file)
fileutils.copyfile(src_file, dest)
dest_file = join(dest, os.listdir(dest)[0])
assert filetype.is_readable(dest_file)
finally:
fileutils.chmod(src_file, fileutils.RW, recurse=True)
fileutils.chmod(dest, fileutils.RW, recurse=True)
def atest_gen(self):
"""
Rename to test xxx to regen tests.
"""
test_dir = self.get_test_loc(u'markup', True)
expected_dir = self.get_test_loc(u'markup_expected')
template = u"""
def test_%(tn)s(self):
test_file = self.get_test_loc(u'markup/%(test_file)s')
expected = self.get_test_loc(u'markup_expected/%(test_file)s')
result = markup.convert_to_text(test_file)
file_cmp(expected, result, ignore_line_endings=True)"""
for test_file in os.listdir(test_dir):
tn = text.python_safe_name(test_file)
location = os.path.join(test_dir, test_file)
result = markup.convert_to_text(location)
expected_file = os.path.join(expected_dir, test_file)
fileutils.copyfile(result, expected_file)
print(template % locals())
def atest_gen(self):
"""
Rename to test xxx to regen tests.
"""
test_dir = self.get_test_loc(u'markup', True)
expected_dir = self.get_test_loc(u'markup_expected')
template = u"""
def test_%(tn)s(self):
test_file = self.get_test_loc(u'markup/%(test_file)s')
expected = self.get_test_loc(u'markup_expected/%(test_file)s')
result = markup.convert_to_text(test_file)
file_cmp(expected, result, ignore_line_endings=True)"""
for test_file in os.listdir(test_dir):
tn = text.python_safe_name(test_file)
location = os.path.join(test_dir, test_file)
result = markup.convert_to_text(location)
expected_file = os.path.join(expected_dir, test_file)
fileutils.copyfile(result, expected_file)
print(template % locals())
def get_test_loc(self, test_path, copy=False):
"""
Given a `test_path` relative to the self.test_data_dir directory, return the
location to a test file or directory for this path. Copy to a temp
test location if `copy` is True.
"""
test_loc = get_test_loc(test_path, self.test_data_dir)
if copy:
base_name = os.path.basename(test_loc)
if filetype.is_file(test_loc):
# target must be an existing dir
target_dir = self.get_temp_dir()
fileutils.copyfile(test_loc, target_dir)
test_loc = os.path.join(target_dir, base_name)
else:
# target must be a NON existing dir
target_dir = os.path.join(self.get_temp_dir(), base_name)
fileutils.copytree(test_loc, target_dir)
# cleanup of VCS that could be left over from checkouts
self.remove_vcs(target_dir)
test_loc = target_dir
return test_loc
def get_test_loc(self, test_path, copy=False):
"""
Given a `test_path` relative to the self.test_data_dir directory, return the
location to a test file or directory for this path. Copy to a temp
test location if `copy` is True.
"""
test_loc = get_test_loc(test_path, self.test_data_dir)
if copy:
base_name = os.path.basename(test_loc)
if filetype.is_file(test_loc):
# target must be an existing dir
target_dir = self.get_temp_dir()
fileutils.copyfile(test_loc, target_dir)
test_loc = os.path.join(target_dir, base_name)
else:
# target must be a NON existing dir
target_dir = os.path.join(self.get_temp_dir(), base_name)
fileutils.copytree(test_loc, target_dir)
# cleanup of VCS that could be left over from checkouts
self.remove_vcs(target_dir)
test_loc = target_dir
return test_loc
def test_copyfile_can_copy_file_to_dir_keeping_full_file_name(self):
test_file = self.get_test_loc('fileutils/exec/subtxt/a.txt', copy=True)
dest = self.get_temp_dir()
expected = os.path.join(dest, 'a.txt')
fileutils.copyfile(test_file, dest)
assert os.path.exists(expected)
def extract_file_by_file(location,
target_dir,
arch_type='*',
skip_symlinks=True):
"""
Extract all files using a one-by-one process from a 7zip-supported archive
file at location in the `target_dir` directory.
Return a list of warning messages if any or an empty list.
Raise exception on errors.
`arch_type` is the type of 7zip archive passed to the -t 7zip option.
Can be None.
"""
abs_location = os.path.abspath(os.path.expanduser(location))
abs_target_dir = os.path.abspath(os.path.expanduser(target_dir))
entries, errors_msgs = list_entries(location, arch_type)
entries = list(entries)
# Determine if we need a one-by-one approach: technically the aproach is to
# check if we have files that are in the same dir and have the same name
# when the case is ignored. We take a simpler approach: we check if all
# paths are unique when we ignore the case: for that we only check that the
# length of two paths sets are the same: one set as-is and the other
# lowercased.
paths_as_is = set(e.path for e in entries)
paths_no_case = set(p.lower() for p in paths_as_is)
need_by_file = len(paths_as_is) != len(paths_no_case)
if not need_by_file:
# use regular extract
return extract_all_files_at_once(location=location,
target_dir=target_dir,
arch_type=arch_type)
# now we are extracting one file at a time. this is a tad painful because we
# are dealing with a full command execution at each time.
errors = {}
warnings = {}
tmp_dir = fileutils.get_temp_dir(prefix='extractcode-extract-')
for i, entry in enumerate(entries):
if not entry.is_file:
continue
tmp_extract_dir = os.path.join(tmp_dir, str(i))
fileutils.create_dir(tmp_extract_dir)
ex_args = build_7z_extract_command(
location=location,
target_dir=tmp_extract_dir,
single_entry=entry,
arch_type=arch_type,
)
rc, stdout, stderr = command.execute2(**ex_args)
error = get_7z_errors(stdout, stderr)
if error or rc != 0:
error = error or UNKNOWN_ERROR
if TRACE:
logger.debug(
'extract: failure: {rc}\n'
'stderr: {stderr}\nstdout: {stdout}'.format(**locals()))
errors[entry.path] = error
continue
# these are all for a single file path
warns = get_7z_warnings(stdout) or {}
wmsg = '\n'.join(warns.values())
if wmsg:
if entry.path in warnings:
warnings[entry.path] += '\n' + wmsg
else:
warnings[entry.path] = wmsg
# finally move that extracted file to its target location, possibly renamed
source_file_name = fileutils.file_name(entry.path)
source_file_loc = os.path.join(tmp_extract_dir, source_file_name)
if not os.path.exists(source_file_loc):
if entry.path in errors:
errors[entry.path] += '\nNo file name extracted.'
else:
errors[entry.path] = 'No file name extracted.'
continue
safe_path = paths.safe_path(entry.path, posix=True)
target_file_loc = os.path.join(target_dir, safe_path)
target_file_dir = os.path.dirname(target_file_loc)
fileutils.create_dir(target_file_dir)
unique_target_file_loc = extractcode.new_name(target_file_loc,
is_dir=False)
if TRACE:
logger.debug(
'extract: unique_target_file_loc: from {} to {}'.format(
target_file_loc, unique_target_file_loc))
if os.path.isfile(source_file_loc):
fileutils.copyfile(source_file_loc, unique_target_file_loc)
else:
fileutils.copytree(source_file_loc, unique_target_file_loc)
extractcode.remove_backslashes_and_dotdots(abs_target_dir)
if errors:
raise ExtractErrorFailedToExtract(errors)
return convert_warnings_to_list(warnings)
