def _build(self):
"""Builds package."""
if not self.working_dir:
raise ValueError('Package doesn\'t have working directory')
# working directory
working_dir = os.path.join(self.project_dir, self.working_dir)
if not dir_exists(working_dir):
raise ValueError('Working directory doesn\'t exist')
# package paths
artifact_dir = self.get_artifact_dir(self.ENV_TMP)
info_path = self.get_artifact_info_path(self.ENV_TMP)
data_dir = self.get_artifact_data_dir(self.ENV_TMP)
# remove artifact directory if it exists
if dir_exists(artifact_dir):
rmtree(artifact_dir)
# create artifact data directory
os.makedirs(data_dir, exist_ok=True)
# copy files to package data directory
if self.files:
for filename in self.files:
src_path = os.path.join(working_dir, filename)
if not file_exists(src_path):
raise FileNotFoundError(
'File "%s" doesn\'t exist in the working directory' %
filename)
copy_file(src_path, os.path.join(data_dir, filename))
else:
copy_dir(working_dir, data_dir)
# get the list of copied files
files = []
for cur_dir, directories, filenames in os.walk(data_dir):
relative_dir = cur_dir.replace(data_dir, '').lstrip(os.sep)
for filename in filenames:
files.append(os.path.join(relative_dir, filename))
# create info.json file
package_info = OrderedDict([('group', self.group),
('artifact', self.artifact),
('version', self.version),
('files', files), ('name', self.name),
('description', self.description),
('hash', get_dir_hash(data_dir))])
with open(info_path, 'w+') as f:
json.dump(package_info, f, indent=2)
return artifact_dir
def get_package_info(self):
"""Returns a package info if package is published locally or downloaded."""
package_info = None
for env in (self.ENV_LOCAL, self.ENV_PRODUCTION):
info_path = self.get_artifact_info_path(env)
if file_exists(info_path):
with open(info_path) as f:
info = json.load(f)
package_info = PackageInfo(info, (env == self.ENV_LOCAL))
break
return package_info
def __init__(self, config_path: str = None, darty_profile: str = None):
if not config_path:
config_path = self.DEFAULT_CONFIG_FILE
if not darty_profile:
darty_profile = self.DEFAULT_DARTY_PROFILE
# check that a configuration file exists
if not file_exists(config_path):
raise ValueError('Configuration file "%s" was not found.' %
config_path)
# get packages directory
settings = get_settings(darty_profile)
packages_dir = os.path.expanduser(settings['packages_dir'])
# read a config file
with open(config_path, 'r') as f:
config = yaml.load(f)
config = validate_dependency_config(config)
project_dir = os.path.dirname(config_path) # project directory
if 'repositories' not in config or not len(config['repositories']):
raise ValueError('Repositories are not specified')
if 'dependencies' not in config or not len(config['dependencies']):
raise ValueError('Dependencies are not specified')
# creating repositories objects
repositories = {}
for rep_name, rep_config in config['repositories'].items():
try:
repositories[rep_name] = Repository(rep_config)
except ValueError as e:
raise ValueError('Repository "%s": %s' % (rep_name, str(e)))
# creating dependency objects
self._dependencies = OrderedDict()
for i, dep_config in enumerate(config['dependencies']):
# get repository object
repository_name = dep_config[
'repository'] if 'repository' in dep_config else 'default'
if repository_name not in repositories:
raise ValueError('Repository "%s" doesn\'t exist' %
repository_name)
# create an object for dependency
try:
dependency = Dependency(dep_config,
repositories[repository_name],
packages_dir, project_dir)
except ValueError as e:
raise ValueError('Dependency #%d: %s' % (i + 1, str(e)))
key = self._get_dependency_key(dependency.group,
dependency.artifact)
if key in self._dependencies:
raise ValueError(
'Config contains two dependencies with the same name: "%s:%s"'
% (dependency.group, dependency.artifact))
self._dependencies[key] = dependency
def package_exists(self, group: str, artifact: str, version: str) -> bool:
artifact_path = self._get_artifact_dir(group, artifact, version)
return file_exists(artifact_path)
def update(self,
rewrite_working_dir: bool = False,
output: AbstractOutputWriter = None):
"""Downloads the package and updates the package's working directory."""
if not output:
output = NullOutputWriter()
package_info = self.download(output)
if not package_info or not self.working_dir:
return
# copy files to a working directory
with output.indent():
output.write('Copying files to the working directory "%s"...' %
self.working_dir)
with output.indent():
# create a working directory if it doesn't exist
working_dir = os.path.join(self.project_dir, self.working_dir)
os.makedirs(working_dir, exist_ok=True)
# get package data directory
env = Dependency.ENV_LOCAL if package_info.local else Dependency.ENV_PRODUCTION
data_dir = self.get_artifact_data_dir(env)
if self.files:
# copy only specified files if they don't exist in a target directory
for filename in self.files:
if filename in package_info.files:
src_path = os.path.join(data_dir, filename)
dst_path = os.path.join(working_dir, filename)
# adding to working directory only files which don't exist
if not file_exists(dst_path):
copy_file(src_path, dst_path)
output.write('[+] "%s": file copied' %
filename)
elif rewrite_working_dir:
copy_file(src_path, dst_path)
output.write('[+] "%s": file rewritten' %
filename)
else:
output.write('[-] "%s": file already exists' %
filename)
else:
output.write(
'[-] "%s": file doesn\'t exist in the package'
% filename)
else:
# copy all files only if a working directory is empty
if is_dir_empty(working_dir):
copy_dir(data_dir, working_dir)
output.write('[+] files copied to the "%s" directory' %
self.working_dir)
elif rewrite_working_dir:
rmtree(working_dir)
copy_dir(data_dir, working_dir)
output.write('[+] directory "%s" was rewritten' %
self.working_dir)
else:
output.write(
'[-] files not changed: directory "%s" is not empty'
% self.working_dir)
def get_path(self, file_path: str = None):
"""Returns a path to the package directory
or to a particular file from the package.
If you are getting a path to a package directory (without specifying "file_path"):
- if a working directory for the dependency is not specified, the absolute path
for the central package directory will be returned
- if the dependency configuration specifies a working directory and that
directory exists and is not empty, the method will return a path to that working directory
- if a working directory is specified, but is empty, the absolute path for the
central package directory will be returned
If you are getting a path to a file within the package ("file_path" is specified):
- if a working directory for the dependency is not specified, the absolute path
for the central directory will be returned
- if the dependency configuration specifies only a working directory (without a files list)
and the file exists in the working directory, the method will return that path
- if the file doesn't exist in the working directory, the method will try to get
the absolute path to the file from the central directory
- if the dependency configuration also specifies a files list, a file list must
contain the requested file path, otherwise the exception will be raised
:param file_path: get a path to a particular file within the package
:return: str
"""
# convert relative path from windows format to linux one
# because only linux format is accepted for file paths in "files"
if file_path:
file_path = convert_path_w2u(file_path)
# return a working directory if "file_path" is not specified and the directory is not empty
if self.working_dir and not file_path and not self.files:
working_dir = os.path.normpath(
os.path.join(self.project_dir, self.working_dir))
if not is_dir_empty(working_dir):
return working_dir
# raise an error if "file_path" specified, but doesn't exist in the list of working files
if self.working_dir and file_path and self.files and (
file_path not in self.files):
raise ValueError('File "%s" is not a part of the package "%s:%s"' %
(file_path, self.group, self.artifact))
# return file path from a working directory if "file_path" is specified and the file exists
if self.working_dir and file_path:
res_path = os.path.normpath(
os.path.join(self.project_dir, self.working_dir, file_path))
if file_exists(res_path):
return res_path
# otherwise return absolute path
package_info = self.get_package_info()
if not package_info:
raise ValueError('Package "%s:%s:%s" is not installed' %
(self.group, self.artifact, self.version))
# check that the file exists in the package
if file_path and (file_path not in package_info.files):
raise FileNotFoundError(
'File "%s" doesn\'t exist in the package "%s:%s:%s"' %
(file_path, self.group, self.artifact, self.version))
# get package data directory
env = Dependency.ENV_LOCAL if package_info.local else Dependency.ENV_PRODUCTION
data_dir = self.get_artifact_data_dir(env)
# package path or file path
res_path = os.path.normpath(os.path.join(
data_dir, file_path)) if file_path else data_dir
return res_path
def test_publish_and_update(self):
dep_without_working_dir = self._get_dependency({
'group': 'group1.subgroup1',
'artifact': 'artifact1',
'version': '1.1',
})
dep_to_publish = self._get_dependency({
'group': 'group1.subgroup1',
'artifact': 'artifact1',
'version': '1.1',
'workingDir': 'working_dir1',
})
dep_to_update = self._get_dependency({
'group':
'group1.subgroup1',
'artifact':
'artifact1',
'version':
'1.1',
'workingDir':
'working_dir_update',
})
# dependency paths
group_dir = os.path.join(self.PACKAGES_DIR, self.REPOSITORY_TYPE,
self.REPOSITORY_ROOT, 'group1', 'subgroup1')
installation_dir = os.path.join(group_dir, '.artifacts',
'artifact1-1.1')
installation_data_dir = os.path.join(installation_dir, 'data')
local_installation_dir = os.path.join(group_dir, '.local-artifacts',
'artifact1-1.1')
local_installation_data_dir = os.path.join(local_installation_dir,
'data')
working_dir = os.path.join(self.PROJECT_DIR, 'working_dir1')
working_dir_update = os.path.join(self.PROJECT_DIR,
'working_dir_update')
# path to local repository (using TestDriver)
rep_root_dir = os.path.join(self.REPOSITORY_DIR, self.REPOSITORY_ROOT)
rep_artifact_dir = os.path.join(rep_root_dir, 'group1', 'subgroup1',
'artifact1-1.1')
""" PREPARE THE TEST """
# clear installed package
if dir_exists(installation_dir):
rmtree(installation_dir)
# clear locally installed package
if dir_exists(local_installation_dir):
rmtree(local_installation_dir)
# clear package in the repository
if dir_exists(rep_root_dir):
rmtree(rep_root_dir)
# clear "update" working directory
if dir_exists(working_dir_update):
rmtree(working_dir_update)
# check that the right paths are returned
self.assertEqual(dep_to_publish.get_path(), working_dir)
with self.assertRaises(ValueError):
dep_without_working_dir.get_path()
with self.assertRaises(ValueError):
dep_to_update.get_path()
""" PUBLISHING LOCALLY """
# publish repository locally
dep_to_publish.publish(local=True)
self.assertEqual(dep_without_working_dir.get_path(),
local_installation_data_dir)
# should return False because the package already published locally
res = dep_to_publish.publish(local=True)
self.assertFalse(res)
# rewrite locally published package
dep_to_publish.publish(local=True, rewrite_local=True)
self.assertEqual(dep_without_working_dir.get_path(),
local_installation_data_dir)
# remove locally installed package
rmtree(local_installation_dir)
with self.assertRaises(ValueError):
dep_without_working_dir.get_path()
""" PUBLISHING TO THE TEST REPOSITORY """
# publish repository
dep_to_publish.publish()
files_to_check = [
'info.json',
os.path.join('data', 'file1.txt'),
os.path.join('data', 'subdir1', 'file1.txt'),
]
for file_path in files_to_check:
self.assertTrue(
file_exists(os.path.join(rep_artifact_dir, file_path)),
'File "%s" doesn\'t exist in the repository' % file_path)
self.assertEqual(dep_without_working_dir.get_path(),
installation_data_dir)
# should return False because the package already exists in the repository
res = dep_to_publish.publish()
self.assertFalse(res)
# check that the right paths are returned
self.assertEqual(dep_to_publish.get_path(), working_dir)
self.assertEqual(dep_to_update.get_path(), installation_data_dir)
# remove the installed package
rmtree(installation_dir)
with self.assertRaises(ValueError):
dep_to_update.get_path()
""" DOWNLOAD THE PACKAGE """
# download the package
dep_to_update.download()
self.assertEqual(dep_to_update.get_path(), installation_data_dir)
# remove the installed package again
rmtree(installation_dir)
with self.assertRaises(ValueError):
dep_to_update.get_path()
""" UPDATE THE DEPENDENCY """
# update the dependency (working directory should be created)
dep_to_update.update()
self.assertEqual(dep_to_update.get_path(), working_dir_update)
# remove the package after test
rmtree(installation_dir)
rmtree(rep_root_dir)
rmtree(working_dir_update)