def test_change_mask_for_routine(self):
default_path_permissions = ['777']
is_linux = sys.platform.startswith('linux')
if is_linux:
default_path_permissions = ['775', '755']
all_permissions = '777'
path = os.path.join(self.tmp_dir, 'test_permission')
shared_path = True
with change_mask_for_routine(shared_path):
ensure_path_exists(path)
st_mode = oct(os.stat(path).st_mode)[-3:]
if is_linux:
self.assertNotIn(st_mode, default_path_permissions)
self.assertEqual(st_mode, all_permissions)
shutil.rmtree(path)
ensure_path_exists(path)
st_mode = oct(os.stat(path).st_mode)[-3:]
self.assertIn(st_mode, default_path_permissions)
shutil.rmtree(path)
shared_path = False
with change_mask_for_routine(shared_path):
ensure_path_exists(path)
st_mode = oct(os.stat(path).st_mode)[-3:]
self.assertIn(st_mode, default_path_permissions)
if is_linux:
self.assertNotEqual(st_mode, all_permissions)
shutil.rmtree(path)
def create_hard_links_in_cache(self, cache_path, index_path,
is_shared_cache, mutability, path, spec):
mf = os.path.join(index_path, 'metadata', spec, MANIFEST_FILE)
with change_mask_for_routine(is_shared_cache):
if mutability in [
Mutability.STRICT.value, Mutability.FLEXIBLE.value
]:
cache = Cache(cache_path, path, mf)
cache.update()
def add(self, spec, file_path, bump_version=False, run_fsck=False):
repo_type = self.__repo_type
is_shared_objects = 'objects_path' in self.__config[repo_type]
is_shared_cache = 'cache_path' in self.__config[repo_type]
if not validate_config_spec_hash(self.__config):
log.error(
'.ml-git/config.yaml invalid. It should look something like this:\n%s'
% get_yaml_str(
get_sample_config_spec('somebucket', 'someprofile',
'someregion')),
class_name=REPOSITORY_CLASS_NAME)
return None
path, file = None, None
try:
refs_path = get_refs_path(self.__config, repo_type)
index_path = get_index_path(self.__config, repo_type)
metadata_path = get_metadata_path(self.__config, repo_type)
cache_path = get_cache_path(self.__config, repo_type)
objects_path = get_objects_path(self.__config, repo_type)
repo = LocalRepository(self.__config, objects_path, repo_type)
mutability, check_mutability = repo.get_mutability_from_spec(
spec, repo_type)
sampling_flag = os.path.exists(
os.path.join(index_path, 'metadata', spec, 'sampling'))
if sampling_flag:
log.error(
'You cannot add new data to an entity that is based on a checkout with the --sampling option.',
class_name=REPOSITORY_CLASS_NAME)
return
if not mutability:
return
if not check_mutability:
log.error('Spec mutability cannot be changed.',
class_name=REPOSITORY_CLASS_NAME)
return
if not self._has_new_data(repo, spec):
return None
ref = Refs(refs_path, spec, repo_type)
tag, sha = ref.branch()
categories_path = get_path_with_categories(tag)
path, file = search_spec_file(self.__repo_type, spec,
categories_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
if path is None:
return
spec_path = os.path.join(path, file)
if not self._is_spec_valid(spec_path):
return None
# Check tag before anything to avoid creating unstable state
log.debug('Repository: check if tag already exists',
class_name=REPOSITORY_CLASS_NAME)
m = Metadata(spec, metadata_path, self.__config, repo_type)
if not m.check_exists():
log.error('The %s has not been initialized' % self.__repo_type,
class_name=REPOSITORY_CLASS_NAME)
return
try:
m.update()
except Exception:
pass
# get version of current manifest file
manifest = self._get_current_manifest_file(m, tag)
try:
# adds chunks to ml-git Index
log.info('%s adding path [%s] to ml-git index' % (repo_type, path),
class_name=REPOSITORY_CLASS_NAME)
with change_mask_for_routine(is_shared_objects):
idx = MultihashIndex(spec, index_path, objects_path,
mutability, cache_path)
idx.add(path, manifest, file_path)
# create hard links in ml-git Cache
self.create_hard_links_in_cache(cache_path, index_path,
is_shared_cache, mutability, path,
spec)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return None
if bump_version and not increment_version_in_spec(
spec_path, self.__repo_type):
return None
idx.add_metadata(path, file)
self._check_corrupted_files(spec, repo)
# Run file check
if run_fsck:
self.fsck()