def create_generation(self):
self._load_data()
self._require_previous_generation_is_finished()
new_generation = GAGeneration()
new_metadata = new_generation.get_file_metadata()
new_metadata.set_blob_store(self._get_blob_store())
if self._generations:
latest = self._generations.get_latest()
new_dict = copy.deepcopy(latest.as_dict())
new_generation.set_from_dict(new_dict)
latest_metadata = latest.get_file_metadata()
new_metadata.set_root_object_id(
latest_metadata.get_root_object_id())
else:
new_metadata.set_root_object_id(None)
new_generation.set_number(self._new_generation_number())
new_generation.set_key(
obnamlib.repo_key_name(obnamlib.REPO_GENERATION_STARTED),
int(self._current_time()))
new_generation.set_key(
obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED), None)
self._generations.append(new_generation)
return obnamlib.GenerationId(self._client_name,
new_generation.get_number())
def create_generation(self):
self._load_data()
self._require_previous_generation_is_finished()
new_generation = GAGeneration()
new_metadata = new_generation.get_file_metadata()
new_metadata.set_blob_store(self._get_blob_store())
if self._generations:
latest = self._generations.get_latest()
new_dict = copy.deepcopy(latest.as_dict())
new_generation.set_from_dict(new_dict)
latest_metadata = latest.get_file_metadata()
new_metadata.set_root_object_id(
latest_metadata.get_root_object_id())
else:
new_metadata.set_root_object_id(None)
new_generation.set_number(self._new_generation_number())
new_generation.set_key(
obnamlib.repo_key_name(obnamlib.REPO_GENERATION_STARTED),
int(self._current_time()))
new_generation.set_key(
obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED),
None)
self._generations.append(new_generation)
return obnamlib.GenerationId(
self._client_name, new_generation.get_number())
def _require_previous_generation_is_finished(self):
if self._generations:
latest = self._generations.get_latest()
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if latest.get_key(key_name) is None:
raise obnamlib.RepositoryClientGenerationUnfinished(
client_name=self._client_name)
def set_client_key(self, client_name, key, value):
if key not in self.get_allowed_client_keys():
raise obnamlib.RepositoryClientKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
self._client_list[client_name].set_key(key, value)
def _finish_current_generation_if_any(self):
generations = self._data.get('generations', [])
if generations:
keys = generations[-1]['keys']
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if keys[key_name] is None:
keys[key_name] = int(self._current_time())
def dump_generation_keys(self, repo, gen_id):
return dict(
(obnamlib.repo_key_name(key),
repo.get_generation_key(gen_id, key))
for key in repo.get_allowed_generation_keys()
if key != obnamlib.REPO_GENERATION_TEST_KEY
)
def dump_file_keys(self, repo, gen_id, filename):
return dict(
(obnamlib.repo_key_name(key),
repo.get_file_key(gen_id, filename, key))
for key in repo.get_allowed_file_keys()
if key != obnamlib.REPO_FILE_TEST_KEY
)
def dump_client_keys(self, repo, client_name):
return dict(
(obnamlib.repo_key_name(key),
repo.get_client_key(client_name, key))
for key in repo.get_allowed_client_keys()
if key != obnamlib.REPO_CLIENT_TEST_KEY
)
def _finish_current_generation_if_any(self):
generations = self._data.get('generations', [])
if generations:
keys = generations[-1]['keys']
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if keys[key_name] is None:
keys[key_name] = int(self._current_time())
def set_generation_key(
self, generation_id, key, value): # pragma: no cover
# FIXME: This is not working for generations other than the currently
# started one. There should at least be an assert about it.
client_name, gen_number = self._unpack_gen_id(generation_id)
self._require_client_lock(client_name)
client = self._open_client(client_name)
if key == obnamlib.REPO_GENERATION_STARTED:
client.set_generation_started(value)
elif key == obnamlib.REPO_GENERATION_ENDED:
client.set_generation_ended(value)
elif key == obnamlib.REPO_GENERATION_IS_CHECKPOINT:
client.set_current_generation_is_checkpoint(value)
elif key == obnamlib.REPO_GENERATION_FILE_COUNT:
client.set_generation_file_count(gen_number, value)
elif key == obnamlib.REPO_GENERATION_TOTAL_DATA:
client.set_generation_total_data(gen_number, value)
elif key == obnamlib.REPO_GENERATION_TEST_KEY:
client.set_generation_test_data(value)
else:
raise obnamlib.RepositoryGenerationKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def _require_previous_generation_is_finished(self):
if self._generations:
latest = self._generations.get_latest()
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if latest.get_key(key_name) is None:
raise obnamlib.RepositoryClientGenerationUnfinished(
client_name=self._client_name)
def dump_generation_keys(self, repo, gen_id):
return dict(
(obnamlib.repo_key_name(key),
repo.get_generation_key(gen_id, key))
for key in repo.get_allowed_generation_keys()
if key != obnamlib.REPO_GENERATION_TEST_KEY
)
def dump_client_keys(self, repo, client_name):
return dict(
(obnamlib.repo_key_name(key),
repo.get_client_key(client_name, key))
for key in repo.get_allowed_client_keys()
if key != obnamlib.REPO_CLIENT_TEST_KEY
)
def dump_file_keys(self, repo, gen_id, filename):
return dict(
(obnamlib.repo_key_name(key),
repo.get_file_key(gen_id, filename, key))
for key in repo.get_allowed_file_keys()
if key != obnamlib.REPO_FILE_TEST_KEY
)
def set_client_key(self, client_name, key, value):
if key not in self.get_allowed_client_keys():
raise obnamlib.RepositoryClientKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
self._client_list[client_name].set_key(key, value)
def set_generation_key(
self, generation_id, key, value): # pragma: no cover
# FIXME: This no worky for generations other than the currently
# started one. There should at least be an assert about it.
client_name, gen_number = self._unpack_gen_id(generation_id)
self._require_client_lock(client_name)
client = self._open_client(client_name)
if key == obnamlib.REPO_GENERATION_STARTED:
client.set_generation_started(value)
elif key == obnamlib.REPO_GENERATION_ENDED:
client.set_generation_ended(value)
elif key == obnamlib.REPO_GENERATION_IS_CHECKPOINT:
client.set_current_generation_is_checkpoint(value)
elif key == obnamlib.REPO_GENERATION_FILE_COUNT:
client.set_generation_file_count(gen_number, value)
elif key == obnamlib.REPO_GENERATION_TOTAL_DATA:
client.set_generation_total_data(gen_number, value)
elif key == obnamlib.REPO_GENERATION_TEST_KEY:
client.set_generation_test_data(value)
else:
raise obnamlib.RepositoryGenerationKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def _require_previous_generation_is_finished(self):
generations = self._data.get('generations', [])
if generations:
keys = generations[-1]['keys']
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if keys[key_name] is None:
raise obnamlib.RepositoryClientGenerationUnfinished(
client_name=self._client_name)
def set_generation_key(self, generation_id, key, value):
if key not in self.get_allowed_generation_keys():
raise obnamlib.RepositoryGenerationKeyNotAllowed(
client_name=generation_id.client_name,
format=self.format,
key_name=obnamlib.repo_key_name(key))
client = self._lookup_client_by_generation(generation_id)
return client.set_generation_key(generation_id.gen_number, key, value)
def set_generation_key(self, generation_id, key, value):
if key not in self.get_allowed_generation_keys():
raise obnamlib.RepositoryGenerationKeyNotAllowed(
client_name=generation_id.client_name,
format=self.format,
key_name=obnamlib.repo_key_name(key))
client = self._lookup_client_by_generation(generation_id)
return client.set_generation_key(generation_id.gen_number, key, value)
def set_file_key(self, generation_id, filename, key, value):
client = self._client_list.get_client_by_generation_id(generation_id)
if key not in self.get_allowed_file_keys():
raise obnamlib.RepositoryFileKeyNotAllowed(
format=self.format,
client_name=client.name,
key_name=obnamlib.repo_key_name(key))
client.set_file_key(generation_id, filename, key, value)
def _require_previous_generation_is_finished(self):
generations = self._data.get('generations', [])
if generations:
keys = generations[-1]['keys']
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if keys[key_name] is None:
raise obnamlib.RepositoryClientGenerationUnfinished(
client_name=self._client_name)
def set_file_key(self, generation_id, filename, key, value):
client = self._client_list.get_client_by_generation_id(generation_id)
if key not in self.get_allowed_file_keys():
raise obnamlib.RepositoryFileKeyNotAllowed(
format=self.format,
client_name=client.name,
key_name=obnamlib.repo_key_name(key))
client.set_file_key(generation_id, filename, key, value)
def get_file_key(self, generation_id, filename, key):
if key not in self.get_allowed_file_keys():
raise obnamlib.RepositoryFileKeyNotAllowed(
client_name=generation_id.client_name,
key_name=obnamlib.repo_key_name(key),
format=self.format)
client = self._lookup_client_by_generation(generation_id)
return client.get_file_key(generation_id.gen_number, filename, key)
def get_file_key(self, generation_id, filename, key):
if key not in self.get_allowed_file_keys():
raise obnamlib.RepositoryFileKeyNotAllowed(
client_name=generation_id.client_name,
key_name=obnamlib.repo_key_name(key),
format=self.format)
client = self._lookup_client_by_generation(generation_id)
return client.get_file_key(generation_id.gen_number, filename, key)
def get_generation_key(self, gen_number, key):
generation = self._lookup_generation_by_gen_number(gen_number)
key_name = obnamlib.repo_key_name(key)
if key in obnamlib.REPO_GENERATION_INTEGER_KEYS:
value = generation['keys'].get(key_name, None)
if value is None:
value = 0
return int(value)
else:
return generation['keys'].get(key_name, '')
def get_generation_key(self, gen_number, key):
generation = self._lookup_generation_by_gen_number(gen_number)
key_name = obnamlib.repo_key_name(key)
if key in obnamlib.REPO_GENERATION_INTEGER_KEYS:
value = generation['keys'].get(key_name, None)
if value is None:
value = 0
return int(value)
else:
return generation['keys'].get(key_name, '')
def get_generation_key(self, gen_number, key):
self._load_data()
generation = self._lookup_generation_by_gen_number(gen_number)
key_name = obnamlib.repo_key_name(key)
if key in obnamlib.REPO_GENERATION_INTEGER_KEYS:
value = generation.get_key(key_name)
if value is None:
value = 0
return int(value)
else:
return generation.get_key(key_name, default='')
def get_generation_key(self, gen_number, key):
self._load_data()
generation = self._lookup_generation_by_gen_number(gen_number)
key_name = obnamlib.repo_key_name(key)
if key in obnamlib.REPO_GENERATION_INTEGER_KEYS:
value = generation.get_key(key_name)
if value is None:
value = 0
return int(value)
else:
return generation.get_key(key_name, default='')
def get_file_key(self, gen_number, filename, key):
self._require_file_exists(gen_number, filename)
generation = self._lookup_generation_by_gen_number(gen_number)
files = generation['files']
key_name = obnamlib.repo_key_name(key)
if key in obnamlib.REPO_FILE_INTEGER_KEYS:
default = 0
else:
default = ''
if key_name not in files[filename]['keys']:
return default
return files[filename]['keys'][key_name] or default
def get_file_key(self, gen_number, filename, key):
self._require_file_exists(gen_number, filename)
generation = self._lookup_generation_by_gen_number(gen_number)
files = generation['files']
key_name = obnamlib.repo_key_name(key)
if key in obnamlib.REPO_FILE_INTEGER_KEYS:
default = 0
else:
default = ''
if key_name not in files[filename]['keys']:
return default
return files[filename]['keys'][key_name] or default
def create_generation(self):
self._require_lock()
self._require_previous_generation_is_finished()
generations = self._data.get('generations', [])
if generations:
previous = copy.deepcopy(generations[-1])
else:
previous = {
'keys': {},
'files': {},
}
new_generation = dict(previous)
new_generation['id'] = self._new_generation_number()
keys = new_generation['keys']
keys[obnamlib.repo_key_name(obnamlib.REPO_GENERATION_STARTED)] = \
int(self._current_time())
keys[obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)] = None
self._data['generations'] = generations + [new_generation]
return GenerationId(self._client_name, new_generation['id'])
def create_generation(self):
self._require_lock()
self._require_previous_generation_is_finished()
generations = self._data.get('generations', [])
if generations:
previous = copy.deepcopy(generations[-1])
else:
previous = {
'keys': {},
'files': {},
}
new_generation = dict(previous)
new_generation['id'] = self._new_generation_number()
keys = new_generation['keys']
keys[obnamlib.repo_key_name(obnamlib.REPO_GENERATION_STARTED)] = \
int(self._current_time())
keys[obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)] = None
self._data['generations'] = generations + [new_generation]
return GenerationId(self._client_name, new_generation['id'])
def set_file_key(self, generation_id, filename, key, value):
client_name, gen_number = self._unpack_gen_id(generation_id)
self._require_client_lock(client_name)
cache_key = self._get_file_key_cache_key(generation_id, filename)
if cache_key not in self._file_key_cache:
self._flush_file_key_cache()
self._require_existing_file(generation_id, filename)
self._cache_file_keys_from_storage(generation_id, filename)
dirty, metadata = self._file_key_cache[cache_key]
if key in self._file_keys:
setattr(metadata, self._file_keys[key], value)
self._file_key_cache[cache_key] = (True, metadata)
else:
raise obnamlib.RepositoryFileKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def set_file_key(self, generation_id, filename, key, value):
client_name, _ = self._unpack_gen_id(generation_id)
self._require_client_lock(client_name)
cache_key = self._get_file_key_cache_key(generation_id, filename)
if cache_key not in self._file_key_cache:
self._flush_file_key_cache()
self._require_existing_file(generation_id, filename)
self._cache_file_keys_from_storage(generation_id, filename)
_, metadata = self._file_key_cache[cache_key]
if key in self._file_keys:
setattr(metadata, self._file_keys[key], value)
self._file_key_cache[cache_key] = (True, metadata)
else:
raise obnamlib.RepositoryFileKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def get_file_key(self, generation_id, filename, key):
cache_key = self._get_file_key_cache_key(generation_id, filename)
if cache_key not in self._file_key_cache:
self._flush_file_key_cache()
self._require_existing_file(generation_id, filename)
self._cache_file_keys_from_storage(generation_id, filename)
_, metadata = self._file_key_cache[cache_key]
if key in self._file_keys:
value = getattr(metadata, self._file_keys[key])
if key in obnamlib.REPO_FILE_INTEGER_KEYS:
return value or 0
else:
return value or ''
else:
client_name, _ = self._unpack_gen_id(generation_id)
raise obnamlib.RepositoryFileKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def get_file_key(self, generation_id, filename, key):
cache_key = self._get_file_key_cache_key(generation_id, filename)
if cache_key not in self._file_key_cache:
self._flush_file_key_cache()
self._require_existing_file(generation_id, filename)
self._cache_file_keys_from_storage(generation_id, filename)
dirty, metadata = self._file_key_cache[cache_key]
if key in self._file_keys:
value = getattr(metadata, self._file_keys[key])
if key in obnamlib.REPO_FILE_INTEGER_KEYS:
return value or 0
else:
return value or ''
else:
client_name, gen_number = self._unpack_gen_id(generation_id)
raise obnamlib.RepositoryFileKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def get_generation_key(self, generation_id, key): # pragma: no cover
client_name, gen_number = self._unpack_gen_id(generation_id)
client = self._open_client(client_name)
if key == obnamlib.REPO_GENERATION_STARTED:
started, ended = client.get_generation_times(gen_number)
return started or 0
elif key == obnamlib.REPO_GENERATION_ENDED:
started, ended = client.get_generation_times(gen_number)
return ended or 0
elif key == obnamlib.REPO_GENERATION_IS_CHECKPOINT:
return client.get_is_checkpoint(gen_number) or 0
elif key == obnamlib.REPO_GENERATION_FILE_COUNT:
return client.get_generation_file_count(gen_number) or 0
elif key == obnamlib.REPO_GENERATION_TOTAL_DATA:
return client.get_generation_data(gen_number) or 0
elif key == obnamlib.REPO_GENERATION_TEST_KEY:
return client.get_generation_test_data() or ''
else:
raise obnamlib.RepositoryGenerationKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def get_generation_key(self, generation_id, key): # pragma: no cover
client_name, gen_number = self._unpack_gen_id(generation_id)
client = self._open_client(client_name)
if key == obnamlib.REPO_GENERATION_STARTED:
started, ended = client.get_generation_times(gen_number)
return started or 0
elif key == obnamlib.REPO_GENERATION_ENDED:
started, ended = client.get_generation_times(gen_number)
return ended or 0
elif key == obnamlib.REPO_GENERATION_IS_CHECKPOINT:
return client.get_is_checkpoint(gen_number) or 0
elif key == obnamlib.REPO_GENERATION_FILE_COUNT:
return client.get_generation_file_count(gen_number) or 0
elif key == obnamlib.REPO_GENERATION_TOTAL_DATA:
return client.get_generation_data(gen_number) or 0
elif key == obnamlib.REPO_GENERATION_TEST_KEY:
return client.get_generation_test_data() or ''
else:
raise obnamlib.RepositoryGenerationKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def get_checksum_algorithm_name(wanted_key):
for name, key, _ in _algorithm_list:
if key == wanted_key:
return name
raise UnknownChecksumAlgorithm(
algorithm=obnamlib.repo_key_name(wanted_key))
def set_generation_key(self, gen_number, key, value):
self._require_lock()
generation = self._lookup_generation_by_gen_number(gen_number)
generation['keys'][obnamlib.repo_key_name(key)] = value
def set_generation_key(self, gen_number, key, value):
self._load_data()
generation = self._lookup_generation_by_gen_number(gen_number)
generation.set_key(obnamlib.repo_key_name(key), value)
def set_file_key(self, basename, key, value):
self._require_mutable()
key_name = obnamlib.repo_key_name(key)
self._dict['metadata'][basename][key_name] = value
def get_client_key(self, client_name, key): # pragma: no cover
raise obnamlib.RepositoryClientKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def get_file_key(self, basename, key):
key_name = obnamlib.repo_key_name(key)
return self._dict['metadata'][basename].get(key_name)
def set_generation_key(self, gen_number, key, value):
self._require_lock()
generation = self._lookup_generation_by_gen_number(gen_number)
generation['keys'][obnamlib.repo_key_name(key)] = value
def _finish_current_generation_if_any(self):
if self._generations:
latest = self._generations.get_latest()
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if latest.get_key(key_name) is None:
latest.set_key(key_name, int(self._current_time()))
def set_generation_key(self, gen_number, key, value):
self._load_data()
generation = self._lookup_generation_by_gen_number(gen_number)
generation.set_key(obnamlib.repo_key_name(key), value)
def set_file_key(self, gen_number, filename, key, value):
self._require_file_exists(gen_number, filename)
generation = self._lookup_generation_by_gen_number(gen_number)
files = generation['files']
key_name = obnamlib.repo_key_name(key)
files[filename]['keys'][key_name] = value
def set_client_key(self, client_name, key, value):
raise obnamlib.RepositoryClientKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def get_client_key(self, client_name, key): # pragma: no cover
raise obnamlib.RepositoryClientKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def set_file_key(self, gen_number, filename, key, value):
self._require_file_exists(gen_number, filename)
generation = self._lookup_generation_by_gen_number(gen_number)
files = generation['files']
key_name = obnamlib.repo_key_name(key)
files[filename]['keys'][key_name] = value
def set_client_key(self, client_name, key, value):
raise obnamlib.RepositoryClientKeyNotAllowed(
format=self.format,
client_name=client_name,
key_name=obnamlib.repo_key_name(key))
def _finish_current_generation_if_any(self):
if self._generations:
latest = self._generations.get_latest()
key_name = obnamlib.repo_key_name(obnamlib.REPO_GENERATION_ENDED)
if latest.get_key(key_name) is None:
latest.set_key(key_name, int(self._current_time()))
