def do(self):
tracing.trace("CheckForest: checking forest %s" % self.name)
for tree in self.fsck.forest.trees:
self.fsck.count(tree.root.id)
root_node = self.get_node(tree.root.id)
tracing.trace('root_node.id=%s' % root_node.id)
yield CheckIndexNode(self.fsck, root_node)
def put_chunk_only(self, data):
'''Put chunk of data into repository.
If the same data is already in the repository, it will be put there
a second time. It is the caller's responsibility to check
that the data is not already in the repository.
Return the unique identifier of the new chunk.
'''
def random_chunkid():
return random.randint(0, obnamlib.MAX_ID)
self.require_started_generation()
if self.prev_chunkid is None:
self.prev_chunkid = random_chunkid()
while True:
chunkid = (self.prev_chunkid + 1) % obnamlib.MAX_ID
filename = self._chunk_filename(chunkid)
try:
self.fs.write_file(filename, data)
except OSError, e: # pragma: no cover
if e.errno == errno.EEXIST:
self.prev_chunkid = random_chunkid()
continue
raise
else:
tracing.trace('chunkid=%s', chunkid)
break
def remove(self, filename):
tracing.trace('filename=%s', filename)
file_id = self.get_file_id(self.tree, filename)
genid = self.get_generation_id(self.tree)
# Remove any children.
minkey = self.fskey(file_id, self.DIR_CONTENTS, 0)
maxkey = self.fskey(file_id, self.DIR_CONTENTS, obnamlib.MAX_ID)
for key, basename in self.tree.lookup_range(minkey, maxkey):
self.remove(os.path.join(filename, basename))
# Remove chunk refs.
for chunkid in self.get_file_chunks(genid, filename):
key = self.chunk_key(chunkid, file_id)
self.tree.remove_range(key, key)
# Remove this file's metadata.
minkey = self.fskey(file_id, 0, 0)
maxkey = self.fskey(file_id, self.TYPE_MAX, self.SUBKEY_MAX)
self.tree.remove_range(minkey, maxkey)
# Remove filename.
default_file_id = self.default_file_id(filename)
key = self.fskey(default_file_id, self.FILE_NAME, file_id)
self.tree.remove_range(key, key)
# Also remove from parent's contents.
parent = os.path.dirname(filename)
if parent != filename: # root dir is its own parent
parent_id = self.set_file_id(parent)
key = self.fskey(parent_id, self.DIR_CONTENTS, file_id)
# The range removal will work even if the key does not exist.
self.tree.remove_range(key, key)
def lock_client(self, client_name):
'''Lock a client for exclusive write access.
Raise obnamlib.LockFail if locking fails. Lock will be released
by commit_client() or unlock_client().
'''
tracing.trace('client_name=%s', client_name)
self.require_no_client_lock()
self.require_no_shared_lock()
self.check_format_version()
client_id = self.clientlist.get_client_id(client_name)
if client_id is None:
raise LockFail('client %s does not exist' % client_name)
client_dir = self.client_dir(client_id)
if not self.fs.exists(client_dir):
self.fs.mkdir(client_dir)
self.hooks.call('repository-toplevel-init', self, client_dir)
self.lockmgr.lock([client_dir])
self.got_client_lock = True
self.current_client = client_name
self.current_client_id = client_id
self.added_generations = []
self.removed_generations = []
self.client = obnamlib.ClientMetadataTree(self.fs, client_dir,
self.node_size,
self.upload_queue_size,
self.lru_size, self)
self.client.init_forest()
def unlock_shared(self):
'''Unlock currently locked shared B-trees.'''
tracing.trace('unlocking shared')
self.require_shared_lock()
self.lockmgr.unlock(self.shared_dirs)
self.got_shared_lock = False
self._open_shared()
def _write_format_version(self, version):
'''Write the desired format version to the repository.'''
tracing.trace('write format version')
if not self.fs.exists('metadata'):
self.fs.mkdir('metadata')
self.fs.overwrite_file('metadata/format', '%s\n' % version,
runfilters=False)
def get_node(self, node_id):
tracing.trace('node_id=%s' % node_id)
try:
return self.fsck.forest.node_store.get_node(node_id)
except larch.NodeMissing:
self.error('forest %s: node %s is missing' %
(self.fsck.forest_name, node_id))
def _raw_unlock_client(self, client_name):
tracing.trace('client_name=%s', client_name)
client_id = self._get_client_id(client_name)
client_dir = self._get_client_dir(client_id)
self._lockmgr.unlock([client_dir])
if client_name in self._open_client_infos:
del self._open_client_infos[client_name]
def get_repository_object(self, create=False, repofs=None):
'''Return an implementation of obnamlib.RepositoryInterface.'''
logging.info('Opening repository: %s', self.settings['repository'])
tracing.trace('create=%s', create)
tracing.trace('repofs=%s', repofs)
repopath = self.settings['repository']
if repofs is None:
repofs = self.fsf.new(repopath, create=create)
if self.settings['crash-limit'] > 0:
repofs.crash_limit = self.settings['crash-limit']
repofs.connect()
else:
repofs.reinit(repopath)
kwargs = {
'lock_timeout': self.settings['lock-timeout'],
'node_size': self.settings['node-size'],
'upload_queue_size': self.settings['upload-queue-size'],
'lru_size': self.settings['lru-size'],
'idpath_depth': self.settings['idpath-depth'],
'idpath_bits': self.settings['idpath-bits'],
'idpath_skip': self.settings['idpath-skip'],
'hooks': self.hooks,
'current_time': self.time,
'chunk_size': self.settings['chunk-size'],
}
if create:
return self.repo_factory.create_repo(
repofs, self.get_default_repository_class(), **kwargs)
else:
return self.repo_factory.open_existing_repo(repofs, **kwargs)
def _raw_lock_client(self, client_name):
tracing.trace('client_name=%s', client_name)
if self.got_client_lock(client_name):
raise obnamlib.RepositoryClientLockingFailed(
client_name=client_name)
client_id = self._get_client_id(client_name)
if client_id is None: # pragma: no cover
raise obnamlib.RepositoryClientDoesNotExist(
client_name=client_name)
# Create and initialise the client's own directory, if needed.
client_dir = self._get_client_dir(client_id)
if not self._fs.exists(client_dir):
self._fs.mkdir(client_dir)
self.hooks.call('repository-toplevel-init', self, client_dir)
# Actually lock the directory.
self._lockmgr.lock([client_dir])
# Remember that we have the lock.
self._open_client(client_name) # Ensure client is open
open_client_info = self._open_client_infos[client_name]
open_client_info.locked = True
def find_files(self, root):
'''Find all files and directories that need to be backed up.
This is a generator. It yields (pathname, metadata) pairs.
The caller should not recurse through directories, just backup
the directory itself (name, metadata, file list).
'''
for pathname, st in self.fs.scan_tree(root, ok=self.can_be_backed_up):
tracing.trace('considering %s' % pathname)
try:
metadata = obnamlib.read_metadata(self.fs, pathname, st=st)
self.progress.update_progress_with_file(pathname, metadata)
if self.needs_backup(pathname, metadata):
yield pathname, metadata
else:
self.progress.update_progress_with_scanned(
metadata.st_size)
except GeneratorExit:
raise
except KeyboardInterrupt:
logging.error('Keyboard interrupt')
raise
except BaseException, e:
msg = 'Cannot back up %s: %s' % (pathname, str(e))
self.progress.error(msg, e)
def commit_client_list(self):
tracing.trace('committing client list')
for client_name in self._added_clients:
self.hooks.call('repository-add-client', self, client_name)
self._added_clients = []
self._client_list.commit()
self._raw_unlock_client_list()
def _write_to_tempfile(self, pathname, contents):
path = self.join(pathname)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
tracing.trace('os.makedirs(%s)' % dirname)
try:
os.makedirs(dirname, mode=obnamlib.NEW_DIR_MODE)
except OSError as e: # pragma: no cover
# This avoids a race condition: another Obnam process
# may have created the directory between our check and
# creation attempt. If so, we ignore it. As long as
# the directory exists, all's good.
if e.errno != errno.EEXIST:
raise
fd, tempname = tempfile.mkstemp(dir=dirname)
os.fchmod(fd, obnamlib.NEW_FILE_MODE)
os.close(fd)
f = self.open(tempname, 'wb')
pos = 0
while pos < len(contents):
chunk = contents[pos:pos + self.chunk_size]
f.write(chunk)
pos += len(chunk)
self.bytes_written += len(chunk)
f.close()
return tempname
def backup_dir_contents(self, root):
'''Back up the list of files in a directory.'''
tracing.trace('backup_dir: %s', root)
if self.pretend:
return
new_basenames = self.fs.listdir(root)
new_pathnames = [os.path.join(root, x) for x in new_basenames]
if self.repo.file_exists(self.new_generation, root):
old_pathnames = self.repo.get_file_children(
self.new_generation, root)
else:
old_pathnames = []
for old in old_pathnames:
if old not in new_pathnames:
self.repo.remove_file(self.new_generation, old)
else:
try:
st = self.fs.lstat(old)
except OSError:
pass
else:
if not self.can_be_backed_up(old, st):
self.repo.remove_file(self.new_generation, old)
def chmod_symlink(self, pathname, mode): # pragma: no cover
tracing.trace('chmod_symlink %s %o', pathname, mode)
if self.got_lchmod:
lchmod = getattr(os, 'lchmod')
lchmod(self.join(pathname), mode)
else:
self.lstat(pathname)
def backup_dir_contents(self, root):
'''Back up the list of files in a directory.'''
tracing.trace('backup_dir: %s', root)
if self.pretend:
return
new_basenames = self.fs.listdir(root)
new_pathnames = [os.path.join(root, x) for x in new_basenames]
if self.repo.file_exists(self.new_generation, root):
old_pathnames = self.repo.get_file_children(
self.new_generation, root)
else:
old_pathnames = []
for old in old_pathnames:
if old not in new_pathnames:
self.repo.remove_file(self.new_generation, old)
else:
try:
st = self.fs.lstat(old)
except OSError:
pass
else:
if not self.can_be_backed_up(old, st):
self.repo.remove_file(self.new_generation, old)
def update_student_dates(self, student_id, course_data):
"""
The following will be copied from the course data:
- Date of every lecture
- Date of every dayoffs
The following will be calculated:
- Date descriptions. This is one string containing all of the lectures and vacations, directly insertable to
emails.
:param student_id: id number of the student as a string.
:type student_id: str
:param course_data: all of the date about a course the student has applied to as stored in the MiniCRM system.
:type course_data: dict
:return: None
"""
data_to_update = {
"Datumleirasok": self._get_date_description(course_data)
}
data_to_update = copy_dates_from_course_data_to_student_data(data_to_update, course_data)
trace("DATA TO BE REPLACED:")
pretty_print(data_to_update)
self._request_handler.fetch(crmrequestfactory.set_project_data(student_id, data_to_update))
def _get_date_description(self, course_data):
date_list = []
date_list.append(course_data[COURSE_FIRST_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_SECOND_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_THIRD_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_FOURTH_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_FIFTH_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_SIXTH_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_SEVENTH_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_EIGTH_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_NINTH_OCCASION_DATE_FIELD][:10])
date_list.append(course_data[COURSE_TENTH_OCCASION_DATE_FIELD][:10])
if course_data[COURSE_FISRST_DAYOFF_FIELD] != "":
date_list.append("{} - {}".format(course_data[COURSE_FISRST_DAYOFF_FIELD][:10], "szünet"))
if course_data[COURSE_SECOND_DAYOFF_FIELD] != "":
date_list.append("{} - {}".format(course_data[COURSE_SECOND_DAYOFF_FIELD][:10], "szünet"))
if course_data[COURSE_THIRD_DAYOFF_FIELD] != "":
date_list.append("{} - {}".format(course_data[COURSE_THIRD_DAYOFF_FIELD][:10], "szünet"))
date_list.sort()
return_string = " - " + "\n - ".join(date_list)
trace("JOINED STRING:\n{}".format(return_string))
return return_string
def get_location_by_name(self, location_name):
"""
Returns the dictionary of a location for a given name.
- returns one of the locations if name is not unique
- returns None if name doesn't exist
:param location_name: Course code of a course
:type location_name: int
:return: containing all information about the given location
:rtype: dict
"""
location_list = self._request_handler.fetch(
crmrequestfactory.get_location_list_by_location_name(location_name)
)
pretty_print(location_list)
for location in location_list[RESULTS_FIELD]:
return self._request_handler.fetch(
crmrequestfactory.get_location(location))
trace("COURSE NOT FOUND: [{}]".format(location_name))
return None
def save_refcounts(self):
'''Save all modified refcounts.'''
tracing.trace('saving refcounts (len(dirty) = %s)' %
(len(self.dirty)))
if self.dirty:
dirname = os.path.join(self.node_store.dirname, self.refcountdir)
if not self.node_store.journal.exists(dirname):
self.node_store.journal.makedirs(dirname)
ids = sorted(self.dirty)
all_ids_in_memory = set(self.refcounts.keys())
for start_id in range(self._start_id(ids[0]),
self._start_id(ids[-1]) + 1,
self.per_group):
all_ids_in_group = set(
range(start_id, start_id + self.per_group))
keys = all_ids_in_group.intersection(all_ids_in_memory)
if keys:
encoded = encode_refcounts(
self.refcounts, start_id, self.per_group, keys)
filename = self._group_filename(start_id)
self.node_store.journal.overwrite_file(filename, encoded)
# We re-initialize these so that they don't grow indefinitely.
self.refcounts = dict()
self.dirty = set()
def __init__(self, allow_writes, fs, storedir):
logging.debug('Initializing Journal for %s' % storedir)
self.allow_writes = allow_writes
self.fs = fs
self.storedir = storedir
if not self.storedir.endswith(os.sep):
self.storedir += os.sep
self.newdir = os.path.join(self.storedir, 'new/')
self.deletedir = os.path.join(self.storedir, 'delete/')
self.flag_file = os.path.join(self.storedir, self.flag_basename)
self.new_flag = os.path.join(self.newdir, self.flag_basename)
self.new_flag_seen = self.fs.exists(self.new_flag)
tracing.trace('self.new_flag_seen: %s' % self.new_flag_seen)
if self.allow_writes:
if self.new_flag_seen:
logging.debug('Automatically committing remaining changes')
self.commit()
else:
logging.debug('Automatically rolling back remaining changes')
self.rollback()
else:
logging.debug('Not committing/rolling back since read-only')
self.new_files = set()
self.deleted_files = set()
def overwrite_file(self, filename, data, runfilters=True):
tracing.trace('overwriting hooked %s' % filename)
toplevel = self._get_toplevel(filename)
if runfilters:
data = self.hooks.filter_write('repository-data', data,
repo=self.repo, toplevel=toplevel)
self.fs.overwrite_file(filename, data)
def do(self):
tracing.trace("CheckForest: checking forest %s" % self.name )
for tree in self.fsck.forest.trees:
self.fsck.count(tree.root.id)
root_node = self.get_node(tree.root.id)
tracing.trace('root_node.id=%s' % root_node.id)
yield CheckIndexNode(self.fsck, root_node)
def needs_backup(self, pathname, current):
'''Does a given file need to be backed up?'''
# Directories always require backing up so that backup_dir_contents
# can remove stuff that no longer exists from them.
if current.isdir():
tracing.trace('%s is directory, so needs backup' % pathname)
return True
if self.pretend:
gens = self.repo.list_generations()
if not gens:
return True
gen = gens[-1]
else:
gen = self.repo.new_generation
tracing.trace('gen=%s' % repr(gen))
try:
old = self.repo.get_metadata(gen, pathname)
except obnamlib.Error, e:
# File does not exist in the previous generation, so it
# does need to be backed up.
tracing.trace('%s not in previous gen, so needs backup' % pathname)
tracing.trace('error: %s' % str(e))
tracing.trace(traceback.format_exc())
return True
def __init__(self,
allow_writes,
node_size,
codec,
dirname=None,
upload_max=1024,
lru_size=500,
vfs=None,
format=None):
tracing.trace('new NodeStoreDisk: %s', dirname)
assert dirname is not None
if format is not None:
tracing.trace('forcing format_base: %s', format)
self.format_base = format
larch.NodeStore.__init__(self,
allow_writes=allow_writes,
node_size=node_size,
codec=codec)
self.dirname = dirname
self.metadata_name = os.path.join(dirname, 'metadata')
self.metadata = None
self.rs = larch.RefcountStore(self)
self.cache_size = lru_size
self.cache = larch.LRUCache(self.cache_size)
self.upload_max = upload_max
self.upload_queue = larch.UploadQueue(self._really_put_node,
self.upload_max)
self.vfs = vfs if vfs != None else LocalFS()
self.journal = larch.Journal(allow_writes, self.vfs, dirname)
self.idpath = larch.IdPath(os.path.join(dirname, self.nodedir),
DIR_DEPTH, DIR_BITS, DIR_SKIP)
def __init__(self, allow_writes, fs, storedir):
logging.debug('Initializing Journal for %s' % storedir)
self.allow_writes = allow_writes
self.fs = fs
self.storedir = storedir
if not self.storedir.endswith(os.sep):
self.storedir += os.sep
self.newdir = os.path.join(self.storedir, 'new/')
self.deletedir = os.path.join(self.storedir, 'delete/')
self.flag_file = os.path.join(self.storedir, self.flag_basename)
self.new_flag = os.path.join(self.newdir, self.flag_basename)
self.new_flag_seen = self.fs.exists(self.new_flag)
tracing.trace('self.new_flag_seen: %s' % self.new_flag_seen)
if self.allow_writes:
if self.new_flag_seen:
logging.debug('Automatically committing remaining changes')
self.commit()
else:
logging.debug('Automatically rolling back remaining changes')
self.rollback()
else:
logging.debug('Not committing/rolling back since read-only')
self.new_files = set()
self.deleted_files = set()
def statfs(self):
tracing.trace('called')
client_name = self.obnam.app.settings['client-name']
total_data = sum(
self.obnam.repo.get_generation_key(
gen, obnamlib.REPO_GENERATION_TOTAL_DATA)
for gen in obnamlib.repo.get_clientgeneration_ids(client_name))
files = sum(
self.obnam.repo.get_generation_key(
gen, obnamlib.REPO_GENERATION_FILE_COUNT)
for gen in self.obnam.repo.get_client_generation_ids(client_name))
stv = fuse.StatVfs()
stv.f_bsize = 65536
stv.f_frsize = 0
stv.f_blocks = blocks/65536
stv.f_bfree = 0
stv.f_bavail = 0
stv.f_files = files
stv.f_ffree = 0
stv.f_favail = 0
stv.f_flag = 0
stv.f_namemax = 255
#raise OSError(errno.ENOSYS, 'Unimplemented')
return stv
def chmod_symlink(self, pathname, mode): # pragma: no cover
tracing.trace('chmod_symlink %s %o', pathname, mode)
if self.got_lchmod:
lchmod = getattr(os, 'lchmod')
lchmod(self.join(pathname), mode)
else:
self.lstat(pathname)
def _raw_unlock_client(self, client_name):
tracing.trace('client_name=%s', client_name)
self._require_client_lock(client_name)
open_client = self._open_clients[client_name]
self._lockmgr.unlock([open_client.client.dirname])
del self._open_clients[client_name]
def remove_chunk(self, chunk_id):
tracing.trace('chunk_id=%s', chunk_id)
filename = self._chunk_filename(chunk_id)
try:
self._fs.remove(filename)
except OSError:
raise obnamlib.RepositoryChunkDoesNotExist(str(chunk_id))
def get_repository_object(self, create=False, repofs=None):
'''Return an implementation of obnamlib.RepositoryInterface.'''
logging.info('Opening repository: %s', self.settings['repository'])
tracing.trace('create=%s', create)
tracing.trace('repofs=%s', repofs)
repopath = self.settings['repository']
if repofs is None:
repofs = self.fsf.new(repopath, create=create)
if self.settings['crash-limit'] > 0:
repofs.crash_limit = self.settings['crash-limit']
repofs.connect()
else:
repofs.reinit(repopath)
kwargs = {
'lock_timeout': self.settings['lock-timeout'],
'node_size': self.settings['node-size'],
'upload_queue_size': self.settings['upload-queue-size'],
'lru_size': self.settings['lru-size'],
'idpath_depth': self.settings['idpath-depth'],
'idpath_bits': self.settings['idpath-bits'],
'idpath_skip': self.settings['idpath-skip'],
'hooks': self.hooks,
'current_time': self.time,
}
if create:
return self.repo_factory.create_repo(repofs,
obnamlib.RepositoryFormat6,
**kwargs)
else:
return self.repo_factory.open_existing_repo(repofs, **kwargs)
def _write_to_tempfile(self, pathname, contents):
path = self.join(pathname)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
tracing.trace('os.makedirs(%s)' % dirname)
try:
os.makedirs(dirname)
except OSError as e: # pragma: no cover
# This avoids a race condition: another Obnam process
# may have created the directory between our check and
# creation attempt. If so, we ignore it. As long as
# the directory exists, all's good.
if e.errno != errno.EEXIST:
raise
fd, tempname = tempfile.mkstemp(dir=dirname)
os.close(fd)
f = self.open(tempname, 'wb')
pos = 0
while pos < len(contents):
chunk = contents[pos:pos+self.chunk_size]
f.write(chunk)
pos += len(chunk)
self.bytes_written += len(chunk)
f.close()
return tempname
def _add_or_merge(self, parent, node, merge):
assert not parent.frozen
assert node.frozen
keys = parent.keys()
key = node.first_key()
i = bisect.bisect_left(keys, key)
new_node = None
if i > 0:
new_node = merge(parent, node, i-1)
if new_node is None and i < len(keys):
new_node = merge(parent, node, i)
if new_node is None:
new_node = node
assert new_node is not None
self._put_node(new_node)
parent.add(new_node.first_key(), new_node.id)
self._increment(new_node.id)
if new_node != node: # pragma: no cover
# We made a new node, so get rid of the old one.
tracing.trace('decrementing unused node id=%s' % node.id)
self._decrement(node.id)
def _new_index(self, keys, values):
'''Create a new index node.'''
index = larch.IndexNode(self._new_id(), keys, values)
for child_id in values:
self._increment(child_id)
tracing.trace('id=%s' % index.id)
return index
def statfs(self):
tracing.trace('called')
client_name = self.obnam.app.settings['client-name']
total_data = sum(
self.obnam.repo.get_generation_key(
gen, obnamlib.REPO_GENERATION_TOTAL_DATA)
for gen in self.obnam.repo.get_client_generation_ids(client_name))
files = sum(
self.obnam.repo.get_generation_key(
gen, obnamlib.REPO_GENERATION_FILE_COUNT)
for gen in self.obnam.repo.get_client_generation_ids(client_name))
stv = fuse.StatVfs()
stv.f_bsize = 65536
stv.f_frsize = 0
stv.f_blocks = total_data / 65536
stv.f_bfree = 0
stv.f_bavail = 0
stv.f_files = files
stv.f_ffree = 0
stv.f_favail = 0
stv.f_flag = 0
stv.f_namemax = 255
# raise OSError(errno.ENOSYS, 'Unimplemented')
return stv
def remove_chunk_from_indexes(self, chunk_id, client_id):
tracing.trace('chunk_id=%s', chunk_id)
self._require_chunk_indexes_lock()
checksum = self._chunklist.get_checksum(chunk_id)
self._chunksums.remove(checksum, chunk_id, client_id)
self._chunklist.remove(chunk_id)
def remove(self, filename):
tracing.trace('filename=%s', filename)
file_id = self.get_file_id(self.tree, filename)
genid = self.get_generation_id(self.tree)
# Remove any children.
minkey = self.fskey(file_id, self.DIR_CONTENTS, 0)
maxkey = self.fskey(file_id, self.DIR_CONTENTS, obnamlib.MAX_ID)
for key, basename in self.tree.lookup_range(minkey, maxkey):
self.remove(os.path.join(filename, basename))
# Remove chunk refs.
for chunkid in self.get_file_chunks(genid, filename):
key = self.chunk_key(chunkid, file_id)
self.tree.remove_range(key, key)
# Remove this file's metadata.
minkey = self.fskey(file_id, 0, 0)
maxkey = self.fskey(file_id, self.TYPE_MAX, self.SUBKEY_MAX)
self.tree.remove_range(minkey, maxkey)
# Remove filename.
default_file_id = self.default_file_id(filename)
key = self.fskey(default_file_id, self.FILE_NAME, file_id)
self.tree.remove_range(key, key)
# Also remove from parent's contents.
parent = os.path.dirname(filename)
if parent != filename: # root dir is its own parent
parent_id = self.set_file_id(parent)
key = self.fskey(parent_id, self.DIR_CONTENTS, file_id)
# The range removal will work even if the key does not exist.
self.tree.remove_range(key, key)
def create_generation(self, client_name):
tracing.trace('client_name=%s', client_name)
self._require_existing_client(client_name)
self._require_client_lock(client_name)
open_client_info = self._open_client_infos[client_name]
if open_client_info.current_generation_number is not None:
raise obnamlib.RepositoryClientGenerationUnfinished(
client_name=client_name)
open_client_info.client.start_generation()
open_client_info.client.set_generation_started(self._current_time())
new_gen_number = open_client_info.client.get_generation_id(
open_client_info.client.tree)
open_client_info.current_generation_number = new_gen_number
self._refresh_open_client_info_cached_generation_ids(
client_name, open_client_info)
new_gen_id = self._construct_gen_id(client_name, new_gen_number)
self._add_to_open_client_info_cached_generation_ids(
open_client_info, new_gen_id)
return self._construct_gen_id(
client_name, open_client_info.current_generation_number)
def _raw_unlock_client(self, client_name):
tracing.trace('client_name=%s', client_name)
client_id = self._get_client_id(client_name)
client_dir = self._get_client_dir(client_id)
self._lockmgr.unlock([client_dir])
if client_name in self._open_client_infos:
del self._open_client_infos[client_name]
def _raw_lock_client(self, client_name):
tracing.trace('client_name=%s', client_name)
if self.got_client_lock(client_name):
raise obnamlib.RepositoryClientLockingFailed(
client_name=client_name)
client_id = self._get_client_id(client_name)
if client_id is None: # pragma: no cover
raise obnamlib.RepositoryClientDoesNotExist(
client_name=client_name)
# Create and initialise the client's own directory, if needed.
client_dir = self._get_client_dir(client_id)
if not self._fs.exists(client_dir):
self._fs.mkdir(client_dir)
self.hooks.call('repository-toplevel-init', self, client_dir)
# Actually lock the directory.
self._lockmgr.lock([client_dir])
# Remember that we have the lock.
self._open_client(client_name) # Ensure client is open
open_client_info = self._open_client_infos[client_name]
open_client_info.locked = True
def create_generation(self, client_name):
tracing.trace('client_name=%s', client_name)
self._require_existing_client(client_name)
self._require_client_lock(client_name)
self._open_client(client_name) # Ensure client is open
open_client_info = self._open_client_infos[client_name]
if open_client_info.current_generation_number is not None:
raise obnamlib.RepositoryClientGenerationUnfinished(
client_name=client_name)
open_client_info.client.start_generation()
open_client_info.client.set_generation_started(self._current_time())
new_gen_number = open_client_info.client.get_generation_id(
open_client_info.client.tree)
open_client_info.current_generation_number = new_gen_number
self._refresh_open_client_info_cached_generation_ids(
client_name, open_client_info)
new_gen_id = self._construct_gen_id(client_name, new_gen_number)
self._add_to_open_client_info_cached_generation_ids(
open_client_info, new_gen_id)
return self._construct_gen_id(
client_name, open_client_info.current_generation_number)
def create(self, filename, encoded_metadata):
tracing.trace('filename=%s', filename)
file_id = self.set_file_id(filename)
gen_id = self.get_generation_id(self.tree)
try:
old_metadata = self.get_metadata(gen_id, filename)
except KeyError:
old_metadata = None
else: # pragma: no cover
old = obnamlib.fmt_6.metadata_codec.decode_metadata(old_metadata)
metadata = obnamlib.fmt_6.metadata_codec.decode_metadata(
encoded_metadata)
if encoded_metadata != old_metadata:
tracing.trace('new or changed metadata')
self.set_metadata(filename, encoded_metadata)
# Add to parent's contents, unless already there.
parent = os.path.dirname(filename)
tracing.trace('parent=%s', parent)
if parent != filename: # root dir is its own parent
basename = os.path.basename(filename)
parent_id = self.set_file_id(parent)
key = self.fskey(parent_id, self.DIR_CONTENTS, file_id)
# We could just insert, but that would cause unnecessary
# churn in the tree if nothing changes.
try:
self.tree.lookup(key)
tracing.trace('was already in parent') # pragma: no cover
except KeyError:
self.tree.insert(key, basename)
tracing.trace('added to parent')
def find_files(self, root):
'''Find all files and directories that need to be backed up.
This is a generator. It yields (pathname, metadata) pairs.
The caller should not recurse through directories, just backup
the directory itself (name, metadata, file list).
'''
for pathname, st in self.fs.scan_tree(root, ok=self.can_be_backed_up):
tracing.trace('considering %s' % pathname)
try:
metadata = obnamlib.read_metadata(self.fs, pathname, st=st)
self.progress.update_progress_with_file(pathname, metadata)
if self.needs_backup(pathname, metadata):
self.progress.backed_up_count += 1
yield pathname, metadata
else:
self.progress.update_progress_with_scanned(
metadata.st_size)
except GeneratorExit:
raise
except KeyboardInterrupt:
logging.error('Keyboard interrupt')
raise
except BaseException, e:
msg = 'Cannot back up %s: %s' % (pathname, str(e))
self.error(msg, e)
def create(self, filename, encoded_metadata):
tracing.trace('filename=%s', filename)
file_id = self.set_file_id(filename)
gen_id = self.get_generation_id(self.tree)
try:
old_metadata = self.get_metadata(gen_id, filename)
except KeyError:
old_metadata = None
if encoded_metadata != old_metadata:
tracing.trace('new or changed metadata')
self.set_metadata(filename, encoded_metadata)
# Add to parent's contents, unless already there.
parent = os.path.dirname(filename)
tracing.trace('parent=%s', parent)
if parent != filename: # root dir is its own parent
basename = os.path.basename(filename)
parent_id = self.set_file_id(parent)
key = self.fskey(parent_id, self.DIR_CONTENTS, file_id)
# We could just insert, but that would cause unnecessary
# churn in the tree if nothing changes.
try:
self.tree.lookup(key)
tracing.trace('was already in parent') # pragma: no cover
except KeyError:
self.tree.insert(key, basename)
tracing.trace('added to parent')
def get_node(self, node_id):
tracing.trace('node_id=%s' % node_id)
try:
return self.fsck.forest.node_store.get_node(node_id)
except larch.NodeMissing:
self.error(
'forest %s: node %s is missing' %
(self.fsck.forest_name, node_id))
def force_chunk_indexes_lock(self):
tracing.trace('forcing chunk indexes lock')
lock_name = os.path.join('lock')
for dirname in self._chunk_index_dirs_to_lock():
lock_name = os.path.join(dirname, 'lock')
if self._real_fs.exists(lock_name):
self._real_fs.remove(lock_name)
self._setup_chunk_indexes()
def commit_client_list(self):
tracing.trace('committing client list')
for client_name in self._added_clients:
self.hooks.call(
'repository-add-client', self._client_list, client_name)
self._added_clients = []
self._client_list.commit()
self._raw_unlock_client_list()
def _clear_directory(self, dirname):
tracing.trace(dirname)
for pathname in self.climb(dirname):
if pathname != dirname:
if self.fs.isdir(pathname):
self.fs.rmdir(pathname)
else:
self.fs.remove(pathname)
def read_pid(self, length, offset):
tracing.trace('length=%r', length)
tracing.trace('offset=%r', offset)
pid = str(os.getpid())
if length < len(pid) or offset != 0:
return ''
else:
return pid
def __init__(self, fs, name, checksum_length, node_size, upload_queue_size,
lru_size, hooks):
tracing.trace('new ChecksumTree name=%s' % name)
self.fmt = '!%dsQQ' % checksum_length
key_bytes = struct.calcsize(self.fmt)
obnamlib.RepositoryTree.__init__(self, fs, name, key_bytes, node_size,
upload_queue_size, lru_size, hooks)
self.keep_just_one_tree = True
def default_file_id(self, filename):
'''Return hash of filename suitable for use as main key.'''
tracing.trace(repr(filename))
def hash(s):
return hashlib.md5(s).digest()[:4]
dirname = os.path.dirname(filename)
basename = os.path.basename(filename)
return hash(dirname) + hash(basename)
def force_chunk_indexes_lock(self):
tracing.trace('forcing chunk indexes lock')
lock_name = os.path.join('lock')
for dirname in self._chunk_index_dirs_to_lock():
lock_name = os.path.join(dirname, 'lock')
if self._real_fs.exists(lock_name):
self._real_fs.remove(lock_name)
self._setup_chunk_indexes()
def _really_put_node(self, node):
tracing.trace('really put node %s' % node.id)
encoded_node = self.codec.encode(node)
if len(encoded_node) > self.node_size:
raise larch.NodeTooBig(node, len(encoded_node))
name = self.pathname(node.id)
tracing.trace('node %s to be stored in %s' % (node.id, name))
self.journal.overwrite_file(name, encoded_node)
def read_pid(self, length, offset):
tracing.trace('length=%r', length)
tracing.trace('offset=%r', offset)
pid = str(os.getpid())
if length < len(pid) or offset != 0:
return ''
else:
return pid
def __init__(self, fs, node_size, upload_queue_size, lru_size, hooks):
tracing.trace('new ChunkList')
self.fmt = '!Q'
self.key_bytes = struct.calcsize(self.fmt)
obnamlib.RepositoryTree.__init__(self, fs, 'chunklist', self.key_bytes,
node_size, upload_queue_size,
lru_size, hooks)
self.keep_just_one_tree = True
def doExperiment():
if not os.path.exists("logs"):
os.makedirs("logs")
objects = dict(
(objectName, [{"top": location[0] * CM_PER_UNIT,
"left": location[1] * CM_PER_UNIT,
"width": CM_PER_UNIT,
"height": CM_PER_UNIT,
"name": featureName}
for location, featureName in objectDict.iteritems()])
for objectName, objectDict in DISCRETE_OBJECTS.iteritems())
objectPlacements = dict(
(objectName, [placement[0] * CM_PER_UNIT,
placement[1] * CM_PER_UNIT])
for objectName, placement in DISCRETE_OBJECT_PLACEMENTS.iteritems())
cellDimensions = (10, 10)
locationConfigs = []
for i in xrange(4):
scale = 10.0 * (math.sqrt(2) ** i)
for _ in xrange(4):
orientation = random.gauss(7.5, 7.5) * math.pi / 180.0
orientation = random.choice([orientation, -orientation])
locationConfigs.append({
"cellDimensions": cellDimensions,
"moduleMapDimensions": (scale, scale),
"orientation": orientation,
})
print("Initializing experiment")
exp = MultiColumn2DExperiment(
featureNames=("A", "B"),
objects=objects,
objectPlacements=objectPlacements,
locationConfigs=locationConfigs,
numCorticalColumns=3,
worldDimensions=(100, 100))
print("Learning objects")
filename = "logs/{}-cells-learn.log".format(np.prod(cellDimensions))
with open(filename, "w") as fileOut:
with trace(exp, fileOut, includeSynapses=True):
print "Logging to", filename
bodyPlacement = [6. * CM_PER_UNIT, 1. * CM_PER_UNIT]
exp.learnObjects(bodyPlacement)
filename = "logs/{}-cells-infer.log".format(np.prod(cellDimensions))
with open(filename, "w") as fileOut:
with trace(exp, fileOut, includeSynapses=True):
print "Logging to", filename
bodyPlacement = [6. * CM_PER_UNIT, 11. * CM_PER_UNIT]
exp.inferObjectsWithTwoTouches(bodyPlacement)
def overwrite_file(self, filename, data, runfilters=True):
tracing.trace('overwriting hooked %s' % filename)
toplevel = self._get_toplevel(filename)
if runfilters:
data = self.hooks.filter_write('repository-data',
data,
repo=self.repo,
toplevel=toplevel)
self.fs.overwrite_file(filename, data)
def get_xattrs_as_blob(fs, filename): # pragma: no cover
tracing.trace('filename=%s' % filename)
try:
names = fs.llistxattr(filename)
except (OSError, IOError), e:
if e.errno in (errno.EOPNOTSUPP, errno.EACCES):
return None
raise
