def do(self): tracing.trace("CheckForest: checking forest %s" % self.name) for tree in self.fsck.forest.trees: self.fsck.count(tree.root.id) root_node = self.get_node(tree.root.id) tracing.trace('root_node.id=%s' % root_node.id) yield CheckIndexNode(self.fsck, root_node)
def put_chunk_only(self, data): '''Put chunk of data into repository. If the same data is already in the repository, it will be put there a second time. It is the caller's responsibility to check that the data is not already in the repository. Return the unique identifier of the new chunk. ''' def random_chunkid(): return random.randint(0, obnamlib.MAX_ID) self.require_started_generation() if self.prev_chunkid is None: self.prev_chunkid = random_chunkid() while True: chunkid = (self.prev_chunkid + 1) % obnamlib.MAX_ID filename = self._chunk_filename(chunkid) try: self.fs.write_file(filename, data) except OSError, e: # pragma: no cover if e.errno == errno.EEXIST: self.prev_chunkid = random_chunkid() continue raise else: tracing.trace('chunkid=%s', chunkid) break
def remove(self, filename): tracing.trace('filename=%s', filename) file_id = self.get_file_id(self.tree, filename) genid = self.get_generation_id(self.tree) # Remove any children. minkey = self.fskey(file_id, self.DIR_CONTENTS, 0) maxkey = self.fskey(file_id, self.DIR_CONTENTS, obnamlib.MAX_ID) for key, basename in self.tree.lookup_range(minkey, maxkey): self.remove(os.path.join(filename, basename)) # Remove chunk refs. for chunkid in self.get_file_chunks(genid, filename): key = self.chunk_key(chunkid, file_id) self.tree.remove_range(key, key) # Remove this file's metadata. minkey = self.fskey(file_id, 0, 0) maxkey = self.fskey(file_id, self.TYPE_MAX, self.SUBKEY_MAX) self.tree.remove_range(minkey, maxkey) # Remove filename. default_file_id = self.default_file_id(filename) key = self.fskey(default_file_id, self.FILE_NAME, file_id) self.tree.remove_range(key, key) # Also remove from parent's contents. parent = os.path.dirname(filename) if parent != filename: # root dir is its own parent parent_id = self.set_file_id(parent) key = self.fskey(parent_id, self.DIR_CONTENTS, file_id) # The range removal will work even if the key does not exist. self.tree.remove_range(key, key)
def lock_client(self, client_name): '''Lock a client for exclusive write access. Raise obnamlib.LockFail if locking fails. Lock will be released by commit_client() or unlock_client(). ''' tracing.trace('client_name=%s', client_name) self.require_no_client_lock() self.require_no_shared_lock() self.check_format_version() client_id = self.clientlist.get_client_id(client_name) if client_id is None: raise LockFail('client %s does not exist' % client_name) client_dir = self.client_dir(client_id) if not self.fs.exists(client_dir): self.fs.mkdir(client_dir) self.hooks.call('repository-toplevel-init', self, client_dir) self.lockmgr.lock([client_dir]) self.got_client_lock = True self.current_client = client_name self.current_client_id = client_id self.added_generations = [] self.removed_generations = [] self.client = obnamlib.ClientMetadataTree(self.fs, client_dir, self.node_size, self.upload_queue_size, self.lru_size, self) self.client.init_forest()
def unlock_shared(self): '''Unlock currently locked shared B-trees.''' tracing.trace('unlocking shared') self.require_shared_lock() self.lockmgr.unlock(self.shared_dirs) self.got_shared_lock = False self._open_shared()
def _write_format_version(self, version): '''Write the desired format version to the repository.''' tracing.trace('write format version') if not self.fs.exists('metadata'): self.fs.mkdir('metadata') self.fs.overwrite_file('metadata/format', '%s\n' % version, runfilters=False)
def get_node(self, node_id): tracing.trace('node_id=%s' % node_id) try: return self.fsck.forest.node_store.get_node(node_id) except larch.NodeMissing: self.error('forest %s: node %s is missing' % (self.fsck.forest_name, node_id))
def _raw_unlock_client(self, client_name): tracing.trace('client_name=%s', client_name) client_id = self._get_client_id(client_name) client_dir = self._get_client_dir(client_id) self._lockmgr.unlock([client_dir]) if client_name in self._open_client_infos: del self._open_client_infos[client_name]
def get_repository_object(self, create=False, repofs=None): '''Return an implementation of obnamlib.RepositoryInterface.''' logging.info('Opening repository: %s', self.settings['repository']) tracing.trace('create=%s', create) tracing.trace('repofs=%s', repofs) repopath = self.settings['repository'] if repofs is None: repofs = self.fsf.new(repopath, create=create) if self.settings['crash-limit'] > 0: repofs.crash_limit = self.settings['crash-limit'] repofs.connect() else: repofs.reinit(repopath) kwargs = { 'lock_timeout': self.settings['lock-timeout'], 'node_size': self.settings['node-size'], 'upload_queue_size': self.settings['upload-queue-size'], 'lru_size': self.settings['lru-size'], 'idpath_depth': self.settings['idpath-depth'], 'idpath_bits': self.settings['idpath-bits'], 'idpath_skip': self.settings['idpath-skip'], 'hooks': self.hooks, 'current_time': self.time, 'chunk_size': self.settings['chunk-size'], } if create: return self.repo_factory.create_repo( repofs, self.get_default_repository_class(), **kwargs) else: return self.repo_factory.open_existing_repo(repofs, **kwargs)
def _raw_lock_client(self, client_name): tracing.trace('client_name=%s', client_name) if self.got_client_lock(client_name): raise obnamlib.RepositoryClientLockingFailed( client_name=client_name) client_id = self._get_client_id(client_name) if client_id is None: # pragma: no cover raise obnamlib.RepositoryClientDoesNotExist( client_name=client_name) # Create and initialise the client's own directory, if needed. client_dir = self._get_client_dir(client_id) if not self._fs.exists(client_dir): self._fs.mkdir(client_dir) self.hooks.call('repository-toplevel-init', self, client_dir) # Actually lock the directory. self._lockmgr.lock([client_dir]) # Remember that we have the lock. self._open_client(client_name) # Ensure client is open open_client_info = self._open_client_infos[client_name] open_client_info.locked = True
def find_files(self, root): '''Find all files and directories that need to be backed up. This is a generator. It yields (pathname, metadata) pairs. The caller should not recurse through directories, just backup the directory itself (name, metadata, file list). ''' for pathname, st in self.fs.scan_tree(root, ok=self.can_be_backed_up): tracing.trace('considering %s' % pathname) try: metadata = obnamlib.read_metadata(self.fs, pathname, st=st) self.progress.update_progress_with_file(pathname, metadata) if self.needs_backup(pathname, metadata): yield pathname, metadata else: self.progress.update_progress_with_scanned( metadata.st_size) except GeneratorExit: raise except KeyboardInterrupt: logging.error('Keyboard interrupt') raise except BaseException, e: msg = 'Cannot back up %s: %s' % (pathname, str(e)) self.progress.error(msg, e)
def commit_client_list(self): tracing.trace('committing client list') for client_name in self._added_clients: self.hooks.call('repository-add-client', self, client_name) self._added_clients = [] self._client_list.commit() self._raw_unlock_client_list()
def _write_to_tempfile(self, pathname, contents): path = self.join(pathname) dirname = os.path.dirname(path) if not os.path.exists(dirname): tracing.trace('os.makedirs(%s)' % dirname) try: os.makedirs(dirname, mode=obnamlib.NEW_DIR_MODE) except OSError as e: # pragma: no cover # This avoids a race condition: another Obnam process # may have created the directory between our check and # creation attempt. If so, we ignore it. As long as # the directory exists, all's good. if e.errno != errno.EEXIST: raise fd, tempname = tempfile.mkstemp(dir=dirname) os.fchmod(fd, obnamlib.NEW_FILE_MODE) os.close(fd) f = self.open(tempname, 'wb') pos = 0 while pos < len(contents): chunk = contents[pos:pos + self.chunk_size] f.write(chunk) pos += len(chunk) self.bytes_written += len(chunk) f.close() return tempname
def backup_dir_contents(self, root): '''Back up the list of files in a directory.''' tracing.trace('backup_dir: %s', root) if self.pretend: return new_basenames = self.fs.listdir(root) new_pathnames = [os.path.join(root, x) for x in new_basenames] if self.repo.file_exists(self.new_generation, root): old_pathnames = self.repo.get_file_children( self.new_generation, root) else: old_pathnames = [] for old in old_pathnames: if old not in new_pathnames: self.repo.remove_file(self.new_generation, old) else: try: st = self.fs.lstat(old) except OSError: pass else: if not self.can_be_backed_up(old, st): self.repo.remove_file(self.new_generation, old)
def chmod_symlink(self, pathname, mode): # pragma: no cover tracing.trace('chmod_symlink %s %o', pathname, mode) if self.got_lchmod: lchmod = getattr(os, 'lchmod') lchmod(self.join(pathname), mode) else: self.lstat(pathname)
def update_student_dates(self, student_id, course_data): """ The following will be copied from the course data: - Date of every lecture - Date of every dayoffs The following will be calculated: - Date descriptions. This is one string containing all of the lectures and vacations, directly insertable to emails. :param student_id: id number of the student as a string. :type student_id: str :param course_data: all of the date about a course the student has applied to as stored in the MiniCRM system. :type course_data: dict :return: None """ data_to_update = { "Datumleirasok": self._get_date_description(course_data) } data_to_update = copy_dates_from_course_data_to_student_data(data_to_update, course_data) trace("DATA TO BE REPLACED:") pretty_print(data_to_update) self._request_handler.fetch(crmrequestfactory.set_project_data(student_id, data_to_update))
def _get_date_description(self, course_data): date_list = [] date_list.append(course_data[COURSE_FIRST_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_SECOND_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_THIRD_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_FOURTH_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_FIFTH_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_SIXTH_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_SEVENTH_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_EIGTH_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_NINTH_OCCASION_DATE_FIELD][:10]) date_list.append(course_data[COURSE_TENTH_OCCASION_DATE_FIELD][:10]) if course_data[COURSE_FISRST_DAYOFF_FIELD] != "": date_list.append("{} - {}".format(course_data[COURSE_FISRST_DAYOFF_FIELD][:10], "szünet")) if course_data[COURSE_SECOND_DAYOFF_FIELD] != "": date_list.append("{} - {}".format(course_data[COURSE_SECOND_DAYOFF_FIELD][:10], "szünet")) if course_data[COURSE_THIRD_DAYOFF_FIELD] != "": date_list.append("{} - {}".format(course_data[COURSE_THIRD_DAYOFF_FIELD][:10], "szünet")) date_list.sort() return_string = " - " + "\n - ".join(date_list) trace("JOINED STRING:\n{}".format(return_string)) return return_string
def get_location_by_name(self, location_name): """ Returns the dictionary of a location for a given name. - returns one of the locations if name is not unique - returns None if name doesn't exist :param location_name: Course code of a course :type location_name: int :return: containing all information about the given location :rtype: dict """ location_list = self._request_handler.fetch( crmrequestfactory.get_location_list_by_location_name(location_name) ) pretty_print(location_list) for location in location_list[RESULTS_FIELD]: return self._request_handler.fetch( crmrequestfactory.get_location(location)) trace("COURSE NOT FOUND: [{}]".format(location_name)) return None
def save_refcounts(self): '''Save all modified refcounts.''' tracing.trace('saving refcounts (len(dirty) = %s)' % (len(self.dirty))) if self.dirty: dirname = os.path.join(self.node_store.dirname, self.refcountdir) if not self.node_store.journal.exists(dirname): self.node_store.journal.makedirs(dirname) ids = sorted(self.dirty) all_ids_in_memory = set(self.refcounts.keys()) for start_id in range(self._start_id(ids[0]), self._start_id(ids[-1]) + 1, self.per_group): all_ids_in_group = set( range(start_id, start_id + self.per_group)) keys = all_ids_in_group.intersection(all_ids_in_memory) if keys: encoded = encode_refcounts( self.refcounts, start_id, self.per_group, keys) filename = self._group_filename(start_id) self.node_store.journal.overwrite_file(filename, encoded) # We re-initialize these so that they don't grow indefinitely. self.refcounts = dict() self.dirty = set()
def __init__(self, allow_writes, fs, storedir): logging.debug('Initializing Journal for %s' % storedir) self.allow_writes = allow_writes self.fs = fs self.storedir = storedir if not self.storedir.endswith(os.sep): self.storedir += os.sep self.newdir = os.path.join(self.storedir, 'new/') self.deletedir = os.path.join(self.storedir, 'delete/') self.flag_file = os.path.join(self.storedir, self.flag_basename) self.new_flag = os.path.join(self.newdir, self.flag_basename) self.new_flag_seen = self.fs.exists(self.new_flag) tracing.trace('self.new_flag_seen: %s' % self.new_flag_seen) if self.allow_writes: if self.new_flag_seen: logging.debug('Automatically committing remaining changes') self.commit() else: logging.debug('Automatically rolling back remaining changes') self.rollback() else: logging.debug('Not committing/rolling back since read-only') self.new_files = set() self.deleted_files = set()
def overwrite_file(self, filename, data, runfilters=True): tracing.trace('overwriting hooked %s' % filename) toplevel = self._get_toplevel(filename) if runfilters: data = self.hooks.filter_write('repository-data', data, repo=self.repo, toplevel=toplevel) self.fs.overwrite_file(filename, data)
def do(self): tracing.trace("CheckForest: checking forest %s" % self.name ) for tree in self.fsck.forest.trees: self.fsck.count(tree.root.id) root_node = self.get_node(tree.root.id) tracing.trace('root_node.id=%s' % root_node.id) yield CheckIndexNode(self.fsck, root_node)
def needs_backup(self, pathname, current): '''Does a given file need to be backed up?''' # Directories always require backing up so that backup_dir_contents # can remove stuff that no longer exists from them. if current.isdir(): tracing.trace('%s is directory, so needs backup' % pathname) return True if self.pretend: gens = self.repo.list_generations() if not gens: return True gen = gens[-1] else: gen = self.repo.new_generation tracing.trace('gen=%s' % repr(gen)) try: old = self.repo.get_metadata(gen, pathname) except obnamlib.Error, e: # File does not exist in the previous generation, so it # does need to be backed up. tracing.trace('%s not in previous gen, so needs backup' % pathname) tracing.trace('error: %s' % str(e)) tracing.trace(traceback.format_exc()) return True
def __init__(self, allow_writes, node_size, codec, dirname=None, upload_max=1024, lru_size=500, vfs=None, format=None): tracing.trace('new NodeStoreDisk: %s', dirname) assert dirname is not None if format is not None: tracing.trace('forcing format_base: %s', format) self.format_base = format larch.NodeStore.__init__(self, allow_writes=allow_writes, node_size=node_size, codec=codec) self.dirname = dirname self.metadata_name = os.path.join(dirname, 'metadata') self.metadata = None self.rs = larch.RefcountStore(self) self.cache_size = lru_size self.cache = larch.LRUCache(self.cache_size) self.upload_max = upload_max self.upload_queue = larch.UploadQueue(self._really_put_node, self.upload_max) self.vfs = vfs if vfs != None else LocalFS() self.journal = larch.Journal(allow_writes, self.vfs, dirname) self.idpath = larch.IdPath(os.path.join(dirname, self.nodedir), DIR_DEPTH, DIR_BITS, DIR_SKIP)
def statfs(self): tracing.trace('called') client_name = self.obnam.app.settings['client-name'] total_data = sum( self.obnam.repo.get_generation_key( gen, obnamlib.REPO_GENERATION_TOTAL_DATA) for gen in obnamlib.repo.get_clientgeneration_ids(client_name)) files = sum( self.obnam.repo.get_generation_key( gen, obnamlib.REPO_GENERATION_FILE_COUNT) for gen in self.obnam.repo.get_client_generation_ids(client_name)) stv = fuse.StatVfs() stv.f_bsize = 65536 stv.f_frsize = 0 stv.f_blocks = blocks/65536 stv.f_bfree = 0 stv.f_bavail = 0 stv.f_files = files stv.f_ffree = 0 stv.f_favail = 0 stv.f_flag = 0 stv.f_namemax = 255 #raise OSError(errno.ENOSYS, 'Unimplemented') return stv
def _raw_unlock_client(self, client_name): tracing.trace('client_name=%s', client_name) self._require_client_lock(client_name) open_client = self._open_clients[client_name] self._lockmgr.unlock([open_client.client.dirname]) del self._open_clients[client_name]
def remove_chunk(self, chunk_id): tracing.trace('chunk_id=%s', chunk_id) filename = self._chunk_filename(chunk_id) try: self._fs.remove(filename) except OSError: raise obnamlib.RepositoryChunkDoesNotExist(str(chunk_id))
def get_repository_object(self, create=False, repofs=None): '''Return an implementation of obnamlib.RepositoryInterface.''' logging.info('Opening repository: %s', self.settings['repository']) tracing.trace('create=%s', create) tracing.trace('repofs=%s', repofs) repopath = self.settings['repository'] if repofs is None: repofs = self.fsf.new(repopath, create=create) if self.settings['crash-limit'] > 0: repofs.crash_limit = self.settings['crash-limit'] repofs.connect() else: repofs.reinit(repopath) kwargs = { 'lock_timeout': self.settings['lock-timeout'], 'node_size': self.settings['node-size'], 'upload_queue_size': self.settings['upload-queue-size'], 'lru_size': self.settings['lru-size'], 'idpath_depth': self.settings['idpath-depth'], 'idpath_bits': self.settings['idpath-bits'], 'idpath_skip': self.settings['idpath-skip'], 'hooks': self.hooks, 'current_time': self.time, } if create: return self.repo_factory.create_repo(repofs, obnamlib.RepositoryFormat6, **kwargs) else: return self.repo_factory.open_existing_repo(repofs, **kwargs)
def _write_to_tempfile(self, pathname, contents): path = self.join(pathname) dirname = os.path.dirname(path) if not os.path.exists(dirname): tracing.trace('os.makedirs(%s)' % dirname) try: os.makedirs(dirname) except OSError as e: # pragma: no cover # This avoids a race condition: another Obnam process # may have created the directory between our check and # creation attempt. If so, we ignore it. As long as # the directory exists, all's good. if e.errno != errno.EEXIST: raise fd, tempname = tempfile.mkstemp(dir=dirname) os.close(fd) f = self.open(tempname, 'wb') pos = 0 while pos < len(contents): chunk = contents[pos:pos+self.chunk_size] f.write(chunk) pos += len(chunk) self.bytes_written += len(chunk) f.close() return tempname
def _add_or_merge(self, parent, node, merge): assert not parent.frozen assert node.frozen keys = parent.keys() key = node.first_key() i = bisect.bisect_left(keys, key) new_node = None if i > 0: new_node = merge(parent, node, i-1) if new_node is None and i < len(keys): new_node = merge(parent, node, i) if new_node is None: new_node = node assert new_node is not None self._put_node(new_node) parent.add(new_node.first_key(), new_node.id) self._increment(new_node.id) if new_node != node: # pragma: no cover # We made a new node, so get rid of the old one. tracing.trace('decrementing unused node id=%s' % node.id) self._decrement(node.id)
def _new_index(self, keys, values): '''Create a new index node.''' index = larch.IndexNode(self._new_id(), keys, values) for child_id in values: self._increment(child_id) tracing.trace('id=%s' % index.id) return index
def statfs(self): tracing.trace('called') client_name = self.obnam.app.settings['client-name'] total_data = sum( self.obnam.repo.get_generation_key( gen, obnamlib.REPO_GENERATION_TOTAL_DATA) for gen in self.obnam.repo.get_client_generation_ids(client_name)) files = sum( self.obnam.repo.get_generation_key( gen, obnamlib.REPO_GENERATION_FILE_COUNT) for gen in self.obnam.repo.get_client_generation_ids(client_name)) stv = fuse.StatVfs() stv.f_bsize = 65536 stv.f_frsize = 0 stv.f_blocks = total_data / 65536 stv.f_bfree = 0 stv.f_bavail = 0 stv.f_files = files stv.f_ffree = 0 stv.f_favail = 0 stv.f_flag = 0 stv.f_namemax = 255 # raise OSError(errno.ENOSYS, 'Unimplemented') return stv
def remove_chunk_from_indexes(self, chunk_id, client_id): tracing.trace('chunk_id=%s', chunk_id) self._require_chunk_indexes_lock() checksum = self._chunklist.get_checksum(chunk_id) self._chunksums.remove(checksum, chunk_id, client_id) self._chunklist.remove(chunk_id)
def create_generation(self, client_name): tracing.trace('client_name=%s', client_name) self._require_existing_client(client_name) self._require_client_lock(client_name) open_client_info = self._open_client_infos[client_name] if open_client_info.current_generation_number is not None: raise obnamlib.RepositoryClientGenerationUnfinished( client_name=client_name) open_client_info.client.start_generation() open_client_info.client.set_generation_started(self._current_time()) new_gen_number = open_client_info.client.get_generation_id( open_client_info.client.tree) open_client_info.current_generation_number = new_gen_number self._refresh_open_client_info_cached_generation_ids( client_name, open_client_info) new_gen_id = self._construct_gen_id(client_name, new_gen_number) self._add_to_open_client_info_cached_generation_ids( open_client_info, new_gen_id) return self._construct_gen_id( client_name, open_client_info.current_generation_number)
def create_generation(self, client_name): tracing.trace('client_name=%s', client_name) self._require_existing_client(client_name) self._require_client_lock(client_name) self._open_client(client_name) # Ensure client is open open_client_info = self._open_client_infos[client_name] if open_client_info.current_generation_number is not None: raise obnamlib.RepositoryClientGenerationUnfinished( client_name=client_name) open_client_info.client.start_generation() open_client_info.client.set_generation_started(self._current_time()) new_gen_number = open_client_info.client.get_generation_id( open_client_info.client.tree) open_client_info.current_generation_number = new_gen_number self._refresh_open_client_info_cached_generation_ids( client_name, open_client_info) new_gen_id = self._construct_gen_id(client_name, new_gen_number) self._add_to_open_client_info_cached_generation_ids( open_client_info, new_gen_id) return self._construct_gen_id( client_name, open_client_info.current_generation_number)
def create(self, filename, encoded_metadata): tracing.trace('filename=%s', filename) file_id = self.set_file_id(filename) gen_id = self.get_generation_id(self.tree) try: old_metadata = self.get_metadata(gen_id, filename) except KeyError: old_metadata = None else: # pragma: no cover old = obnamlib.fmt_6.metadata_codec.decode_metadata(old_metadata) metadata = obnamlib.fmt_6.metadata_codec.decode_metadata( encoded_metadata) if encoded_metadata != old_metadata: tracing.trace('new or changed metadata') self.set_metadata(filename, encoded_metadata) # Add to parent's contents, unless already there. parent = os.path.dirname(filename) tracing.trace('parent=%s', parent) if parent != filename: # root dir is its own parent basename = os.path.basename(filename) parent_id = self.set_file_id(parent) key = self.fskey(parent_id, self.DIR_CONTENTS, file_id) # We could just insert, but that would cause unnecessary # churn in the tree if nothing changes. try: self.tree.lookup(key) tracing.trace('was already in parent') # pragma: no cover except KeyError: self.tree.insert(key, basename) tracing.trace('added to parent')
def find_files(self, root): '''Find all files and directories that need to be backed up. This is a generator. It yields (pathname, metadata) pairs. The caller should not recurse through directories, just backup the directory itself (name, metadata, file list). ''' for pathname, st in self.fs.scan_tree(root, ok=self.can_be_backed_up): tracing.trace('considering %s' % pathname) try: metadata = obnamlib.read_metadata(self.fs, pathname, st=st) self.progress.update_progress_with_file(pathname, metadata) if self.needs_backup(pathname, metadata): self.progress.backed_up_count += 1 yield pathname, metadata else: self.progress.update_progress_with_scanned( metadata.st_size) except GeneratorExit: raise except KeyboardInterrupt: logging.error('Keyboard interrupt') raise except BaseException, e: msg = 'Cannot back up %s: %s' % (pathname, str(e)) self.error(msg, e)
def create(self, filename, encoded_metadata): tracing.trace('filename=%s', filename) file_id = self.set_file_id(filename) gen_id = self.get_generation_id(self.tree) try: old_metadata = self.get_metadata(gen_id, filename) except KeyError: old_metadata = None if encoded_metadata != old_metadata: tracing.trace('new or changed metadata') self.set_metadata(filename, encoded_metadata) # Add to parent's contents, unless already there. parent = os.path.dirname(filename) tracing.trace('parent=%s', parent) if parent != filename: # root dir is its own parent basename = os.path.basename(filename) parent_id = self.set_file_id(parent) key = self.fskey(parent_id, self.DIR_CONTENTS, file_id) # We could just insert, but that would cause unnecessary # churn in the tree if nothing changes. try: self.tree.lookup(key) tracing.trace('was already in parent') # pragma: no cover except KeyError: self.tree.insert(key, basename) tracing.trace('added to parent')
def get_node(self, node_id): tracing.trace('node_id=%s' % node_id) try: return self.fsck.forest.node_store.get_node(node_id) except larch.NodeMissing: self.error( 'forest %s: node %s is missing' % (self.fsck.forest_name, node_id))
def force_chunk_indexes_lock(self): tracing.trace('forcing chunk indexes lock') lock_name = os.path.join('lock') for dirname in self._chunk_index_dirs_to_lock(): lock_name = os.path.join(dirname, 'lock') if self._real_fs.exists(lock_name): self._real_fs.remove(lock_name) self._setup_chunk_indexes()
def commit_client_list(self): tracing.trace('committing client list') for client_name in self._added_clients: self.hooks.call( 'repository-add-client', self._client_list, client_name) self._added_clients = [] self._client_list.commit() self._raw_unlock_client_list()
def _clear_directory(self, dirname): tracing.trace(dirname) for pathname in self.climb(dirname): if pathname != dirname: if self.fs.isdir(pathname): self.fs.rmdir(pathname) else: self.fs.remove(pathname)
def read_pid(self, length, offset): tracing.trace('length=%r', length) tracing.trace('offset=%r', offset) pid = str(os.getpid()) if length < len(pid) or offset != 0: return '' else: return pid
def __init__(self, fs, name, checksum_length, node_size, upload_queue_size, lru_size, hooks): tracing.trace('new ChecksumTree name=%s' % name) self.fmt = '!%dsQQ' % checksum_length key_bytes = struct.calcsize(self.fmt) obnamlib.RepositoryTree.__init__(self, fs, name, key_bytes, node_size, upload_queue_size, lru_size, hooks) self.keep_just_one_tree = True
def default_file_id(self, filename): '''Return hash of filename suitable for use as main key.''' tracing.trace(repr(filename)) def hash(s): return hashlib.md5(s).digest()[:4] dirname = os.path.dirname(filename) basename = os.path.basename(filename) return hash(dirname) + hash(basename)
def _really_put_node(self, node): tracing.trace('really put node %s' % node.id) encoded_node = self.codec.encode(node) if len(encoded_node) > self.node_size: raise larch.NodeTooBig(node, len(encoded_node)) name = self.pathname(node.id) tracing.trace('node %s to be stored in %s' % (node.id, name)) self.journal.overwrite_file(name, encoded_node)
def __init__(self, fs, node_size, upload_queue_size, lru_size, hooks): tracing.trace('new ChunkList') self.fmt = '!Q' self.key_bytes = struct.calcsize(self.fmt) obnamlib.RepositoryTree.__init__(self, fs, 'chunklist', self.key_bytes, node_size, upload_queue_size, lru_size, hooks) self.keep_just_one_tree = True
def doExperiment(): if not os.path.exists("logs"): os.makedirs("logs") objects = dict( (objectName, [{"top": location[0] * CM_PER_UNIT, "left": location[1] * CM_PER_UNIT, "width": CM_PER_UNIT, "height": CM_PER_UNIT, "name": featureName} for location, featureName in objectDict.iteritems()]) for objectName, objectDict in DISCRETE_OBJECTS.iteritems()) objectPlacements = dict( (objectName, [placement[0] * CM_PER_UNIT, placement[1] * CM_PER_UNIT]) for objectName, placement in DISCRETE_OBJECT_PLACEMENTS.iteritems()) cellDimensions = (10, 10) locationConfigs = [] for i in xrange(4): scale = 10.0 * (math.sqrt(2) ** i) for _ in xrange(4): orientation = random.gauss(7.5, 7.5) * math.pi / 180.0 orientation = random.choice([orientation, -orientation]) locationConfigs.append({ "cellDimensions": cellDimensions, "moduleMapDimensions": (scale, scale), "orientation": orientation, }) print("Initializing experiment") exp = MultiColumn2DExperiment( featureNames=("A", "B"), objects=objects, objectPlacements=objectPlacements, locationConfigs=locationConfigs, numCorticalColumns=3, worldDimensions=(100, 100)) print("Learning objects") filename = "logs/{}-cells-learn.log".format(np.prod(cellDimensions)) with open(filename, "w") as fileOut: with trace(exp, fileOut, includeSynapses=True): print "Logging to", filename bodyPlacement = [6. * CM_PER_UNIT, 1. * CM_PER_UNIT] exp.learnObjects(bodyPlacement) filename = "logs/{}-cells-infer.log".format(np.prod(cellDimensions)) with open(filename, "w") as fileOut: with trace(exp, fileOut, includeSynapses=True): print "Logging to", filename bodyPlacement = [6. * CM_PER_UNIT, 11. * CM_PER_UNIT] exp.inferObjectsWithTwoTouches(bodyPlacement)
def get_xattrs_as_blob(fs, filename): # pragma: no cover tracing.trace('filename=%s' % filename) try: names = fs.llistxattr(filename) except (OSError, IOError), e: if e.errno in (errno.EOPNOTSUPP, errno.EACCES): return None raise