def test_get_pathinfo(self): basedir = "util/FileUtil/test_get_pathinfo" fileutil.make_dirs(basedir) # create a directory self.mkdir(basedir, "a") dirinfo = fileutil.get_pathinfo(basedir) self.failUnlessTrue(dirinfo.isdir) self.failUnlessTrue(dirinfo.exists) self.failUnlessFalse(dirinfo.isfile) self.failUnlessFalse(dirinfo.islink) # create a file f = os.path.join(basedir, "1.txt") fileutil.write(f, b"a" * 10) fileinfo = fileutil.get_pathinfo(f) self.failUnlessTrue(fileinfo.isfile) self.failUnlessTrue(fileinfo.exists) self.failUnlessFalse(fileinfo.isdir) self.failUnlessFalse(fileinfo.islink) self.failUnlessEqual(fileinfo.size, 10) # path at which nothing exists dnename = os.path.join(basedir, "doesnotexist") now_ns = fileutil.seconds_to_ns(time.time()) dneinfo = fileutil.get_pathinfo(dnename, now_ns=now_ns) self.failUnlessFalse(dneinfo.exists) self.failUnlessFalse(dneinfo.isfile) self.failUnlessFalse(dneinfo.isdir) self.failUnlessFalse(dneinfo.islink) self.failUnlessEqual(dneinfo.size, None) self.failUnlessEqual(dneinfo.mtime_ns, now_ns) self.failUnlessEqual(dneinfo.ctime_ns, now_ns)
def test_get_pathinfo_symlink(self): if not hasattr(os, 'symlink'): raise unittest.SkipTest("can't create symlinks on this platform") basedir = "util/FileUtil/test_get_pathinfo" fileutil.make_dirs(basedir) f = os.path.join(basedir, "1.txt") fileutil.write(f, b"a" * 10) # create a symlink pointing to 1.txt slname = os.path.join(basedir, "linkto1.txt") os.symlink(f, slname) symlinkinfo = fileutil.get_pathinfo(slname) self.failUnlessTrue(symlinkinfo.islink) self.failUnlessTrue(symlinkinfo.exists) self.failUnlessFalse(symlinkinfo.isfile) self.failUnlessFalse(symlinkinfo.isdir)
def do_update_db(written_abspath_u): filecap = item.file_node.get_uri() last_uploaded_uri = item.metadata.get('last_uploaded_uri', None) self._log("DOUPDATEDB %r" % written_abspath_u) last_downloaded_uri = filecap last_downloaded_timestamp = now written_pathinfo = get_pathinfo(written_abspath_u) if not written_pathinfo.exists and not item.metadata.get('deleted', False): raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u)) self._db.did_upload_version( item.relpath_u, item.metadata['version'], last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, written_pathinfo, ) self._count('objects_downloaded') item.set_status('success', self._clock.seconds()) return True
def do_update_db(written_abspath_u): filecap = item.file_node.get_uri() last_uploaded_uri = item.metadata.get('last_uploaded_uri', None) self._log("DOUPDATEDB %r" % written_abspath_u) last_downloaded_uri = filecap last_downloaded_timestamp = now written_pathinfo = get_pathinfo(written_abspath_u) if not written_pathinfo.exists and not item.metadata.get('deleted', False): raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u)) self._db.did_upload_version( item.relpath_u, item.metadata['version'], last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, written_pathinfo, ) self._count('objects_downloaded') item.set_status('success', self._clock.seconds()) return True
def _maybe_upload(ign, now=None): self._log("_maybe_upload: relpath_u=%r, now=%r" % (relpath_u, now)) if now is None: now = time.time() fp = self._get_filepath(relpath_u) pathinfo = get_pathinfo(unicode_from_filepath(fp)) self._log("about to remove %r from pending set %r" % (relpath_u, self._pending)) try: self._pending.remove(relpath_u) except KeyError: self._log("WRONG that %r wasn't in pending" % (relpath_u,)) encoded_path_u = magicpath.path2magic(relpath_u) if not pathinfo.exists: # FIXME merge this with the 'isfile' case. self._log("notified object %s disappeared (this is normal)" % quote_filepath(fp)) self._count('objects_disappeared') db_entry = self._db.get_db_entry(relpath_u) if db_entry is None: return False last_downloaded_timestamp = now # is this correct? if is_new_file(pathinfo, db_entry): new_version = db_entry.version + 1 else: self._log("Not uploading %r" % (relpath_u,)) self._count('objects_not_uploaded') return False metadata = { 'version': new_version, 'deleted': True, 'last_downloaded_timestamp': last_downloaded_timestamp, } if db_entry.last_downloaded_uri is not None: metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri empty_uploadable = Data("", self._client.convergence) d2 = self._upload_dirnode.add_file( encoded_path_u, empty_uploadable, metadata=metadata, overwrite=True, progress=item.progress, ) def _add_db_entry(filenode): filecap = filenode.get_uri() last_downloaded_uri = metadata.get('last_downloaded_uri', None) self._db.did_upload_version(relpath_u, new_version, filecap, last_downloaded_uri, last_downloaded_timestamp, pathinfo) self._count('files_uploaded') d2.addCallback(_add_db_entry) d2.addCallback(lambda ign: True) return d2 elif pathinfo.islink: self.warn("WARNING: cannot upload symlink %s" % quote_filepath(fp)) return False elif pathinfo.isdir: self._log("ISDIR") if not getattr(self._notifier, 'recursive_includes_new_subdirectories', False): self._notifier.watch(fp, mask=self.mask, callbacks=[self._notify], recursive=True) db_entry = self._db.get_db_entry(relpath_u) self._log("isdir dbentry %r" % (db_entry,)) if not is_new_file(pathinfo, db_entry): self._log("NOT A NEW FILE") return False uploadable = Data("", self._client.convergence) encoded_path_u += magicpath.path2magic(u"/") self._log("encoded_path_u = %r" % (encoded_path_u,)) upload_d = self._upload_dirnode.add_file( encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True, progress=item.progress, ) def _dir_succeeded(ign): self._log("created subdirectory %r" % (relpath_u,)) self._count('directories_created') def _dir_failed(f): self._log("failed to create subdirectory %r" % (relpath_u,)) return f upload_d.addCallbacks(_dir_succeeded, _dir_failed) upload_d.addCallback(lambda ign: self._scan(relpath_u)) upload_d.addCallback(lambda ign: True) return upload_d elif pathinfo.isfile: db_entry = self._db.get_db_entry(relpath_u) last_downloaded_timestamp = now if db_entry is None: new_version = 0 elif is_new_file(pathinfo, db_entry): new_version = db_entry.version + 1 else: self._log("Not uploading %r" % (relpath_u,)) self._count('objects_not_uploaded') return False metadata = { 'version': new_version, 'last_downloaded_timestamp': last_downloaded_timestamp, } if db_entry is not None and db_entry.last_downloaded_uri is not None: metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri uploadable = FileName(unicode_from_filepath(fp), self._client.convergence) d2 = self._upload_dirnode.add_file( encoded_path_u, uploadable, metadata=metadata, overwrite=True, progress=item.progress, ) def _add_db_entry(filenode): filecap = filenode.get_uri() last_downloaded_uri = metadata.get('last_downloaded_uri', None) self._db.did_upload_version(relpath_u, new_version, filecap, last_downloaded_uri, last_downloaded_timestamp, pathinfo) self._count('files_uploaded') return True d2.addCallback(_add_db_entry) return d2 else: self.warn("WARNING: cannot process special file %s" % quote_filepath(fp)) return False
def _maybe_upload(ign, now=None): self._log("_maybe_upload: relpath_u=%r, now=%r" % (relpath_u, now)) if now is None: now = time.time() fp = self._get_filepath(relpath_u) pathinfo = get_pathinfo(unicode_from_filepath(fp)) self._log("about to remove %r from pending set %r" % (relpath_u, self._pending)) try: self._pending.remove(relpath_u) except KeyError: self._log("WRONG that %r wasn't in pending" % (relpath_u,)) encoded_path_u = magicpath.path2magic(relpath_u) if not pathinfo.exists: # FIXME merge this with the 'isfile' case. self._log("notified object %s disappeared (this is normal)" % quote_filepath(fp)) self._count('objects_disappeared') db_entry = self._db.get_db_entry(relpath_u) if db_entry is None: return False last_downloaded_timestamp = now # is this correct? if is_new_file(pathinfo, db_entry): new_version = db_entry.version + 1 else: self._log("Not uploading %r" % (relpath_u,)) self._count('objects_not_uploaded') return False metadata = { 'version': new_version, 'deleted': True, 'last_downloaded_timestamp': last_downloaded_timestamp, } if db_entry.last_downloaded_uri is not None: metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri empty_uploadable = Data("", self._client.convergence) d2 = self._upload_dirnode.add_file( encoded_path_u, empty_uploadable, metadata=metadata, overwrite=True, progress=item.progress, ) def _add_db_entry(filenode): filecap = filenode.get_uri() last_downloaded_uri = metadata.get('last_downloaded_uri', None) self._db.did_upload_version(relpath_u, new_version, filecap, last_downloaded_uri, last_downloaded_timestamp, pathinfo) self._count('files_uploaded') d2.addCallback(_add_db_entry) d2.addCallback(lambda ign: True) return d2 elif pathinfo.islink: self.warn("WARNING: cannot upload symlink %s" % quote_filepath(fp)) return False elif pathinfo.isdir: self._log("ISDIR") if not getattr(self._notifier, 'recursive_includes_new_subdirectories', False): self._notifier.watch(fp, mask=self.mask, callbacks=[self._notify], recursive=True) db_entry = self._db.get_db_entry(relpath_u) self._log("isdir dbentry %r" % (db_entry,)) if not is_new_file(pathinfo, db_entry): self._log("NOT A NEW FILE") return False uploadable = Data("", self._client.convergence) encoded_path_u += magicpath.path2magic(u"/") self._log("encoded_path_u = %r" % (encoded_path_u,)) upload_d = self._upload_dirnode.add_file( encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True, progress=item.progress, ) def _dir_succeeded(ign): self._log("created subdirectory %r" % (relpath_u,)) self._count('directories_created') def _dir_failed(f): self._log("failed to create subdirectory %r" % (relpath_u,)) return f upload_d.addCallbacks(_dir_succeeded, _dir_failed) upload_d.addCallback(lambda ign: self._scan(relpath_u)) upload_d.addCallback(lambda ign: True) return upload_d elif pathinfo.isfile: db_entry = self._db.get_db_entry(relpath_u) last_downloaded_timestamp = now if db_entry is None: new_version = 0 elif is_new_file(pathinfo, db_entry): new_version = db_entry.version + 1 else: self._log("Not uploading %r" % (relpath_u,)) self._count('objects_not_uploaded') return False metadata = { 'version': new_version, 'last_downloaded_timestamp': last_downloaded_timestamp, } if db_entry is not None and db_entry.last_downloaded_uri is not None: metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri uploadable = FileName(unicode_from_filepath(fp), self._client.convergence) d2 = self._upload_dirnode.add_file( encoded_path_u, uploadable, metadata=metadata, overwrite=True, progress=item.progress, ) def _add_db_entry(filenode): filecap = filenode.get_uri() last_downloaded_uri = metadata.get('last_downloaded_uri', None) self._db.did_upload_version(relpath_u, new_version, filecap, last_downloaded_uri, last_downloaded_timestamp, pathinfo) self._count('files_uploaded') return True d2.addCallback(_add_db_entry) return d2 else: self.warn("WARNING: cannot process special file %s" % quote_filepath(fp)) return False