def add_stream(self, name, stream): key, version = self._parse_name(name) existing_verion = self.versions[key] if version is not None and existing_verion > version: return versioned_name(key, existing_verion) if version is None: version = max(self.versions[key] + 1, int(time.time())) data = b'' while True: record = stream.read() if not record: break data += record self.data[key] = data self.versions[key] = version return versioned_name(key, version)
def add_stream(self, name, stream): key, version = self._parse_name(name) existing_verion = self.versions[key] if version is not None and existing_verion > version: return versioned_name(key, existing_verion) if version is None: version = max(self.versions[key] + 1, int(time.time())) data = b'' while True: record = stream.read() if not record: break data += record self.data[key] = data self.versions[key] = version return versioned_name(key, version)
def get_stream(self, name): url, version = self._parse_name(name) response = requests.get(url, stream=True) response.raise_for_status() remote_version = self._parse_last_modified(response) if version is not None and remote_version is not None \ and version != remote_version: raise FiletrackerError("Version %s not available. Server has %s" \ % (name, remote_version)) name, version = split_name(name) stream = _FileLikeFromResponse(response) return stream, versioned_name(name, remote_version)
def get_stream(self, name): url, version = self._parse_name(name) response = requests.get(url, stream=True) response.raise_for_status() remote_version = self._parse_last_modified(response) if version is not None and remote_version is not None \ and version != remote_version: raise FiletrackerError("Version %s not available. Server has %s" \ % (name, remote_version)) name, version = split_name(name) stream = _FileLikeFromResponse(response) return stream, versioned_name(name, remote_version)
def list_files(self): result = [] for root, _dirs, files in os.walk(self.dir): for basename in files: relative_dir = os.path.relpath(root, self.dir) store_dir = os.path.normpath(os.path.join('/', relative_dir)) name = os.path.join(store_dir, basename) path, version = self._parse_name(name) file_stat = os.lstat(path) vname = versioned_name(name, self.file_version(name)) result.append( DataStore.FileInfoEntry(name=vname, mtime=file_stat.st_mtime, size=file_stat.st_size)) return result
def list_files(self): result = [] for root, _dirs, files in os.walk(self.dir): for basename in files: relative_dir = os.path.relpath(root, self.dir) store_dir = os.path.normpath( os.path.join('/', relative_dir)) name = os.path.join(store_dir, basename) path, version = self._parse_name(name) file_stat = os.lstat(path) vname = versioned_name(name, self.file_version(name)) result.append( DataStore.FileInfoEntry(name=vname, mtime=file_stat.st_mtime, size=file_stat.st_size)) return result
def get_stream(self, name, force_refresh=False, serve_from_cache=False): """Retrieves file identified by ``name`` in streaming mode. Works like :meth:`get_file`, except that returns a tuple (file-like object, versioned name). When both remote_store and local_store are present, serve_from_cache can be used to ensure that the file will be downloaded and served from a local cache. If a full version is specified and the file exists in the cache a file will be always served locally. """ uname, version = split_name(name) lock = None if self.local_store: lock = self.lock_manager.lock_for(uname) lock.lock_shared() try: if not self.remote_store or (version is not None and not force_refresh): try: if self.local_store and self.local_store.exists(name): return self.local_store.get_stream(name) except Exception: if self.remote_store: logger.warning("Error getting '%s' from local store", name, exc_info=True) else: raise if self.remote_store: if self.local_store and serve_from_cache: if version is None: version = self.remote_store.file_version(name) if version: name = versioned_name(uname, version) if force_refresh or not self.local_store.exists(name): (stream, vname) = self.remote_store.get_stream(name) name = self.local_store.add_stream(vname, stream) return self.local_store.get_stream(name) return self.remote_store.get_stream(name) raise FiletrackerError("File not available: %s" % name) finally: if lock: lock.close()
def get_stream(self, name, force_refresh=False, serve_from_cache=False): """Retrieves file identified by ``name`` in streaming mode. Works like :meth:`get_file`, except that returns a tuple (file-like object, versioned name). When both remote_store and local_store are present, serve_from_cache can be used to ensure that the file will be downloaded and served from a local cache. If a full version is specified and the file exists in the cache a file will be always served locally. """ uname, version = split_name(name) lock = None if self.local_store: lock = self.lock_manager.lock_for(uname) lock.lock_shared() try: if not self.remote_store or (version is not None and not force_refresh): try: if self.local_store and self.local_store.exists(name): return self.local_store.get_stream(name) except Exception: if self.remote_store: logger.warning("Error getting '%s' from local store", name, exc_info=True) else: raise if self.remote_store: if self.local_store and serve_from_cache: if version is None: version = self.remote_store.file_version(name) if version: name = versioned_name(uname, version) if force_refresh or not self.local_store.exists(name): (stream, vname) = self.remote_store.get_stream(name) name = self.local_store.add_stream(vname, stream) return self.local_store.get_stream(name) return self.remote_store.get_stream(name) raise FiletrackerError("File not available: %s" % name) finally: if lock: lock.close()
def add_file(self, name, filename, compress_hint=True): url, version = self._parse_name(name) if self._has_capability(SERVER_ACCEPTS_SHA256_DIGEST): sha = file_digest(filename) else: sha = '' headers = {'SHA256-Checksum': sha} # Important detail: this upload is streaming. # http://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads with open(filename, 'rb') as f: if (compress_hint and self._has_capability(SERVER_ACCEPTS_GZIP)): # Unfortunately it seems a temporary file is required here. # Our server requires Content-Length to be present, because # some WSGI implementations (among others the one used in # our tests) are not required to support EOF (instead the # user is required to not read beyond content length, # but that cannot be done if we don't know the content # length). As content length is required for the tests to # work, we need to send it, and to be able to compute it we # need to temporarily store the compressed data before # sending. It can be stored in memory or in a temporary file # and a temporary file seems to be a more suitable choice. with tempfile.TemporaryFile() as tmp: with gzip.GzipFile(fileobj=tmp, mode='wb') as gz: shutil.copyfileobj(f, gz) tmp.seek(0) headers['Content-Encoding'] = 'gzip' headers['Logical-Size'] = str(os.stat(filename).st_size) response = self._put_file(url, version, tmp, headers) else: response = self._put_file(url, version, f, headers) name, version = split_name(name) return versioned_name(name, self._parse_last_modified(response))
def add_file(self, name, filename, compress_hint=True): url, version = self._parse_name(name) headers = {} if (compress_hint and self._has_capability(SERVER_ACCEPTS_SHA256_DIGEST)): headers['SHA256-Checksum'] = file_digest(filename) # Important detail: this upload is streaming. # http://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads with open(filename, 'rb') as f: if (compress_hint and self._has_capability(SERVER_ACCEPTS_GZIP)): # Unfortunately it seems a temporary file is required here. # Our server requires Content-Length to be present, because # some WSGI implementations (among others the one used in # our tests) are not required to support EOF (instead the # user is required to not read beyond content length, # but that cannot be done if we don't know the content # length). As content length is required for the tests to # work, we need to send it, and to be able to compute it we # need to temporarily store the compressed data before # sending. It can be stored in memory or in a temporary file # and a temporary file seems to be a more suitable choice. with tempfile.TemporaryFile() as tmp: with gzip.GzipFile(fileobj=tmp, mode='wb') as gz: shutil.copyfileobj(f, gz) tmp.seek(0) headers['Content-Encoding'] = 'gzip' headers['Logical-Size'] = str(os.stat(filename).st_size) response = self._put_file(url, version, tmp, headers) else: response = self._put_file(url, version, f, headers) name, version = split_name(name) return versioned_name(name, self._parse_last_modified(response))
def get_stream(self, name): path, version = self._parse_name(name) if not os.path.exists(path): raise FiletrackerError("File not found: " + path) return open(path, 'rb'), versioned_name(name, _file_version(path))
def add_stream(self, name, stream): path, version = self._parse_name(name) return versioned_name( split_name(name)[0], _save_stream(path, stream, version))
def get_stream(self, name): key, version = self._parse_name(name) if version is not None and self.versions[key] != version: raise KeyError("Version %s of %s not found" % (version, key)) return BytesIO(self.data[key]), versioned_name(key, self.versions[key])
def get_stream(self, name): key, version = self._parse_name(name) if version is not None and self.versions[key] != version: raise KeyError("Version %s of %s not found" % (version, key)) return BytesIO(self.data[key]), \ versioned_name(key, self.versions[key])
def get_stream(self, name): path, version = self._parse_name(name) if not os.path.exists(path): raise FiletrackerError("File not found: " + path) return open(path, 'rb'), versioned_name(name, _file_version(path))
def add_stream(self, name, stream): path, version = self._parse_name(name) return versioned_name( split_name(name)[0], _save_stream(path, stream, version))