def Get(self): tarball_result = self._tarball.Get() if tarball_result.status_code != 200: raise RietveldPatcherError( 'Failed to download tarball for issue %s patchset %s. Status: %s' % (self._issue, self._patchset, tarball_result.status_code)) try: tar = tarfile.open(fileobj=StringIO(tarball_result.content)) except tarfile.TarError as e: raise RietveldPatcherError( 'Error loading tarball for issue %s patchset %s.' % (self._issue, self._patchset)) self._value = {} for path in self._files: if self._base_path: tar_path = 'b/%s/%s' % (self._base_path, path) else: tar_path = 'b/%s' % path patched_file = None try: patched_file = tar.extractfile(tar_path) data = patched_file.read() except tarfile.TarError as e: # Show appropriate error message in the unlikely case that the tarball # is corrupted. raise RietveldPatcherError( 'Error extracting tarball for issue %s patchset %s file %s.' % (self._issue, self._patchset, tar_path)) except KeyError as e: raise FileNotFoundError( 'File %s not found in the tarball for issue %s patchset %s' % (tar_path, self._issue, self._patchset)) finally: if patched_file: patched_file.close() if self._binary: self._value[path] = data else: self._value[path] = ToUnicode(data) return self._value
def _CreateStatInfo(path): try: path_mtime = os.stat(path).st_mtime if os.path.isdir(path): child_versions = dict( (_ConvertFromFilepath(filename), os.stat(os.path.join(path, filename)).st_mtime) for filename in os.listdir(path)) # This file system stat mimics subversion, where the stat of directories # is max(file stats). That means we need to recursively check the whole # file system tree :\ so approximate that by just checking this dir. version = max([path_mtime] + child_versions.values()) else: child_versions = None version = path_mtime return StatInfo(version, child_versions) except OSError as e: raise FileNotFoundError('os.stat failed for %s: %s' % (path, e))
def get_content(result): if result.status_code == 404: if skip_not_found: return None raise FileNotFoundError( 'Got 404 when fetching %s for Get from %s' % (path, self._base_url)) if result.status_code == 429: logging.warning( 'Access throttled when fetching %s for Get from %s' % (path, self._base_url)) raise FileSystemThrottledError( 'Access throttled when fetching %s for Get from %s' % (path, self._base_url)) if result.status_code != 200: raise FileSystemError( 'Got %s when fetching %s for Get from %s, content %s' % (result.status_code, path, self._base_url, result.content)) return result.content
def ListDir(path, revision='HEAD'): '''Retrieves a directory listing for the specified path and optional revision (default is HEAD.) Returns a list of objects with the following properties: |type|: the type of entry; 'blob' (file) or 'tree' (directory) |id|: the hash of the directory entry. This is either tree hash or blob hash |name|: the name of the directory entry. ''' from subprocess import CalledProcessError try: ref = '%s:%s' % (ParseRevision(revision), path.lstrip('/')) listing = RunGit('ls-tree', [ref]).rstrip().splitlines() except CalledProcessError: raise FileNotFoundError('%s not found in revision %s' % (path, revision)) def parse_line(line): return re.match(_LS_TREE_REGEX, line).groupdict() return map(parse_line, listing)
def fetch(self, url): path = os.path.join(os.pardir, self._base_pattern.match(url).group(1)) if self._IsDir(path): html = ['<table><tbody><tr>...</tr>'] for f in self._ListDir(path): if f.startswith('.'): continue html.append('<tr>') html.append(' <td><a>%s%s</a></td>' % ( f, '/' if self._IsDir(os.path.join(path, f)) else '')) stat = self._Stat(os.path.join(path, f)) html.append(' <td><a><strong>%s</strong></a></td>' % stat) html.append('<td></td><td></td><td></td>') html.append('</tr>') html.append('</tbody></table>') return '\n'.join(html) try: return self._ReadFile(path) except IOError as e: raise FileNotFoundError('Reading %s failed: %s' % (path, e))
def Stat(self, path): version = self._patcher.GetVersion() assert version is not None version = 'patched_%s' % version directory, filename = path.rsplit('/', 1) added, deleted, modified = self._GetDirectoryListingFromPatch( directory + '/') if len(added) > 0: # There are new files added. It's possible (if |directory| is new) that # self._base_file_system.Stat will throw an exception. try: stat_info = self._PatchStat( self._base_file_system.Stat(directory + '/'), version, added, deleted, modified) except FileNotFoundError: stat_info = StatInfo( version, dict((child, version) for child in added + modified)) elif len(deleted) + len(modified) > 0: # No files were added. stat_info = self._PatchStat(self._base_file_system.Stat(directory + '/'), version, added, deleted, modified) else: # No changes are made in this directory. return self._base_file_system.Stat(path) if stat_info.child_versions is not None: if filename: if filename in stat_info.child_versions: stat_info = StatInfo(stat_info.child_versions[filename]) else: raise FileNotFoundError('%s was not in child versions' % filename) return stat_info
def fetch(self, url): path = os.path.join(os.pardir, self._base_pattern.match(url).group(1)) if self._IsDir(path): html = [ '<html><td>Directory revision:</td><td><a>%s</a></td>' % self._Stat(path) ] for f in self._ListDir(path): if f.startswith('.'): continue html.append('<td><a name="%s"></a></td>' % f) stat = self._Stat(os.path.join(path, f)) html.append( '<td><a title="%s"><strong>%s</strong></a></td>' % ('dir' if self._IsDir(os.path.join(path, f)) else 'file', stat)) html.append('</html>') return '\n'.join(html) try: return self._ReadFile(path) except IOError: raise FileNotFoundError(path)
def apply_(tarball_result): if tarball_result.status_code != 200: raise RietveldPatcherError( 'Failed to download tarball for issue %s patchset %s. Status: %s' % (self._issue, version, tarball_result.status_code)) try: tar = tarfile.open(fileobj=StringIO(tarball_result.content)) except tarfile.TarError as e: raise RietveldPatcherError( 'Error loading tarball for issue %s patchset %s.' % (self._issue, version)) value = {} for path in paths: tar_path = 'b/%s' % path patched_file = None try: patched_file = tar.extractfile(tar_path) data = patched_file.read() except tarfile.TarError as e: # Show appropriate error message in the unlikely case that the tarball # is corrupted. raise RietveldPatcherError( 'Error extracting tarball for issue %s patchset %s file %s.' % (self._issue, version, tar_path)) except KeyError as e: raise FileNotFoundError( 'File %s not found in the tarball for issue %s patchset %s' % (tar_path, self._issue, version)) finally: if patched_file: patched_file.close() value[path] = data return value
def resolve(): value = {} for path, future in fetches: try: result = future.Get() except Exception as e: exc_type = FileNotFoundError if IsDownloadError( e) else FileSystemError raise exc_type( '%s fetching %s for Get: %s' % (type(e).__name__, path, traceback.format_exc())) if result.status_code == 404: raise FileNotFoundError( 'Got 404 when fetching %s for Get, content %s' % (path, result.content)) if result.status_code != 200: raise FileSystemError( 'Got %s when fetching %s for Get, content %s' % (result.status_code, path, result.content)) if path.endswith('/'): value[path] = list_dir(result.content) else: value[path] = result.content return value
def Get(self): for path, future in self._fetches: try: result = future.Get() except Exception as e: raise FileSystemError('Error fetching %s for Get: %s' % (path, traceback.format_exc())) if result.status_code == 404: raise FileNotFoundError( 'Got 404 when fetching %s for Get, content %s' % (path, result.content)) if result.status_code != 200: raise FileSystemError( 'Got %s when fetching %s for Get, content %s' % (result.status_code, path, result.content)) if path.endswith('/'): self._value[path] = self._ListDir(result.content) else: self._value[path] = result.content if self._error is not None: raise self._error return self._value
def raise_file_not_found(): raise FileNotFoundError('No paths can be found, repository is empty')
def Stat(self, path): if not path.endswith('/'): raise FileNotFoundError('EmptyDirFileSystem cannot stat %s' % path) return StatInfo(0, child_versions=[])
def get(self, key, disable_refs=False): if key not in self._json: raise FileNotFoundError(key) return self._json[key]
def Stat(self, path): raise FileNotFoundError('File system is offline, cannot read %s' % path)
def get(self, key, disable_refs=False): assert disable_refs, 'ReferenceResolve should be disabling refs' if key not in self._json: raise FileNotFoundError(key) return self._json[key]
def raise_file_not_found(): raise FileNotFoundError('File system is offline, cannot read %s' % paths)
def get(self, key): if key not in self._json: raise FileNotFoundError(key) return self._json[key]
def Stat(self, path): if not IsDirectory(path): raise FileNotFoundError('EmptyDirFileSystem cannot stat %s' % path) return StatInfo(0, child_versions=[])
def raise_cache_miss(path): raise FileNotFoundError('Got cache miss when trying to stat %s' % path)
def raise_file_not_found(): raise FileNotFoundError('Files are removed from the patch.')
def _ReadFile(filename): try: with open(filename, 'rb') as f: return f.read() except IOError as e: raise FileNotFoundError('Read failed for %s: %s' % (filename, e))
def Read(self, paths, binary=False): raise FileNotFoundError('File system is offline, cannot read %s' % paths)
def _GetSuccessResponse(self, request_path, server_instance): '''Returns the Response from trying to render |path| with |server_instance|. If |path| isn't found then a FileNotFoundError will be raised, such that the only responses that will be returned from this method are Ok and Redirect. ''' content_provider, serve_from, path = ( server_instance.content_providers.GetByServeFrom(request_path)) assert content_provider, 'No ContentProvider found for %s' % path redirect = Redirector(server_instance.compiled_fs_factory, content_provider.file_system).Redirect( self._request.host, path) if redirect is not None: # Absolute redirects stay absolute, relative redirects are relative to # |serve_from|; all redirects eventually need to be *served* as absolute. if not redirect.startswith('/'): redirect = '/' + posixpath.join(serve_from, redirect) return Response.Redirect(redirect, permanent=False) canonical_path = content_provider.GetCanonicalPath(path) if canonical_path != path: redirect_path = posixpath.join(serve_from, canonical_path) return Response.Redirect('/' + redirect_path, permanent=False) if request_path.endswith('/'): # Directory request hasn't been redirected by now. Default behaviour is # to redirect as though it were a file. return Response.Redirect('/' + request_path.rstrip('/'), permanent=False) if not path: # Empty-path request hasn't been redirected by now. It doesn't exist. raise FileNotFoundError('Empty path') content_and_type = content_provider.GetContentAndType(path).Get() if not content_and_type.content: logging.error('%s had empty content' % path) content = content_and_type.content if isinstance(content, Handlebar): template_content, template_warnings = ( server_instance.template_renderer.Render( content, self._request)) # HACK: the site verification file (google2ed...) doesn't have a title. content, doc_warnings = server_instance.document_renderer.Render( template_content, path, render_title=path != SITE_VERIFICATION_FILE) warnings = template_warnings + doc_warnings if warnings: sep = '\n - ' logging.warning('Rendering %s:%s%s' % (path, sep, sep.join(warnings))) # Content was dynamic. The new etag is a hash of the content. etag = None elif content_and_type.version is not None: # Content was static. The new etag is the version of the content. Hash it # to make sure it's valid. etag = '"%s"' % hashlib.md5(str( content_and_type.version)).hexdigest() else: # Sometimes non-dynamic content does not have a version, for example # .zip files. The new etag is a hash of the content. etag = None content_type = content_and_type.content_type if isinstance(content, unicode): content = content.encode('utf-8') content_type += '; charset=utf-8' if etag is None: # Note: we're using md5 as a convenient and fast-enough way to identify # content. It's not intended to be cryptographic in any way, and this # is *not* what etags is for. That's what SSL is for, this is unrelated. etag = '"%s"' % hashlib.md5(content).hexdigest() headers = _MakeHeaders(content_type, etag=etag) if etag == self._request.headers.get('If-None-Match'): return Response.NotModified('Not Modified', headers=headers) return Response.Ok(content, headers=headers)
def _CreateAPIModel(path, data): schema = ProcessSchema(path, data)[0] if not schema: raise FileNotFoundError('No schema for %s' % path) return Namespace(schema, schema['namespace'])