def Get(self):
tarball_result = self._tarball.Get()
if tarball_result.status_code != 200:
raise RietveldPatcherError(
'Failed to download tarball for issue %s patchset %s. Status: %s'
% (self._issue, self._patchset, tarball_result.status_code))
try:
tar = tarfile.open(fileobj=StringIO(tarball_result.content))
except tarfile.TarError as e:
raise RietveldPatcherError(
'Error loading tarball for issue %s patchset %s.' %
(self._issue, self._patchset))
self._value = {}
for path in self._files:
if self._base_path:
tar_path = 'b/%s/%s' % (self._base_path, path)
else:
tar_path = 'b/%s' % path
patched_file = None
try:
patched_file = tar.extractfile(tar_path)
data = patched_file.read()
except tarfile.TarError as e:
# Show appropriate error message in the unlikely case that the tarball
# is corrupted.
raise RietveldPatcherError(
'Error extracting tarball for issue %s patchset %s file %s.'
% (self._issue, self._patchset, tar_path))
except KeyError as e:
raise FileNotFoundError(
'File %s not found in the tarball for issue %s patchset %s'
% (tar_path, self._issue, self._patchset))
finally:
if patched_file:
patched_file.close()
if self._binary:
self._value[path] = data
else:
self._value[path] = ToUnicode(data)
return self._value
def _CreateStatInfo(path):
try:
path_mtime = os.stat(path).st_mtime
if os.path.isdir(path):
child_versions = dict(
(_ConvertFromFilepath(filename),
os.stat(os.path.join(path, filename)).st_mtime)
for filename in os.listdir(path))
# This file system stat mimics subversion, where the stat of directories
# is max(file stats). That means we need to recursively check the whole
# file system tree :\ so approximate that by just checking this dir.
version = max([path_mtime] + child_versions.values())
else:
child_versions = None
version = path_mtime
return StatInfo(version, child_versions)
except OSError as e:
raise FileNotFoundError('os.stat failed for %s: %s' % (path, e))
def get_content(result):
if result.status_code == 404:
if skip_not_found:
return None
raise FileNotFoundError(
'Got 404 when fetching %s for Get from %s' %
(path, self._base_url))
if result.status_code == 429:
logging.warning(
'Access throttled when fetching %s for Get from %s' %
(path, self._base_url))
raise FileSystemThrottledError(
'Access throttled when fetching %s for Get from %s' %
(path, self._base_url))
if result.status_code != 200:
raise FileSystemError(
'Got %s when fetching %s for Get from %s, content %s' %
(result.status_code, path, self._base_url, result.content))
return result.content
def ListDir(path, revision='HEAD'):
'''Retrieves a directory listing for the specified path and optional revision
(default is HEAD.) Returns a list of objects with the following properties:
|type|: the type of entry; 'blob' (file) or 'tree' (directory)
|id|: the hash of the directory entry. This is either tree hash or blob hash
|name|: the name of the directory entry.
'''
from subprocess import CalledProcessError
try:
ref = '%s:%s' % (ParseRevision(revision), path.lstrip('/'))
listing = RunGit('ls-tree', [ref]).rstrip().splitlines()
except CalledProcessError:
raise FileNotFoundError('%s not found in revision %s' %
(path, revision))
def parse_line(line):
return re.match(_LS_TREE_REGEX, line).groupdict()
return map(parse_line, listing)
def fetch(self, url):
path = os.path.join(os.pardir, self._base_pattern.match(url).group(1))
if self._IsDir(path):
html = ['<table><tbody><tr>...</tr>']
for f in self._ListDir(path):
if f.startswith('.'):
continue
html.append('<tr>')
html.append(' <td><a>%s%s</a></td>' % (
f, '/' if self._IsDir(os.path.join(path, f)) else ''))
stat = self._Stat(os.path.join(path, f))
html.append(' <td><a><strong>%s</strong></a></td>' % stat)
html.append('<td></td><td></td><td></td>')
html.append('</tr>')
html.append('</tbody></table>')
return '\n'.join(html)
try:
return self._ReadFile(path)
except IOError as e:
raise FileNotFoundError('Reading %s failed: %s' % (path, e))
def Stat(self, path):
version = self._patcher.GetVersion()
assert version is not None
version = 'patched_%s' % version
directory, filename = path.rsplit('/', 1)
added, deleted, modified = self._GetDirectoryListingFromPatch(
directory + '/')
if len(added) > 0:
# There are new files added. It's possible (if |directory| is new) that
# self._base_file_system.Stat will throw an exception.
try:
stat_info = self._PatchStat(
self._base_file_system.Stat(directory + '/'),
version,
added,
deleted,
modified)
except FileNotFoundError:
stat_info = StatInfo(
version,
dict((child, version) for child in added + modified))
elif len(deleted) + len(modified) > 0:
# No files were added.
stat_info = self._PatchStat(self._base_file_system.Stat(directory + '/'),
version,
added,
deleted,
modified)
else:
# No changes are made in this directory.
return self._base_file_system.Stat(path)
if stat_info.child_versions is not None:
if filename:
if filename in stat_info.child_versions:
stat_info = StatInfo(stat_info.child_versions[filename])
else:
raise FileNotFoundError('%s was not in child versions' % filename)
return stat_info
def fetch(self, url):
path = os.path.join(os.pardir, self._base_pattern.match(url).group(1))
if self._IsDir(path):
html = [
'<html><td>Directory revision:</td><td><a>%s</a></td>' %
self._Stat(path)
]
for f in self._ListDir(path):
if f.startswith('.'):
continue
html.append('<td><a name="%s"></a></td>' % f)
stat = self._Stat(os.path.join(path, f))
html.append(
'<td><a title="%s"><strong>%s</strong></a></td>' %
('dir' if self._IsDir(os.path.join(path, f)) else 'file',
stat))
html.append('</html>')
return '\n'.join(html)
try:
return self._ReadFile(path)
except IOError:
raise FileNotFoundError(path)
def apply_(tarball_result):
if tarball_result.status_code != 200:
raise RietveldPatcherError(
'Failed to download tarball for issue %s patchset %s. Status: %s'
% (self._issue, version, tarball_result.status_code))
try:
tar = tarfile.open(fileobj=StringIO(tarball_result.content))
except tarfile.TarError as e:
raise RietveldPatcherError(
'Error loading tarball for issue %s patchset %s.' %
(self._issue, version))
value = {}
for path in paths:
tar_path = 'b/%s' % path
patched_file = None
try:
patched_file = tar.extractfile(tar_path)
data = patched_file.read()
except tarfile.TarError as e:
# Show appropriate error message in the unlikely case that the tarball
# is corrupted.
raise RietveldPatcherError(
'Error extracting tarball for issue %s patchset %s file %s.'
% (self._issue, version, tar_path))
except KeyError as e:
raise FileNotFoundError(
'File %s not found in the tarball for issue %s patchset %s'
% (tar_path, self._issue, version))
finally:
if patched_file:
patched_file.close()
value[path] = data
return value
def resolve():
value = {}
for path, future in fetches:
try:
result = future.Get()
except Exception as e:
exc_type = FileNotFoundError if IsDownloadError(
e) else FileSystemError
raise exc_type(
'%s fetching %s for Get: %s' %
(type(e).__name__, path, traceback.format_exc()))
if result.status_code == 404:
raise FileNotFoundError(
'Got 404 when fetching %s for Get, content %s' %
(path, result.content))
if result.status_code != 200:
raise FileSystemError(
'Got %s when fetching %s for Get, content %s' %
(result.status_code, path, result.content))
if path.endswith('/'):
value[path] = list_dir(result.content)
else:
value[path] = result.content
return value
def Get(self):
for path, future in self._fetches:
try:
result = future.Get()
except Exception as e:
raise FileSystemError('Error fetching %s for Get: %s' %
(path, traceback.format_exc()))
if result.status_code == 404:
raise FileNotFoundError(
'Got 404 when fetching %s for Get, content %s' %
(path, result.content))
if result.status_code != 200:
raise FileSystemError(
'Got %s when fetching %s for Get, content %s' %
(result.status_code, path, result.content))
if path.endswith('/'):
self._value[path] = self._ListDir(result.content)
else:
self._value[path] = result.content
if self._error is not None:
raise self._error
return self._value
def raise_file_not_found():
raise FileNotFoundError('No paths can be found, repository is empty')
def Stat(self, path):
if not path.endswith('/'):
raise FileNotFoundError('EmptyDirFileSystem cannot stat %s' % path)
return StatInfo(0, child_versions=[])
def get(self, key, disable_refs=False):
if key not in self._json:
raise FileNotFoundError(key)
return self._json[key]
def Stat(self, path):
raise FileNotFoundError('File system is offline, cannot read %s' %
path)
def get(self, key, disable_refs=False):
assert disable_refs, 'ReferenceResolve should be disabling refs'
if key not in self._json:
raise FileNotFoundError(key)
return self._json[key]
def raise_file_not_found():
raise FileNotFoundError('File system is offline, cannot read %s' %
paths)
def get(self, key):
if key not in self._json:
raise FileNotFoundError(key)
return self._json[key]
def Stat(self, path):
if not IsDirectory(path):
raise FileNotFoundError('EmptyDirFileSystem cannot stat %s' % path)
return StatInfo(0, child_versions=[])
def raise_cache_miss(path):
raise FileNotFoundError('Got cache miss when trying to stat %s' % path)
def raise_file_not_found():
raise FileNotFoundError('Files are removed from the patch.')
def _ReadFile(filename):
try:
with open(filename, 'rb') as f:
return f.read()
except IOError as e:
raise FileNotFoundError('Read failed for %s: %s' % (filename, e))
def Read(self, paths, binary=False):
raise FileNotFoundError('File system is offline, cannot read %s' %
paths)
def _GetSuccessResponse(self, request_path, server_instance):
'''Returns the Response from trying to render |path| with
|server_instance|. If |path| isn't found then a FileNotFoundError will be
raised, such that the only responses that will be returned from this method
are Ok and Redirect.
'''
content_provider, serve_from, path = (
server_instance.content_providers.GetByServeFrom(request_path))
assert content_provider, 'No ContentProvider found for %s' % path
redirect = Redirector(server_instance.compiled_fs_factory,
content_provider.file_system).Redirect(
self._request.host, path)
if redirect is not None:
# Absolute redirects stay absolute, relative redirects are relative to
# |serve_from|; all redirects eventually need to be *served* as absolute.
if not redirect.startswith('/'):
redirect = '/' + posixpath.join(serve_from, redirect)
return Response.Redirect(redirect, permanent=False)
canonical_path = content_provider.GetCanonicalPath(path)
if canonical_path != path:
redirect_path = posixpath.join(serve_from, canonical_path)
return Response.Redirect('/' + redirect_path, permanent=False)
if request_path.endswith('/'):
# Directory request hasn't been redirected by now. Default behaviour is
# to redirect as though it were a file.
return Response.Redirect('/' + request_path.rstrip('/'),
permanent=False)
if not path:
# Empty-path request hasn't been redirected by now. It doesn't exist.
raise FileNotFoundError('Empty path')
content_and_type = content_provider.GetContentAndType(path).Get()
if not content_and_type.content:
logging.error('%s had empty content' % path)
content = content_and_type.content
if isinstance(content, Handlebar):
template_content, template_warnings = (
server_instance.template_renderer.Render(
content, self._request))
# HACK: the site verification file (google2ed...) doesn't have a title.
content, doc_warnings = server_instance.document_renderer.Render(
template_content,
path,
render_title=path != SITE_VERIFICATION_FILE)
warnings = template_warnings + doc_warnings
if warnings:
sep = '\n - '
logging.warning('Rendering %s:%s%s' %
(path, sep, sep.join(warnings)))
# Content was dynamic. The new etag is a hash of the content.
etag = None
elif content_and_type.version is not None:
# Content was static. The new etag is the version of the content. Hash it
# to make sure it's valid.
etag = '"%s"' % hashlib.md5(str(
content_and_type.version)).hexdigest()
else:
# Sometimes non-dynamic content does not have a version, for example
# .zip files. The new etag is a hash of the content.
etag = None
content_type = content_and_type.content_type
if isinstance(content, unicode):
content = content.encode('utf-8')
content_type += '; charset=utf-8'
if etag is None:
# Note: we're using md5 as a convenient and fast-enough way to identify
# content. It's not intended to be cryptographic in any way, and this
# is *not* what etags is for. That's what SSL is for, this is unrelated.
etag = '"%s"' % hashlib.md5(content).hexdigest()
headers = _MakeHeaders(content_type, etag=etag)
if etag == self._request.headers.get('If-None-Match'):
return Response.NotModified('Not Modified', headers=headers)
return Response.Ok(content, headers=headers)
def _CreateAPIModel(path, data):
schema = ProcessSchema(path, data)[0]
if not schema:
raise FileNotFoundError('No schema for %s' % path)
return Namespace(schema, schema['namespace'])