def send(self, req, **kwargs): # pylint: disable=unused-argument
"""Return the file specified by the given request
@type req: C{PreparedRequest}
@todo: Should I bother filling `response.headers` and processing
If-Modified-Since and friends using `os.stat`?
"""
path = os.path.normcase(os.path.normpath(url2pathname(req.path_url)))
response = requests.Response()
response.status_code, response.reason = self._chkpath(req.method, path)
if response.status_code == 200 and req.method.lower() != 'head':
try:
response.raw = open(path, 'rb')
except (OSError, IOError) as err:
response.status_code = 500
response.reason = str(err)
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
response.request = req
response.connection = self
return response
def make_best_resolver(self, param):
if isinstance(param, list):
path = param[0]
arg = param[1]
else:
path = param
arg = None
url_parts = urlsplit(path)
if url_parts.scheme == 'redis':
logging.debug('Adding Redis Index: ' + path)
return RedisResolver(path, arg)
if url_parts.scheme == 'file':
path = url_parts.path
path = url2pathname(path)
if os.path.isfile(path):
logging.debug('Adding Path Index: ' + path)
return PathIndexResolver(path)
# non-file paths always treated as prefix for now
else:
logging.debug('Adding Archive Path Source: ' + path)
return PrefixResolver(path, arg)
def get_filename(self, basename):
"""
Returns full path to a file, for example:
get_filename('css/one.css') -> '/full/path/to/static/css/one.css'
"""
filename = None
# First try finding the file using the storage class.
# This is skipped in DEBUG mode as files might be outdated in
# compressor's final destination (COMPRESS_ROOT) during development
if not settings.DEBUG:
try:
# call path first so remote storages don't make it to exists,
# which would cause network I/O
filename = self.storage.path(basename)
if not self.storage.exists(basename):
filename = None
except NotImplementedError:
# remote storages don't implement path, access the file locally
if compressor_file_storage.exists(basename):
filename = compressor_file_storage.path(basename)
# secondly try to find it with staticfiles
if not filename and self.finders:
filename = self.finders.find(url2pathname(basename))
if filename:
return filename
# or just raise an exception as the last resort
raise UncompressableFileError(
"'%s' could not be found in the COMPRESS_ROOT '%s'%s" %
(basename, settings.COMPRESS_ROOT,
self.finders and " or with staticfiles." or "."))
def load(self, url, offset=0, length=-1):
"""
Load a file-like reader from the local file system
"""
# if starting with . or /, can only be a file path..
file_only = url.startswith(('/', '.'))
# convert to filename
if url.startswith('file://'):
file_only = True
url = url2pathname(url[len('file://'):])
try:
# first, try as file
afile = open(url, 'rb')
except IOError:
if file_only:
raise
# then, try as package.path/file
pkg_split = url.split('/', 1)
if len(pkg_split) == 1:
raise
afile = pkg_resources.resource_stream(pkg_split[0], pkg_split[1])
if offset > 0:
afile.seek(offset)
if length >= 0:
return LimitReader(afile, length)
else:
return afile
def send(self, req, **kwargs): # pylint: disable=unused-argument
"""Return the file specified by the given request
@type req: C{PreparedRequest}
@todo: Should I bother filling `response.headers` and processing
If-Modified-Since and friends using `os.stat`?
"""
path = os.path.normcase(os.path.normpath(url2pathname(req.path_url)))
response = requests.Response()
response.status_code, response.reason = self._chkpath(req.method, path)
if response.status_code == 200 and req.method.lower() != 'head':
try:
response.raw = open(path, 'rb')
except (OSError, IOError) as err:
response.status_code = 500
response.reason = str(err)
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
response.request = req
response.connection = self
return response
def make_best_resolver(self, param):
if isinstance(param, list):
path = param[0]
arg = param[1]
else:
path = param
arg = None
url_parts = urlsplit(path)
if url_parts.scheme == 'redis':
logging.debug('Adding Redis Index: ' + path)
return RedisResolver(path, arg)
if url_parts.scheme == 'file':
path = url_parts.path
path = url2pathname(path)
if os.path.isfile(path):
logging.debug('Adding Path Index: ' + path)
return PathIndexResolver(path)
# non-file paths always treated as prefix for now
else:
logging.debug('Adding Archive Path Source: ' + path)
return PrefixResolver(path, arg)
def test_url_parsing(self):
obj = urls.URLTransformer('http://some-site.com:80/path/#frag?query')
obj.default_filename = 'index.html'
obj._unique_fn_required = False
self.assertEqual(obj.original_url,
'http://some-site.com:80/path/#frag?query')
self.assertEqual(obj.url, 'http://some-site.com:80/path/#frag?query')
self.assertEqual(obj.parsed_url,
urlparse.urlsplit('http://some-site.com:80/path/'))
self.assertEqual(obj.parsed_url.port, 80)
self.assertEqual(obj.hostname, 'some-site.com')
self.assertEqual(obj.url_path, '/path/')
self.assertEqual(obj.file_name, 'index.html')
self.assertEqual(obj.to_path, url2pathname('some-site.com/path/'))
self.assertEqual(obj.file_path,
url2pathname('some-site.com/path/index.html'))
def read(self, request, vendor, name, version):
resource = get_object_or_404(CatalogueResource, vendor=vendor, short_name=name, version=version)
resource_info = json.loads(resource.json_description)
if resource_info['doc'] == '':
raise Http404
doc_base_url = None
if resource_info['doc'].startswith(('http://', 'https://')):
doc_code = _('You can find the userguide of this component in this external <a target="_blank" href="%s">link</a>') % resource_info['doc']
doc_code = '<div style="margin-top: 10px"><p>%s</p></div>' % doc_code
else:
doc_relative_path = url2pathname(resource_info['doc'])
doc_base_url = force_trailing_slash(urljoin(resource.get_template_url(request=request, for_base=True), pathname2url(os.path.dirname(doc_relative_path))))
doc_path = os.path.join(catalogue_utils.wgt_deployer.get_base_dir(vendor, name, version), doc_relative_path)
(doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
localized_doc_path = doc_filename_root + '.' + get_language() + doc_filename_ext
try:
doc_code = download_local_file(localized_doc_path).decode('utf-8')
except:
try:
doc_code = download_local_file(doc_path).decode('utf-8')
except:
msg = _('Error opening the userguide file')
doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg
doc_pre_html = markdown.markdown(doc_code, output_format='xhtml5', extensions=['codehilite', 'fenced_code'])
doc = clean_html(doc_pre_html, base_url=doc_base_url)
return HttpResponse(doc, content_type='application/xhtml+xml; charset=UTF-8')
def s3_open(self, req):
server_name = req.host
bucket_name = url2pathname(req.selector).split('/')[1]
key_name = req.selector.replace(
"/{bucket_name}/".format(bucket_name=bucket_name), '')
if not bucket_name or not key_name:
raise URLError('url must be in the format '
's3://<server_name>/<bucket>/<object>')
try:
credential = self.settings[server_name]
except KeyError:
raise ServerNameError('missing credential')
s3 = boto3.client(
's3',
endpoint_url="https://{server}".format(server=server_name),
aws_access_key_id=credential['access_key'],
aws_secret_access_key=credential['secret_key'],
config=Config(signature_version='s3v4'),
region_name=credential.get('region_name', 'us-east-1'))
url = s3.generate_presigned_url(ClientMethod='get_object',
Params={
'Bucket': bucket_name,
'Key': key_name
},
ExpiresIn=3600)
req.full_url = url
return self.https_open(req)
def get_html(self, url, retries=1, base_url=None):
"""
Makes request for a URL and returns HTML as a BeautifulSoup object.
"""
# Put together the full URL
full_url = urljoin(base_url or self.base_url, url)
if self.verbosity > 2:
self.log(" Retrieving data for {}".format(url))
# Pull a cached version of the file, if it exists
cache_path = os.path.join(
self.cache_dir,
url2pathname(url.strip("/"))
)
if os.path.exists(cache_path) and not self.force_download:
# Make a HEAD request for the file size of the live page
if self.update_cache:
cache_file_size = os.path.getsize(cache_path)
head = self.get_headers(full_url)
web_file_size = head['content-length']
if self.verbosity > 2:
msg = " Cached file sized {}. Web file size {}."
self.log(msg.format(
cache_file_size,
web_file_size
))
# If our cache is the same size as the live page, return the cache
if not self.update_cache or cache_file_size == web_file_size:
if self.verbosity > 2:
self.log(" Returning cached {}".format(cache_path))
html = open(cache_path, 'r').read()
return BeautifulSoup(html, "html.parser")
# Otherwise, retrieve the full page and cache it
try:
response = self.get_url(full_url)
except requests.exceptions.HTTPError as e:
# If web requests fails, fall back to cached file, if it exists
if os.path.exists(cache_path):
if self.verbosity > 2:
self.log(" Returning cached {}".format(cache_path))
html = open(cache_path, 'r').read()
return BeautifulSoup(html, "html.parser")
else:
raise e
# Grab the HTML and cache it
html = response.text
if self.verbosity > 2:
self.log(" Writing to cache {}".format(cache_path))
cache_subdir = os.path.dirname(cache_path)
os.path.exists(cache_subdir) or os.makedirs(cache_subdir)
with open(cache_path, 'w') as f:
f.write(html)
# Finally return the HTML ready to parse with BeautifulSoup
return BeautifulSoup(html, "html.parser")
def get_html(self, url, retries=1, base_url=None):
"""
Makes request for a URL and returns HTML as a BeautifulSoup object.
"""
# Put together the full URL
full_url = urljoin(base_url or self.base_url, url)
if self.verbosity > 2:
self.log(" Retrieving data for {}".format(url))
# Pull a cached version of the file, if it exists
cache_path = os.path.join(
self.cache_dir,
url2pathname(url.strip("/"))
)
if os.path.exists(cache_path) and not self.force_download:
# Make a HEAD request for the file size of the live page
if self.update_cache:
cache_file_size = os.path.getsize(cache_path)
head = self.get_headers(full_url)
web_file_size = head['content-length']
if self.verbosity > 2:
msg = " Cached file sized {}. Web file size {}."
self.log(msg.format(
cache_file_size,
web_file_size
))
# If our cache is the same size as the live page, return the cache
if not self.update_cache or cache_file_size == web_file_size:
if self.verbosity > 2:
self.log(" Returning cached {}".format(cache_path))
html = open(cache_path, 'r').read()
return BeautifulSoup(html, "html.parser")
# Otherwise, retrieve the full page and cache it
try:
response = self.get_url(full_url)
except requests.exceptions.HTTPError as e:
# If web requests fails, fall back to cached file, if it exists
if os.path.exists(cache_path):
if self.verbosity > 2:
self.log(" Returning cached {}".format(cache_path))
html = open(cache_path, 'r').read()
return BeautifulSoup(html, "html.parser")
else:
raise e
# Grab the HTML and cache it
html = response.text
if self.verbosity > 2:
self.log(" Writing to cache {}".format(cache_path))
cache_subdir = os.path.dirname(cache_path)
os.path.exists(cache_subdir) or os.makedirs(cache_subdir)
with open(cache_path, 'w') as f:
f.write(html)
# Finally return the HTML ready to parse with BeautifulSoup
return BeautifulSoup(html, "html.parser")
def cached_file(url):
fname = os.path.basename(url2pathname(url.split(':', 1)[1]))
fpath = os.path.join(CACHE_DIR, fname)
if not os.path.exists(fpath):
with open(fpath, 'w') as f:
furl = urlopen(url)
f.write(furl.read())
furl.close()
return fpath
def on_file():
try:
filename = url2pathname(uri[5:].encode('utf-8') if isinstance(uri, text_type) else uri[5:])
if filename.endswith(BLOB_EXTENSION):
return on_mdblob(filename)
metainfo_deferred.callback(bdecode(fix_torrent(filename)))
return NOT_DONE_YET
except TypeError:
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
return json.dumps({"error": "error while decoding torrent file"})
def start_download_from_uri(self, uri, dconfig=None):
if uri.startswith("http"):
return self.start_download_from_url(bytes(uri), dconfig=dconfig)
if uri.startswith("magnet:"):
return succeed(self.start_download_from_magnet(uri, dconfig=dconfig))
if uri.startswith("file:"):
argument = url2pathname(uri[5:])
return succeed(self.start_download(torrentfilename=argument, dconfig=dconfig))
return fail(Failure(Exception("invalid uri")))
def get_repo(uri):
parts = urlparse(uri, "file", allow_fragments=0)
path = url2pathname(parts[2])
try:
return repo.Repository(root=path)
except cfg.ConfigError as e:
raise repo.RepositoryError(
_("The specified "
"repository's configuration data is not "
"valid:\n{0}").format(e))
def check_invalid_image(wgt_file, resource_info, key):
image_url = resource_info[key]
if image_url != '' and not image_url.startswith(('http://', 'https://')):
image_path = url2pathname(image_url)
try:
wgt_file.read(image_path)
except KeyError:
raise InvalidContents('missing image file: %s' % image_path)
def read(self, request, vendor, name, version):
from_version = request.GET.get('from')
if from_version is not None:
try:
from_version = Version(from_version)
except:
return build_error_response(
request, 422, _("Missing parameter: template_uri or file"))
resource = get_object_or_404(CatalogueResource,
vendor=vendor,
short_name=name,
version=version)
resource_info = resource.get_processed_info(process_urls=False)
if resource_info['changelog'] == '':
raise Http404
doc_relative_path = url2pathname(resource_info['changelog'])
doc_base_url = force_trailing_slash(
urljoin(resource.get_template_url(request=request, for_base=True),
pathname2url(os.path.dirname(doc_relative_path))))
doc_path = os.path.join(
catalogue_utils.wgt_deployer.get_base_dir(vendor, name, version),
doc_relative_path)
(doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
localized_doc_path = doc_filename_root + '.' + get_language(
) + doc_filename_ext
try:
doc_code = download_local_file(localized_doc_path).decode('utf-8')
except:
try:
doc_code = download_local_file(doc_path).decode('utf-8')
except:
msg = _('Error opening the changelog file')
doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg
doc_pre_html = markdown.markdown(doc_code,
output_format='xhtml5',
extensions=[
'markdown.extensions.codehilite',
'markdown.extensions.fenced_code'
])
if from_version:
doc_pre_html = filter_changelog(doc_pre_html, from_version)
if doc_pre_html.strip() == '':
raise Http404
doc = clean_html(doc_pre_html, base_url=doc_base_url)
return HttpResponse(
doc, content_type='application/xhtml+xml; charset=UTF-8')
def check_invalid_image(wgt_file, resource_info, key):
image_url = resource_info[key]
if image_url != '' and not image_url.startswith(('http://', 'https://')):
image_path = url2pathname(image_url)
try:
wgt_file.read(image_path)
except KeyError:
raise InvalidContents('missing image file: %s' % image_path)
def uri(param):
# type: (str) -> str
if urlsplit(param).scheme:
return param
path = expanduser(expandvars(url2pathname(param)))
if exists(path):
return urljoin('file:', pathname2url(abspath(path)))
raise ArgumentTypeError(
'`{param}` is not an existing file and either a valid URI'.format(
param=param))
def start_download_from_uri(self, uri, dconfig=None):
if uri.startswith("http"):
return self.start_download_from_url(bytes(uri), dconfig=dconfig)
if uri.startswith("magnet:"):
return succeed(
self.start_download_from_magnet(uri, dconfig=dconfig))
if uri.startswith("file:"):
argument = url2pathname(uri[5:])
return succeed(
self.start_download(torrentfilename=argument, dconfig=dconfig))
return fail(Failure(Exception("invalid uri")))
def _create_run_in_store(store):
config = {
'experiment_id': '0',
'user_id': 'Anderson',
'start_time': int(time.time()),
'tags': {}
}
run = store.create_run(**config)
artifact_path = url2pathname(unquote(urlparse(run.info.artifact_uri).path))
if not os.path.exists(artifact_path):
os.makedirs(artifact_path)
return run
def on_file():
try:
filename = url2pathname(uri[5:].encode('utf-8') if isinstance(
uri, text_type) else uri[5:])
if filename.endswith(BLOB_EXTENSION):
return on_mdblob(filename)
metainfo_deferred.callback(bdecode(fix_torrent(filename)))
return NOT_DONE_YET
except TypeError:
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
return json.dumps(
{"error": "error while decoding torrent file"})
def _create_run_in_store(store):
config = {
"experiment_id": "0",
"user_id": "Anderson",
"start_time": int(time.time()),
"tags": {},
}
run = store.create_run(**config)
artifact_path = url2pathname(unquote(urlparse(run.info.artifact_uri).path))
if not os.path.exists(artifact_path):
os.makedirs(artifact_path)
return run
def url_to_path(url):
"""Convert a valid file url to a local filesystem path
Follows logic taken from pip's equivalent function
"""
assert is_file_url(url), "Only file: urls can be converted to local paths"
_, netloc, path, _, _ = urllib_parse.urlsplit(url)
# Netlocs are UNC paths
if netloc:
netloc = "\\\\" + netloc
path = urllib_request.url2pathname(netloc + path)
return to_bytes(path, encoding="utf-8")
def topath(self):
"""Convert a file: URL to a path."""
assert url.startswith('file:'), (
"You can only turn file: urls into filenames (not %r)" % url)
_, netloc, path, _, _ = urllib_parse.urlsplit(url)
# if we have a UNC path, prepend UNC share notation
if netloc:
netloc = '\\\\' + netloc
path = urllib_request.url2pathname(netloc + path)
return path
def file_path(self):
"""Make a unique path from the url.
:rtype: str
:return: disk compatible path
"""
upath, _ = self._refactor_filename(self.url_path)
# clean the url and prepend hostname to make it complete
upath = url2pathname(self.hostname + upath)
if self.base_path:
upath = os.path.join(self.base_path, upath)
return upath
def read(self, request, vendor, name, version):
resource = get_object_or_404(CatalogueResource,
vendor=vendor,
short_name=name,
version=version)
resource_info = resource.json_description
if resource_info['doc'] == '':
raise Http404
doc_base_url = None
if resource_info['doc'].startswith(('http://', 'https://')):
doc_code = _(
'You can find the userguide of this component in this external <a target="_blank" href="%s">link</a>'
) % resource_info['doc']
doc_code = '<div style="margin-top: 10px"><p>%s</p></div>' % doc_code
else:
doc_relative_path = url2pathname(resource_info['doc'])
doc_base_url = force_trailing_slash(
urljoin(
resource.get_template_url(request=request, for_base=True),
pathname2url(os.path.dirname(doc_relative_path))))
doc_path = os.path.join(
catalogue_utils.wgt_deployer.get_base_dir(
vendor, name, version), doc_relative_path)
(doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
localized_doc_path = doc_filename_root + '.' + get_language(
) + doc_filename_ext
try:
doc_code = download_local_file(localized_doc_path).decode(
'utf-8')
except:
try:
doc_code = download_local_file(doc_path).decode('utf-8')
except:
msg = _('Error opening the userguide file')
doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg
doc_pre_html = markdown.markdown(doc_code,
output_format='xhtml5',
extensions=[
'markdown.extensions.codehilite',
'markdown.extensions.fenced_code'
])
doc = clean_html(doc_pre_html, base_url=doc_base_url)
return HttpResponse(
doc, content_type='application/xhtml+xml; charset=UTF-8')
def url_to_path(url):
# type: (str) -> str
"""Convert a valid file url to a local filesystem path.
Follows logic taken from pip's equivalent function
"""
assert is_file_url(url), "Only file: urls can be converted to local paths"
_, netloc, path, _, _ = urllib_parse.urlsplit(url)
# Netlocs are UNC paths
if netloc:
netloc = "\\\\" + netloc
path = urllib_request.url2pathname(netloc + path)
return urllib_parse.unquote(path)
def cached_file(url):
fname = url2pathname(relative(url))
fpath = os.path.join(CACHE_DIR, fname)
if not os.path.exists(fpath):
print("%s does not exist, fetching from %s" % (fpath, url))
folder = os.path.dirname(fpath)
if not os.path.exists(folder):
os.makedirs(folder)
f = open(fpath, 'w')
try:
f.write(urlopen(url).read())
finally:
f.close()
return fpath
def check_invalid_doc_content(wgt_file, resource_info, key):
doc_url = resource_info[key]
if doc_url != '' and not doc_url.startswith(('http://', 'https://')):
doc_path = url2pathname(doc_url)
# Check default version
check_invalid_doc_entry(wgt_file, doc_path)
# Check localized versions
(doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
pattern = re.escape(doc_filename_root) + '\.\w\w(?:-\w\w)?' + re.escape(doc_filename_ext)
for filename in wgt_file.namelist():
if re.match(pattern, filename):
check_invalid_doc_entry(wgt_file, filename)
def cached_file(url):
fname = url2pathname(relative(url))
fpath = os.path.join(CACHE_DIR, fname)
if not os.path.exists(fpath):
print("%s does not exist, fetching from %s" % (fpath, url))
folder = os.path.dirname(fpath)
if not os.path.exists(folder):
os.makedirs(folder)
f = open(fpath, 'w')
try:
f.write(urlopen(url).read())
finally:
f.close()
return fpath
def __init__(self, url=None, *args, **kwargs):
# Works around an apparent Git bug
# (see http://article.gmane.org/gmane.comp.version-control.git/146500)
if url:
scheme, netloc, path, query, fragment = urlsplit(url)
if scheme.endswith('file'):
initial_slashes = path[:-len(path.lstrip('/'))]
newpath = (initial_slashes +
urllib_request.url2pathname(path).replace(
'\\', '/').lstrip('/'))
url = urlunsplit((scheme, netloc, newpath, query, fragment))
after_plus = scheme.find('+') + 1
url = scheme[:after_plus] + urlunsplit(
(scheme[after_plus:], netloc, newpath, query, fragment), )
super(Git, self).__init__(url, *args, **kwargs)
def check_invalid_doc_content(wgt_file, resource_info, key):
doc_url = resource_info[key]
if doc_url != '' and not doc_url.startswith(('http://', 'https://')):
doc_path = url2pathname(doc_url)
# Check default version
check_invalid_doc_entry(wgt_file, doc_path)
# Check localized versions
(doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
pattern = re.escape(
doc_filename_root) + '\.\w\w(?:-\w\w)?' + re.escape(
doc_filename_ext)
for filename in wgt_file.namelist():
if re.match(pattern, filename):
check_invalid_doc_entry(wgt_file, filename)
def file_path(self):
"""Make a unique path from the url.
:rtype: str
:return: disk compatible path
"""
path, _ = self._refactor_filename(self.url_path)
hostname = self.hostname
assert isinstance(path, basestring)
assert isinstance(hostname, basestring)
# clean the url and prepend hostname to make it complete
path = url2pathname(hostname + path)
if self.base_path:
path = os.path.join(self.base_path, path)
return path
def open_local_file(self, req):
try:
host = req.host
filename = req.selector
# if that bombs, then go on with original method
localfile = url.url2pathname(host + filename)
stats = os.stat(localfile)
size = stats.st_size
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(filename)[0]
headers = email.message_from_string(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n'
% (mtype or 'text/plain', size, modified))
origurl = 'file://' + host + filename
return addinfourl(open(localfile, 'rb'), headers, origurl)
except:
pass
return url.FileHandler.open_local_file(self, req)
def _handle_user_code_url(self, code):
"""Gets the S3 URL containing the user's code.
Inspects the scheme the customer passed in ("s3://" for code in S3, "file://" or nothing
for absolute or local file paths. Uploads the code to S3 if the code is a local file.
Args:
code (str): A URL to the customer's code.
Returns:
str: The S3 URL to the customer's code.
Raises:
ValueError: if the code isn't found, is a directory, or
does not have a valid URL scheme.
"""
code_url = urlparse(code)
if code_url.scheme == "s3":
user_code_s3_uri = code
elif code_url.scheme == "" or code_url.scheme == "file":
# Validate that the file exists locally and is not a directory.
code_path = url2pathname(code_url.path)
if not os.path.exists(code_path):
raise ValueError(
"""code {} wasn't found. Please make sure that the file exists.
""".format(
code
)
)
if not os.path.isfile(code_path):
raise ValueError(
"""code {} must be a file, not a directory. Please pass a path to a file.
""".format(
code
)
)
user_code_s3_uri = self._upload_code(code_path)
else:
raise ValueError(
"code {} url scheme {} is not recognized. Please pass a file path or S3 url".format(
code, code_url.scheme
)
)
return user_code_s3_uri
def test_url_parsing_after_set_base(self):
obj = urls.URLTransformer('../some/rel/path/')
obj.base_url = "http://some-site.com:80"
obj.base_path = "e:\\tests\\"
obj.default_filename = 'index.html'
obj._unique_fn_required = False
self.assertEqual(obj.original_url, '../some/rel/path/')
self.assertEqual(obj.url, 'http://some-site.com:80/some/rel/path/')
self.assertEqual(
obj.parsed_url,
urlparse.urlsplit('http://some-site.com:80/some/rel/path/',
allow_fragments=False))
self.assertEqual(obj.parsed_url.port, 80)
self.assertEqual(obj.hostname, 'some-site.com')
self.assertEqual(obj.url_path, '/some/rel/path/')
self.assertEqual(obj.file_name, 'index.html')
self.assertEqual(obj.to_path,
'e:\\tests\\some-site.com\\some\\rel\\path\\')
self.assertEqual(
obj.file_path,
url2pathname(
'e://tests/some-site.com/some/rel/path/index.html').lower())
def read(self, request, vendor, name, version):
from_version = request.GET.get('from')
if from_version is not None:
try:
from_version = Version(from_version)
except:
return build_error_response(request, 422, _("Missing parameter: template_uri or file"))
resource = get_object_or_404(CatalogueResource, vendor=vendor, short_name=name, version=version)
resource_info = resource.get_processed_info(process_urls=False)
if resource_info['changelog'] == '':
raise Http404
doc_relative_path = url2pathname(resource_info['changelog'])
doc_base_url = force_trailing_slash(urljoin(resource.get_template_url(request=request, for_base=True), pathname2url(os.path.dirname(doc_relative_path))))
doc_path = os.path.join(catalogue_utils.wgt_deployer.get_base_dir(vendor, name, version), doc_relative_path)
(doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
localized_doc_path = doc_filename_root + '.' + get_language() + doc_filename_ext
try:
doc_code = download_local_file(localized_doc_path).decode('utf-8')
except:
try:
doc_code = download_local_file(doc_path).decode('utf-8')
except:
msg = _('Error opening the changelog file')
doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg
doc_pre_html = markdown.markdown(doc_code, output_format='xhtml5', extensions=['codehilite', 'fenced_code'])
if from_version:
doc_pre_html = filter_changelog(doc_pre_html, from_version)
if doc_pre_html.strip() == '':
raise Http404
doc = clean_html(doc_pre_html, base_url=doc_base_url)
return HttpResponse(doc, content_type='application/xhtml+xml; charset=UTF-8')
def load(self, url, offset=0, length=-1):
"""
Load a file-like reader from the local file system
"""
# if starting with . or /, can only be a file path..
file_only = url.startswith(('/', '.'))
# convert to filename
if url.startswith('file://'):
file_only = True
url = url2pathname(url[len('file://'):])
try:
# first, try as file
afile = open(url, 'rb')
except IOError:
if file_only:
raise
# then, try as package.path/file
pkg_split = url.split('/', 1)
if len(pkg_split) == 1:
raise
afile = pkg_resources.resource_stream(pkg_split[0],
pkg_split[1])
if offset > 0:
afile.seek(offset)
if length >= 0:
return LimitReader(afile, length)
else:
return afile
def do_GET(self):
up = urlparse( self.path )
try:
if up.path=='/favicon.ico' and favicon:
content = favicon
content_type = 'image/x-icon'
else:
file = url2pathname(os.path.basename(up.path.split('#')[0]))
fname = os.path.join( Utils.getHelpFolder(), file )
with io.open(fname, 'r', encoding='utf-8') as fp:
content = fp.read()
content_type = self.html_content
except Exception as e:
self.send_error(404,'File Not Found: {} {}\n{}'.format(self.path, e, traceback.format_exc()))
return
self.send_response( 200 )
self.send_header('Content-type',content_type)
if content_type == self.html_content:
self.send_header( 'Cache-Control', 'no-cache, no-store, must-revalidate' )
self.send_header( 'Pragma', 'no-cache' )
self.send_header( 'Expires', '0' )
self.end_headers()
self.wfile.write( content.encode() )
def url_to_path(url): # type: (str) -> Path
"""
Convert an RFC8089 file URI to path.
The logic used here is borrowed from pip
https://github.com/pypa/pip/blob/4d1932fcdd1974c820ea60b3286984ebb0c3beaa/src/pip/_internal/utils/urls.py#L31
"""
if not url.startswith("file:"):
raise ValueError("{} is not a valid file URI".format(url))
_, netloc, path, _, _ = urlsplit(url)
if not netloc or netloc == "localhost":
# According to RFC 8089, same as empty authority.
netloc = ""
elif netloc not in {".", ".."} and sys.platform == "win32":
# If we have a UNC path, prepend UNC share notation.
netloc = "\\\\" + netloc
else:
raise ValueError(
"non-local file URIs are not supported on this platform: {}".
format(url))
return Path(url2pathname(netloc + unquote(path)))
def file_uri_to_path(uri):
"""Convert File URI to local filesystem path according to:
http://en.wikipedia.org/wiki/File_URI_scheme
"""
uri_path = urlparse(uri).path
return url2pathname(uri_path)
def read(self, request, vendor, name, version):
resource = get_object_or_404(CatalogueResource.objects.select_related('widget__xhtml'), vendor=vendor, short_name=name, version=version)
# For now, all widgets are freely accessible/distributable
#if not resource.is_available_for(request.user):
# return build_error_response(request, 403, "Forbidden")
if resource.resource_type() != 'widget':
raise Http404()
mode = request.GET.get('mode', 'classic')
widget_info = json.loads(resource.json_description)
# check if the xhtml code has been cached
if widget_info['contents']['cacheable'] is True:
cache_key = resource.widget.xhtml.get_cache_key(get_current_domain(request), mode)
cache_entry = cache.get(cache_key)
if cache_entry is not None:
response = HttpResponse(cache_entry['code'], content_type=cache_entry['content_type'])
patch_cache_headers(response, cache_entry['timestamp'], cache_entry['timeout'])
return response
# process xhtml
xhtml = resource.widget.xhtml
content_type = widget_info['contents'].get('contenttype', 'text/html')
charset = widget_info['contents'].get('charset', 'utf-8')
force_base = False
base_url = xhtml.url
if not base_url.startswith(('http://', 'https://')):
# Newer versions of Django urlencode urls created using reverse
# Fix double encoding
base_url = urlunquote(base_url)
base_url = get_absolute_reverse_url('wirecloud.showcase_media', args=(base_url.split('/', 4)), request=request)
force_base = True
code = xhtml.code
if not xhtml.cacheable or code == '':
try:
if xhtml.url.startswith(('http://', 'https://')):
code = download_http_content(urljoin(base_url, xhtml.url), user=request.user)
else:
code = download_local_file(os.path.join(showcase_utils.wgt_deployer.root_dir, url2pathname(xhtml.url)))
except Exception as e:
return build_response(request, 502, {'error_msg': _("(X)HTML code is not accessible"), 'details': "%s" % e}, WIDGET_ERROR_FORMATTERS)
else:
# Code contents comes as unicode from persistence, we need bytes
code = code.encode(charset)
if xhtml.cacheable and (xhtml.code == '' or xhtml.code_timestamp is None):
try:
xhtml.code = code.decode(charset)
except UnicodeDecodeError:
msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)
xhtml.code_timestamp = time.time() * 1000
xhtml.save()
elif not xhtml.cacheable and xhtml.code != '':
xhtml.code = ''
xhtml.code_timestamp = None
xhtml.save()
try:
code = fix_widget_code(code, base_url, content_type, request, charset, xhtml.use_platform_style, process_requirements(widget_info['requirements']), force_base, mode)
except UnicodeDecodeError:
msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)
except Exception as e:
msg = _('Error processing widget code')
return build_response(request, 502, {'error_msg': msg, 'details':"%s" % e}, WIDGET_ERROR_FORMATTERS)
if xhtml.cacheable:
cache_timeout = 31536000 # 1 year
cache_entry = {
'code': code,
'content_type': '%s; charset=%s' % (content_type, charset),
'timestamp': xhtml.code_timestamp,
'timeout': cache_timeout,
}
cache.set(cache_key, cache_entry, cache_timeout)
else:
cache_timeout = 0
response = HttpResponse(code, content_type='%s; charset=%s' % (content_type, charset))
patch_cache_headers(response, xhtml.code_timestamp, cache_timeout)
return response
def render_PUT(self, request):
"""
.. http:put:: /downloads
A PUT request to this endpoint will start a download from a provided URI. This URI can either represent a file
location, a magnet link or a HTTP(S) url.
- anon_hops: the number of hops for the anonymous download. 0 hops is equivalent to a plain download
- safe_seeding: whether the seeding of the download should be anonymous or not (0 = off, 1 = on)
- destination: the download destination path of the torrent
- torrent: the URI of the torrent file that should be downloaded. This parameter is required.
**Example request**:
.. sourcecode:: none
curl -X PUT http://localhost:8085/downloads
--data "anon_hops=2&safe_seeding=1&destination=/my/dest/on/disk/&uri=file:/home/me/test.torrent
**Example response**:
.. sourcecode:: javascript
{"started": True, "infohash": "4344503b7e797ebf31582327a5baae35b11bda01"}
"""
parameters = http.parse_qs(request.content.read(), 1)
if 'uri' not in parameters or len(parameters['uri']) == 0:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": "uri parameter missing"})
download_config, error = DownloadsEndpoint.create_dconfig_from_params(parameters)
if error:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": error})
def download_added(download):
request.write(json.dumps({"started": True,
"infohash": hexlify(download.get_def().get_infohash())}))
request.finish()
def on_error(error):
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
request.write(json.dumps({"error": unichar_string(error.getErrorMessage())}))
request.finish()
uri = parameters['uri'][0]
if uri.startswith("file:"):
if uri.endswith(".mdblob"):
filename = url2pathname(uri[5:].encode('utf-8') if isinstance(uri, six.text_type) else uri[5:])
try:
payload = ChannelMetadataPayload.from_file(filename)
except IOError:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": "file not found"})
except InvalidSignatureException:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": "Metadata has invalid signature"})
with db_session:
result = self.session.lm.mds.process_payload(payload)
if result:
node, status = result[0]
if (status == UNKNOWN_CHANNEL or
(status == UPDATED_OUR_VERSION and node.metadata_type == CHANNEL_TORRENT)):
node.subscribed = True
return json.dumps(
{"started": True, "infohash": hexlify(node.infohash)})
return json.dumps({"error": "Already subscribed"})
else:
download_uri = u"file:%s" % url2pathname(uri[5:]).decode('utf-8')
else:
download_uri = unquote_plus(cast_to_unicode_utf8(uri))
download_deferred = self.session.start_download_from_uri(download_uri, download_config)
download_deferred.addCallback(download_added)
download_deferred.addErrback(on_error)
return NOT_DONE_YET
def get_resource_data(resource, user, request=None):
"""Gets all the information related to the given resource."""
resource_info = resource.get_processed_info(request)
template_uri = get_absolute_reverse_url(
"wirecloud_catalogue.media",
kwargs={
"vendor": resource.vendor,
"name": resource.short_name,
"version": resource.version,
"file_path": resource.template_uri,
},
request=request,
)
wgt_path = os.path.join(
wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version), resource.template_uri
)
size = os.path.getsize(wgt_path)
cdate = resource.creation_date
creation_timestamp = time.mktime(cdate.timetuple()) * 1e3 + cdate.microsecond / 1e3
longdescription = resource_info["longdescription"]
if longdescription != "":
longdescription_relative_path = url2pathname(longdescription)
longdescription_base_url = force_trailing_slash(
urljoin(
resource.get_template_url(request=request, for_base=True),
pathname2url(os.path.dirname(longdescription_relative_path)),
)
)
longdescription_path = os.path.join(
wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version),
longdescription_relative_path,
)
(filename_root, filename_ext) = os.path.splitext(longdescription_path)
localized_longdescription_path = filename_root + "." + get_language() + filename_ext
try:
description_code = download_local_file(localized_longdescription_path)
longdescription = clean_html(
markdown.markdown(description_code, output_format="xhtml5"), base_url=longdescription_base_url
)
except:
try:
description_code = download_local_file(longdescription_path)
longdescription = clean_html(
markdown.markdown(description_code, output_format="xhtml5"), base_url=longdescription_base_url
)
except:
longdescription = resource_info["description"]
else:
longdescription = resource_info["description"]
return {
"id": resource.pk,
"vendor": resource.vendor,
"name": resource.short_name,
"version": resource.version,
"type": resource_info["type"],
"date": creation_timestamp,
"permissions": {
"delete": user.is_superuser,
"uninstall": resource.public is False and resource.users.filter(pk=user.pk).exists(),
},
"authors": resource_info["authors"],
"contributors": resource_info["contributors"],
"title": resource_info["title"],
"description": resource_info["description"],
"longdescription": longdescription,
"email": resource_info["email"],
"image": resource_info["image"],
"homepage": resource_info["homepage"],
"doc": resource_info["doc"],
"changelog": resource_info["changelog"],
"size": size,
"uriTemplate": template_uri,
"license": resource_info["license"],
"licenseurl": resource_info["licenseurl"],
"issuetracker": resource_info["issuetracker"],
}
def process_widget_code(request, resource):
mode = request.GET.get('mode', 'classic')
theme = request.GET.get('theme', get_active_theme_name())
widget_info = json.loads(resource.json_description)
# check if the xhtml code has been cached
if widget_info['contents']['cacheable'] is True:
cache_key = resource.widget.xhtml.get_cache_key(get_current_domain(request), mode, theme)
cache_entry = cache.get(cache_key)
if cache_entry is not None:
response = HttpResponse(cache_entry['code'], content_type=cache_entry['content_type'])
patch_cache_headers(response, cache_entry['timestamp'], cache_entry['timeout'])
return response
# process xhtml
xhtml = resource.widget.xhtml
content_type = widget_info['contents'].get('contenttype', 'text/html')
charset = widget_info['contents'].get('charset', 'utf-8')
code = xhtml.code
if not xhtml.cacheable or code == '':
try:
code = download_local_file(os.path.join(showcase_utils.wgt_deployer.root_dir, url2pathname(xhtml.url)))
except Exception as e:
if isinstance(e, IOError) and e.errno == errno.ENOENT:
return build_response(request, 404, {'error_msg': _("Widget code not found"), 'details': "%s" % e}, WIDGET_ERROR_FORMATTERS)
else:
return build_response(request, 500, {'error_msg': _("Error reading widget code"), 'details': "%s" % e}, WIDGET_ERROR_FORMATTERS)
else:
# Code contents comes as unicode from persistence, we need bytes
code = code.encode(charset)
if xhtml.cacheable and (xhtml.code == '' or xhtml.code_timestamp is None):
try:
xhtml.code = code.decode(charset)
except UnicodeDecodeError:
msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)
xhtml.code_timestamp = time.time() * 1000
xhtml.save()
try:
code = fix_widget_code(code, content_type, request, charset, xhtml.use_platform_style, process_requirements(widget_info['requirements']), mode, theme)
except UnicodeDecodeError:
msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)
except Exception as e:
msg = _('Error processing widget code')
return build_response(request, 502, {'error_msg': msg, 'details':"%s" % e}, WIDGET_ERROR_FORMATTERS)
if xhtml.cacheable:
cache_timeout = 31536000 # 1 year
cache_entry = {
'code': code,
'content_type': '%s; charset=%s' % (content_type, charset),
'timestamp': xhtml.code_timestamp,
'timeout': cache_timeout,
}
cache.set(cache_key, cache_entry, cache_timeout)
else:
cache_timeout = 0
response = HttpResponse(code, content_type='%s; charset=%s' % (content_type, charset))
patch_cache_headers(response, xhtml.code_timestamp, cache_timeout)
return response
def create_input_source(source=None, publicID=None,
location=None, file=None, data=None, format=None):
"""
Return an appropriate InputSource instance for the given
parameters.
"""
# test that exactly one of source, location, file, and data is not None.
if sum((
source is not None,
location is not None,
file is not None,
data is not None,
)) != 1:
raise ValueError(
'exactly one of source, location, file or data must be given'
)
input_source = None
if source is not None:
if isinstance(source, InputSource):
input_source = source
else:
if isinstance(source, string_types):
location = source
elif hasattr(source, "read") and not isinstance(source, Namespace):
f = source
input_source = InputSource()
input_source.setByteStream(f)
if f is sys.stdin:
input_source.setSystemId("file:///dev/stdin")
elif hasattr(f, "name"):
input_source.setSystemId(f.name)
else:
raise Exception("Unexpected type '%s' for source '%s'" %
(type(source), source))
absolute_location = None # Further to fix for issue 130
auto_close = False # make sure we close all file handles we open
if location is not None:
# Fix for Windows problem https://github.com/RDFLib/rdflib/issues/145
if os.path.exists(location):
location = pathname2url(location)
base = urljoin("file:", "%s/" % pathname2url(os.getcwd()))
absolute_location = URIRef(location, base=base).defrag()
if absolute_location.startswith("file:///"):
filename = url2pathname(absolute_location.replace("file:///", "/"))
file = open(filename, "rb")
else:
input_source = URLInputSource(absolute_location, format)
auto_close = True
# publicID = publicID or absolute_location # Further to fix
# for issue 130
if file is not None:
input_source = FileInputSource(file)
if data is not None:
if isinstance(data, text_type):
data = data.encode('utf-8')
input_source = StringInputSource(data)
auto_close = True
if input_source is None:
raise Exception("could not create InputSource")
else:
input_source.auto_close |= auto_close
if publicID is not None: # Further to fix for issue 130
input_source.setPublicId(publicID)
# Further to fix for issue 130
elif input_source.getPublicId() is None:
input_source.setPublicId(absolute_location or "")
return input_source
def do_GET(self):
up = urllib.parse.urlparse( self.path )
content, gzip_content = None, None
try:
if up.path=='/':
content = getIndexPage()
content_type = self.html_content
assert isinstance( content, bytes )
elif up.path=='/favicon.ico':
content = favicon
content_type = 'image/x-icon'
assert isinstance( content, bytes )
elif self.reLapCounterHtml.match( up.path ):
content = getLapCounterHtml()
content_type = self.html_content
assert isinstance( content, bytes )
elif up.path=='/Announcer.html':
content = getAnnouncerHtml()
content_type = self.html_content
assert isinstance( content, bytes )
elif up.path=='/qrcode.html':
urlPage = GetCrossMgrHomePage()
content = getQRCodePage( urlPage )
content_type = self.html_content
assert isinstance( content, bytes )
elif up.path=='/servertimestamp.html':
content = Utils.ToJSon( {
'servertime':time.time()*1000.0,
'requesttimestamp':float(up.query),
}
).encode()
content_type = self.json_content;
assert isinstance( content, bytes )
else:
file = None
if up.path == '/CurrentResults.html':
try:
file = os.path.splitext(Model.race.getFileName())[0] + '.html'
except:
pass
elif up.path == '/PreviousResults.html':
file = GetPreviousFileName()
if file is None:
file = url2pathname(os.path.basename(up.path))
content, gzip_content = contentBuffer.getContent( file )
content_type = self.html_content
assert isinstance( content, bytes )
except Exception as e:
self.send_error(404,'File Not Found: {} {}\n{}'.format(self.path, e, traceback.format_exc()))
return
self.send_response( 200 )
self.send_header('Content-Type',content_type)
if content_type == self.html_content:
if gzip_content and 'Accept-Encoding' in self.headers and 'gzip' in self.headers['Accept-Encoding']:
content = gzip_content
self.send_header( 'Content-Encoding', 'gzip' )
self.send_header( 'Cache-Control', 'no-cache, no-store, must-revalidate' )
self.send_header( 'Pragma', 'no-cache' )
self.send_header( 'Expires', '0' )
self.send_header( 'Content-Length', len(content) )
self.end_headers()
self.wfile.write( content )
def get_file(init, mode='r', uri=None):
"""
Returns a `GenericFile` instance suitable for wrapping the given
object `init`.
If passed an already open file-like object, it must be opened for
reading/writing in binary mode. It is the caller's responsibility
to close it.
Parameters
----------
init : object
`init` may be:
- A `bytes` or `unicode` file path or ``file:`` or ``http:``
url.
- A Python 2 `file` object.
- An `io.IOBase` object (the default file object on Python 3).
- A ducktyped object that looks like a file object. If `mode`
is ``"r"``, it must have a ``read`` method. If `mode` is
``"w"``, it must have a ``write`` method. If `mode` is
``"rw"`` it must have the ``read``, ``write``, ``tell`` and
``seek`` methods.
- A `GenericFile` instance, in which case it is wrapped in a
`GenericWrapper` instance, so that the file is closed when
only when the final layer is unwrapped.
mode : str
Must be one of ``"r"``, ``"w"`` or ``"rw"``.
uri : str
Sets the base URI of the file object. This will be used to
resolve any relative URIs contained in the file. This is
redundant if `init` is a `bytes` or `unicode` object (since it
will be the uri), and it may be determined automatically if
`init` refers to a regular filesystem file. It is not required
if URI resolution is not used in the file.
Returns
-------
fd : GenericFile
Raises
------
ValueError, TypeError, IOError
"""
if mode not in ('r', 'w', 'rw'):
raise ValueError("mode must be 'r', 'w' or 'rw'")
if init in (sys.__stdout__, sys.__stdin__, sys.__stderr__):
if six.PY3:
init = init.buffer
else:
init = os.fdopen(init.fileno(), init.mode + 'b')
if isinstance(init, (GenericFile, GenericWrapper)):
if mode not in init.mode:
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
return GenericWrapper(init)
elif isinstance(init, six.string_types):
parsed = urlparse.urlparse(init)
if parsed.scheme == 'http':
if 'w' in mode:
raise ValueError(
"HTTP connections can not be opened for writing")
return _make_http_connection(init, mode, uri=uri)
elif parsed.scheme in _local_file_schemes:
if mode == 'rw':
realmode = 'r+b'
else:
realmode = mode + 'b'
realpath = url2pathname(parsed.path)
if mode == 'w':
fd = atomicfile.atomic_open(realpath, realmode)
else:
fd = open(realpath, realmode)
fd = fd.__enter__()
return RealFile(fd, mode, close=True, uri=uri)
elif isinstance(init, io.BytesIO):
return MemoryIO(init, mode, uri=uri)
elif isinstance(init, io.StringIO):
raise TypeError(
"io.StringIO objects are not supported. Use io.BytesIO instead.")
elif six.PY2 and isinstance(init, file):
if init.mode[0] not in mode:
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
try:
init.tell()
except IOError:
if mode == 'w':
return OutputStream(init, uri=uri)
elif mode == 'r':
return InputStream(init, mode, uri=uri)
else:
raise ValueError(
"File '{0}' could not be opened in 'rw' mode".format(init))
else:
return RealFile(init, mode, uri=uri)
elif isinstance(init, io.IOBase):
if sys.version_info[:2] == (2, 6): # pragma: no cover
raise ValueError(
"io.open file objects are not supported on Python 2.6")
if (('r' in mode and not init.readable()) or
('w' in mode and not init.writable())):
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
if init.seekable():
if isinstance(init, (io.BufferedReader,
io.BufferedWriter,
io.BufferedRandom)):
init2 = init.raw
else:
init2 = init
if isinstance(init2, io.RawIOBase):
result = RealFile(init2, mode, uri=uri)
else:
result = MemoryIO(init2, mode, uri=uri)
result._secondary_fd = init
return result
else:
if mode == 'w':
return OutputStream(init, uri=uri)
elif mode == 'r':
return InputStream(init, mode, uri=uri)
else:
raise ValueError(
"File '{0}' could not be opened in 'rw' mode".format(init))
elif mode == 'w' and (
hasattr(init, 'write') and
hasattr(init, 'seek') and
hasattr(init, 'tell')):
return MemoryIO(init, mode, uri=uri)
elif mode == 'r' and (
hasattr(init, 'read') and
hasattr(init, 'seek') and
hasattr(init, 'tell')):
return MemoryIO(init, mode, uri=uri)
elif mode == 'rw' and (
hasattr(init, 'read') and
hasattr(init, 'write') and
hasattr(init, 'seek') and
hasattr(init, 'tell')):
return MemoryIO(init, mode, uri=uri)
elif mode == 'w' and hasattr(init, 'write'):
return OutputStream(init, uri=uri)
elif mode == 'r' and hasattr(init, 'read'):
return InputStream(init, mode, uri=uri)
raise ValueError("Can't handle '{0}' as a file for mode '{1}'".format(
init, mode))
def mapper(self, key, line):
for record in self.read_warc(line.strip()):
payload = self.get_payload(record)
for value in self.process_record(payload):
yield ((url2pathname(record.url), value), 1)
def get_resource_data(resource, user, request=None):
"""Gets all the information related to the given resource."""
resource_info = resource.get_processed_info(request)
template_uri = get_absolute_reverse_url('wirecloud_catalogue.media', kwargs={
'vendor': resource.vendor,
'name': resource.short_name,
'version': resource.version,
'file_path': resource.template_uri
}, request=request)
wgt_path = os.path.join(wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version), resource.template_uri)
size = os.path.getsize(wgt_path)
cdate = resource.creation_date
creation_timestamp = time.mktime(cdate.timetuple()) * 1e3 + cdate.microsecond / 1e3
longdescription = resource_info['longdescription']
if longdescription != '':
longdescription_relative_path = url2pathname(longdescription)
longdescription_base_url = force_trailing_slash(urljoin(resource.get_template_url(request=request, for_base=True), pathname2url(os.path.dirname(longdescription_relative_path))))
longdescription_path = os.path.join(wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version), longdescription_relative_path)
(filename_root, filename_ext) = os.path.splitext(longdescription_path)
localized_longdescription_path = filename_root + '.' + get_language() + filename_ext
try:
description_code = download_local_file(localized_longdescription_path).decode('utf-8')
longdescription = clean_html(markdown.markdown(description_code, output_format='xhtml5'), base_url=longdescription_base_url)
except:
try:
description_code = download_local_file(longdescription_path).decode('utf-8')
longdescription = clean_html(markdown.markdown(description_code, output_format='xhtml5'), base_url=longdescription_base_url)
except:
longdescription = resource_info['description']
else:
longdescription = resource_info['description']
return {
'id': resource.pk,
'vendor': resource.vendor,
'name': resource.short_name,
'version': resource.version,
'type': resource_info['type'],
'date': creation_timestamp,
'permissions': {
'delete': user.is_superuser,
'uninstall': resource.public is False and resource.users.filter(pk=user.pk).exists(),
},
'authors': resource_info['authors'],
'contributors': resource_info['contributors'],
'title': resource_info['title'],
'description': resource_info['description'],
'longdescription': longdescription,
'email': resource_info['email'],
'image': resource_info['image'],
'homepage': resource_info['homepage'],
'doc': resource_info['doc'],
'changelog': resource_info['changelog'],
'size': size,
'uriTemplate': template_uri,
'license': resource_info['license'],
'licenseurl': resource_info['licenseurl'],
'issuetracker': resource_info['issuetracker'],
}
