コード例 #1
0
    def send(self, req, **kwargs):  # pylint: disable=unused-argument
        """Return the file specified by the given request

        @type req: C{PreparedRequest}
        @todo: Should I bother filling `response.headers` and processing
               If-Modified-Since and friends using `os.stat`?
        """
        path = os.path.normcase(os.path.normpath(url2pathname(req.path_url)))
        response = requests.Response()

        response.status_code, response.reason = self._chkpath(req.method, path)
        if response.status_code == 200 and req.method.lower() != 'head':
            try:
                response.raw = open(path, 'rb')
            except (OSError, IOError) as err:
                response.status_code = 500
                response.reason = str(err)

        if isinstance(req.url, bytes):
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        response.request = req
        response.connection = self

        return response
コード例 #2
0
    def make_best_resolver(self, param):
        if isinstance(param, list):
            path = param[0]
            arg = param[1]
        else:
            path = param
            arg = None

        url_parts = urlsplit(path)

        if url_parts.scheme == 'redis':
            logging.debug('Adding Redis Index: ' + path)
            return RedisResolver(path, arg)

        if url_parts.scheme == 'file':
            path = url_parts.path
            path = url2pathname(path)

        if os.path.isfile(path):
            logging.debug('Adding Path Index: ' + path)
            return PathIndexResolver(path)

        # non-file paths always treated as prefix for now
        else:
            logging.debug('Adding Archive Path Source: ' + path)
            return PrefixResolver(path, arg)
コード例 #3
0
ファイル: base.py プロジェクト: Ofey521/AMP
    def get_filename(self, basename):
        """
        Returns full path to a file, for example:

        get_filename('css/one.css') -> '/full/path/to/static/css/one.css'
        """
        filename = None
        # First try finding the file using the storage class.
        # This is skipped in DEBUG mode as files might be outdated in
        # compressor's final destination (COMPRESS_ROOT) during development
        if not settings.DEBUG:
            try:
                # call path first so remote storages don't make it to exists,
                # which would cause network I/O
                filename = self.storage.path(basename)
                if not self.storage.exists(basename):
                    filename = None
            except NotImplementedError:
                # remote storages don't implement path, access the file locally
                if compressor_file_storage.exists(basename):
                    filename = compressor_file_storage.path(basename)
        # secondly try to find it with staticfiles
        if not filename and self.finders:
            filename = self.finders.find(url2pathname(basename))
        if filename:
            return filename
        # or just raise an exception as the last resort
        raise UncompressableFileError(
            "'%s' could not be found in the COMPRESS_ROOT '%s'%s" %
            (basename, settings.COMPRESS_ROOT,
             self.finders and " or with staticfiles." or "."))
コード例 #4
0
ファイル: loaders.py プロジェクト: yumatchlab/pywb
    def load(self, url, offset=0, length=-1):
        """
        Load a file-like reader from the local file system
        """

        # if starting with . or /, can only be a file path..
        file_only = url.startswith(('/', '.'))

        # convert to filename
        if url.startswith('file://'):
            file_only = True
            url = url2pathname(url[len('file://'):])

        try:
            # first, try as file
            afile = open(url, 'rb')

        except IOError:
            if file_only:
                raise

            # then, try as package.path/file
            pkg_split = url.split('/', 1)
            if len(pkg_split) == 1:
                raise

            afile = pkg_resources.resource_stream(pkg_split[0], pkg_split[1])

        if offset > 0:
            afile.seek(offset)

        if length >= 0:
            return LimitReader(afile, length)
        else:
            return afile
コード例 #5
0
    def send(self, req, **kwargs):  # pylint: disable=unused-argument
        """Return the file specified by the given request

        @type req: C{PreparedRequest}
        @todo: Should I bother filling `response.headers` and processing
               If-Modified-Since and friends using `os.stat`?
        """
        path = os.path.normcase(os.path.normpath(url2pathname(req.path_url)))
        response = requests.Response()

        response.status_code, response.reason = self._chkpath(req.method, path)
        if response.status_code == 200 and req.method.lower() != 'head':
            try:
                response.raw = open(path, 'rb')
            except (OSError, IOError) as err:
                response.status_code = 500
                response.reason = str(err)

        if isinstance(req.url, bytes):
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        response.request = req
        response.connection = self

        return response
コード例 #6
0
ファイル: pathresolvers.py プロジェクト: chdorner/pywb
    def make_best_resolver(self, param):
        if isinstance(param, list):
            path = param[0]
            arg = param[1]
        else:
            path = param
            arg = None

        url_parts = urlsplit(path)

        if url_parts.scheme == 'redis':
            logging.debug('Adding Redis Index: ' + path)
            return RedisResolver(path, arg)

        if url_parts.scheme == 'file':
            path = url_parts.path
            path = url2pathname(path)

        if os.path.isfile(path):
            logging.debug('Adding Path Index: ' + path)
            return PathIndexResolver(path)

        # non-file paths always treated as prefix for now
        else:
            logging.debug('Adding Archive Path Source: ' + path)
            return PrefixResolver(path, arg)
コード例 #7
0
ファイル: structures_test.py プロジェクト: maxpearl/pywebcopy
 def test_url_parsing(self):
     obj = urls.URLTransformer('http://some-site.com:80/path/#frag?query')
     obj.default_filename = 'index.html'
     obj._unique_fn_required = False
     self.assertEqual(obj.original_url,
                      'http://some-site.com:80/path/#frag?query')
     self.assertEqual(obj.url, 'http://some-site.com:80/path/#frag?query')
     self.assertEqual(obj.parsed_url,
                      urlparse.urlsplit('http://some-site.com:80/path/'))
     self.assertEqual(obj.parsed_url.port, 80)
     self.assertEqual(obj.hostname, 'some-site.com')
     self.assertEqual(obj.url_path, '/path/')
     self.assertEqual(obj.file_name, 'index.html')
     self.assertEqual(obj.to_path, url2pathname('some-site.com/path/'))
     self.assertEqual(obj.file_path,
                      url2pathname('some-site.com/path/index.html'))
コード例 #8
0
ファイル: views.py プロジェクト: rachmadagitam/apps.Wirecloud
    def read(self, request, vendor, name, version):

        resource = get_object_or_404(CatalogueResource, vendor=vendor, short_name=name, version=version)
        resource_info = json.loads(resource.json_description)
        if resource_info['doc'] == '':
            raise Http404

        doc_base_url = None
        if resource_info['doc'].startswith(('http://', 'https://')):
            doc_code = _('You can find the userguide of this component in this external <a target="_blank" href="%s">link</a>') % resource_info['doc']
            doc_code = '<div style="margin-top: 10px"><p>%s</p></div>' % doc_code
        else:
            doc_relative_path = url2pathname(resource_info['doc'])
            doc_base_url = force_trailing_slash(urljoin(resource.get_template_url(request=request, for_base=True), pathname2url(os.path.dirname(doc_relative_path))))
            doc_path = os.path.join(catalogue_utils.wgt_deployer.get_base_dir(vendor, name, version), doc_relative_path)

            (doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
            localized_doc_path = doc_filename_root + '.' + get_language() + doc_filename_ext

            try:
                doc_code = download_local_file(localized_doc_path).decode('utf-8')
            except:
                try:
                    doc_code = download_local_file(doc_path).decode('utf-8')
                except:
                    msg = _('Error opening the userguide file')
                    doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg

        doc_pre_html = markdown.markdown(doc_code, output_format='xhtml5', extensions=['codehilite', 'fenced_code'])
        doc = clean_html(doc_pre_html, base_url=doc_base_url)
        return HttpResponse(doc, content_type='application/xhtml+xml; charset=UTF-8')
コード例 #9
0
ファイル: handler.py プロジェクト: WaYdotNET/urllib_s3
    def s3_open(self, req):
        server_name = req.host
        bucket_name = url2pathname(req.selector).split('/')[1]
        key_name = req.selector.replace(
            "/{bucket_name}/".format(bucket_name=bucket_name), '')

        if not bucket_name or not key_name:
            raise URLError('url must be in the format '
                           's3://<server_name>/<bucket>/<object>')
        try:
            credential = self.settings[server_name]
        except KeyError:
            raise ServerNameError('missing credential')

        s3 = boto3.client(
            's3',
            endpoint_url="https://{server}".format(server=server_name),
            aws_access_key_id=credential['access_key'],
            aws_secret_access_key=credential['secret_key'],
            config=Config(signature_version='s3v4'),
            region_name=credential.get('region_name', 'us-east-1'))
        url = s3.generate_presigned_url(ClientMethod='get_object',
                                        Params={
                                            'Bucket': bucket_name,
                                            'Key': key_name
                                        },
                                        ExpiresIn=3600)
        req.full_url = url
        return self.https_open(req)
コード例 #10
0
    def get_html(self, url, retries=1, base_url=None):
        """
        Makes request for a URL and returns HTML as a BeautifulSoup object.
        """
        # Put together the full URL
        full_url = urljoin(base_url or self.base_url, url)
        if self.verbosity > 2:
            self.log(" Retrieving data for {}".format(url))

        # Pull a cached version of the file, if it exists
        cache_path = os.path.join(
            self.cache_dir,
            url2pathname(url.strip("/"))
        )
        if os.path.exists(cache_path) and not self.force_download:
            # Make a HEAD request for the file size of the live page
            if self.update_cache:
                cache_file_size = os.path.getsize(cache_path)
                head = self.get_headers(full_url)
                web_file_size = head['content-length']

                if self.verbosity > 2:
                    msg = " Cached file sized {}. Web file size {}."
                    self.log(msg.format(
                        cache_file_size,
                        web_file_size
                    ))

            # If our cache is the same size as the live page, return the cache
            if not self.update_cache or cache_file_size == web_file_size:
                if self.verbosity > 2:
                    self.log(" Returning cached {}".format(cache_path))
                html = open(cache_path, 'r').read()
                return BeautifulSoup(html, "html.parser")

        # Otherwise, retrieve the full page and cache it
        try:
            response = self.get_url(full_url)
        except requests.exceptions.HTTPError as e:
            # If web requests fails, fall back to cached file, if it exists
            if os.path.exists(cache_path):
                if self.verbosity > 2:
                    self.log(" Returning cached {}".format(cache_path))
                html = open(cache_path, 'r').read()
                return BeautifulSoup(html, "html.parser")
            else:
                raise e

        # Grab the HTML and cache it
        html = response.text
        if self.verbosity > 2:
            self.log(" Writing to cache {}".format(cache_path))
        cache_subdir = os.path.dirname(cache_path)
        os.path.exists(cache_subdir) or os.makedirs(cache_subdir)
        with open(cache_path, 'w') as f:
            f.write(html)

        # Finally return the HTML ready to parse with BeautifulSoup
        return BeautifulSoup(html, "html.parser")
    def get_html(self, url, retries=1, base_url=None):
        """
        Makes request for a URL and returns HTML as a BeautifulSoup object.
        """
        # Put together the full URL
        full_url = urljoin(base_url or self.base_url, url)
        if self.verbosity > 2:
            self.log(" Retrieving data for {}".format(url))

        # Pull a cached version of the file, if it exists
        cache_path = os.path.join(
            self.cache_dir,
            url2pathname(url.strip("/"))
        )
        if os.path.exists(cache_path) and not self.force_download:
            # Make a HEAD request for the file size of the live page
            if self.update_cache:
                cache_file_size = os.path.getsize(cache_path)
                head = self.get_headers(full_url)
                web_file_size = head['content-length']

                if self.verbosity > 2:
                    msg = " Cached file sized {}. Web file size {}."
                    self.log(msg.format(
                        cache_file_size,
                        web_file_size
                    ))

            # If our cache is the same size as the live page, return the cache
            if not self.update_cache or cache_file_size == web_file_size:
                if self.verbosity > 2:
                    self.log(" Returning cached {}".format(cache_path))
                html = open(cache_path, 'r').read()
                return BeautifulSoup(html, "html.parser")

        # Otherwise, retrieve the full page and cache it
        try:
            response = self.get_url(full_url)
        except requests.exceptions.HTTPError as e:
            # If web requests fails, fall back to cached file, if it exists
            if os.path.exists(cache_path):
                if self.verbosity > 2:
                    self.log(" Returning cached {}".format(cache_path))
                html = open(cache_path, 'r').read()
                return BeautifulSoup(html, "html.parser")
            else:
                raise e

        # Grab the HTML and cache it
        html = response.text
        if self.verbosity > 2:
            self.log(" Writing to cache {}".format(cache_path))
        cache_subdir = os.path.dirname(cache_path)
        os.path.exists(cache_subdir) or os.makedirs(cache_subdir)
        with open(cache_path, 'w') as f:
            f.write(html)

        # Finally return the HTML ready to parse with BeautifulSoup
        return BeautifulSoup(html, "html.parser")
コード例 #12
0
def cached_file(url):
    fname = os.path.basename(url2pathname(url.split(':', 1)[1]))
    fpath = os.path.join(CACHE_DIR, fname)
    if not os.path.exists(fpath):
        with open(fpath, 'w') as f:
            furl = urlopen(url)
            f.write(furl.read())
            furl.close()
    return fpath
コード例 #13
0
 def on_file():
     try:
         filename = url2pathname(uri[5:].encode('utf-8') if isinstance(uri, text_type) else uri[5:])
         if filename.endswith(BLOB_EXTENSION):
             return on_mdblob(filename)
         metainfo_deferred.callback(bdecode(fix_torrent(filename)))
         return NOT_DONE_YET
     except TypeError:
         request.setResponseCode(http.INTERNAL_SERVER_ERROR)
         return json.dumps({"error": "error while decoding torrent file"})
コード例 #14
0
ファイル: LibtorrentMgr.py プロジェクト: Tribler/tribler
    def start_download_from_uri(self, uri, dconfig=None):
        if uri.startswith("http"):
            return self.start_download_from_url(bytes(uri), dconfig=dconfig)
        if uri.startswith("magnet:"):
            return succeed(self.start_download_from_magnet(uri, dconfig=dconfig))
        if uri.startswith("file:"):
            argument = url2pathname(uri[5:])
            return succeed(self.start_download(torrentfilename=argument, dconfig=dconfig))

        return fail(Failure(Exception("invalid uri")))
コード例 #15
0
    def get_repo(uri):
        parts = urlparse(uri, "file", allow_fragments=0)
        path = url2pathname(parts[2])

        try:
            return repo.Repository(root=path)
        except cfg.ConfigError as e:
            raise repo.RepositoryError(
                _("The specified "
                  "repository's configuration data is not "
                  "valid:\n{0}").format(e))
コード例 #16
0
def check_invalid_image(wgt_file, resource_info, key):

    image_url = resource_info[key]
    if image_url != '' and not image_url.startswith(('http://', 'https://')):

        image_path = url2pathname(image_url)

        try:
            wgt_file.read(image_path)
        except KeyError:
            raise InvalidContents('missing image file: %s' % image_path)
コード例 #17
0
    def read(self, request, vendor, name, version):

        from_version = request.GET.get('from')
        if from_version is not None:
            try:
                from_version = Version(from_version)
            except:
                return build_error_response(
                    request, 422, _("Missing parameter: template_uri or file"))

        resource = get_object_or_404(CatalogueResource,
                                     vendor=vendor,
                                     short_name=name,
                                     version=version)
        resource_info = resource.get_processed_info(process_urls=False)
        if resource_info['changelog'] == '':
            raise Http404

        doc_relative_path = url2pathname(resource_info['changelog'])
        doc_base_url = force_trailing_slash(
            urljoin(resource.get_template_url(request=request, for_base=True),
                    pathname2url(os.path.dirname(doc_relative_path))))
        doc_path = os.path.join(
            catalogue_utils.wgt_deployer.get_base_dir(vendor, name, version),
            doc_relative_path)

        (doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
        localized_doc_path = doc_filename_root + '.' + get_language(
        ) + doc_filename_ext

        try:
            doc_code = download_local_file(localized_doc_path).decode('utf-8')
        except:
            try:
                doc_code = download_local_file(doc_path).decode('utf-8')
            except:
                msg = _('Error opening the changelog file')
                doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg

        doc_pre_html = markdown.markdown(doc_code,
                                         output_format='xhtml5',
                                         extensions=[
                                             'markdown.extensions.codehilite',
                                             'markdown.extensions.fenced_code'
                                         ])

        if from_version:
            doc_pre_html = filter_changelog(doc_pre_html, from_version)
            if doc_pre_html.strip() == '':
                raise Http404

        doc = clean_html(doc_pre_html, base_url=doc_base_url)
        return HttpResponse(
            doc, content_type='application/xhtml+xml; charset=UTF-8')
コード例 #18
0
ファイル: utils.py プロジェクト: Wirecloud/wirecloud
def check_invalid_image(wgt_file, resource_info, key):

    image_url = resource_info[key]
    if image_url != '' and not image_url.startswith(('http://', 'https://')):

        image_path = url2pathname(image_url)

        try:
            wgt_file.read(image_path)
        except KeyError:
            raise InvalidContents('missing image file: %s' % image_path)
コード例 #19
0
def uri(param):
    # type: (str) -> str
    if urlsplit(param).scheme:
        return param

    path = expanduser(expandvars(url2pathname(param)))
    if exists(path):
        return urljoin('file:', pathname2url(abspath(path)))

    raise ArgumentTypeError(
        '`{param}` is not an existing file and either a valid URI'.format(
            param=param))
コード例 #20
0
    def start_download_from_uri(self, uri, dconfig=None):
        if uri.startswith("http"):
            return self.start_download_from_url(bytes(uri), dconfig=dconfig)
        if uri.startswith("magnet:"):
            return succeed(
                self.start_download_from_magnet(uri, dconfig=dconfig))
        if uri.startswith("file:"):
            argument = url2pathname(uri[5:])
            return succeed(
                self.start_download(torrentfilename=argument, dconfig=dconfig))

        return fail(Failure(Exception("invalid uri")))
コード例 #21
0
def _create_run_in_store(store):
    config = {
        'experiment_id': '0',
        'user_id': 'Anderson',
        'start_time': int(time.time()),
        'tags': {}
    }
    run = store.create_run(**config)
    artifact_path = url2pathname(unquote(urlparse(run.info.artifact_uri).path))
    if not os.path.exists(artifact_path):
        os.makedirs(artifact_path)
    return run
コード例 #22
0
 def on_file():
     try:
         filename = url2pathname(uri[5:].encode('utf-8') if isinstance(
             uri, text_type) else uri[5:])
         if filename.endswith(BLOB_EXTENSION):
             return on_mdblob(filename)
         metainfo_deferred.callback(bdecode(fix_torrent(filename)))
         return NOT_DONE_YET
     except TypeError:
         request.setResponseCode(http.INTERNAL_SERVER_ERROR)
         return json.dumps(
             {"error": "error while decoding torrent file"})
コード例 #23
0
ファイル: test_cli.py プロジェクト: yourivandenberg/mlflow
def _create_run_in_store(store):
    config = {
        "experiment_id": "0",
        "user_id": "Anderson",
        "start_time": int(time.time()),
        "tags": {},
    }
    run = store.create_run(**config)
    artifact_path = url2pathname(unquote(urlparse(run.info.artifact_uri).path))
    if not os.path.exists(artifact_path):
        os.makedirs(artifact_path)
    return run
コード例 #24
0
ファイル: path.py プロジェクト: alastairwp/yugendigital
def url_to_path(url):
    """Convert a valid file url to a local filesystem path

    Follows logic taken from pip's equivalent function
    """
    assert is_file_url(url), "Only file: urls can be converted to local paths"
    _, netloc, path, _, _ = urllib_parse.urlsplit(url)
    # Netlocs are UNC paths
    if netloc:
        netloc = "\\\\" + netloc

    path = urllib_request.url2pathname(netloc + path)
    return to_bytes(path, encoding="utf-8")
コード例 #25
0
    def topath(self):
        """Convert a file: URL to a path."""
        assert url.startswith('file:'), (
            "You can only turn file: urls into filenames (not %r)" % url)

        _, netloc, path, _, _ = urllib_parse.urlsplit(url)

        # if we have a UNC path, prepend UNC share notation
        if netloc:
            netloc = '\\\\' + netloc

        path = urllib_request.url2pathname(netloc + path)
        return path
コード例 #26
0
ファイル: urls.py プロジェクト: maxpearl/pywebcopy
    def file_path(self):
        """Make a unique path from the url.

        :rtype: str
        :return: disk compatible path
        """
        upath, _ = self._refactor_filename(self.url_path)

        # clean the url and prepend hostname to make it complete
        upath = url2pathname(self.hostname + upath)

        if self.base_path:
            upath = os.path.join(self.base_path, upath)
        return upath
コード例 #27
0
    def read(self, request, vendor, name, version):

        resource = get_object_or_404(CatalogueResource,
                                     vendor=vendor,
                                     short_name=name,
                                     version=version)
        resource_info = resource.json_description
        if resource_info['doc'] == '':
            raise Http404

        doc_base_url = None
        if resource_info['doc'].startswith(('http://', 'https://')):
            doc_code = _(
                'You can find the userguide of this component in this external <a target="_blank" href="%s">link</a>'
            ) % resource_info['doc']
            doc_code = '<div style="margin-top: 10px"><p>%s</p></div>' % doc_code
        else:
            doc_relative_path = url2pathname(resource_info['doc'])
            doc_base_url = force_trailing_slash(
                urljoin(
                    resource.get_template_url(request=request, for_base=True),
                    pathname2url(os.path.dirname(doc_relative_path))))
            doc_path = os.path.join(
                catalogue_utils.wgt_deployer.get_base_dir(
                    vendor, name, version), doc_relative_path)

            (doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
            localized_doc_path = doc_filename_root + '.' + get_language(
            ) + doc_filename_ext

            try:
                doc_code = download_local_file(localized_doc_path).decode(
                    'utf-8')
            except:
                try:
                    doc_code = download_local_file(doc_path).decode('utf-8')
                except:
                    msg = _('Error opening the userguide file')
                    doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg

        doc_pre_html = markdown.markdown(doc_code,
                                         output_format='xhtml5',
                                         extensions=[
                                             'markdown.extensions.codehilite',
                                             'markdown.extensions.fenced_code'
                                         ])
        doc = clean_html(doc_pre_html, base_url=doc_base_url)
        return HttpResponse(
            doc, content_type='application/xhtml+xml; charset=UTF-8')
コード例 #28
0
def url_to_path(url):
    # type: (str) -> str
    """Convert a valid file url to a local filesystem path.

    Follows logic taken from pip's equivalent function
    """

    assert is_file_url(url), "Only file: urls can be converted to local paths"
    _, netloc, path, _, _ = urllib_parse.urlsplit(url)
    # Netlocs are UNC paths
    if netloc:
        netloc = "\\\\" + netloc

    path = urllib_request.url2pathname(netloc + path)
    return urllib_parse.unquote(path)
コード例 #29
0
def cached_file(url):
    fname = url2pathname(relative(url))

    fpath = os.path.join(CACHE_DIR, fname)
    if not os.path.exists(fpath):
        print("%s does not exist, fetching from %s" % (fpath, url))
        folder = os.path.dirname(fpath)
        if not os.path.exists(folder):
            os.makedirs(folder)
        f = open(fpath, 'w')
        try:
            f.write(urlopen(url).read())
        finally:
            f.close()
    return fpath
コード例 #30
0
ファイル: utils.py プロジェクト: Wirecloud/wirecloud
def check_invalid_doc_content(wgt_file, resource_info, key):

    doc_url = resource_info[key]
    if doc_url != '' and not doc_url.startswith(('http://', 'https://')):
        doc_path = url2pathname(doc_url)

        # Check default version
        check_invalid_doc_entry(wgt_file, doc_path)

        # Check localized versions
        (doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
        pattern = re.escape(doc_filename_root) + '\.\w\w(?:-\w\w)?' + re.escape(doc_filename_ext)
        for filename in wgt_file.namelist():
            if re.match(pattern, filename):
                check_invalid_doc_entry(wgt_file, filename)
コード例 #31
0
ファイル: test_rdfxml.py プロジェクト: RDFLib/rdflib
def cached_file(url):
    fname = url2pathname(relative(url))

    fpath = os.path.join(CACHE_DIR, fname)
    if not os.path.exists(fpath):
        print("%s does not exist, fetching from %s" % (fpath, url))
        folder = os.path.dirname(fpath)
        if not os.path.exists(folder):
            os.makedirs(folder)
        f = open(fpath, 'w')
        try:
            f.write(urlopen(url).read())
        finally:
            f.close()
    return fpath
コード例 #32
0
    def __init__(self, url=None, *args, **kwargs):

        # Works around an apparent Git bug
        # (see http://article.gmane.org/gmane.comp.version-control.git/146500)
        if url:
            scheme, netloc, path, query, fragment = urlsplit(url)
            if scheme.endswith('file'):
                initial_slashes = path[:-len(path.lstrip('/'))]
                newpath = (initial_slashes +
                           urllib_request.url2pathname(path).replace(
                               '\\', '/').lstrip('/'))
                url = urlunsplit((scheme, netloc, newpath, query, fragment))
                after_plus = scheme.find('+') + 1
                url = scheme[:after_plus] + urlunsplit(
                    (scheme[after_plus:], netloc, newpath, query, fragment), )

        super(Git, self).__init__(url, *args, **kwargs)
コード例 #33
0
def check_invalid_doc_content(wgt_file, resource_info, key):

    doc_url = resource_info[key]
    if doc_url != '' and not doc_url.startswith(('http://', 'https://')):
        doc_path = url2pathname(doc_url)

        # Check default version
        check_invalid_doc_entry(wgt_file, doc_path)

        # Check localized versions
        (doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
        pattern = re.escape(
            doc_filename_root) + '\.\w\w(?:-\w\w)?' + re.escape(
                doc_filename_ext)
        for filename in wgt_file.namelist():
            if re.match(pattern, filename):
                check_invalid_doc_entry(wgt_file, filename)
コード例 #34
0
    def file_path(self):
        """Make a unique path from the url.

        :rtype: str
        :return: disk compatible path
        """
        path, _ = self._refactor_filename(self.url_path)
        hostname = self.hostname

        assert isinstance(path, basestring)
        assert isinstance(hostname, basestring)

        # clean the url and prepend hostname to make it complete
        path = url2pathname(hostname + path)

        if self.base_path:
            path = os.path.join(self.base_path, path)
        return path
コード例 #35
0
 def open_local_file(self, req):
     try:
         host = req.host
         filename = req.selector
         # if that bombs, then go on with original method
         localfile = url.url2pathname(host + filename)
         stats = os.stat(localfile)
         size = stats.st_size
         modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
         mtype = mimetypes.guess_type(filename)[0]
         headers = email.message_from_string(
             'Content-type: %s\nContent-length: %d\nLast-modified: %s\n'
             % (mtype or 'text/plain', size, modified))
         origurl = 'file://' + host + filename
         return addinfourl(open(localfile, 'rb'), headers, origurl)
     except:
         pass
     return url.FileHandler.open_local_file(self, req)
コード例 #36
0
    def _handle_user_code_url(self, code):
        """Gets the S3 URL containing the user's code.

           Inspects the scheme the customer passed in ("s3://" for code in S3, "file://" or nothing
           for absolute or local file paths. Uploads the code to S3 if the code is a local file.

        Args:
            code (str): A URL to the customer's code.

        Returns:
            str: The S3 URL to the customer's code.

        Raises:
            ValueError: if the code isn't found, is a directory, or
                does not have a valid URL scheme.
        """
        code_url = urlparse(code)
        if code_url.scheme == "s3":
            user_code_s3_uri = code
        elif code_url.scheme == "" or code_url.scheme == "file":
            # Validate that the file exists locally and is not a directory.
            code_path = url2pathname(code_url.path)
            if not os.path.exists(code_path):
                raise ValueError(
                    """code {} wasn't found. Please make sure that the file exists.
                    """.format(
                        code
                    )
                )
            if not os.path.isfile(code_path):
                raise ValueError(
                    """code {} must be a file, not a directory. Please pass a path to a file.
                    """.format(
                        code
                    )
                )
            user_code_s3_uri = self._upload_code(code_path)
        else:
            raise ValueError(
                "code {} url scheme {} is not recognized. Please pass a file path or S3 url".format(
                    code, code_url.scheme
                )
            )
        return user_code_s3_uri
コード例 #37
0
ファイル: structures_test.py プロジェクト: maxpearl/pywebcopy
 def test_url_parsing_after_set_base(self):
     obj = urls.URLTransformer('../some/rel/path/')
     obj.base_url = "http://some-site.com:80"
     obj.base_path = "e:\\tests\\"
     obj.default_filename = 'index.html'
     obj._unique_fn_required = False
     self.assertEqual(obj.original_url, '../some/rel/path/')
     self.assertEqual(obj.url, 'http://some-site.com:80/some/rel/path/')
     self.assertEqual(
         obj.parsed_url,
         urlparse.urlsplit('http://some-site.com:80/some/rel/path/',
                           allow_fragments=False))
     self.assertEqual(obj.parsed_url.port, 80)
     self.assertEqual(obj.hostname, 'some-site.com')
     self.assertEqual(obj.url_path, '/some/rel/path/')
     self.assertEqual(obj.file_name, 'index.html')
     self.assertEqual(obj.to_path,
                      'e:\\tests\\some-site.com\\some\\rel\\path\\')
     self.assertEqual(
         obj.file_path,
         url2pathname(
             'e://tests/some-site.com/some/rel/path/index.html').lower())
コード例 #38
0
ファイル: views.py プロジェクト: rachmadagitam/apps.Wirecloud
    def read(self, request, vendor, name, version):

        from_version = request.GET.get('from')
        if from_version is not None:
            try:
                from_version = Version(from_version)
            except:
                return build_error_response(request, 422, _("Missing parameter: template_uri or file"))

        resource = get_object_or_404(CatalogueResource, vendor=vendor, short_name=name, version=version)
        resource_info = resource.get_processed_info(process_urls=False)
        if resource_info['changelog'] == '':
            raise Http404

        doc_relative_path = url2pathname(resource_info['changelog'])
        doc_base_url = force_trailing_slash(urljoin(resource.get_template_url(request=request, for_base=True), pathname2url(os.path.dirname(doc_relative_path))))
        doc_path = os.path.join(catalogue_utils.wgt_deployer.get_base_dir(vendor, name, version), doc_relative_path)

        (doc_filename_root, doc_filename_ext) = os.path.splitext(doc_path)
        localized_doc_path = doc_filename_root + '.' + get_language() + doc_filename_ext

        try:
            doc_code = download_local_file(localized_doc_path).decode('utf-8')
        except:
            try:
                doc_code = download_local_file(doc_path).decode('utf-8')
            except:
                msg = _('Error opening the changelog file')
                doc_code = '<div class="margin-top: 10px"><p>%s</p></div>' % msg

        doc_pre_html = markdown.markdown(doc_code, output_format='xhtml5', extensions=['codehilite', 'fenced_code'])

        if from_version:
            doc_pre_html = filter_changelog(doc_pre_html, from_version)
            if doc_pre_html.strip() == '':
                raise Http404

        doc = clean_html(doc_pre_html, base_url=doc_base_url)
        return HttpResponse(doc, content_type='application/xhtml+xml; charset=UTF-8')
コード例 #39
0
ファイル: loaders.py プロジェクト: gwu-libraries/pywb
    def load(self, url, offset=0, length=-1):
        """
        Load a file-like reader from the local file system
        """

        # if starting with . or /, can only be a file path..
        file_only = url.startswith(('/', '.'))

        # convert to filename
        if url.startswith('file://'):
            file_only = True
            url = url2pathname(url[len('file://'):])

        try:
            # first, try as file
            afile = open(url, 'rb')

        except IOError:
            if file_only:
                raise

            # then, try as package.path/file
            pkg_split = url.split('/', 1)
            if len(pkg_split) == 1:
                raise

            afile = pkg_resources.resource_stream(pkg_split[0],
                                                  pkg_split[1])

        if offset > 0:
            afile.seek(offset)

        if length >= 0:
            return LimitReader(afile, length)
        else:
            return afile
コード例 #40
0
	def do_GET(self):
		up = urlparse( self.path )
		try:
			if up.path=='/favicon.ico' and favicon:
				content = favicon
				content_type = 'image/x-icon'
			else:
				file = url2pathname(os.path.basename(up.path.split('#')[0]))
				fname = os.path.join( Utils.getHelpFolder(), file )
				with io.open(fname, 'r', encoding='utf-8') as fp:
					content = fp.read()
				content_type = self.html_content
		except Exception as e:
			self.send_error(404,'File Not Found: {} {}\n{}'.format(self.path, e, traceback.format_exc()))
			return
		
		self.send_response( 200 )
		self.send_header('Content-type',content_type)
		if content_type == self.html_content:
			self.send_header( 'Cache-Control', 'no-cache, no-store, must-revalidate' )
			self.send_header( 'Pragma', 'no-cache' )
			self.send_header( 'Expires', '0' )
		self.end_headers()
		self.wfile.write( content.encode() )
コード例 #41
0
ファイル: utils.py プロジェクト: peterdeme/poetry-core
def url_to_path(url):  # type: (str) -> Path
    """
    Convert an RFC8089 file URI to path.

    The logic used here is borrowed from pip
    https://github.com/pypa/pip/blob/4d1932fcdd1974c820ea60b3286984ebb0c3beaa/src/pip/_internal/utils/urls.py#L31
    """
    if not url.startswith("file:"):
        raise ValueError("{} is not a valid file URI".format(url))

    _, netloc, path, _, _ = urlsplit(url)

    if not netloc or netloc == "localhost":
        # According to RFC 8089, same as empty authority.
        netloc = ""
    elif netloc not in {".", ".."} and sys.platform == "win32":
        # If we have a UNC path, prepend UNC share notation.
        netloc = "\\\\" + netloc
    else:
        raise ValueError(
            "non-local file URIs are not supported on this platform: {}".
            format(url))

    return Path(url2pathname(netloc + unquote(path)))
コード例 #42
0
ファイル: url.py プロジェクト: Preetwinder/w3lib
def file_uri_to_path(uri):
    """Convert File URI to local filesystem path according to:
    http://en.wikipedia.org/wiki/File_URI_scheme
    """
    uri_path = urlparse(uri).path
    return url2pathname(uri_path)
コード例 #43
0
ファイル: views.py プロジェクト: GreenIDer-Donati/wirecloud
    def read(self, request, vendor, name, version):

        resource = get_object_or_404(CatalogueResource.objects.select_related('widget__xhtml'), vendor=vendor, short_name=name, version=version)
        # For now, all widgets are freely accessible/distributable
        #if not resource.is_available_for(request.user):
        #    return build_error_response(request, 403, "Forbidden")

        if resource.resource_type() != 'widget':
            raise Http404()

        mode = request.GET.get('mode', 'classic')
        widget_info = json.loads(resource.json_description)

        # check if the xhtml code has been cached
        if widget_info['contents']['cacheable'] is True:

            cache_key = resource.widget.xhtml.get_cache_key(get_current_domain(request), mode)
            cache_entry = cache.get(cache_key)
            if cache_entry is not None:
                response = HttpResponse(cache_entry['code'], content_type=cache_entry['content_type'])
                patch_cache_headers(response, cache_entry['timestamp'], cache_entry['timeout'])
                return response

        # process xhtml
        xhtml = resource.widget.xhtml
        content_type = widget_info['contents'].get('contenttype', 'text/html')
        charset = widget_info['contents'].get('charset', 'utf-8')

        force_base = False
        base_url = xhtml.url
        if not base_url.startswith(('http://', 'https://')):
            # Newer versions of Django urlencode urls created using reverse
            # Fix double encoding
            base_url = urlunquote(base_url)
            base_url = get_absolute_reverse_url('wirecloud.showcase_media', args=(base_url.split('/', 4)), request=request)
            force_base = True

        code = xhtml.code
        if not xhtml.cacheable or code == '':
            try:
                if xhtml.url.startswith(('http://', 'https://')):
                    code = download_http_content(urljoin(base_url, xhtml.url), user=request.user)
                else:
                    code = download_local_file(os.path.join(showcase_utils.wgt_deployer.root_dir, url2pathname(xhtml.url)))

            except Exception as e:
                return build_response(request, 502, {'error_msg': _("(X)HTML code is not accessible"), 'details': "%s" % e}, WIDGET_ERROR_FORMATTERS)
        else:
            # Code contents comes as unicode from persistence, we need bytes
            code = code.encode(charset)

        if xhtml.cacheable and (xhtml.code == '' or xhtml.code_timestamp is None):
            try:
                xhtml.code = code.decode(charset)
            except UnicodeDecodeError:
                msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
                return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)

            xhtml.code_timestamp = time.time() * 1000
            xhtml.save()
        elif not xhtml.cacheable and xhtml.code != '':
            xhtml.code = ''
            xhtml.code_timestamp = None
            xhtml.save()

        try:
            code = fix_widget_code(code, base_url, content_type, request, charset, xhtml.use_platform_style, process_requirements(widget_info['requirements']), force_base, mode)
        except UnicodeDecodeError:
            msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
            return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)
        except Exception as e:
            msg = _('Error processing widget code')
            return build_response(request, 502, {'error_msg': msg, 'details':"%s" % e}, WIDGET_ERROR_FORMATTERS)

        if xhtml.cacheable:
            cache_timeout = 31536000  # 1 year
            cache_entry = {
                'code': code,
                'content_type': '%s; charset=%s' % (content_type, charset),
                'timestamp': xhtml.code_timestamp,
                'timeout': cache_timeout,
            }
            cache.set(cache_key, cache_entry, cache_timeout)
        else:
            cache_timeout = 0

        response = HttpResponse(code, content_type='%s; charset=%s' % (content_type, charset))
        patch_cache_headers(response, xhtml.code_timestamp, cache_timeout)
        return response
コード例 #44
0
ファイル: downloads_endpoint.py プロジェクト: Tribler/tribler
    def render_PUT(self, request):
        """
        .. http:put:: /downloads

        A PUT request to this endpoint will start a download from a provided URI. This URI can either represent a file
        location, a magnet link or a HTTP(S) url.
        - anon_hops: the number of hops for the anonymous download. 0 hops is equivalent to a plain download
        - safe_seeding: whether the seeding of the download should be anonymous or not (0 = off, 1 = on)
        - destination: the download destination path of the torrent
        - torrent: the URI of the torrent file that should be downloaded. This parameter is required.

            **Example request**:

                .. sourcecode:: none

                    curl -X PUT http://localhost:8085/downloads
                    --data "anon_hops=2&safe_seeding=1&destination=/my/dest/on/disk/&uri=file:/home/me/test.torrent

            **Example response**:

                .. sourcecode:: javascript

                    {"started": True, "infohash": "4344503b7e797ebf31582327a5baae35b11bda01"}
        """
        parameters = http.parse_qs(request.content.read(), 1)

        if 'uri' not in parameters or len(parameters['uri']) == 0:
            request.setResponseCode(http.BAD_REQUEST)
            return json.dumps({"error": "uri parameter missing"})

        download_config, error = DownloadsEndpoint.create_dconfig_from_params(parameters)
        if error:
            request.setResponseCode(http.BAD_REQUEST)
            return json.dumps({"error": error})

        def download_added(download):
            request.write(json.dumps({"started": True,
                                      "infohash": hexlify(download.get_def().get_infohash())}))
            request.finish()

        def on_error(error):
            request.setResponseCode(http.INTERNAL_SERVER_ERROR)
            request.write(json.dumps({"error": unichar_string(error.getErrorMessage())}))
            request.finish()

        uri = parameters['uri'][0]
        if uri.startswith("file:"):
            if uri.endswith(".mdblob"):
                filename = url2pathname(uri[5:].encode('utf-8') if isinstance(uri, six.text_type) else uri[5:])
                try:
                    payload = ChannelMetadataPayload.from_file(filename)
                except IOError:
                    request.setResponseCode(http.BAD_REQUEST)
                    return json.dumps({"error": "file not found"})
                except InvalidSignatureException:
                    request.setResponseCode(http.BAD_REQUEST)
                    return json.dumps({"error": "Metadata has invalid signature"})

                with db_session:
                    result = self.session.lm.mds.process_payload(payload)
                    if result:
                        node, status = result[0]
                        if (status == UNKNOWN_CHANNEL or
                                (status == UPDATED_OUR_VERSION and node.metadata_type == CHANNEL_TORRENT)):
                            node.subscribed = True
                            return json.dumps(
                                {"started": True, "infohash": hexlify(node.infohash)})
                    return json.dumps({"error": "Already subscribed"})
            else:
                download_uri = u"file:%s" % url2pathname(uri[5:]).decode('utf-8')
        else:
            download_uri = unquote_plus(cast_to_unicode_utf8(uri))
        download_deferred = self.session.start_download_from_uri(download_uri, download_config)
        download_deferred.addCallback(download_added)
        download_deferred.addErrback(on_error)

        return NOT_DONE_YET
コード例 #45
0
ファイル: utils.py プロジェクト: GreenIDer-Donati/wirecloud
def get_resource_data(resource, user, request=None):
    """Gets all the information related to the given resource."""
    resource_info = resource.get_processed_info(request)

    template_uri = get_absolute_reverse_url(
        "wirecloud_catalogue.media",
        kwargs={
            "vendor": resource.vendor,
            "name": resource.short_name,
            "version": resource.version,
            "file_path": resource.template_uri,
        },
        request=request,
    )

    wgt_path = os.path.join(
        wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version), resource.template_uri
    )
    size = os.path.getsize(wgt_path)

    cdate = resource.creation_date
    creation_timestamp = time.mktime(cdate.timetuple()) * 1e3 + cdate.microsecond / 1e3

    longdescription = resource_info["longdescription"]
    if longdescription != "":
        longdescription_relative_path = url2pathname(longdescription)
        longdescription_base_url = force_trailing_slash(
            urljoin(
                resource.get_template_url(request=request, for_base=True),
                pathname2url(os.path.dirname(longdescription_relative_path)),
            )
        )
        longdescription_path = os.path.join(
            wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version),
            longdescription_relative_path,
        )

        (filename_root, filename_ext) = os.path.splitext(longdescription_path)
        localized_longdescription_path = filename_root + "." + get_language() + filename_ext

        try:
            description_code = download_local_file(localized_longdescription_path)
            longdescription = clean_html(
                markdown.markdown(description_code, output_format="xhtml5"), base_url=longdescription_base_url
            )
        except:
            try:
                description_code = download_local_file(longdescription_path)
                longdescription = clean_html(
                    markdown.markdown(description_code, output_format="xhtml5"), base_url=longdescription_base_url
                )
            except:
                longdescription = resource_info["description"]

    else:
        longdescription = resource_info["description"]

    return {
        "id": resource.pk,
        "vendor": resource.vendor,
        "name": resource.short_name,
        "version": resource.version,
        "type": resource_info["type"],
        "date": creation_timestamp,
        "permissions": {
            "delete": user.is_superuser,
            "uninstall": resource.public is False and resource.users.filter(pk=user.pk).exists(),
        },
        "authors": resource_info["authors"],
        "contributors": resource_info["contributors"],
        "title": resource_info["title"],
        "description": resource_info["description"],
        "longdescription": longdescription,
        "email": resource_info["email"],
        "image": resource_info["image"],
        "homepage": resource_info["homepage"],
        "doc": resource_info["doc"],
        "changelog": resource_info["changelog"],
        "size": size,
        "uriTemplate": template_uri,
        "license": resource_info["license"],
        "licenseurl": resource_info["licenseurl"],
        "issuetracker": resource_info["issuetracker"],
    }
コード例 #46
0
ファイル: views.py プロジェクト: Fiware/apps.Wirecloud
def process_widget_code(request, resource):

    mode = request.GET.get('mode', 'classic')
    theme = request.GET.get('theme', get_active_theme_name())
    widget_info = json.loads(resource.json_description)

    # check if the xhtml code has been cached
    if widget_info['contents']['cacheable'] is True:

        cache_key = resource.widget.xhtml.get_cache_key(get_current_domain(request), mode, theme)
        cache_entry = cache.get(cache_key)
        if cache_entry is not None:
            response = HttpResponse(cache_entry['code'], content_type=cache_entry['content_type'])
            patch_cache_headers(response, cache_entry['timestamp'], cache_entry['timeout'])
            return response

    # process xhtml
    xhtml = resource.widget.xhtml
    content_type = widget_info['contents'].get('contenttype', 'text/html')
    charset = widget_info['contents'].get('charset', 'utf-8')

    code = xhtml.code
    if not xhtml.cacheable or code == '':
        try:
            code = download_local_file(os.path.join(showcase_utils.wgt_deployer.root_dir, url2pathname(xhtml.url)))

        except Exception as e:
            if isinstance(e, IOError) and e.errno == errno.ENOENT:
                return build_response(request, 404, {'error_msg': _("Widget code not found"), 'details': "%s" % e}, WIDGET_ERROR_FORMATTERS)
            else:
                return build_response(request, 500, {'error_msg': _("Error reading widget code"), 'details': "%s" % e}, WIDGET_ERROR_FORMATTERS)
    else:
        # Code contents comes as unicode from persistence, we need bytes
        code = code.encode(charset)

    if xhtml.cacheable and (xhtml.code == '' or xhtml.code_timestamp is None):
        try:
            xhtml.code = code.decode(charset)
        except UnicodeDecodeError:
            msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
            return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)

        xhtml.code_timestamp = time.time() * 1000
        xhtml.save()

    try:
        code = fix_widget_code(code, content_type, request, charset, xhtml.use_platform_style, process_requirements(widget_info['requirements']), mode, theme)
    except UnicodeDecodeError:
        msg = _('Widget code was not encoded using the specified charset (%(charset)s as stated in the widget description file).') % {'charset': charset}
        return build_response(request, 502, {'error_msg': msg}, WIDGET_ERROR_FORMATTERS)
    except Exception as e:
        msg = _('Error processing widget code')
        return build_response(request, 502, {'error_msg': msg, 'details':"%s" % e}, WIDGET_ERROR_FORMATTERS)

    if xhtml.cacheable:
        cache_timeout = 31536000  # 1 year
        cache_entry = {
            'code': code,
            'content_type': '%s; charset=%s' % (content_type, charset),
            'timestamp': xhtml.code_timestamp,
            'timeout': cache_timeout,
        }
        cache.set(cache_key, cache_entry, cache_timeout)
    else:
        cache_timeout = 0

    response = HttpResponse(code, content_type='%s; charset=%s' % (content_type, charset))
    patch_cache_headers(response, xhtml.code_timestamp, cache_timeout)
    return response
コード例 #47
0
ファイル: parser.py プロジェクト: drewp/rdflib
def create_input_source(source=None, publicID=None,
                        location=None, file=None, data=None, format=None):
    """
    Return an appropriate InputSource instance for the given
    parameters.
    """

    # test that exactly one of source, location, file, and data is not None.
    if sum((
        source is not None,
        location is not None,
        file is not None,
        data is not None,
    )) != 1:
        raise ValueError(
            'exactly one of source, location, file or data must be given'
        )

    input_source = None

    if source is not None:
        if isinstance(source, InputSource):
            input_source = source
        else:
            if isinstance(source, string_types):
                location = source
            elif hasattr(source, "read") and not isinstance(source, Namespace):
                f = source
                input_source = InputSource()
                input_source.setByteStream(f)
                if f is sys.stdin:
                    input_source.setSystemId("file:///dev/stdin")
                elif hasattr(f, "name"):
                    input_source.setSystemId(f.name)
            else:
                raise Exception("Unexpected type '%s' for source '%s'" %
                                (type(source), source))

    absolute_location = None  # Further to fix for issue 130

    auto_close = False  # make sure we close all file handles we open
    if location is not None:
        # Fix for Windows problem https://github.com/RDFLib/rdflib/issues/145
        if os.path.exists(location):
            location = pathname2url(location)
        base = urljoin("file:", "%s/" % pathname2url(os.getcwd()))
        absolute_location = URIRef(location, base=base).defrag()
        if absolute_location.startswith("file:///"):
            filename = url2pathname(absolute_location.replace("file:///", "/"))
            file = open(filename, "rb")
        else:
            input_source = URLInputSource(absolute_location, format)
        auto_close = True
        # publicID = publicID or absolute_location  # Further to fix
                                                    # for issue 130

    if file is not None:
        input_source = FileInputSource(file)

    if data is not None:
        if isinstance(data, text_type):
            data = data.encode('utf-8')
        input_source = StringInputSource(data)
        auto_close = True

    if input_source is None:
        raise Exception("could not create InputSource")
    else:
        input_source.auto_close |= auto_close
        if publicID is not None:  # Further to fix for issue 130
            input_source.setPublicId(publicID)
        # Further to fix for issue 130
        elif input_source.getPublicId() is None:
            input_source.setPublicId(absolute_location or "")
        return input_source
コード例 #48
0
ファイル: WebServer.py プロジェクト: esitarski/CrossMgr
	def do_GET(self):
		up = urllib.parse.urlparse( self.path )		
		content, gzip_content = None,  None
		try:
			if up.path=='/':
				content = getIndexPage()
				content_type = self.html_content
				assert isinstance( content, bytes )
			elif up.path=='/favicon.ico':
				content = favicon
				content_type = 'image/x-icon'
				assert isinstance( content, bytes )
			elif self.reLapCounterHtml.match( up.path ):
				content = getLapCounterHtml()
				content_type = self.html_content
				assert isinstance( content, bytes )
			elif up.path=='/Announcer.html':
				content = getAnnouncerHtml()
				content_type = self.html_content
				assert isinstance( content, bytes )
			elif up.path=='/qrcode.html':
				urlPage = GetCrossMgrHomePage()
				content = getQRCodePage( urlPage )
				content_type = self.html_content
				assert isinstance( content, bytes )
			elif up.path=='/servertimestamp.html':
				content = Utils.ToJSon( {
						'servertime':time.time()*1000.0,
						'requesttimestamp':float(up.query),
					}
				).encode()
				content_type = self.json_content;
				assert isinstance( content, bytes )
			else:
				file = None
				
				if up.path == '/CurrentResults.html':
					try:
						file = os.path.splitext(Model.race.getFileName())[0] + '.html'
					except:
						pass
				
				elif up.path == '/PreviousResults.html':
					file = GetPreviousFileName()
				
				if file is None: 
					file = url2pathname(os.path.basename(up.path))
				content, gzip_content = contentBuffer.getContent( file )
				content_type = self.html_content
				assert isinstance( content, bytes )
		except Exception as e:
			self.send_error(404,'File Not Found: {} {}\n{}'.format(self.path, e, traceback.format_exc()))
			return
		
		self.send_response( 200 )
		self.send_header('Content-Type',content_type)
		if content_type == self.html_content:
			if gzip_content and 'Accept-Encoding' in self.headers and 'gzip' in self.headers['Accept-Encoding']:
				content = gzip_content
				self.send_header( 'Content-Encoding', 'gzip' )
			self.send_header( 'Cache-Control', 'no-cache, no-store, must-revalidate' )
			self.send_header( 'Pragma', 'no-cache' )
			self.send_header( 'Expires', '0' )
		self.send_header( 'Content-Length', len(content) )
		self.end_headers()
		self.wfile.write( content )
コード例 #49
0
ファイル: generic_io.py プロジェクト: brechmos-stsci/asdf
def get_file(init, mode='r', uri=None):
    """
    Returns a `GenericFile` instance suitable for wrapping the given
    object `init`.

    If passed an already open file-like object, it must be opened for
    reading/writing in binary mode.  It is the caller's responsibility
    to close it.

    Parameters
    ----------
    init : object
        `init` may be:

        - A `bytes` or `unicode` file path or ``file:`` or ``http:``
          url.

        - A Python 2 `file` object.

        - An `io.IOBase` object (the default file object on Python 3).

        - A ducktyped object that looks like a file object.  If `mode`
          is ``"r"``, it must have a ``read`` method.  If `mode` is
          ``"w"``, it must have a ``write`` method.  If `mode` is
          ``"rw"`` it must have the ``read``, ``write``, ``tell`` and
          ``seek`` methods.

        - A `GenericFile` instance, in which case it is wrapped in a
          `GenericWrapper` instance, so that the file is closed when
          only when the final layer is unwrapped.

    mode : str
        Must be one of ``"r"``, ``"w"`` or ``"rw"``.

    uri : str
        Sets the base URI of the file object.  This will be used to
        resolve any relative URIs contained in the file.  This is
        redundant if `init` is a `bytes` or `unicode` object (since it
        will be the uri), and it may be determined automatically if
        `init` refers to a regular filesystem file.  It is not required
        if URI resolution is not used in the file.

    Returns
    -------
    fd : GenericFile

    Raises
    ------
    ValueError, TypeError, IOError
    """
    if mode not in ('r', 'w', 'rw'):
        raise ValueError("mode must be 'r', 'w' or 'rw'")

    if init in (sys.__stdout__, sys.__stdin__, sys.__stderr__):
        if six.PY3:
            init = init.buffer
        else:
            init = os.fdopen(init.fileno(), init.mode + 'b')

    if isinstance(init, (GenericFile, GenericWrapper)):
        if mode not in init.mode:
            raise ValueError(
                "File is opened as '{0}', but '{1}' was requested".format(
                    init.mode, mode))
        return GenericWrapper(init)

    elif isinstance(init, six.string_types):
        parsed = urlparse.urlparse(init)
        if parsed.scheme == 'http':
            if 'w' in mode:
                raise ValueError(
                    "HTTP connections can not be opened for writing")
            return _make_http_connection(init, mode, uri=uri)
        elif parsed.scheme in _local_file_schemes:
            if mode == 'rw':
                realmode = 'r+b'
            else:
                realmode = mode + 'b'
            realpath = url2pathname(parsed.path)
            if mode == 'w':
                fd = atomicfile.atomic_open(realpath, realmode)
            else:
                fd = open(realpath, realmode)
            fd = fd.__enter__()
            return RealFile(fd, mode, close=True, uri=uri)

    elif isinstance(init, io.BytesIO):
        return MemoryIO(init, mode, uri=uri)

    elif isinstance(init, io.StringIO):
        raise TypeError(
            "io.StringIO objects are not supported.  Use io.BytesIO instead.")

    elif six.PY2 and isinstance(init, file):
        if init.mode[0] not in mode:
            raise ValueError(
                "File is opened as '{0}', but '{1}' was requested".format(
                    init.mode, mode))

        try:
            init.tell()
        except IOError:
            if mode == 'w':
                return OutputStream(init, uri=uri)
            elif mode == 'r':
                return InputStream(init, mode, uri=uri)
            else:
                raise ValueError(
                    "File '{0}' could not be opened in 'rw' mode".format(init))
        else:
            return RealFile(init, mode, uri=uri)

    elif isinstance(init, io.IOBase):
        if sys.version_info[:2] == (2, 6):  # pragma: no cover
            raise ValueError(
                "io.open file objects are not supported on Python 2.6")

        if (('r' in mode and not init.readable()) or
            ('w' in mode and not init.writable())):
            raise ValueError(
                "File is opened as '{0}', but '{1}' was requested".format(
                    init.mode, mode))

        if init.seekable():
            if isinstance(init, (io.BufferedReader,
                                 io.BufferedWriter,
                                 io.BufferedRandom)):
                init2 = init.raw
            else:
                init2 = init
            if isinstance(init2, io.RawIOBase):
                result = RealFile(init2, mode, uri=uri)
            else:
                result = MemoryIO(init2, mode, uri=uri)
            result._secondary_fd = init
            return result
        else:
            if mode == 'w':
                return OutputStream(init, uri=uri)
            elif mode == 'r':
                return InputStream(init, mode, uri=uri)
            else:
                raise ValueError(
                    "File '{0}' could not be opened in 'rw' mode".format(init))

    elif mode == 'w' and (
          hasattr(init, 'write') and
          hasattr(init, 'seek') and
          hasattr(init, 'tell')):
        return MemoryIO(init, mode, uri=uri)

    elif mode == 'r' and (
          hasattr(init, 'read') and
          hasattr(init, 'seek') and
          hasattr(init, 'tell')):
        return MemoryIO(init, mode, uri=uri)

    elif mode == 'rw' and (
          hasattr(init, 'read') and
          hasattr(init, 'write') and
          hasattr(init, 'seek') and
          hasattr(init, 'tell')):
        return MemoryIO(init, mode, uri=uri)

    elif mode == 'w' and hasattr(init, 'write'):
        return OutputStream(init, uri=uri)

    elif mode == 'r' and hasattr(init, 'read'):
        return InputStream(init, mode, uri=uri)

    raise ValueError("Can't handle '{0}' as a file for mode '{1}'".format(
        init, mode))
コード例 #50
0
 def mapper(self, key, line):
     for record in self.read_warc(line.strip()):
         payload = self.get_payload(record)
         for value in self.process_record(payload):
             yield ((url2pathname(record.url), value), 1)
コード例 #51
0
ファイル: utils.py プロジェクト: Wirecloud/wirecloud
def get_resource_data(resource, user, request=None):
    """Gets all the information related to the given resource."""
    resource_info = resource.get_processed_info(request)

    template_uri = get_absolute_reverse_url('wirecloud_catalogue.media', kwargs={
        'vendor': resource.vendor,
        'name': resource.short_name,
        'version': resource.version,
        'file_path': resource.template_uri
    }, request=request)

    wgt_path = os.path.join(wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version), resource.template_uri)
    size = os.path.getsize(wgt_path)

    cdate = resource.creation_date
    creation_timestamp = time.mktime(cdate.timetuple()) * 1e3 + cdate.microsecond / 1e3

    longdescription = resource_info['longdescription']
    if longdescription != '':
        longdescription_relative_path = url2pathname(longdescription)
        longdescription_base_url = force_trailing_slash(urljoin(resource.get_template_url(request=request, for_base=True), pathname2url(os.path.dirname(longdescription_relative_path))))
        longdescription_path = os.path.join(wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version), longdescription_relative_path)

        (filename_root, filename_ext) = os.path.splitext(longdescription_path)
        localized_longdescription_path = filename_root + '.' + get_language() + filename_ext

        try:
            description_code = download_local_file(localized_longdescription_path).decode('utf-8')
            longdescription = clean_html(markdown.markdown(description_code, output_format='xhtml5'), base_url=longdescription_base_url)
        except:
            try:
                description_code = download_local_file(longdescription_path).decode('utf-8')
                longdescription = clean_html(markdown.markdown(description_code, output_format='xhtml5'), base_url=longdescription_base_url)
            except:
                longdescription = resource_info['description']

    else:
        longdescription = resource_info['description']

    return {
        'id': resource.pk,
        'vendor': resource.vendor,
        'name': resource.short_name,
        'version': resource.version,
        'type': resource_info['type'],
        'date': creation_timestamp,
        'permissions': {
            'delete': user.is_superuser,
            'uninstall': resource.public is False and resource.users.filter(pk=user.pk).exists(),
        },
        'authors': resource_info['authors'],
        'contributors': resource_info['contributors'],
        'title': resource_info['title'],
        'description': resource_info['description'],
        'longdescription': longdescription,
        'email': resource_info['email'],
        'image': resource_info['image'],
        'homepage': resource_info['homepage'],
        'doc': resource_info['doc'],
        'changelog': resource_info['changelog'],
        'size': size,
        'uriTemplate': template_uri,
        'license': resource_info['license'],
        'licenseurl': resource_info['licenseurl'],
        'issuetracker': resource_info['issuetracker'],
    }