示例#1
0
 def get_repos(self, account, page_url=None, sess=None, refresh=True):
     if not page_url:
         page_url = self.api_repos_path.format(
             user_id=urlquote(account.user_id),
             user_name=urlquote(account.user_name or ''),
         )
     if not getattr(self.api_repos_path, 'use_session', True):
         sess = None
     r = self.api_get(account.domain, page_url, sess=sess)
     repos, count, pages_urls = self.api_paginator(r, self.api_parser(r))
     repos = [self.extract_repo_info(repo, account.domain) for repo in repos]
     if '{user_name}' in self.api_repos_path and repos and repos[0].owner_id != account.user_id:
         # https://hackerone.com/reports/452920
         if not refresh:
             raise TokenExpiredError()
         from liberapay.models.account_elsewhere import UnableToRefreshAccount
         try:
             account = account.refresh_user_info()
         except UnableToRefreshAccount:
             raise TokenExpiredError()
         # Note: we can't pass the page_url below, because it contains the old user_name
         return self.get_repos(account, page_url=None, sess=sess, refresh=False)
     if count == -1 and hasattr(self, 'x_repos_count'):
         count = self.x_repos_count(None, account.extra_info, -1)
     return repos, count, pages_urls
示例#2
0
	def sentiment_targeted(self, flavor, data, target, options={}):
		"""
		Calculates the targeted sentiment for text, a URL or HTML.
		For an overview, please refer to: http://www.alchemyapi.com/products/features/sentiment-analysis/
		For the docs, please refer to: http://www.alchemyapi.com/api/sentiment-analysis/
		
		INPUT:
		flavor -> which version of the call, i.e. text, url or html.
		data -> the data to analyze, either the text, the url or html code.
		target -> the word or phrase to run sentiment analysis on.
		options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.
		
		Available Options:
		showSourceText	-> 0: disabled, 1: enabled

		OUTPUT:
		The response, already converted from JSON to a Python object. 
		"""

		#Make sure the target is valid
		if target is None or target == '':
			return { 'status':'ERROR', 'statusInfo':'targeted sentiment requires a non-null target' }

		#Make sure this request supports this flavor
		if flavor not in AlchemyAPI.ENDPOINTS['sentiment_targeted']:
			return { 'status':'ERROR', 'statusInfo':'targeted sentiment analysis for ' + flavor + ' not available' }
			
		#add the URL encoded data and target to the options and analyze 			
		options[flavor] = urlquote(data)
		options['target'] = urlquote(target)
		return self.__analyze(AlchemyAPI.ENDPOINTS['sentiment_targeted'][flavor], options)
 def _create_table(self, db, table, type, params=None):
     params = {} if params is None else params
     with self.post("/v3/table/create/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table)), urlquote(str(type))), params) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Create %s table failed" % (type), res, body)
         return True
示例#4
0
文件: urls.py 项目: bwhmather/verktyg
def _encode_netloc(components):
    host = ''
    if components.hostname:
        host = encode_idna(components.hostname).decode('ascii')
    if ':' in host:
        host = '[%s]' % host

    netloc = host

    if components.port:
        if not 0 <= int(components.port) <= 65535:
            raise ValueError('Invalid port')
        netloc = '%s:%s' % (netloc, components.port)

    if components.username or components.password:
        if components.username:
            username = urlquote(
                components.username, safe='/:%'
            )
        else:
            username = ''

        if components.password:
            password = urlquote(
                components.password, safe='/:%'
            )
            auth = '%s:%s' % (username, password)
        else:
            auth = username

        netloc = '%s@%s' % (auth, netloc)
    return netloc
示例#5
0
文件: client.py 项目: jagguli/trombi
    def attach(self, name, data, callback, type='text/plain'):
        def _really_callback(response):
            if  response.code != 201:
                callback(_error_response(response))
                return
            data = json.loads(response.body.decode('utf-8'))
            assert data['id'] == self.id
            self.rev = data['rev']
            self.attachments[name] = {
                'content_type': type,
                'length': len(data),
                'stub': True,
            }
            callback(self)

        headers = {'Content-Type': type, 'Expect': ''}

        self.db._fetch(
            '%s/%s?rev=%s' % (
                urlquote(self.id, safe=''),
                urlquote(name, safe=''),
                self.rev),
            _really_callback,
            method='PUT',
            body=data,
            headers=headers,
            )
示例#6
0
    def import_data(self, db, table, format, bytes_or_stream, size, unique_id=None):
        """Import data into Treasure Data Service

        This method expects data from a file-like object formatted with "msgpack.gz".

        Params:
            db (str): name of a database
            table (str): name of a table
            format (str): format of data type (e.g. "msgpack.gz")
            bytes_or_stream (str or file-like): a byte string or a file-like object contains the data
            size (int): the length of the data
            unique_id (str): a unique identifier of the data

        Returns: float represents the elapsed time to import data
        """
        if unique_id is not None:
            path = "/v3/table/import_with_id/%s/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table)), urlquote(str(unique_id)), urlquote(str(format)))
        else:
            path = "/v3/table/import/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table)), urlquote(str(format)))

        kwargs = {}
        with self.put(path, bytes_or_stream, size, **kwargs) as res:
            code, body = res.status, res.read()
            if code / 100 != 2:
                self.raise_error("Import failed", res, body)
            js = self.checked_json(body, ["elapsed_time"])
            time = float(js["elapsed_time"])
            return time
示例#7
0
 def query(self, q, type="hive", db=None, result_url=None, priority=None, retry_limit=None, **kwargs):
     """
     TODO: add docstring
     => jobId:str
     """
     params = {"query": q}
     params.update(kwargs)
     if result_url is not None:
         params["result"] = result_url
     if priority is not None:
         if not isinstance(priority, int):
             priority_name = str(priority).upper()
             if priority_name in self.JOB_PRIORITY:
                 priority = self.JOB_PRIORITY[priority_name]
             else:
                 raise(ValueError("unknown job priority: %s" % (priority_name,)))
         params["priority"] = priority
     if retry_limit is not None:
         params["retry_limit"] = retry_limit
     with self.post("/v3/job/issue/%s/%s" % (urlquote(str(type)), urlquote(str(db))), params) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Query failed", res, body)
         js = self.checked_json(body, ["job_id"])
         return str(js["job_id"])
示例#8
0
文件: wsgi.py 项目: bwhmather/verktyg
def get_current_url(
    environ, root_only=False, strip_querystring=False,
    host_only=False, trusted_hosts=None,
):
    """A handy helper function that recreates the full URL as IRI for the
    current request or parts of it.  Here an example:

    >>> from verktyg.test import create_environ
    >>> env = create_environ("/?param=foo", "http://localhost/script")
    >>> get_current_url(env)
    'http://localhost/script/?param=foo'
    >>> get_current_url(env, root_only=True)
    'http://localhost/script/'
    >>> get_current_url(env, host_only=True)
    'http://localhost/'
    >>> get_current_url(env, strip_querystring=True)
    'http://localhost/script/'

    This optionally it verifies that the host is in a list of trusted hosts.
    If the host is not in there it will raise a
    :exc:`~verktyg.exceptions.SecurityError`.

    Note that the string returned might contain unicode characters as the
    representation is an IRI not an URI.  If you need an ASCII only
    representation you can use the :func:`~verktyg.urls.iri_to_uri`
    function:

    >>> from verktyg.urls import iri_to_uri
    >>> iri_to_uri(get_current_url(env))
    'http://localhost/script/?param=foo'

    :param environ:
        The WSGI environment to get the current URL from.
    :param root_only:
        Set `True` if you only want the root URL.
    :param strip_querystring:
        Set to `True` if you don't want the querystring.
    :param host_only:
        Set to `True` if the host URL should be returned.
    :param trusted_hosts:
        A list of trusted hosts, see :func:`host_is_trusted` for more
        information.
    """
    tmp = [environ['wsgi.url_scheme'], '://', get_host(environ, trusted_hosts)]
    cat = tmp.append
    if host_only:
        return uri_to_iri(''.join(tmp) + '/')
    cat(urlquote(wsgi_get_bytes(environ.get('SCRIPT_NAME', ''))).rstrip('/'))
    cat('/')
    if not root_only:
        cat(urlquote(
            wsgi_get_bytes(environ.get('PATH_INFO', '')).lstrip(b'/')
        ))
        if not strip_querystring:
            qs = get_query_string(environ)
            if qs:
                cat('?' + qs)
    return uri_to_iri(''.join(tmp))
 def update_expire(self, db, table, expire_days):
     """
     TODO: add docstring
     """
     with self.post("/v3/table/update/%s/%s" % (urlquote(str(db)), urlquote(str(table))), {"expire_days": expire_days}) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Update table expiration failed", res, body)
         return True
示例#10
0
    def __init__(self, plugin_name, init_schema, db_file_name=None, upgrade_mod=None):
        """
        Runs the initialisation process that includes creating the connection to the database and the tables if they do
        not exist.

        ``plugin_name``
            The name to setup paths and settings section names

        ``init_schema``
            The init_schema function for this database

        ``upgrade_schema``
            The upgrade_schema function for this database

        ``db_file_name``
            The file name to use for this database. Defaults to None resulting in the plugin_name being used.
        """
        settings = Settings()
        settings.beginGroup(plugin_name)
        self.db_url = ''
        self.is_dirty = False
        self.session = None
        db_type = settings.value('db type')
        if db_type == 'sqlite':
            if db_file_name:
                self.db_url = 'sqlite:///%s/%s' % (AppLocation.get_section_data_path(plugin_name), db_file_name)
            else:
                self.db_url = 'sqlite:///%s/%s.sqlite' % (AppLocation.get_section_data_path(plugin_name), plugin_name)
        else:
            self.db_url = '%s://%s:%s@%s/%s' % (db_type,
                urlquote(settings.value('db username')),
                urlquote(settings.value('db password')),
                urlquote(settings.value('db hostname')),
                urlquote(settings.value('db database')))
            if db_type == 'mysql':
                db_encoding = settings.value('db encoding')
                self.db_url += '?charset=%s' % urlquote(db_encoding)
        settings.endGroup()
        if upgrade_mod:
            db_ver, up_ver = upgrade_db(self.db_url, upgrade_mod)
            if db_ver > up_ver:
                critical_error_message_box(
                    translate('OpenLP.Manager', 'Database Error'),
                    translate('OpenLP.Manager', 'The database being loaded was created in a more recent version of '
                        'OpenLP. The database is version %d, while OpenLP expects version %d. The database will not '
                        'be loaded.\n\nDatabase: %s') % (db_ver, up_ver, self.db_url)
                )
                return
        try:
            self.session = init_schema(self.db_url)
        except (SQLAlchemyError, DBAPIError):
            log.exception('Error loading database: %s', self.db_url)
            critical_error_message_box(
                translate('OpenLP.Manager', 'Database Error'),
                translate('OpenLP.Manager', 'OpenLP cannot load your database.\n\nDatabase: %s') % self.db_url
            )
 def bulk_import_upload_part(self, name, part_name, stream, size):
     """
     TODO: add docstring
     => None
     """
     self.validate_part_name(part_name)
     with self.put("/v3/bulk_import/upload_part/%s/%s" % (urlquote(str(name)), urlquote(str(part_name))), stream, size) as res:
         code, body = res.status, res.read()
         if code / 100 != 2:
             self.raise_error("Upload a part failed", res, body)
示例#12
0
 def update_schema(self, db, table, schema_json):
     """
     TODO: add docstring
     => True
     """
     with self.post("/v3/table/update-schema/%s/%s" % (urlquote(str(db)), urlquote(str(table))), {"schema": schema_json}) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Create schema table failed", res, body)
         return True
示例#13
0
 def swap_table(self, db, table1, table2):
     """
     TODO: add docstring
     => True
     """
     with self.post("/v3/table/swap/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table1)), urlquote(str(table2)))) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Swap tables failed", res, body)
         return True
示例#14
0
 def addFile(self, newFile, path):
   """
   This adds the given file to the owncloud server. newFile is a string path to a local file and that file name
   will be used as its name.
   """
   self.log.debug("Adding New File: %s/%s" % (path, os.path.basename(newFile)))
   data = open(newFile, "rb").read()
   if path not in self.DIRS:
     self.mkdir(path)
   self.http(
     str("{}/{}/{}".format(self.url, urlquote(path), urlquote(os.path.basename(newFile)))), "PUT", data=data)
 def bulk_import_upload_part(self, name, part_name, stream, size):
     """
     TODO: add docstring
     => None
     """
     if 0 < part_name.find("/"):
         raise ValueError("part name must not contain '/': %s" % (repr(part_name,)))
     with self.put("/v3/bulk_import/upload_part/%s/%s" % (urlquote(str(name)), urlquote(str(part_name))), stream, size) as res:
         code, body = res.status, res.read()
         if code / 100 != 2:
             self.raise_error("Upload a part failed", res, body)
 def bulk_import_delete_part(self, name, part_name, params=None):
     """
     TODO: add docstring
     => True
     """
     params = {} if params is None else params
     with self.post("/v3/bulk_import/delete_part/%s/%s" % (urlquote(str(name)), urlquote(str(part_name))), params) as res:
         code, body = res.status, res.read()
         if code / 100 != 2:
             self.raise_error("Delete a part failed", res, body)
         return True
 def create_bulk_import(self, name, db, table, params=None):
     """
     TODO: add docstring
     => True
     """
     params = {} if params is None else params
     with self.post("/v3/bulk_import/create/%s/%s/%s" % (urlquote(str(name)), urlquote(str(db)), urlquote(str(table))), params) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Create bulk import failed", res, body)
         return True
示例#18
0
文件: __init__.py 项目: namely/crab
    def _get_url(self, action):
        """Creates the URL to be used to perform the given server action."""

        url = ('/api/0/' + action +
               '/' + urlquote(self.config.get('client', 'hostname'), '') +
               '/' + urlquote(self.config.get('client', 'username'), ''))

        if self.crabid is not None:
            url = url + '/' + urlquote(self.crabid, '')

        return url
示例#19
0
 def get_friends_for(self, account, page_url=None, sess=None):
     if not page_url:
         page_url = self.api_friends_path.format(
             user_id=urlquote(account.user_id),
             user_name=urlquote(account.user_name or ''),
         )
     r = self.api_get(account.domain, page_url, sess=sess)
     friends, count, pages_urls = self.api_paginator(r, self.api_parser(r))
     friends = [self.extract_user_info(f, account.domain) for f in friends]
     if count == -1 and hasattr(self, 'x_friends_count'):
         count = self.x_friends_count(None, account.extra_info, -1)
     return friends, count, pages_urls
示例#20
0
 def export_data(self, db, table, storage_type, params={}):
     """
     TODO: add docstring
     => jobId:str
     """
     params["storage_type"] = storage_type
     with self.post("/v3/export/run/%s/%s" % (urlquote(str(db)), urlquote(str(table))), params) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Export failed", res, body)
         js = self.checked_json(body, ["job_id"])
         return str(js["job_id"])
示例#21
0
 def delete_table(self, db, table):
     """
     TODO: add docstring
     => type:str
     """
     with self.post("/v3/table/delete/%s/%s" % (urlquote(str(db)), urlquote(str(table)))) as res:
         code, body = res.status, res.read()
         if code != 200:
             self.raise_error("Delete table failed", res, body)
         js = self.checked_json(body, [])
         t = js.get("type", "?")
         return t
示例#22
0
 def get_team_members(self, account, page_url=None):
     """Given an AccountElsewhere, return its membership list from the API.
     """
     if not page_url:
         page_url = self.api_team_members_path.format(
             user_id=urlquote(account.user_id),
             user_name=urlquote(account.user_name or ''),
         )
     domain = account.domain
     r = self.api_get(domain, page_url)
     members, count, pages_urls = self.api_paginator(r, self.api_parser(r))
     members = [self.extract_user_info(m, domain) for m in members]
     return members, count, pages_urls
示例#23
0
文件: client.py 项目: relic-/trombi
    def get_attachment(self, doc_id, attachment_name, callback):
        doc_id = urlquote(doc_id, safe='')
        attachment_name = urlquote(attachment_name, safe='')

        response = yield gen.Task(self._fetch, '%s/%s' % (doc_id, attachment_name))
 
        if response.code == 200:
            callback(response.body)
            #elif response.code == 404:
                # Document or attachment doesn't exist
                #callback(None)
        else:
            raise _exception(response)
示例#24
0
 def get_starred_repos(self, account, sess, page_url=None):
     if not page_url:
         page_url = self.api_starred_path.format(
             user_id=urlquote(account.user_id),
             user_name=urlquote(account.user_name or ''),
         )
     assert page_url[:1] == '/'
     if not getattr(self.api_starred_path, 'use_session', True):
         sess = None
     r = self.api_get(account.domain, page_url, sess=sess)
     repos, count, pages_urls = self.api_paginator(r, self.api_parser(r))
     repos = [self.extract_repo_info(repo, account.domain) for repo in repos]
     return repos, count, pages_urls
示例#25
0
def base_string_request_params(uri, oauth_params = None, data = None):
    from urllib.parse import urlsplit, parse_qs, urlencode 
    from collections import Mapping
    query = dict((k, v[0]) for k, v in parse_qs(urlsplit(uri).query).items())
    if oauth_params: 
        query.update(oauth_params)
    if data and isinstance(data, Mapping):
        query.update(data)
    if "oauth_signature" in query: 
        del query["oauth_signature"]
    #Parameter normalization
    params = [(urlquote(str(k)), urlquote(str(v))) for k, v in query.items()]
    params.sort()
    return '&'.join(['{0}={1}'.format(k, v) for k, v in params])
示例#26
0
文件: __init__.py 项目: mnot/redbot
 def iri_to_uri(iri: str) -> str:
     "Takes a unicode string that can contain an IRI and emits a unicode URI."
     scheme, authority, path, query, frag = urlsplit(iri)
     scheme = scheme
     if ":" in authority:
         host, port = authority.split(":", 1)
         authority = host.encode('idna').decode('ascii') + ":%s" % port
     else:
         authority = authority.encode('idna').decode('ascii')
     sub_delims = "!$&'()*+,;="
     pchar = "-.+~" + sub_delims + ":@" + "%"
     path = urlquote(path, safe=pchar+"/")
     quer = urlquote(query, safe=pchar+"/?")
     frag = urlquote(frag, safe=pchar+"/?")
     return urlunsplit((scheme, authority, path, quer, frag))
示例#27
0
def rule34(phenny, input):
    """.rule34 <query> - Rule 34: If it exists there is p**n of it."""

    if input.nick in phenny.config.user_ignore:
        return

    if check_nsfw(phenny, input.sender, None, input.nick):
        return
    q = input.group(2)
    if not q:
        phenny.say(rule34.__doc__.strip())
        return

    try:
        req = web.get(
            "https://rule34.xxx/index.php?page=dapi&s=post&q=index&tags={0}".format(urlquote(q))
        )  # Lets use XML!
    except (HTTPError, IOError):
        raise GrumbleError("THE INTERNET IS F*****G BROKEN. Please try again later.")

    results = ET.fromstring(req)

    if len(results) <= 0:
        phenny.reply("Huh. rule34.xxx is missing {0}".format(q))
        return

    try:
        link = "https:" + (choice(results).attrib["file_url"])
    except AttributeError:
        raise GrumbleError("THE INTERNET IS BROKEN. Please try again later.")

    response = "!!NSFW!! -> {0} <- !!NSFW!!".format(link)
    phenny.reply(response)
示例#28
0
	def relations(self, flavor, data, options={}):
		"""
		Extracts the relations for text, a URL or HTML.
		For an overview, please refer to: http://www.alchemyapi.com/products/features/relation-extraction/ 
		For the docs, please refer to: http://www.alchemyapi.com/api/relation-extraction/
		
		INPUT:
		flavor -> which version of the call, i.e. text, url or html.
		data -> the data to analyze, either the text, the url or html code.
		options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.
		
		Available Options:
		sentiment -> 0: disabled (default), 1: enabled. Requires one additional API transaction if enabled.
		keywords -> extract keywords from the subject and object. 0: disabled (default), 1: enabled. Requires one additional API transaction if enabled.
		entities -> extract entities from the subject and object. 0: disabled (default), 1: enabled. Requires one additional API transaction if enabled.
		requireEntities -> only extract relations that have entities. 0: disabled (default), 1: enabled.
		sentimentExcludeEntities -> exclude full entity name in sentiment analysis. 0: disabled, 1: enabled (default)
		disambiguate -> disambiguate entities (i.e. Apple the company vs. apple the fruit). 0: disabled, 1: enabled (default)
		linkedData -> include linked data with disambiguated entities. 0: disabled, 1: enabled (default).
		coreference -> resolve entity coreferences. 0: disabled, 1: enabled (default)  
		showSourceText -> 0: disabled (default), 1: enabled.
		maxRetrieve -> the maximum number of relations to extract (default: 50, max: 100)

		OUTPUT:
		The response, already converted from JSON to a Python object. 
		"""
		
		#Make sure this request supports this flavor
		if flavor not in AlchemyAPI.ENDPOINTS['relations']:
			return { 'status':'ERROR', 'statusInfo':'relation extraction for ' + flavor + ' not available' }
		
		#add the URL encoded data to the options and analyze 			
		options[flavor] = urlquote(data)
		return self.__analyze(AlchemyAPI.ENDPOINTS['relations'][flavor], options)
示例#29
0
	def feeds(self, flavor, data, options={}):
		"""
		Detects the RSS/ATOM feeds for a URL or HTML.
		For an overview, please refer to: http://www.alchemyapi.com/products/features/feed-detection/ 
		For the docs, please refer to: http://www.alchemyapi.com/api/feed-detection/
		
		INPUT:
		flavor -> which version of the call, i.e.  url or html.
		data -> the data to analyze, either the the url or html code.
		options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.

		Available Options:
		none

		OUTPUT:
		The response, already converted from JSON to a Python object. 
		"""
		
		#Make sure this request supports this flavor
		if flavor not in AlchemyAPI.ENDPOINTS['feeds']:
			return { 'status':'ERROR', 'statusInfo':'feed detection for ' + flavor + ' not available' }
		
		#add the URL encoded data to the options and analyze 			
		options[flavor] = urlquote(data)
		return self.__analyze(AlchemyAPI.ENDPOINTS['feeds'][flavor], options)
示例#30
0
	def text_title(self, flavor, data, options={}):
		"""
		Extracts the title for a URL or HTML.
		For an overview, please refer to: http://www.alchemyapi.com/products/features/text-extraction/ 
		For the docs, please refer to: http://www.alchemyapi.com/api/text-extraction/
		
		INPUT:
		flavor -> which version of the call, i.e. text, url or html.
		data -> the data to analyze, either the text, the url or html code.
		options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.
		
		Available Options:
		useMetadata -> utilize title info embedded in meta data, 0: disabled, 1: enabled (default) 

		OUTPUT:
		The response, already converted from JSON to a Python object. 
		"""
		
		#Make sure this request supports this flavor
		if flavor not in AlchemyAPI.ENDPOINTS['text_title']:
			return { 'status':'ERROR', 'statusInfo':'title extraction for ' + flavor + ' not available' }
		
		#add the URL encoded data to the options and analyze 			
		options[flavor] = urlquote(data)
		return self.__analyze(AlchemyAPI.ENDPOINTS['text_title'][flavor], options)
示例#31
0
ELASTICSEARCH_CONNECTIONS = {
    'default': {
        'hosts': [
            {
                'host': elasticsearch_url.hostname,
                'port': elasticsearch_url.port,
            },
        ],
        'timeout': 60,
        'max_retries': 1,
    }
}
if elasticsearch_url.username is not None and elasticsearch_url.password is not None:
    ELASTICSEARCH_CONNECTIONS['default']['http_auth'] = (
        elasticsearch_url.username + ':' + urlquote(elasticsearch_url.password))
if elasticsearch_url.scheme == 'https':
    ELASTICSEARCH_CONNECTIONS['default']['hosts'][0]['use_ssl'] = True

try:
    from local_essarch_settings import ELASTICSEARCH_TEST_URL
except ImportError:
    ELASTICSEARCH_TEST_URL = os.environ.get('ELASTICSEARCH_TEST_URL', 'http://localhost:19200')
elasticsearch_test_url = urlparse(ELASTICSEARCH_TEST_URL)

ELASTICSEARCH_TEST_CONNECTIONS = {
    'default': {
        'hosts': [
            {
                'host': elasticsearch_test_url.hostname,
                'port': elasticsearch_test_url.port,
示例#32
0
def _get_storage_entities(task_factory, storage_settings):
    # type: (dict, settings.TaskFactoryStorageSettings) -> TaskSettings
    """Generate a task given a config
    :param dict task_factory: task factory object
    :param settings.TaskFactoryStorageSettings storage_settings:
        storage settings
    :rtype: FileInfo
    :return: file info
    """
    if not storage_settings.is_file_share:
        # create blob client
        blob_client = azureblob.BlockBlobService(
            account_name=storage_settings.storage_settings.account,
            account_key=storage_settings.storage_settings.account_key,
            endpoint_suffix=storage_settings.storage_settings.endpoint)
        # list blobs in container with include/exclude
        blobs = blob_client.list_blobs(
            container_name=storage_settings.container)
        for blob in blobs:
            if not _inclusion_check(blob.name, storage_settings.include,
                                    storage_settings.exclude):
                continue
            file_path_with_container = '{}/{}'.format(
                storage_settings.container, blob.name)
            file_name = blob.name.split('/')[-1]
            file_name_no_extension = file_name.split('.')[0]
            if task_factory['file']['task_filepath'] == 'file_path':
                task_filepath = blob.name
            elif (task_factory['file']['task_filepath'] ==
                  'file_path_with_container'):
                task_filepath = file_path_with_container
            elif task_factory['file']['task_filepath'] == 'file_name':
                task_filepath = file_name
            elif (task_factory['file']['task_filepath'] ==
                  'file_name_no_extension'):
                task_filepath = file_name_no_extension
            else:
                raise ValueError(
                    'invalid task_filepath specification: {}'.format(
                        task_factory['file']['task_filepath']))
            # create blob url
            url = 'https://{}.blob.{}/{}/{}'.format(
                storage_settings.storage_settings.account,
                storage_settings.storage_settings.endpoint,
                storage_settings.container, urlquote(blob.name))
            # create blob sas
            sas = blob_client.generate_blob_shared_access_signature(
                storage_settings.container,
                blob.name,
                permission=azureblob.BlobPermissions.READ,
                expiry=datetime.datetime.utcnow() +
                datetime.timedelta(days=_DEFAULT_SAS_EXPIRY_DAYS))
            yield FileInfo(
                is_blob=True,
                url=url,
                sas=sas,
                file_path=blob.name,
                file_path_with_container=file_path_with_container,
                file_name=file_name,
                file_name_no_extension=file_name_no_extension,
                task_filepath=task_filepath,
            )
    else:
        # create file share client
        file_client = azurefile.FileService(
            account_name=storage_settings.storage_settings.account,
            account_key=storage_settings.storage_settings.account_key,
            endpoint_suffix=storage_settings.storage_settings.endpoint)
        # list files in share with include/exclude
        for file in _list_all_files_in_fileshare(file_client,
                                                 storage_settings.container):
            if not _inclusion_check(file, storage_settings.include,
                                    storage_settings.exclude):
                continue
            file_path_with_container = '{}/{}'.format(
                storage_settings.container, file)
            file_name = file.split('/')[-1]
            file_name_no_extension = file_name.split('.')[0]
            if task_factory['file']['task_filepath'] == 'file_path':
                task_filepath = file
            elif (task_factory['file']['task_filepath'] ==
                  'file_path_with_container'):
                task_filepath = file_path_with_container
            elif task_factory['file']['task_filepath'] == 'file_name':
                task_filepath = file_name
            elif (task_factory['file']['task_filepath'] ==
                  'file_name_no_extension'):
                task_filepath = file_name_no_extension
            else:
                raise ValueError(
                    'invalid task_filepath specification: {}'.format(
                        task_factory['file']['task_filepath']))
            yield FileInfo(
                is_blob=False,
                url=None,
                sas=None,
                file_path=file,
                file_path_with_container=file_path_with_container,
                file_name=file_name,
                file_name_no_extension=file_name_no_extension,
                task_filepath=task_filepath,
            )
示例#33
0
 def image_request_response(self, path):
     """Parse image request and create response."""
     # Parse the request in path
     if (len(path) > 1024):
         raise IIIFError(code=414,
                         text="URI Too Long: Max 1024 chars, got %d\n" %
                         len(path))
     # print "GET " + path
     try:
         self.iiif.identifier = self.identifier
         self.iiif.parse_url(path)
     except IIIFRequestPathError as e:
         # Reraise as IIIFError with code=404 because we can't tell
         # whether there was an encoded slash in the identifier or
         # whether there was a bad number of path segments.
         raise IIIFError(code=404, text=e.text)
     except IIIFRequestBaseURI as e:
         info_uri = self.server_and_prefix + '/' + \
             urlquote(self.iiif.identifier) + '/info.json'
         raise IIIFError(code=303, headers={'Location': info_uri})
     except IIIFError as e:
         # Pass through
         raise e
     except Exception as e:
         # Something completely unexpected => 500
         raise IIIFError(
             code=500,
             text=
             "Internal Server Error: unexpected exception parsing request ("
             + str(e) + ")")
     dr = degraded_request(self.identifier)
     if (dr):
         self.logger.info("image_request: degraded %s -> %s" %
                          (self.identifier, dr))
         self.degraded = self.identifier
         self.identifier = dr
         self.iiif.quality = 'gray'
     else:
         # Parsed request OK, attempt to fulfill
         self.logger.info("image_request: %s" % (self.identifier))
     file = self.file
     self.manipulator.srcfile = file
     self.manipulator.do_first()
     if (self.api_version < '2.0' and self.iiif.format is None
             and 'Accept' in request.headers):
         # In 1.0 and 1.1 conneg was specified as an alternative to format, see:
         # http://iiif.io/api/image/1.0/#format
         # http://iiif.io/api/image/1.1/#parameters-format
         formats = {
             'image/jpeg': 'jpg',
             'image/tiff': 'tif',
             'image/png': 'png',
             'image/gif': 'gif',
             'image/jp2': 'jps',
             'application/pdf': 'pdf'
         }
         accept = do_conneg(request.headers['Accept'], list(formats.keys()))
         # Ignore Accept header if not recognized, should this be an error
         # instead?
         if (accept in formats):
             self.iiif.format = formats[accept]
     (outfile, mime_type) = self.manipulator.derive(file, self.iiif)
     # FIXME - find efficient way to serve file with headers
     self.add_compliance_header()
     return send_file(outfile, mimetype=mime_type)
示例#34
0
def main(argv):
    parser = argparse.ArgumentParser()
    sub_parser = parser.add_subparsers(title='actions')

    # common options
    parser.add_argument('-c', '--config', default='api.conf', help='Specify which contains api key (format json)')
    parser.add_argument('-K', '--insecure', action='store_true', default=False, help='Do not check hostname when using https')
    parser.add_argument('-k', '--api-key', default=None, help='Specify the api key')
    parser.add_argument('--url', default=None, help='specify url')
    parser.add_argument('-l', '--admin', default=None, help='Specify the login of admin user')
    parser.add_argument('--admin_password', default=None, help='Specify the admin\'s password')
    parser.add_argument('-P', '--pwprompt', action='store_true', default=False, help='Prompt for password')

    # sub parser for create_config command
    parser_cf = sub_parser.add_parser('create_config', aliases=['cf'], help='Create a config file')
    parser_cf.set_defaults(action='create_config')

    # sub parser for create_user command
    parser_cu = sub_parser.add_parser('create_user', aliases=['cu'], help='Create a user')
    parser_cu.add_argument('--homedirectory', default='/var/www/nextcloud/data/<login>/files/', help='specify a homedirectory ("<login>" will be replaced by login)')
    parser_cu.add_argument('--pool_template', help='specify a pool template')
    parser_cu.add_argument('--vtl_id', help='specify vtl id')
    parser_cu.add_argument('login', help='specify login of new user')
    parser_cu.add_argument('password', nargs='?', default=None, help='specify password of new user')
    parser_cu.set_defaults(action='create_user')

    # sub parser for enable_user command
    parser_eu = sub_parser.add_parser('enable_user', aliases=['eu'], help='enable a user')
    parser_eu.add_argument('key', help='specify key')
    parser_eu.set_defaults(action='enable_user')

    # sub parser for disable_user command
    parser_du = sub_parser.add_parser('disable_user', aliases=['du'], help='disable a user')
    parser_du.add_argument('--key', help='specify key')
    parser_du.add_argument('login', help='specify login of user')
    parser_du.set_defaults(action='disable_user')

    # sub parser for list_pool command
    parser_lp = sub_parser.add_parser('list_pool', aliases=['lp'], help='List pools')
    parser_lp.set_defaults(action='list_pool')

    # sub parser for list_user command
    parser_lu = sub_parser.add_parser('list_user', aliases=['lu'], help='List users')
    parser_lu.set_defaults(action='list_user')

    args = parser.parse_args(argv)


    # check default options
    base_url = None
    if args.url is not None:
        base_url = args.url
    api_key = None
    if args.api_key is not None:
        api_key = args.api_key
    admin_user = None
    if args.admin is not None:
        admin_user = args.admin
    admin_password = None
    if args.admin_password is not None:
        admin_password = args.admin_password

    # open config file if needed
    if None in [base_url, api_key, admin_user, admin_password]:
        try:
            fd = open(args.config)
            config = json.load(fd)
            fd.close()

            if 'api key' in config:
                api_key = config['api key']
            if 'base url' in config:
                base_url = config['base url']
            if 'login' in config:
                admin_user = config['login']
            if 'password' in config:
                admin_password = config['password']
        except OSError as err:
            print('Error occured while reading file "%s" because %s' % (args.config, err), file=sys.stderr)
            sys.exit(1)

    if admin_password is None and args.pwprompt:
        try:
            admin_password = getpass('Admin\'s password: '******'Error, you should specify an api key', file=sys.stderr)
    if base_url is None:
        print('Error, you should specify an url', file=sys.stderr)
    if admin_user is None:
        print('Error, you should specify a login for admin user', file=sys.stderr)
    if admin_password is None:
        print('Error, you should specify a password for admin userl', file=sys.stderr)
    if None in [base_url, api_key, admin_user, admin_password]:
        sys.exit(1)


    def authentication(info):
        print('Login to "%s"... ' % base_url, end='', flush=True)
        connection = newHttpConnection(info)
        headers = {"Content-type": "application/json"}
        params = json.dumps({'login': admin_user, 'password': admin_password, 'apikey': api_key})
        connection.request('POST', base_url + '/auth/', params, headers)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        if response.status == 201:
            print('connected')
            return (connection, {'Cookie': response.getheader('Set-Cookie').split(';')[0]})
        else:
            print('connection failure because %s' % message)
            connection.close()
            sys.exit(2)

    def newHttpConnection(info):
        try:
            if info.scheme == 'http':
                return http.client.HTTPConnection(info.hostname, info.port)
            elif info.scheme == 'https' and args.insecure:
                ssl_context = ssl.SSLContext() # older python (< v3.5.3) need ssl.PROTOCOL_TLSv1 as parameter
                ssl_context.verify_mode = False
                ssl_context.check_hostname = False
                return http.client.HTTPSConnection(info.hostname, info.port, context=ssl_context)
            elif info.scheme == 'https':
                return http.client.HTTPSConnection(info.hostname, info.port)
        except Exception as err:
            print('Error occured while creating http(s) connection because %s' % (err), file=sys.stderr)
            sys.exit(1)

    pp = pprint.PrettyPrinter(indent=4)

    if args.action == 'create_config':
        try:
            fd = open(args.config, 'w')
            json.dump({'api key': api_key, 'base url': base_url, 'login': admin_user, 'password': admin_password}, fd, sort_keys=True)
            fd.close()
        except OSError as err:
            print('Error occured while writing info file "%s" because %s' % (args.config, err), file=sys.stderr)
            sys.exit(1)
    elif args.action == 'create_user':
        user_password = args.password
        if user_password is None and args.pwprompt:
            try:
                user_password = getpass('User\'s password: '******'Error, you should specify a password for user or use option --pwprompt', file=sys.stderr)
            sys.exit(1)

        url_info = urlparse(base_url)
        (connection, cookie) = authentication(url_info)

        # search user
        print('Search user "%s"... ' % args.login, end='', flush=True)
        connection.request('GET', '%s/user/search/?login=%s' % (base_url, args.login), headers=cookie)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        if response.status == 404:
            print('not found')
        else:
            print('found, user id: %d' % message['users'][0])

        # search archive format
        print('Search archive format "Storiq One (TAR)"... ', end='', flush=True)
        connection.request('GET', '%s/archiveformat/?name=%s' % (base_url, urlquote('Storiq One (TAR)')), headers=cookie)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        archive_format_id = None
        if response.status == 200:
            archive_format_id = message['archiveformat id']
            print('found, id: %d' % archive_format_id)
        else:
            print('not found')
            sys.exit(1)

        vtl_id = args.vtl_id
        if vtl_id is None:
            # search vtl
            print('Search vtl "%s"... ' % args.login, end='', flush=True)
            connection.request('GET', '%s/vtl/' % (base_url), headers=cookie)

            response = connection.getresponse()
            message = json.loads(response.read().decode("utf-8"))

            if message['total_rows'] == 1:
                vtl_id = int(message['vtls'][0]['vtlid'])
                print('found, select vtl (id: %d)' % vtl_id)
            elif message['total_rows'] > 1:
                print('found, there is more than one vtl, you should specify vtl_id')
                connection.close()
                sys.exit(2)
            else:
                print('no vtl found')
                connection.close()
                sys.exit(2)

        # get vtl information
        print('Get vtl information (id: %d)... ' % vtl_id, end='', flush=True)
        connection.request('GET', '%s/vtl/?id=%d' % (base_url, vtl_id), headers=cookie)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        mediaformat_id = None
        if response.status == 200:
            mediaformat_id = message['vtl']['mediaformat']
            print('ok, will use mediaformat(%d)' % mediaformat_id)
        else:
            print('failed, no vtl found with id = %d' % vtl_id)
            connection.close()
            sys.exit(2)

        # find blank media
        print('Find blank media (mediaformat: %d)... ' % mediaformat_id, end='', flush=True)
        connection.request('GET', '%s/media/search/?status=new&mediaformat=%d&order_by=id' % (base_url, mediaformat_id), headers=cookie)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        media_id = None
        if response.status == 200:
            if message['total_rows'] > 0:
                media_id = message['medias'][0]
                print('found %d media(s), will use media (id: %d)' % (message['total_rows'], media_id))
            else:
                print('no medias found')
                connection.close()
                sys.exit(2)
        else:
            print('error while finding blank media because %s' % message)
            connection.close()
            sys.exit(2)

        # creating pool
        pool = {
            'name': 'pool_' + args.login,
            'archiveformat': archive_format_id,
            'mediaformat': mediaformat_id
        }

        if args.pool_template is not None:
            pool['pooltemplate'] = args.pool_template

        pool_header = { 'Content-type': 'application/json' }
        pool_header.update(cookie)

        print('Create new pool (name: %s)... ' % pool['name'], end='', flush=True)
        connection.request('POST', '%s/pool/' % (base_url), json.dumps(pool), headers=pool_header)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        pool_id = None
        if response.status == 201:
            pool_id = message['pool_id']
            print('created, new id: %d' % pool_id)
        else:
            print('failed because (%d, %s)' % (response.status, message['message']))
            connection.close()
            sys.exit(2)

        # creating poolgroup
        poolgroup = {
            'name': 'poolgroup_' + args.login,
            'pools': [pool_id]
        }

        print('Create new pool group (name: %s)... ' % poolgroup['name'], end='', flush=True)
        connection.request('POST', '%s/poolgroup/' % (base_url), json.dumps(poolgroup), headers=pool_header)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        poolgroup_id = None
        if response.status == 201:
            poolgroup_id = message['poolgroup_id']
            print('created, new id: %d' % poolgroup_id)
        else:
            print('failed because (%d, %s)' % (response.status, message['message']))
            connection.close()
            sys.exit(2)

        # creating user
        homedirectory = args.homedirectory
        if homedirectory.find('<login>') > -1:
            homedirectory = homedirectory.replace('<login>', args.login)

        user = {
            'login': args.login,
            'password': user_password,
            'fullname': args.login,
            'email': args.login,
            'homedirectory': homedirectory,
            'isadmin': False,
            'canarchive': True,
            'canrestore': True,
            'poolgroup': poolgroup_id,
            'disabled': False
        }

        print('Create new user (name: %s)... ' % args.login, end='', flush=True)
        connection.request('POST', '%s/user/' % (base_url), json.dumps(user), headers=pool_header)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        user_id = None
        if response.status == 201:
            user_id = message['user_id']
            print('created, new id: %d' % user_id)
        else:
            print('failed because (%d, %s)' % (response.status, message['message']))
            connection.close()
            sys.exit(2)

        # format media
        task_info = {
            'media': media_id,
            'pool': pool_id
        }

        print('Create formatting task (media: %d, pool: %d)... ' % (media_id, pool_id), end='', flush=True)
        connection.request('POST', '%s/media/format/' % (base_url), json.dumps(task_info), headers=pool_header)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        task_id = None
        if response.status == 201:
            task_id = message['job_id']
            print('created, new task: %d' % task_id)
            connection.close()
            sys.exit(0)
        else:
            print('failed because (%d, %s)' % (response.status, message['message']))
            connection.close()
            sys.exit(2)



    elif args.action == 'list_pool':
        url_info = urlparse(base_url)
        (connection, cookie) = authentication(url_info)

        print('Getting pool list... ', end='', flush=True)
        connection.request('GET', base_url + '/pool/', headers=cookie)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        print(message['total_rows'])

        pools = message['pools']
        for pool_id in pools:
            connection.request('GET', '%s/pool/?id=%d' % (base_url, pool_id), headers=cookie)

            response = connection.getresponse()
            sub_message = json.loads(response.read().decode("utf-8"))

            pool = sub_message['pool']
            pp.pprint(pool)


        connection.close()

    elif args.action == 'list_user':
        url_info = urlparse(base_url)
        (connection, cookie) = authentication(url_info)

        print('Getting user list... ', end='', flush=True)
        connection.request('GET', base_url + '/user/', headers=cookie)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        print(message['total_rows'])

        users = message['users']
        for user_id in users:
            connection.request('GET', '%s/user/?id=%d' % (base_url, user_id), headers=cookie)

            response = connection.getresponse()
            sub_message = json.loads(response.read().decode("utf-8"))

            user = sub_message['user']
            pp.pprint(user)


        connection.close()

    elif args.action == 'enable_user':
        key = args.key
        print('enable user : %s' % key)

        url_info = urlparse(base_url)
        (connection, cookie) = authentication(url_info)

        print('Activating user... ', end='', flush=True)
        connection.request('GET', base_url + '/user/update/?action=activate&key=' + key, headers=cookie)

        response = connection.getresponse()
        message = json.loads(response.read().decode("utf-8"))

        if response.status == 200:
            print('activated')
            connection.close()
            sys.exit(0)

        else:
            print('not activated')
            connection.close()
            sys.exit(1)


    elif args.action == 'disable_user':
        print('disable user : %s' % args.login)

        url_info = urlparse(base_url)
        (connection, cookie) = authentication(url_info)

        print('Deactivating user... ', end='', flush=True)
        connection.request('GET', base_url + '/user/update/?action=deactivate&login='******'deactivated')
        else:
            print('not deactivated')

        if args.key is not None:
            url_info = urlparse(base_url)
            (connection, cookie) = authentication(url_info)

            print('Adding key... ', end='', flush=True)
            connection.request('GET', base_url + '/user/update/?action=key&login='******'&key=' + args.key, headers=cookie)

            response = connection.getresponse()
            message = json.loads(response.read().decode("utf-8"))

            if response.status == 200:
                print('added')
            else:
                print('not added')

        connection.close()

    else:
        print('Error, you should specify one action from ("create_config", "create_user")', file=sys.stderr)
        sys.exit(1)
示例#35
0
    MYSQL_DATABASE = environ.get("MYSQL_DATABASE", MYSQL_DATABASE)
    MYSQL_USER = environ.get("MYSQL_USER", MYSQL_USER)
    MYSQL_PASSWORD = environ.get("MYSQL_PASSWORD", MYSQL_PASSWORD)

    DHCP_DDNS_IP = environ.get("DHCP_DDNS_IP", DHCP_DDNS_IP)
    DHCP_DDNS_PORT = int(environ.get("DHCP_DDNS_PORT", DHCP_DDNS_PORT))

    CUSTOMER = environ.get("CUSTOMER", CUSTOMER)

    APPS = environ.get("APPS", APPS)
    if type(APPS) == str:
        APPS = APPS.split(",")

MYSQL_URL = "mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8mb4".format(
    MYSQL_USER,
    urlquote(MYSQL_PASSWORD),
    MYSQL_HOST,
    MYSQL_PORT,
    MYSQL_DATABASE,
)

MYSQL_URL = environ.get("MYSQL_URL", MYSQL_URL)

SCOPES = None
AUDIT_MODULES = None


def set_scopes(scopes):
    global SCOPES
    SCOPES = scopes
示例#36
0
    def load(self, bundle_id, bundle_version=None):
        '''
        Loads a bundle by downloading an index file, looking up the bundle location, and
        then downloading the bundle
        '''
        import requests
        self._setup_index()
        binfo = self._index.get(bundle_id)
        if not binfo:
            raise LoadFailed(bundle_id, self, 'Bundle is not in the index')
        if not isinstance(binfo, dict):
            raise LoadFailed(bundle_id, self,
                             'Unexpected type of bundle info in the index')

        if bundle_version is None:
            max_vn = 0
            for k in binfo.keys():
                try:
                    val = int(k)
                except ValueError:
                    L.warning(
                        "Got unexpected non-version-number key '%s' in bundle index info",
                        k)
                else:
                    if max_vn < val:
                        max_vn = val
            if not max_vn:
                raise LoadFailed(bundle_id, self, 'No releases found')
            bundle_version = max_vn

        versioned_binfo = binfo.get(str(bundle_version))

        if not versioned_binfo or not isinstance(versioned_binfo, dict):
            raise LoadFailed(bundle_id, self,
                             f'No bundle info for version {bundle_version}')

        bundle_url = versioned_binfo.get('url')

        if not self._bundle_url_is_ok(bundle_url):
            raise LoadFailed(
                bundle_id, self, 'Did not find a valid URL for "%s" at'
                ' version %s' % (bundle_id, bundle_version))

        hashes = versioned_binfo.get('hashes')
        if not isinstance(hashes, dict) or not hashes:
            raise LoadFailed(bundle_id, self,
                             f'No hash info for version {bundle_version}')

        for hash_name in self.hash_preference:
            bundle_hash = hashes.get(hash_name)
            if bundle_hash:
                break
        else:  # no break
            for hash_name, bundle_hash in hashes.items():
                if hash_name in hashlib.algorithms_available:
                    break
            else:  # no break
                raise LoadFailed(
                    bundle_id, self,
                    f'No supported hash for version {bundle_version}')

        try:
            hsh = hashlib.new(hash_name)
        except ValueError:
            L.warning(
                'Hash in hashlib.algorithms_available unsupported in hashlib.new'
            )
            raise LoadFailed(
                bundle_id, self,
                f'Unsupported hash {hash_name} for version {bundle_version}')

        response = requests.get(bundle_url, stream=True)
        if self.cachedir is not None:
            bfn = urlquote(bundle_id)
            with open(p(self.cachedir, bfn), 'wb') as f:
                for chunk in response.iter_content(chunk_size=1024):
                    hsh.update(chunk)
                    f.write(chunk)
            if bundle_hash != hsh.hexdigest():
                raise LoadFailed(
                    bundle_id, self,
                    f'Failed to verify {hash_name} hash for version {bundle_version}'
                )
            with open(p(self.cachedir, bfn), 'rb') as f:
                Unarchiver().unpack(f, self.base_directory)
        else:
            bio = io.BytesIO()
            bundle_bytes = response.raw.read()
            hsh.update(bundle_bytes)
            if bundle_hash != hsh.hexdigest():
                raise LoadFailed(
                    bundle_id, self,
                    f'Failed to verify {hash_name} hash for version {bundle_version}'
                )
            bio.write(bundle_bytes)
            bio.seek(0)
            Unarchiver().unpack(bio, self.base_directory)
示例#37
0
def tasteometer(phenny, input):
    input1 = input.group(2)
    if not input1 or len(input1) == 0:
        phenny.say("tasteometer: compares two users' musical taste")
        phenny.say("syntax: .taste user1 user2")
        return
    input2 = input.group(3)
    user1 = resolve_username(input1)
    if not user1:
        user1 = input1
    user2 = resolve_username(input2)
    if not user2:
        user2 = input2
    if not user2 or len(user2) == 0:
        user2 = resolve_username(input.nick)
        if not user2:
            user2 = input.nick
    try:
        req = urlopen(
            "%smethod=tasteometer.compare&type1=user&type2=user&value1=%s&value2=%s"
            % (APIURL, urlquote(user1), urlquote(user2)))
    except (HTTPError, http.client.BadStatusLine) as e:
        if e.code == 400:
            phenny.say(
                "uhoh, someone doesn't exist on last.fm, perhaps they need to set user"
            )
            return
        else:
            phenny.say("uhoh. try again later, mmkay?")
            return
    doc = etree.parse(req)
    root = doc.getroot()
    score = root.xpath('comparison/result/score')
    if len(score) == 0:
        phenny.say("something isn't right. have those users scrobbled?")
        return

    score = float(score[0].text)
    rating = ""
    if score >= 0.9:
        rating = "Super"
    elif score >= 0.7:
        rating = "Very High"
    elif score >= 0.5:
        rating = "High"
    elif score >= 0.3:
        rating = "Medium"
    elif score >= 0.1:
        rating = "Low"
    else:
        rating = "Very Low"

    artists = root.xpath("comparison/result/artists/artist/name")
    common_artists = ""
    names = []
    if len(artists) == 0:
        common_artists = ". they don't have any artists in common."
    else:
        list(map(lambda a: names.append(a.text), artists))
        common_artists = "and music they have in common includes: %s" % ", ".join(
            names)

    phenny.say("%s's and %s's musical compatibility rating is %s %s" %
               (user1, user2, rating, common_artists))
示例#38
0
 def _trackerBranch(self, branch):
     return "refs/buildbot/{}/{}".format(urlquote(self.repourl, ''),
                                         self._removeHeads(branch))
示例#39
0
def gcodeFileCommand(filename, target):
    if target not in [FileDestinations.LOCAL, FileDestinations.SDCARD]:
        abort(404)

    if not _validate(target, filename):
        abort(404)

    # valid file commands, dict mapping command name to mandatory parameters
    valid_commands = {
        "select": [],
        "unselect": [],
        "slice": [],
        "analyse": [],
        "copy": ["destination"],
        "move": ["destination"],
    }

    command, data, response = get_json_command_from_request(
        request, valid_commands)
    if response is not None:
        return response

    user = current_user.get_name()

    if command == "select":
        with Permissions.FILES_SELECT.require(403):
            if not _verifyFileExists(target, filename):
                abort(404)

            # selects/loads a file
            if not octoprint.filemanager.valid_file_type(filename,
                                                         type="machinecode"):
                abort(
                    415,
                    description=
                    "Cannot select file for printing, not a machinecode file",
                )

            if not printer.is_ready():
                abort(
                    409,
                    description=
                    "Printer is already printing, cannot select a new file",
                )

            printAfterLoading = False
            if "print" in data and data["print"] in valid_boolean_trues:
                with Permissions.PRINT.require(403):
                    if not printer.is_operational():
                        abort(
                            409,
                            description=
                            "Printer is not operational, cannot directly start printing",
                        )
                    printAfterLoading = True

            sd = False
            if target == FileDestinations.SDCARD:
                filenameToSelect = filename
                sd = True
            else:
                filenameToSelect = fileManager.path_on_disk(target, filename)
            printer.select_file(filenameToSelect, sd, printAfterLoading, user)

    elif command == "unselect":
        with Permissions.FILES_SELECT.require(403):
            if not printer.is_ready():
                return make_response(
                    "Printer is already printing, cannot unselect current file",
                    409)

            _, currentFilename = _getCurrentFile()
            if currentFilename is None:
                return make_response(
                    "Cannot unselect current file when there is no file selected",
                    409)

            if filename != currentFilename and filename != "current":
                return make_response(
                    "Only the currently selected file can be unselected", 400)

            printer.unselect_file()

    elif command == "slice":
        with Permissions.SLICE.require(403):
            if not _verifyFileExists(target, filename):
                abort(404)

            try:
                if "slicer" in data:
                    slicer = data["slicer"]
                    del data["slicer"]
                    slicer_instance = slicingManager.get_slicer(slicer)

                elif "cura" in slicingManager.registered_slicers:
                    slicer = "cura"
                    slicer_instance = slicingManager.get_slicer("cura")

                else:
                    abort(415,
                          description="Cannot slice file, no slicer available")
            except octoprint.slicing.UnknownSlicer:
                abort(404)

            if not any([
                    octoprint.filemanager.valid_file_type(
                        filename, type=source_file_type)
                    for source_file_type in slicer_instance.
                    get_slicer_properties().get("source_file_types", ["model"])
            ]):
                abort(415, description="Cannot slice file, not a model file")

            cores = psutil.cpu_count()
            if (slicer_instance.get_slicer_properties().get(
                    "same_device", True)
                    and (printer.is_printing() or printer.is_paused())
                    and (cores is None or cores < 2)):
                # slicer runs on same device as OctoPrint, slicing while printing is hence disabled
                abort(
                    409,
                    description=
                    "Cannot slice on this slicer while printing on single core systems or systems of unknown core count due to performance reasons",
                )

            if "destination" in data and data["destination"]:
                destination = data["destination"]
                del data["destination"]
            elif "gcode" in data and data["gcode"]:
                destination = data["gcode"]
                del data["gcode"]
            else:
                import os

                name, _ = os.path.splitext(filename)
                destination = (
                    name + "." + slicer_instance.get_slicer_properties().get(
                        "destination_extensions", ["gco", "gcode", "g"])[0])

            full_path = destination
            if "path" in data and data["path"]:
                full_path = fileManager.join_path(target, data["path"],
                                                  destination)
            else:
                path, _ = fileManager.split_path(target, filename)
                if path:
                    full_path = fileManager.join_path(target, path,
                                                      destination)

            canon_path, canon_name = fileManager.canonicalize(
                target, full_path)
            sanitized_name = fileManager.sanitize_name(target, canon_name)

            if canon_path:
                full_path = fileManager.join_path(target, canon_path,
                                                  sanitized_name)
            else:
                full_path = sanitized_name

            # prohibit overwriting the file that is currently being printed
            currentOrigin, currentFilename = _getCurrentFile()
            if (currentFilename == full_path and currentOrigin == target
                    and (printer.is_printing() or printer.is_paused())):
                abort(
                    409,
                    description=
                    "Trying to slice into file that is currently being printed",
                )

            if "profile" in data and data["profile"]:
                profile = data["profile"]
                del data["profile"]
            else:
                profile = None

            if "printerProfile" in data and data["printerProfile"]:
                printerProfile = data["printerProfile"]
                del data["printerProfile"]
            else:
                printerProfile = None

            if ("position" in data and data["position"]
                    and isinstance(data["position"], dict)
                    and "x" in data["position"] and "y" in data["position"]):
                position = data["position"]
                del data["position"]
            else:
                position = None

            select_after_slicing = False
            if "select" in data and data["select"] in valid_boolean_trues:
                if not printer.is_operational():
                    abort(
                        409,
                        description=
                        "Printer is not operational, cannot directly select for printing",
                    )
                select_after_slicing = True

            print_after_slicing = False
            if "print" in data and data["print"] in valid_boolean_trues:
                if not printer.is_operational():
                    abort(
                        409,
                        description=
                        "Printer is not operational, cannot directly start printing",
                    )
                select_after_slicing = print_after_slicing = True

            override_keys = [
                k for k in data
                if k.startswith("profile.") and data[k] is not None
            ]
            overrides = {}
            for key in override_keys:
                overrides[key[len("profile."):]] = data[key]

            def slicing_done(target, path, select_after_slicing,
                             print_after_slicing):
                if select_after_slicing or print_after_slicing:
                    sd = False
                    if target == FileDestinations.SDCARD:
                        filenameToSelect = path
                        sd = True
                    else:
                        filenameToSelect = fileManager.path_on_disk(
                            target, path)
                    printer.select_file(filenameToSelect, sd,
                                        print_after_slicing, user)

            try:
                fileManager.slice(
                    slicer,
                    target,
                    filename,
                    target,
                    full_path,
                    profile=profile,
                    printer_profile_id=printerProfile,
                    position=position,
                    overrides=overrides,
                    display=canon_name,
                    callback=slicing_done,
                    callback_args=(
                        target,
                        full_path,
                        select_after_slicing,
                        print_after_slicing,
                    ),
                )
            except octoprint.slicing.UnknownProfile:
                abort(404, description="Unknown profile")

            location = url_for(
                ".readGcodeFile",
                target=target,
                filename=full_path,
                _external=True,
            )
            result = {
                "name": destination,
                "path": full_path,
                "display": canon_name,
                "origin": FileDestinations.LOCAL,
                "refs": {
                    "resource":
                    location,
                    "download":
                    url_for("index", _external=True) + "downloads/files/" +
                    target + "/" + urlquote(full_path),
                },
            }

            r = make_response(jsonify(result), 202)
            r.headers["Location"] = location
            return r

    elif command == "analyse":
        with Permissions.FILES_UPLOAD.require(403):
            if not _verifyFileExists(target, filename):
                abort(404)

            printer_profile = None
            if "printerProfile" in data and data["printerProfile"]:
                printer_profile = data["printerProfile"]

            if not fileManager.analyse(
                    target, filename, printer_profile_id=printer_profile):
                abort(400, description="No analysis possible")

    elif command == "copy" or command == "move":
        with Permissions.FILES_UPLOAD.require(403):
            # Copy and move are only possible on local storage
            if target not in [FileDestinations.LOCAL]:
                abort(400,
                      description="Unsupported target for {}".format(command))

            if not _verifyFileExists(target,
                                     filename) and not _verifyFolderExists(
                                         target, filename):
                abort(404)

            path, name = fileManager.split_path(target, filename)

            destination = data["destination"]
            dst_path, dst_name = fileManager.split_path(target, destination)
            sanitized_destination = fileManager.join_path(
                target, dst_path, fileManager.sanitize_name(target, dst_name))

            # Check for exception thrown by _verifyFolderExists, if outside the root directory
            try:
                if (_verifyFolderExists(target, destination)
                        and sanitized_destination != filename):
                    # destination is an existing folder and not ourselves (= display rename), we'll assume we are supposed
                    # to move filename to this folder under the same name
                    destination = fileManager.join_path(
                        target, destination, name)

                if _verifyFileExists(target,
                                     destination) or _verifyFolderExists(
                                         target, destination):
                    abort(409, description="File or folder does already exist")

            except Exception:
                abort(409,
                      description=
                      "Exception thrown by storage, bad folder/file name?")

            is_file = fileManager.file_exists(target, filename)
            is_folder = fileManager.folder_exists(target, filename)

            if not (is_file or is_folder):
                abort(400,
                      description="Neither file nor folder, can't {}".format(
                          command))

            if command == "copy":
                # destination already there? error...
                if _verifyFileExists(target,
                                     destination) or _verifyFolderExists(
                                         target, destination):
                    abort(409, description="File or folder does already exist")

                if is_file:
                    fileManager.copy_file(target, filename, destination)
                else:
                    fileManager.copy_folder(target, filename, destination)

            elif command == "move":
                with Permissions.FILES_DELETE.require(403):
                    if _isBusy(target, filename):
                        abort(
                            409,
                            description=
                            "Trying to move a file or folder that is currently in use",
                        )

                    # destination already there AND not ourselves (= display rename)? error...
                    if (_verifyFileExists(target, destination)
                            or _verifyFolderExists(target, destination)
                        ) and sanitized_destination != filename:
                        abort(409,
                              description="File or folder does already exist")

                    # deselect the file if it's currently selected
                    currentOrigin, currentFilename = _getCurrentFile()
                    if currentFilename is not None and filename == currentFilename:
                        printer.unselect_file()

                    if is_file:
                        fileManager.move_file(target, filename, destination)
                    else:
                        fileManager.move_folder(target, filename, destination)

            location = url_for(
                ".readGcodeFile",
                target=target,
                filename=destination,
                _external=True,
            )
            result = {
                "name": name,
                "path": destination,
                "origin": FileDestinations.LOCAL,
                "refs": {
                    "resource": location
                },
            }
            if is_file:
                result["refs"]["download"] = (
                    url_for("index", _external=True) + "downloads/files/" +
                    target + "/" + urlquote(destination))

            r = make_response(jsonify(result), 201)
            r.headers["Location"] = location
            return r

    return NO_CONTENT
示例#40
0
def uploadGcodeFile(target):
    input_name = "file"
    input_upload_name = (input_name + "." +
                         settings().get(["server", "uploads", "nameSuffix"]))
    input_upload_path = (input_name + "." +
                         settings().get(["server", "uploads", "pathSuffix"]))
    if input_upload_name in request.values and input_upload_path in request.values:
        if target not in [FileDestinations.LOCAL, FileDestinations.SDCARD]:
            abort(404)

        upload = octoprint.filemanager.util.DiskFileWrapper(
            request.values[input_upload_name],
            request.values[input_upload_path])

        # Store any additional user data the caller may have passed.
        userdata = None
        if "userdata" in request.values:
            import json

            try:
                userdata = json.loads(request.values["userdata"])
            except Exception:
                abort(400, description="userdata contains invalid JSON")

        if target == FileDestinations.SDCARD and not settings().getBoolean(
            ["feature", "sdSupport"]):
            abort(404)

        sd = target == FileDestinations.SDCARD
        selectAfterUpload = ("select" in request.values and
                             request.values["select"] in valid_boolean_trues
                             and Permissions.FILES_SELECT.can())
        printAfterSelect = ("print" in request.values
                            and request.values["print"] in valid_boolean_trues
                            and Permissions.PRINT.can())

        if sd:
            # validate that all preconditions for SD upload are met before attempting it
            if not (printer.is_operational()
                    and not (printer.is_printing() or printer.is_paused())):
                abort(
                    409,
                    description=
                    "Can not upload to SD card, printer is either not operational or already busy",
                )
            if not printer.is_sd_ready():
                abort(
                    409,
                    description="Can not upload to SD card, not yet initialized"
                )

        # determine future filename of file to be uploaded, abort if it can't be uploaded
        try:
            # FileDestinations.LOCAL = should normally be target, but can't because SDCard handling isn't implemented yet
            canonPath, canonFilename = fileManager.canonicalize(
                FileDestinations.LOCAL, upload.filename)
            if request.values.get("path"):
                canonPath = request.values.get("path")
            if request.values.get("filename"):
                canonFilename = request.values.get("filename")

            futurePath = fileManager.sanitize_path(FileDestinations.LOCAL,
                                                   canonPath)
            futureFilename = fileManager.sanitize_name(FileDestinations.LOCAL,
                                                       canonFilename)
        except Exception:
            canonFilename = None
            futurePath = None
            futureFilename = None

        if futureFilename is None:
            abort(415, description="Can not upload file, wrong format?")

        # prohibit overwriting currently selected file while it's being printed
        futureFullPath = fileManager.join_path(FileDestinations.LOCAL,
                                               futurePath, futureFilename)
        futureFullPathInStorage = fileManager.path_in_storage(
            FileDestinations.LOCAL, futureFullPath)

        if not printer.can_modify_file(futureFullPathInStorage, sd):
            abort(
                409,
                description=
                "Trying to overwrite file that is currently being printed",
            )

        if (fileManager.file_exists(FileDestinations.LOCAL,
                                    futureFullPathInStorage)
                and request.values.get("noOverwrite") in valid_boolean_trues):
            abort(409,
                  description="File already exists and noOverwrite was set")

        reselect = printer.is_current_file(futureFullPathInStorage, sd)

        user = current_user.get_name()

        def fileProcessingFinished(filename, absFilename, destination):
            """
            Callback for when the file processing (upload, optional slicing, addition to analysis queue) has
            finished.

            Depending on the file's destination triggers either streaming to SD card or directly calls selectAndOrPrint.
            """

            if (destination == FileDestinations.SDCARD
                    and octoprint.filemanager.valid_file_type(
                        filename, "machinecode")):
                return filename, printer.add_sd_file(
                    filename,
                    absFilename,
                    on_success=selectAndOrPrint,
                    tags={"source:api", "api:files.sd"},
                )
            else:
                selectAndOrPrint(filename, absFilename, destination)
                return filename

        def selectAndOrPrint(filename, absFilename, destination):
            """
            Callback for when the file is ready to be selected and optionally printed. For SD file uploads this is only
            the case after they have finished streaming to the printer, which is why this callback is also used
            for the corresponding call to addSdFile.

            Selects the just uploaded file if either selectAfterUpload or printAfterSelect are True, or if the
            exact file is already selected, such reloading it.
            """
            if octoprint.filemanager.valid_file_type(
                    added_file, "gcode") and (selectAfterUpload
                                              or printAfterSelect or reselect):
                printer.select_file(
                    absFilename,
                    destination == FileDestinations.SDCARD,
                    printAfterSelect,
                    user,
                )

        try:
            added_file = fileManager.add_file(
                FileDestinations.LOCAL,
                futureFullPathInStorage,
                upload,
                allow_overwrite=True,
                display=canonFilename,
            )
        except octoprint.filemanager.storage.StorageError as e:
            if e.code == octoprint.filemanager.storage.StorageError.INVALID_FILE:
                abort(400, description="Could not upload file, invalid type")
            else:
                abort(500, description="Could not upload file")

        if octoprint.filemanager.valid_file_type(added_file, "stl"):
            filename = added_file
            done = True
        else:
            filename = fileProcessingFinished(
                added_file,
                fileManager.path_on_disk(FileDestinations.LOCAL, added_file),
                target,
            )
            done = not sd

        if userdata is not None:
            # upload included userdata, add this now to the metadata
            fileManager.set_additional_metadata(FileDestinations.LOCAL,
                                                added_file, "userdata",
                                                userdata)

        sdFilename = None
        if isinstance(filename, tuple):
            filename, sdFilename = filename

        payload = {
            "name": futureFilename,
            "path": filename,
            "target": target,
            "select": selectAfterUpload,
            "print": printAfterSelect,
        }
        if userdata is not None:
            payload["userdata"] = userdata
        eventManager.fire(Events.UPLOAD, payload)

        files = {}
        location = url_for(
            ".readGcodeFile",
            target=FileDestinations.LOCAL,
            filename=filename,
            _external=True,
        )
        files.update({
            FileDestinations.LOCAL: {
                "name": futureFilename,
                "path": filename,
                "origin": FileDestinations.LOCAL,
                "refs": {
                    "resource":
                    location,
                    "download":
                    url_for("index", _external=True) + "downloads/files/" +
                    FileDestinations.LOCAL + "/" + urlquote(filename),
                },
            }
        })

        if sd and sdFilename:
            location = url_for(
                ".readGcodeFile",
                target=FileDestinations.SDCARD,
                filename=sdFilename,
                _external=True,
            )
            files.update({
                FileDestinations.SDCARD: {
                    "name": sdFilename,
                    "path": sdFilename,
                    "origin": FileDestinations.SDCARD,
                    "refs": {
                        "resource": location
                    },
                }
            })

        r = make_response(jsonify(files=files, done=done), 201)
        r.headers["Location"] = location
        return r

    elif "foldername" in request.values:
        foldername = request.values["foldername"]

        if target not in [FileDestinations.LOCAL]:
            abort(400, description="target is invalid")

        canonPath, canonName = fileManager.canonicalize(target, foldername)
        futurePath = fileManager.sanitize_path(target, canonPath)
        futureName = fileManager.sanitize_name(target, canonName)
        if not futureName or not futurePath:
            abort(400, description="folder name is empty")

        if "path" in request.values and request.values["path"]:
            futurePath = fileManager.sanitize_path(FileDestinations.LOCAL,
                                                   request.values["path"])

        futureFullPath = fileManager.join_path(target, futurePath, futureName)
        if octoprint.filemanager.valid_file_type(futureName):
            abort(409,
                  description="Can't create folder, please try another name")

        try:
            added_folder = fileManager.add_folder(target,
                                                  futureFullPath,
                                                  display=canonName)
        except octoprint.filemanager.storage.StorageError as e:
            if e.code == octoprint.filemanager.storage.StorageError.INVALID_DIRECTORY:
                abort(400,
                      description="Could not create folder, invalid directory")
            else:
                abort(500, description="Could not create folder")

        location = url_for(
            ".readGcodeFile",
            target=FileDestinations.LOCAL,
            filename=added_folder,
            _external=True,
        )
        folder = {
            "name": futureName,
            "path": added_folder,
            "origin": target,
            "refs": {
                "resource": location
            },
        }

        r = make_response(jsonify(folder=folder, done=True), 201)
        r.headers["Location"] = location
        return r

    else:
        abort(400, description="No file to upload and no folder to create")
示例#41
0
        def analyse_recursively(files, path=None):
            if path is None:
                path = ""

            result = []
            for file_or_folder in files:
                # make a shallow copy in order to not accidentally modify the cached data
                file_or_folder = dict(file_or_folder)

                file_or_folder["origin"] = FileDestinations.LOCAL

                if file_or_folder["type"] == "folder":
                    if "children" in file_or_folder:
                        file_or_folder["children"] = analyse_recursively(
                            file_or_folder["children"].values(),
                            path + file_or_folder["name"] + "/",
                        )

                    file_or_folder["refs"] = {
                        "resource":
                        url_for(
                            ".readGcodeFile",
                            target=FileDestinations.LOCAL,
                            filename=path + file_or_folder["name"],
                            _external=True,
                        )
                    }
                else:
                    if ("analysis" in file_or_folder
                            and octoprint.filemanager.valid_file_type(
                                file_or_folder["name"], type="gcode")):
                        file_or_folder["gcodeAnalysis"] = file_or_folder[
                            "analysis"]
                        del file_or_folder["analysis"]

                    if ("history" in file_or_folder
                            and octoprint.filemanager.valid_file_type(
                                file_or_folder["name"], type="gcode")):
                        # convert print log
                        history = file_or_folder["history"]
                        del file_or_folder["history"]
                        success = 0
                        failure = 0
                        last = None
                        for entry in history:
                            success += 1 if "success" in entry and entry[
                                "success"] else 0
                            failure += (1 if "success" in entry
                                        and not entry["success"] else 0)
                            if not last or ("timestamp" in entry
                                            and "timestamp" in last
                                            and entry["timestamp"] >
                                            last["timestamp"]):
                                last = entry
                        if last:
                            prints = {
                                "success": success,
                                "failure": failure,
                                "last": {
                                    "success": last["success"],
                                    "date": last["timestamp"],
                                },
                            }
                            if "printTime" in last:
                                prints["last"]["printTime"] = last["printTime"]
                            file_or_folder["prints"] = prints

                    file_or_folder["refs"] = {
                        "resource":
                        url_for(
                            ".readGcodeFile",
                            target=FileDestinations.LOCAL,
                            filename=file_or_folder["path"],
                            _external=True,
                        ),
                        "download":
                        url_for("index", _external=True) + "downloads/files/" +
                        FileDestinations.LOCAL + "/" +
                        urlquote(file_or_folder["path"]),
                    }

                result.append(file_or_folder)

            return result
示例#42
0
 def _trackerBranch(self, branch):
     # manually quote tilde for Python 3.7
     url = urlquote(self.repourl, '').replace('~', '%7E')
     return "refs/buildbot/{}/{}".format(url, self._removeHeads(branch))
示例#43
0
    def build_url(self, local_path, **kwargs):

        # Make the path relative.
        local_path = local_path.strip('/')

        # We complain when we see non-normalized paths, as it is a good
        # indicator that unsanitized data may be getting through.
        # Mutating the scheme syntax to match is a little gross, but it works
        # for today.
        norm_path = os.path.normpath(local_path)
        if local_path.replace(
                '://', ':/') != norm_path or norm_path.startswith('../'):
            raise ValueError('path is not normalized')

        external = kwargs.pop('external', None) or kwargs.pop(
            '_external', None)
        scheme = kwargs.pop('scheme', None)
        if scheme and not external:
            raise ValueError('cannot specify scheme without external=True')
        if kwargs.get('_anchor'):
            raise ValueError('images have no _anchor')
        if kwargs.get('_method'):
            raise ValueError('images have no _method')

        # Remote URLs are encoded into the query.
        parsed = urlparse(local_path)
        if parsed.scheme or parsed.netloc:
            if parsed.scheme not in ALLOWED_SCHEMES:
                raise ValueError('scheme %r is not allowed' % parsed.scheme)
            kwargs['url'] = local_path
            local_path = '_'  # Must be something.

        # Local ones are not.
        else:
            abs_path = self.find_img(local_path)
            if abs_path:
                kwargs['version'] = encode_int(int(os.path.getmtime(abs_path)))

        # Prep the cache flag, which defaults to True.
        cache = kwargs.pop('cache', True)
        if not cache:
            kwargs['cache'] = ''

        # Prep the enlarge flag, which defaults to False.
        enlarge = kwargs.pop('enlarge', False)
        if enlarge:
            kwargs['enlarge'] = '1'

        # Prep the transform, which is a set of delimited strings.
        transform = kwargs.get('transform')
        if transform:
            if isinstance(transform, string_types):
                transform = re.split(r'[,;:_ ]', transform)
            # We replace delimiters with underscores, and percent with p, since
            # these won't need escaping.
            kwargs['transform'] = '_'.join(
                str(x).replace('%', 'p') for x in transform)

        # Sign the query.
        # Collapse to a dict first so that if we accidentally have two of the
        # same kwarg (e.g. used `hidpi_sharpen` and `usm` which both turn into `usm`).
        public_kwargs = {
            LONG_TO_SHORT.get(k, k): v
            for k, v in iteritems(kwargs)
            if v is not None and not k.startswith('_')
        }
        query = urlencode(sorted(iteritems(public_kwargs)), True)
        signer = Signer(current_app.secret_key)
        sig = signer.get_signature('%s?%s' % (local_path, query))

        url = '%s/%s?%s&s=%s' % (
            current_app.config['IMAGES_URL'],
            urlquote(local_path, "/$-_.+!*'(),"),
            query,
            sig,
        )

        if external:
            url = '%s://%s%s/%s' % (scheme or request.scheme, request.host,
                                    request.script_root, url.lstrip('/'))

        return url
示例#44
0
    async def _perform_request(self, req: QueuedRequest):
        def _put_back(_delta):
            """
            Put request back into the queue after delta seconds
            Resets the request expiration
            """
            req.time_added = time.perf_counter()
            return self.loop.call_later(_delta, self.queue.put, req)

        req.tries += 1
        route = req.route
        delta = await self._ratelimits.get_delta(route)
        if delta > 0:
            return _put_back(delta)

        options = req.options
        options["headers"] = headers = {
            "User-Agent": self.user_agent,
            "X-Ratelimit-Precision": "millisecond",
            "Authorization": f"Bot {self.token}",
        }

        if "json" in options:
            headers["Content-Type"] = "application/json"
            options["data"] = orjson.dumps(options.pop("json")).decode("utf-8")

        if "reason" in options:
            headers["X-Audit-Log-Reason"] = urlquote(options.pop("reason"), safe="/ ")

        async with self.session.request(
            method=route.method, url=route.url, raise_for_status=False, **options
        ) as resp:
            data = await json_or_text(resp)

            rl_bucket = resp.headers.get("X-Ratelimit-Remaining")
            if rl_bucket is not None:
                await self._ratelimits.set_bucket(route, rl_bucket)

            rl_remaining = resp.headers.get("X-Ratelimit-Remaining")
            if rl_remaining == 0:
                reset = datetime.fromtimestamp(
                    resp.headers["X-Ratelimit-Reset"], timezone.utc
                )
                delta = (reset - datetime.utcnow()).total_seconds()
                await self._ratelimits.set_delta(route, delta)

            if 300 > resp.status >= 200:
                return req.future.set_result(data)

            if resp.status == 429 and resp.headers.get("Via"):
                retry_after = data["retry_after"] / 1000.0
                is_global = data.get("global", False)
                if is_global:
                    await self._ratelimits.set_global(retry_after)

                else:
                    await self._ratelimits.set_delta(route, retry_after)

                return _put_back(retry_after)

            if resp.status in {500, 502} and req.tries <= 5:
                return _put_back(req.tries * 2)

            resp.raise_for_status()
示例#45
0
    def request(
        self,
        route: Route,
        session: Session,
        *,
        payload: Optional[Dict[str, Any]] = None,
        multipart: Optional[List[Dict[str, Any]]] = None,
        files: Optional[List[File]] = None,
        reason: Optional[str] = None,
        auth_token: Optional[str] = None,
        params: Optional[Dict[str, Any]] = None,
    ) -> Any:
        headers: Dict[str, str] = {}
        files = files or []
        to_send: Optional[Union[str, Dict[str, Any]]] = None
        bucket = (route.webhook_id, route.webhook_token)

        try:
            lock = self._locks[bucket]
        except KeyError:
            self._locks[bucket] = lock = threading.Lock()

        if payload is not None:
            headers['Content-Type'] = 'application/json'
            to_send = utils._to_json(payload)

        if auth_token is not None:
            headers['Authorization'] = f'Bot {auth_token}'

        if reason is not None:
            headers['X-Audit-Log-Reason'] = urlquote(reason, safe='/ ')

        response: Optional[Response] = None
        data: Optional[Union[Dict[str, Any], str]] = None
        file_data: Optional[Dict[str, Any]] = None
        method = route.method
        url = route.url
        webhook_id = route.webhook_id

        with DeferredLock(lock) as lock:
            for attempt in range(5):
                for file in files:
                    file.reset(seek=attempt)

                if multipart:
                    file_data = {}
                    for p in multipart:
                        name = p['name']
                        if name == 'payload_json':
                            to_send = {'payload_json': p['value']}
                        else:
                            file_data[name] = (p['filename'], p['value'],
                                               p['content_type'])

                try:
                    with session.request(method,
                                         url,
                                         data=to_send,
                                         files=file_data,
                                         headers=headers,
                                         params=params) as response:
                        _log.debug(
                            'Webhook ID %s with %s %s has returned status code %s',
                            webhook_id,
                            method,
                            url,
                            response.status_code,
                        )
                        response.encoding = 'utf-8'
                        # Compatibility with aiohttp
                        response.status = response.status_code  # type: ignore

                        data = response.text or None
                        if data and response.headers[
                                'Content-Type'] == 'application/json':
                            data = json.loads(data)

                        remaining = response.headers.get(
                            'X-Ratelimit-Remaining')
                        if remaining == '0' and response.status_code != 429:
                            delta = utils._parse_ratelimit_header(response)
                            _log.debug(
                                'Webhook ID %s has been pre-emptively rate limited, waiting %.2f seconds',
                                webhook_id, delta)
                            lock.delay_by(delta)

                        if 300 > response.status_code >= 200:
                            return data

                        if response.status_code == 429:
                            if not response.headers.get('Via'):
                                raise HTTPException(response, data)

                            retry_after: float = data[
                                'retry_after']  # type: ignore
                            _log.warning(
                                'Webhook ID %s is rate limited. Retrying in %.2f seconds',
                                webhook_id, retry_after)
                            time.sleep(retry_after)
                            continue

                        if response.status_code >= 500:
                            time.sleep(1 + attempt * 2)
                            continue

                        if response.status_code == 403:
                            raise Forbidden(response, data)
                        elif response.status_code == 404:
                            raise NotFound(response, data)
                        else:
                            raise HTTPException(response, data)

                except OSError as e:
                    if attempt < 4 and e.errno in (54, 10054):
                        time.sleep(1 + attempt * 2)
                        continue
                    raise

            if response:
                if response.status_code >= 500:
                    raise DiscordServerError(response, data)
                raise HTTPException(response, data)

            raise RuntimeError('Unreachable code in HTTP handling.')
示例#46
0
def scm_link(package, operation):
    parts = (operation, package.pkgbase)
    linkbase = ("https://github.com/z3ntu/archhurd_packages/%s/master/%s")
    return linkbase % tuple(urlquote(part.encode('utf-8')) for part in parts)
示例#47
0
def now_playing(phenny, input):
    nick = input.nick
    user = ""
    arg = input.group(2)
    if not arg or len(arg.strip()) == 0:
        user = resolve_username(nick)  # use the sender
        if not user:  #nick didnt resolve
            user = nick
    else:  # use the argument
        user = resolve_username(arg.strip())
        if not user:  # user didnt resolve
            user = arg
    user = user.strip()
    try:
        req = urlopen("%smethod=user.getrecenttracks&user=%s" %
                      (APIURL, urlquote(user)))
    except HTTPError as e:
        if e.code == 400:
            phenny.say(
                "%s doesn't exist on last.fm, perhaps they need to set user" %
                (user))
            return
        else:
            phenny.say("uhoh. try again later, mmkay?")
            return
    except http.client.BadStatusLine:
        phenny.say("uhoh. try again later, mmkay?")
        return
    doc = etree.parse(req)
    root = doc.getroot()
    recenttracks = list(root)
    if len(recenttracks) == 0:
        phenny.say(
            "%s hasn't played anything recently. this isn't you? try lastfm-set"
            % (user))
        return
    tracks = list(recenttracks[0])
    #print etree.tostring(recenttracks[0])
    if len(tracks) == 0:
        phenny.say(
            "%s hasn't played anything recently. this isn't you? try lastfm-set"
            % (user))
        return
    first = tracks[0]
    now = True if first.get("nowplaying") == "true" else False
    tags = {}
    for e in first.getiterator():
        tags[e.tag] = e

    track = tags['name'].text.strip()
    artist = tags['artist'].text.strip()

    album = "unknown"
    if tags['album'].text:
        album = tags['album'].text

    date = None
    stamp = None
    if not now:
        date = tags['date'].get("uts")
        stamp = int(date)

    if now:
        present = get_verb(nick)[1]
        phenny.say("%s %s \"%s\" by %s on %s" %
                   (user.strip(), present.strip(), track, artist, album))
        return
    else:
        past = get_verb(nick)[0]
        phenny.say("%s %s \"%s\" by %s on %s %s" %
                   (user.strip(), past.strip(), track, artist, album,
                    pretty_date(stamp)))
示例#48
0
 def quote(uri):
     return urlquote(uri, "!$&'()*+,-./:=@_~", encoding="latin-1")
示例#49
0
    async def request(
        self,
        route: Route,
        session: aiohttp.ClientSession,
        *,
        payload: Optional[Dict[str, Any]] = None,
        multipart: Optional[List[Dict[str, Any]]] = None,
        files: Optional[List[File]] = None,
        reason: Optional[str] = None,
        auth_token: Optional[str] = None,
        params: Optional[Dict[str, Any]] = None,
    ) -> Any:
        headers: Dict[str, str] = {}
        files = files or []
        to_send: Optional[Union[str, aiohttp.FormData]] = None
        bucket = (route.webhook_id, route.webhook_token)

        try:
            lock = self._locks[bucket]
        except KeyError:
            self._locks[bucket] = lock = asyncio.Lock()

        if payload is not None:
            headers['Content-Type'] = 'application/json'
            to_send = utils.to_json(payload)

        if auth_token is not None:
            headers['Authorization'] = auth_token # Removed bot authorization here [OVERRIDEN BY BRNI]

        if reason is not None:
            headers['X-Audit-Log-Reason'] = urlquote(reason, safe='/ ')

        response: Optional[aiohttp.ClientResponse] = None
        data: Optional[Union[Dict[str, Any], str]] = None
        method = route.method
        url = route.url
        webhook_id = route.webhook_id

        async with AsyncDeferredLock(lock) as lock:
            for attempt in range(5):
                for file in files:
                    file.reset(seek=attempt)

                if multipart:
                    form_data = aiohttp.FormData()
                    for p in multipart:
                        form_data.add_field(**p)
                    to_send = form_data

                try:
                    async with session.request(method, url, data=to_send, headers=headers, params=params) as response:
                        log.debug(
                            'Webhook ID %s with %s %s has returned status code %s',
                            webhook_id,
                            method,
                            url,
                            response.status,
                        )
                        data = (await response.text(encoding='utf-8')) or None
                        if data and response.headers['Content-Type'] == 'application/json':
                            data = json.loads(data)

                        remaining = response.headers.get('X-Ratelimit-Remaining')
                        if remaining == '0' and response.status != 429:
                            delta = utils._parse_ratelimit_header(response)
                            log.debug(
                                'Webhook ID %s has been pre-emptively rate limited, waiting %.2f seconds', webhook_id, delta
                            )
                            lock.delay_by(delta)

                        if 300 > response.status >= 200:
                            return data

                        if response.status == 429:
                            if not response.headers.get('Via'):
                                raise HTTPException(response, data)

                            retry_after: float = data['retry_after']  # type: ignore
                            log.warning('Webhook ID %s is rate limited. Retrying in %.2f seconds', webhook_id, retry_after)
                            await asyncio.sleep(retry_after)
                            continue

                        if response.status >= 500:
                            await asyncio.sleep(1 + attempt * 2)
                            continue

                        if response.status == 403:
                            raise Forbidden(response, data)
                        elif response.status == 404:
                            raise NotFound(response, data)
                        else:
                            raise HTTPException(response, data)

                except OSError as e:
                    if attempt < 4 and e.errno in (54, 10054):
                        await asyncio.sleep(1 + attempt * 2)
                        continue
                    raise

            if response:
                if response.status >= 500:
                    raise DiscordServerError(response, data)
                raise HTTPException(response, data)

            raise RuntimeError('Unreachable code in HTTP handling.')
示例#50
0
def get_digikey_part_html_tree(dist,
                               pn,
                               extra_search_terms='',
                               url=None,
                               descend=2,
                               local_part_html=None):
    '''Find the Digikey HTML page for a part number and return the URL and parse tree.'''
    def merge_price_tiers(main_tree, alt_tree):
        '''Merge the price tiers from the alternate-packaging tree into the main tree.'''
        try:
            insertion_point = main_tree.find('table',
                                             id='product-dollars').find('tr')
            for tr in alt_tree.find('table',
                                    id='product-dollars').find_all('tr'):
                insertion_point.insert_after(tr)
        except AttributeError:
            logger.log(
                DEBUG_OBSESSIVE,
                'Problem merging price tiers for Digikey part {} with alternate packaging!'
                .format(pn))
            pass

    def merge_qty_avail(main_tree, alt_tree):
        '''Merge the quantities from the alternate-packaging tree into the main tree.'''
        try:
            main_qty = get_digikey_qty_avail(main_tree)
            alt_qty = get_digikey_qty_avail(alt_tree)
            if main_qty is None:
                merged_qty = alt_qty
            elif alt_qty is None:
                merged_qty = main_qty
            else:
                merged_qty = max(main_qty, alt_qty)
            if merged_qty is not None:
                insertion_point = main_tree.find(
                    'td', id='quantityAvailable').find('span', id='dkQty')
                insertion_point.string = '{}'.format(merged_qty)
        except AttributeError:
            logger.log(
                DEBUG_OBSESSIVE,
                'Problem merging available quantities for Digikey part {} with alternate packaging!'
                .format(pn))
            pass

    # Use the part number to lookup the part using the site search function, unless a starting url was given.
    if url is None:
        url = 'http://www.digikey.com/scripts/DkSearch/dksus.dll?WT.z_header=search_go&lang=en&keywords=' + urlquote(
            pn + ' ' + extra_search_terms, safe='')
        #url = 'http://www.digikey.com/product-search/en?KeyWords=' + urlquote(pn,safe='') + '&WT.z_header=search_go'
    elif url[0] == '/':
        url = 'http://www.digikey.com' + url

    # Open the URL, read the HTML from it, and parse it into a tree structure.
    req = FakeBrowser(url)
    for _ in range(HTML_RESPONSE_RETRIES):
        try:
            response = urlopen(req)
            html = response.read()
            break
        except WEB_SCRAPE_EXCEPTIONS:
            logger.log(
                DEBUG_DETAILED,
                'Exception while web-scraping {} from {}'.format(pn, dist))
            pass
    else:  # Couldn't get a good read from the website.
        logger.log(DEBUG_OBSESSIVE,
                   'No HTML page for {} from {}'.format(pn, dist))
        raise PartHtmlError

    # Abort if the part number isn't in the HTML somewhere.
    # (Only use the numbers and letters to compare PN to HTML.)
    if re.sub('[\W_]', '',
              str.lower(pn)) not in re.sub('[\W_]', '', str.lower(str(html))):
        logger.log(DEBUG_OBSESSIVE,
                   'No part number {} in HTML page from {}'.format(pn, dist))
        raise PartHtmlError

    # Use the following code if Javascript challenge pages are used to block scrapers.
    # try:
    # ghst = Ghost()
    # sess = ghst.start(plugins_enabled=False, download_images=False, show_scrollbars=False, javascript_enabled=False)
    # html, resources = sess.open(url)
    # print('type of HTML is {}'.format(type(html.content)))
    # html = html.content
    # except Exception as e:
    # print('Exception reading with Ghost: {}'.format(e))

    try:
        tree = BeautifulSoup(html, 'lxml')
    except Exception:
        logger.log(DEBUG_OBSESSIVE,
                   'No HTML tree for {} from {}'.format(pn, dist))
        raise PartHtmlError

    # If the tree contains the tag for a product page, then return it.
    if tree.find('div', class_='product-top-section') is not None:

        # Digikey separates cut-tape and reel packaging, so we need to examine more pages
        # to get all the pricing info. But don't descend any further if limit has been reached.
        if descend > 0:
            try:
                # Find all the URLs to alternate-packaging pages for this part.
                ap_urls = [
                    ap.find('li',
                            class_='lnkAltPack').find_all('a')[-1].get('href')
                    for ap in tree.find(
                        'div', class_='bota', id='additionalPackaging').
                    find_all('ul', class_='more-expander-item')
                ]
                logger.log(
                    DEBUG_OBSESSIVE,
                    'Found {} alternate packagings for {} from {}'.format(
                        len(ap_urls), pn, dist))
                try:
                    ap_trees_and_urls = [
                        get_digikey_part_html_tree(dist,
                                                   pn,
                                                   extra_search_terms,
                                                   ap_url,
                                                   descend=0)
                        for ap_url in ap_urls
                    ]
                except Exception:
                    logger.log(
                        DEBUG_OBSESSIVE,
                        'Failed to find alternate packagings for {} from {}'.
                        format(pn, dist))

                # Put the main tree on the list as well and then look through
                # the entire list for one that's non-reeled. Use this as the
                # main page for the part.
                ap_trees_and_urls.append((tree, url))
                if digikey_part_is_reeled(tree):
                    for ap_tree, ap_url in ap_trees_and_urls:
                        if not digikey_part_is_reeled(ap_tree):
                            # Found a non-reeled part, so use it as the main page.
                            tree = ap_tree
                            url = ap_url
                            break  # Done looking.

                # Now go through the other pages, merging their pricing and quantity
                # info into the main page.
                for ap_tree, ap_url in ap_trees_and_urls:
                    if ap_tree is tree:
                        continue  # Skip examining the main tree. It already contains its info.
                    try:
                        # Merge the pricing info from that into the main parse tree to make
                        # a single, unified set of price tiers...
                        merge_price_tiers(tree, ap_tree)
                        # and merge available quantity, using the maximum found.
                        merge_qty_avail(tree, ap_tree)
                    except AttributeError:
                        logger.log(
                            DEBUG_OBSESSIVE,
                            'Problem merging price/qty for {} from {}'.format(
                                pn, dist))
                        continue
            except AttributeError:
                logger.log(
                    DEBUG_OBSESSIVE,
                    'Problem parsing URLs from product page for {} from {}'.
                    format(pn, dist))
                pass
        return tree, url  # Return the parse tree and the URL where it came from.

    # If the tree is for a list of products, then examine the links to try to find the part number.
    if tree.find('table', id='productTable') is not None:
        logger.log(DEBUG_OBSESSIVE,
                   'Found product table for {} from {}'.format(pn, dist))
        if descend <= 0:
            logger.log(DEBUG_OBSESSIVE,
                       'Passed descent limit for {} from {}'.format(pn, dist))
            raise PartHtmlError
        else:
            # Look for the table of products.
            products = tree.find(
                'table', id='productTable').find('tbody').find_all('tr')

            # Extract the product links for the part numbers from the table.
            # Extract links for both manufacturer and catalog numbers.
            product_links = [
                p.find('td', class_='tr-mfgPartNumber').a for p in products
            ]
            product_links.extend(
                [p.find('td', class_='tr-dkPartNumber').a for p in products])

            # Extract all the part numbers from the text portion of the links.
            part_numbers = [l.text for l in product_links]

            # Look for the part number in the list that most closely matches the requested part number.
            match = difflib.get_close_matches(pn, part_numbers, 1, 0.0)[0]

            # Now look for the link that goes with the closest matching part number.
            for l in product_links:
                if l.text == match:
                    # Get the tree for the linked-to page and return that.
                    logger.log(
                        DEBUG_OBSESSIVE,
                        'Selecting {} from product table for {} from {}'.
                        format(l.text, pn, dist))
                    return get_digikey_part_html_tree(dist,
                                                      pn,
                                                      extra_search_terms,
                                                      url=l['href'],
                                                      descend=descend - 1)

    # If the HTML contains a list of part categories, then give up.
    if tree.find('form', id='keywordSearchForm') is not None:
        logger.log(
            DEBUG_OBSESSIVE,
            'Found high-level part categories for {} from {}'.format(pn, dist))
        raise PartHtmlError

    # I don't know what happened here, so give up.
    logger.log(DEBUG_OBSESSIVE,
               'Unknown error for {} from {}'.format(pn, dist))
    raise PartHtmlError
示例#51
0
def encode_path(*components):
    'Encode the path specified as a list of path components using URL encoding'
    return '/' + '/'.join(
        urlquote(x.encode('utf-8'), '').decode('ascii') for x in components)
示例#52
0
def file_predict(contents, filename):
    '''
    Signature   file_predict(contens=None, filename=None)
    Docstring   return a dash table and plot fig
    Parameters  contents: upload file 
                filename: string, file name
    '''
    content_type, content_string = contents.split(',')

    decoded = base64.b64decode(content_string)
    try:
        if 'csv' in filename:
            # Assume that the user uploaded a CSV file
            df = pd.read_csv(io.StringIO(decoded.decode('utf-8')))
        elif 'xls' in filename:
            # Assume that the user uploaded an excel file
            df = pd.read_excel(io.BytesIO(decoded))

        # get predictions
        start = time.time()
        pred = get_pred(df, nlp, tokenizer)
        # append predictions
        df_pred = append_pred(df, pred)

        # create download link
        csv_str = df_pred.to_csv(index=False, encoding='utf-8')
        csv_str = 'data:text/csv;charset=utf-8, ' + urlquote(csv_str)

        # plot precision score if label is included in the upload file
        if (df.columns == 'label').sum() > 0:
            fig = plot_dash_precision(df, pred)
        else:
            fig = {}

        # convert df to dash table
        dash_tbl = html.Div([
            dash_table.DataTable(data=df_pred.to_dict('records'),
                                 columns=[{
                                     'name': i,
                                     'id': i
                                 } for i in df_pred.columns],
                                 page_size=20,
                                 sort_action='native',
                                 style_table={'overflowy': 'auto'},
                                 style_header=css_tbl_header,
                                 style_cell=css_tbl_cell,
                                 style_data_conditional=css_tbl_condl)
        ],
                            style={
                                'font-family': 'Verdana',
                                'font-size': '1.4em'
                            })

        end = time.time()
        total_time = str(datetime.timedelta(seconds=end - start))

        return dash_tbl, fig, csv_str, total_time

    except Exception as e:
        print(e)
        err_msg = html.Div(
            ['There was an error processing this file.'],
            style={
                'font-size': '1.4em',
                'font-weight': 'bold',
                'color': '#CC4848 ',
                'text-align': 'center'
            })
        return err_msg, {}, None
示例#53
0
def get_mouser_part_html_tree(dist,
                              pn,
                              extra_search_terms='',
                              url=None,
                              descend=2,
                              local_part_html=None):
    '''Find the Mouser HTML page for a part number and return the URL and parse tree.'''

    # Use the part number to lookup the part using the site search function, unless a starting url was given.
    if url is None:
        url = 'http://www.mouser.com/Search/Refine.aspx?Keyword=' + urlquote(
            pn + ' ' + extra_search_terms, safe='')
    elif url[0] == '/':
        url = 'http://www.mouser.com' + url
    elif url.startswith('..'):
        url = 'http://www.mouser.com/Search/' + url

    # Open the URL, read the HTML from it, and parse it into a tree structure.
    req = FakeBrowser(url)
    req.add_header('Cookie', 'preferences=ps=www2&pl=en-US&pc_www2=USDe')
    for _ in range(HTML_RESPONSE_RETRIES):
        try:
            response = urlopen(req)
            html = response.read()
            break
        except WEB_SCRAPE_EXCEPTIONS:
            logger.log(
                DEBUG_DETAILED,
                'Exception while web-scraping {} from {}'.format(pn, dist))
            pass
    else:  # Couldn't get a good read from the website.
        logger.log(DEBUG_OBSESSIVE,
                   'No HTML page for {} from {}'.format(pn, dist))
        raise PartHtmlError

    # Abort if the part number isn't in the HTML somewhere.
    # (Only use the numbers and letters to compare PN to HTML.)
    if re.sub('[\W_]', '',
              str.lower(pn)) not in re.sub('[\W_]', '', str.lower(str(html))):
        logger.log(DEBUG_OBSESSIVE,
                   'No part number {} in HTML page from {}'.format(pn, dist))
        raise PartHtmlError

    try:
        tree = BeautifulSoup(html, 'lxml')
    except Exception:
        logger.log(DEBUG_OBSESSIVE,
                   'No HTML tree for {} from {}'.format(pn, dist))
        raise PartHtmlError

    # If the tree contains the tag for a product page, then just return it.
    if tree.find('div', id='product-details') is not None:
        return tree, url

    # If the tree is for a list of products, then examine the links to try to find the part number.
    if tree.find('table', class_='SearchResultsTable') is not None:
        logger.log(DEBUG_OBSESSIVE,
                   'Found product table for {} from {}'.format(pn, dist))
        if descend <= 0:
            logger.log(DEBUG_OBSESSIVE,
                       'Passed descent limit for {} from {}'.format(pn, dist))
            raise PartHtmlError
        else:
            # Look for the table of products.
            products = tree.find('table',
                                 class_='SearchResultsTable').find_all(
                                     'tr',
                                     class_=('SearchResultsRowOdd',
                                             'SearchResultsRowEven'))

            # Extract the product links for the part numbers from the table.
            product_links = [
                p.find('div', class_='mfrDiv').a for p in products
            ]

            # Extract all the part numbers from the text portion of the links.
            part_numbers = [l.text for l in product_links]

            # Look for the part number in the list that most closely matches the requested part number.
            match = difflib.get_close_matches(pn, part_numbers, 1, 0.0)[0]

            # Now look for the link that goes with the closest matching part number.
            for l in product_links:
                if l.text == match:
                    # Get the tree for the linked-to page and return that.
                    logger.log(
                        DEBUG_OBSESSIVE,
                        'Selecting {} from product table for {} from {}'.
                        format(l.text, pn, dist))
                    return get_mouser_part_html_tree(dist,
                                                     pn,
                                                     extra_search_terms,
                                                     url=l['href'],
                                                     descend=descend - 1)

    # I don't know what happened here, so give up.
    logger.log(DEBUG_OBSESSIVE,
               'Unknown error for {} from {}'.format(pn, dist))
    raise PartHtmlError
示例#54
0
def file_download_link(filename):
    """Create a Plotly Dash 'A' element that downloads a file from the app."""
    location = "/download/{}".format(urlquote(filename))
    return html.Div([html.H6(filename), html.A("download", href=location)])
示例#55
0
 def _build_raw_url(self, path, revision):
     url = self.raw_file_url
     url = url.replace("<revision>", revision)
     url = url.replace("<filename>", urlquote(path))
     return url
 def data_url(self, content):
     """Encodes the content of an ignition file using RFC 2397."""
     data = "," + urlquote(content)
     return "data:" + data
示例#57
0
if args.headercell is not None:
    print('Reading from headercell: ' + args.headercell.name)
    header = nf.read(args.headercell, nf.NO_CONVERT)
    plain['cells'].insert(0, *header['cells'])
    args.headercell.close()

for c in markdownlist:
    line = c['source'].split('\n')
    if 'Prerequisites' in line[0]:
        colour = 'green'
        symbol = 'star'
        title = line[0].lstrip('#')
        body = '\n'.join(line[1:])
        safetitle = title.replace(' ', '-')
        safetitle = safetitle.replace('`', '')
        index = urlquote(safetitle, safe='?!$\\') + '%0A'
    elif 'Overview' in line[0]:
        colour = 'green'
        symbol = 'file-o'
        title = line[0].lstrip('#')
        body = '\n'.join(line[1:])
        safetitle = title.replace(' ', '-')
        safetitle = safetitle.replace('`', '')
        index = urlquote(safetitle, safe='?!$\\') + '%0A'
    elif 'Info' in line[0]:
        colour = 'blue'
        symbol = 'info-circle'
        subtitle = line[0].split(':')
        title = ':'.join(subtitle[1:])
        body = '\n'.join(line[1:])
        safetitle = title.replace(' ', '-')
示例#58
0
def escape_url_path(url):
    urlparts = urlsplit(url)
    path = urlquote(urlunquote(urlparts.path), '/')
    query = urlquote(urlunquote(urlparts.query))
    fragment = urlquote(urlunquote(urlparts.fragment))
    return urlunsplit((urlparts.scheme, urlparts.netloc, path, query, fragment))
示例#59
0
    def __init__(
        self,
        *,
        port=None,
        user=None,
        password=None,
        host=None,
        database=None,
        pool_size=5,
        overflow=10,
        conn_str=None,
    ):
        if conn_str is None:
            if ((port is None) or (user is None) or (password is None)
                    or (host is None) or (database is None)):
                raise ValueError(
                    "If conn_str is not given then all of the arguments port, user, password, host, database must be provided."
                )
        else:
            if ((port is not None) or (user is not None)
                    or (password is not None) or (host is not None)
                    or (database is not None)):
                raise ValueError(
                    "If conn_str is given, none of the arguments port, user, password, host, database are allowed."
                )

        if conn_str is None:
            conn_str = "postgresql://{user}:{password}@{host}:{port}/{database}".format(
                user=user,
                password=urlquote(password),
                host=host,
                port=port,
                database=database,
            )
        else:
            pass

        app_name = "flowmachine"
        try:
            app_name = "-".join((app_name, os.getlogin()))
        except (FileNotFoundError, OSError):
            logger.info(
                "Couldn't get username for application name, using 'flowmachine'"
            )
        connect_args = {"application_name": app_name}
        self.engine = sqlalchemy.create_engine(
            conn_str,
            echo=False,
            strategy="threadlocal",
            pool_size=pool_size,
            max_overflow=overflow,
            pool_timeout=None,
            connect_args=connect_args,
        )

        self.inspector = sqlalchemy.inspect(self.engine)
        self.max_connections = pool_size + overflow
        if self.max_connections > os.cpu_count():
            warnings.warn(
                "Maximum number of connections (pool size + overflow = {}) is "
                .format(self.max_connections) +
                "greater than the available cpu cores ({}).".format(
                    os.cpu_count()))
        self.__check_flowdb_version()
示例#60
0
 def open_db(self, dbname, **kwargs):
     endpoint = '/'.join(['db', urlquote(dbname, safe='')])
     return self._call('POST', endpoint, **kwargs)