def get_repos(self, account, page_url=None, sess=None, refresh=True):
if not page_url:
page_url = self.api_repos_path.format(
user_id=urlquote(account.user_id),
user_name=urlquote(account.user_name or ''),
)
if not getattr(self.api_repos_path, 'use_session', True):
sess = None
r = self.api_get(account.domain, page_url, sess=sess)
repos, count, pages_urls = self.api_paginator(r, self.api_parser(r))
repos = [self.extract_repo_info(repo, account.domain) for repo in repos]
if '{user_name}' in self.api_repos_path and repos and repos[0].owner_id != account.user_id:
# https://hackerone.com/reports/452920
if not refresh:
raise TokenExpiredError()
from liberapay.models.account_elsewhere import UnableToRefreshAccount
try:
account = account.refresh_user_info()
except UnableToRefreshAccount:
raise TokenExpiredError()
# Note: we can't pass the page_url below, because it contains the old user_name
return self.get_repos(account, page_url=None, sess=sess, refresh=False)
if count == -1 and hasattr(self, 'x_repos_count'):
count = self.x_repos_count(None, account.extra_info, -1)
return repos, count, pages_urls
def sentiment_targeted(self, flavor, data, target, options={}):
"""
Calculates the targeted sentiment for text, a URL or HTML.
For an overview, please refer to: http://www.alchemyapi.com/products/features/sentiment-analysis/
For the docs, please refer to: http://www.alchemyapi.com/api/sentiment-analysis/
INPUT:
flavor -> which version of the call, i.e. text, url or html.
data -> the data to analyze, either the text, the url or html code.
target -> the word or phrase to run sentiment analysis on.
options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.
Available Options:
showSourceText -> 0: disabled, 1: enabled
OUTPUT:
The response, already converted from JSON to a Python object.
"""
#Make sure the target is valid
if target is None or target == '':
return { 'status':'ERROR', 'statusInfo':'targeted sentiment requires a non-null target' }
#Make sure this request supports this flavor
if flavor not in AlchemyAPI.ENDPOINTS['sentiment_targeted']:
return { 'status':'ERROR', 'statusInfo':'targeted sentiment analysis for ' + flavor + ' not available' }
#add the URL encoded data and target to the options and analyze
options[flavor] = urlquote(data)
options['target'] = urlquote(target)
return self.__analyze(AlchemyAPI.ENDPOINTS['sentiment_targeted'][flavor], options)
def _create_table(self, db, table, type, params=None):
params = {} if params is None else params
with self.post("/v3/table/create/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table)), urlquote(str(type))), params) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Create %s table failed" % (type), res, body)
return True
def _encode_netloc(components):
host = ''
if components.hostname:
host = encode_idna(components.hostname).decode('ascii')
if ':' in host:
host = '[%s]' % host
netloc = host
if components.port:
if not 0 <= int(components.port) <= 65535:
raise ValueError('Invalid port')
netloc = '%s:%s' % (netloc, components.port)
if components.username or components.password:
if components.username:
username = urlquote(
components.username, safe='/:%'
)
else:
username = ''
if components.password:
password = urlquote(
components.password, safe='/:%'
)
auth = '%s:%s' % (username, password)
else:
auth = username
netloc = '%s@%s' % (auth, netloc)
return netloc
def attach(self, name, data, callback, type='text/plain'):
def _really_callback(response):
if response.code != 201:
callback(_error_response(response))
return
data = json.loads(response.body.decode('utf-8'))
assert data['id'] == self.id
self.rev = data['rev']
self.attachments[name] = {
'content_type': type,
'length': len(data),
'stub': True,
}
callback(self)
headers = {'Content-Type': type, 'Expect': ''}
self.db._fetch(
'%s/%s?rev=%s' % (
urlquote(self.id, safe=''),
urlquote(name, safe=''),
self.rev),
_really_callback,
method='PUT',
body=data,
headers=headers,
)
def import_data(self, db, table, format, bytes_or_stream, size, unique_id=None):
"""Import data into Treasure Data Service
This method expects data from a file-like object formatted with "msgpack.gz".
Params:
db (str): name of a database
table (str): name of a table
format (str): format of data type (e.g. "msgpack.gz")
bytes_or_stream (str or file-like): a byte string or a file-like object contains the data
size (int): the length of the data
unique_id (str): a unique identifier of the data
Returns: float represents the elapsed time to import data
"""
if unique_id is not None:
path = "/v3/table/import_with_id/%s/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table)), urlquote(str(unique_id)), urlquote(str(format)))
else:
path = "/v3/table/import/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table)), urlquote(str(format)))
kwargs = {}
with self.put(path, bytes_or_stream, size, **kwargs) as res:
code, body = res.status, res.read()
if code / 100 != 2:
self.raise_error("Import failed", res, body)
js = self.checked_json(body, ["elapsed_time"])
time = float(js["elapsed_time"])
return time
def query(self, q, type="hive", db=None, result_url=None, priority=None, retry_limit=None, **kwargs):
"""
TODO: add docstring
=> jobId:str
"""
params = {"query": q}
params.update(kwargs)
if result_url is not None:
params["result"] = result_url
if priority is not None:
if not isinstance(priority, int):
priority_name = str(priority).upper()
if priority_name in self.JOB_PRIORITY:
priority = self.JOB_PRIORITY[priority_name]
else:
raise(ValueError("unknown job priority: %s" % (priority_name,)))
params["priority"] = priority
if retry_limit is not None:
params["retry_limit"] = retry_limit
with self.post("/v3/job/issue/%s/%s" % (urlquote(str(type)), urlquote(str(db))), params) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Query failed", res, body)
js = self.checked_json(body, ["job_id"])
return str(js["job_id"])
def get_current_url(
environ, root_only=False, strip_querystring=False,
host_only=False, trusted_hosts=None,
):
"""A handy helper function that recreates the full URL as IRI for the
current request or parts of it. Here an example:
>>> from verktyg.test import create_environ
>>> env = create_environ("/?param=foo", "http://localhost/script")
>>> get_current_url(env)
'http://localhost/script/?param=foo'
>>> get_current_url(env, root_only=True)
'http://localhost/script/'
>>> get_current_url(env, host_only=True)
'http://localhost/'
>>> get_current_url(env, strip_querystring=True)
'http://localhost/script/'
This optionally it verifies that the host is in a list of trusted hosts.
If the host is not in there it will raise a
:exc:`~verktyg.exceptions.SecurityError`.
Note that the string returned might contain unicode characters as the
representation is an IRI not an URI. If you need an ASCII only
representation you can use the :func:`~verktyg.urls.iri_to_uri`
function:
>>> from verktyg.urls import iri_to_uri
>>> iri_to_uri(get_current_url(env))
'http://localhost/script/?param=foo'
:param environ:
The WSGI environment to get the current URL from.
:param root_only:
Set `True` if you only want the root URL.
:param strip_querystring:
Set to `True` if you don't want the querystring.
:param host_only:
Set to `True` if the host URL should be returned.
:param trusted_hosts:
A list of trusted hosts, see :func:`host_is_trusted` for more
information.
"""
tmp = [environ['wsgi.url_scheme'], '://', get_host(environ, trusted_hosts)]
cat = tmp.append
if host_only:
return uri_to_iri(''.join(tmp) + '/')
cat(urlquote(wsgi_get_bytes(environ.get('SCRIPT_NAME', ''))).rstrip('/'))
cat('/')
if not root_only:
cat(urlquote(
wsgi_get_bytes(environ.get('PATH_INFO', '')).lstrip(b'/')
))
if not strip_querystring:
qs = get_query_string(environ)
if qs:
cat('?' + qs)
return uri_to_iri(''.join(tmp))
def update_expire(self, db, table, expire_days):
"""
TODO: add docstring
"""
with self.post("/v3/table/update/%s/%s" % (urlquote(str(db)), urlquote(str(table))), {"expire_days": expire_days}) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Update table expiration failed", res, body)
return True
def __init__(self, plugin_name, init_schema, db_file_name=None, upgrade_mod=None):
"""
Runs the initialisation process that includes creating the connection to the database and the tables if they do
not exist.
``plugin_name``
The name to setup paths and settings section names
``init_schema``
The init_schema function for this database
``upgrade_schema``
The upgrade_schema function for this database
``db_file_name``
The file name to use for this database. Defaults to None resulting in the plugin_name being used.
"""
settings = Settings()
settings.beginGroup(plugin_name)
self.db_url = ''
self.is_dirty = False
self.session = None
db_type = settings.value('db type')
if db_type == 'sqlite':
if db_file_name:
self.db_url = 'sqlite:///%s/%s' % (AppLocation.get_section_data_path(plugin_name), db_file_name)
else:
self.db_url = 'sqlite:///%s/%s.sqlite' % (AppLocation.get_section_data_path(plugin_name), plugin_name)
else:
self.db_url = '%s://%s:%s@%s/%s' % (db_type,
urlquote(settings.value('db username')),
urlquote(settings.value('db password')),
urlquote(settings.value('db hostname')),
urlquote(settings.value('db database')))
if db_type == 'mysql':
db_encoding = settings.value('db encoding')
self.db_url += '?charset=%s' % urlquote(db_encoding)
settings.endGroup()
if upgrade_mod:
db_ver, up_ver = upgrade_db(self.db_url, upgrade_mod)
if db_ver > up_ver:
critical_error_message_box(
translate('OpenLP.Manager', 'Database Error'),
translate('OpenLP.Manager', 'The database being loaded was created in a more recent version of '
'OpenLP. The database is version %d, while OpenLP expects version %d. The database will not '
'be loaded.\n\nDatabase: %s') % (db_ver, up_ver, self.db_url)
)
return
try:
self.session = init_schema(self.db_url)
except (SQLAlchemyError, DBAPIError):
log.exception('Error loading database: %s', self.db_url)
critical_error_message_box(
translate('OpenLP.Manager', 'Database Error'),
translate('OpenLP.Manager', 'OpenLP cannot load your database.\n\nDatabase: %s') % self.db_url
)
def bulk_import_upload_part(self, name, part_name, stream, size):
"""
TODO: add docstring
=> None
"""
self.validate_part_name(part_name)
with self.put("/v3/bulk_import/upload_part/%s/%s" % (urlquote(str(name)), urlquote(str(part_name))), stream, size) as res:
code, body = res.status, res.read()
if code / 100 != 2:
self.raise_error("Upload a part failed", res, body)
def update_schema(self, db, table, schema_json):
"""
TODO: add docstring
=> True
"""
with self.post("/v3/table/update-schema/%s/%s" % (urlquote(str(db)), urlquote(str(table))), {"schema": schema_json}) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Create schema table failed", res, body)
return True
def swap_table(self, db, table1, table2):
"""
TODO: add docstring
=> True
"""
with self.post("/v3/table/swap/%s/%s/%s" % (urlquote(str(db)), urlquote(str(table1)), urlquote(str(table2)))) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Swap tables failed", res, body)
return True
def addFile(self, newFile, path):
"""
This adds the given file to the owncloud server. newFile is a string path to a local file and that file name
will be used as its name.
"""
self.log.debug("Adding New File: %s/%s" % (path, os.path.basename(newFile)))
data = open(newFile, "rb").read()
if path not in self.DIRS:
self.mkdir(path)
self.http(
str("{}/{}/{}".format(self.url, urlquote(path), urlquote(os.path.basename(newFile)))), "PUT", data=data)
def bulk_import_upload_part(self, name, part_name, stream, size):
"""
TODO: add docstring
=> None
"""
if 0 < part_name.find("/"):
raise ValueError("part name must not contain '/': %s" % (repr(part_name,)))
with self.put("/v3/bulk_import/upload_part/%s/%s" % (urlquote(str(name)), urlquote(str(part_name))), stream, size) as res:
code, body = res.status, res.read()
if code / 100 != 2:
self.raise_error("Upload a part failed", res, body)
def bulk_import_delete_part(self, name, part_name, params=None):
"""
TODO: add docstring
=> True
"""
params = {} if params is None else params
with self.post("/v3/bulk_import/delete_part/%s/%s" % (urlquote(str(name)), urlquote(str(part_name))), params) as res:
code, body = res.status, res.read()
if code / 100 != 2:
self.raise_error("Delete a part failed", res, body)
return True
def create_bulk_import(self, name, db, table, params=None):
"""
TODO: add docstring
=> True
"""
params = {} if params is None else params
with self.post("/v3/bulk_import/create/%s/%s/%s" % (urlquote(str(name)), urlquote(str(db)), urlquote(str(table))), params) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Create bulk import failed", res, body)
return True
def _get_url(self, action):
"""Creates the URL to be used to perform the given server action."""
url = ('/api/0/' + action +
'/' + urlquote(self.config.get('client', 'hostname'), '') +
'/' + urlquote(self.config.get('client', 'username'), ''))
if self.crabid is not None:
url = url + '/' + urlquote(self.crabid, '')
return url
def get_friends_for(self, account, page_url=None, sess=None):
if not page_url:
page_url = self.api_friends_path.format(
user_id=urlquote(account.user_id),
user_name=urlquote(account.user_name or ''),
)
r = self.api_get(account.domain, page_url, sess=sess)
friends, count, pages_urls = self.api_paginator(r, self.api_parser(r))
friends = [self.extract_user_info(f, account.domain) for f in friends]
if count == -1 and hasattr(self, 'x_friends_count'):
count = self.x_friends_count(None, account.extra_info, -1)
return friends, count, pages_urls
def export_data(self, db, table, storage_type, params={}):
"""
TODO: add docstring
=> jobId:str
"""
params["storage_type"] = storage_type
with self.post("/v3/export/run/%s/%s" % (urlquote(str(db)), urlquote(str(table))), params) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Export failed", res, body)
js = self.checked_json(body, ["job_id"])
return str(js["job_id"])
def delete_table(self, db, table):
"""
TODO: add docstring
=> type:str
"""
with self.post("/v3/table/delete/%s/%s" % (urlquote(str(db)), urlquote(str(table)))) as res:
code, body = res.status, res.read()
if code != 200:
self.raise_error("Delete table failed", res, body)
js = self.checked_json(body, [])
t = js.get("type", "?")
return t
def get_team_members(self, account, page_url=None):
"""Given an AccountElsewhere, return its membership list from the API.
"""
if not page_url:
page_url = self.api_team_members_path.format(
user_id=urlquote(account.user_id),
user_name=urlquote(account.user_name or ''),
)
domain = account.domain
r = self.api_get(domain, page_url)
members, count, pages_urls = self.api_paginator(r, self.api_parser(r))
members = [self.extract_user_info(m, domain) for m in members]
return members, count, pages_urls
def get_attachment(self, doc_id, attachment_name, callback):
doc_id = urlquote(doc_id, safe='')
attachment_name = urlquote(attachment_name, safe='')
response = yield gen.Task(self._fetch, '%s/%s' % (doc_id, attachment_name))
if response.code == 200:
callback(response.body)
#elif response.code == 404:
# Document or attachment doesn't exist
#callback(None)
else:
raise _exception(response)
def get_starred_repos(self, account, sess, page_url=None):
if not page_url:
page_url = self.api_starred_path.format(
user_id=urlquote(account.user_id),
user_name=urlquote(account.user_name or ''),
)
assert page_url[:1] == '/'
if not getattr(self.api_starred_path, 'use_session', True):
sess = None
r = self.api_get(account.domain, page_url, sess=sess)
repos, count, pages_urls = self.api_paginator(r, self.api_parser(r))
repos = [self.extract_repo_info(repo, account.domain) for repo in repos]
return repos, count, pages_urls
def base_string_request_params(uri, oauth_params = None, data = None):
from urllib.parse import urlsplit, parse_qs, urlencode
from collections import Mapping
query = dict((k, v[0]) for k, v in parse_qs(urlsplit(uri).query).items())
if oauth_params:
query.update(oauth_params)
if data and isinstance(data, Mapping):
query.update(data)
if "oauth_signature" in query:
del query["oauth_signature"]
#Parameter normalization
params = [(urlquote(str(k)), urlquote(str(v))) for k, v in query.items()]
params.sort()
return '&'.join(['{0}={1}'.format(k, v) for k, v in params])
def iri_to_uri(iri: str) -> str:
"Takes a unicode string that can contain an IRI and emits a unicode URI."
scheme, authority, path, query, frag = urlsplit(iri)
scheme = scheme
if ":" in authority:
host, port = authority.split(":", 1)
authority = host.encode('idna').decode('ascii') + ":%s" % port
else:
authority = authority.encode('idna').decode('ascii')
sub_delims = "!$&'()*+,;="
pchar = "-.+~" + sub_delims + ":@" + "%"
path = urlquote(path, safe=pchar+"/")
quer = urlquote(query, safe=pchar+"/?")
frag = urlquote(frag, safe=pchar+"/?")
return urlunsplit((scheme, authority, path, quer, frag))
def rule34(phenny, input):
""".rule34 <query> - Rule 34: If it exists there is p**n of it."""
if input.nick in phenny.config.user_ignore:
return
if check_nsfw(phenny, input.sender, None, input.nick):
return
q = input.group(2)
if not q:
phenny.say(rule34.__doc__.strip())
return
try:
req = web.get(
"https://rule34.xxx/index.php?page=dapi&s=post&q=index&tags={0}".format(urlquote(q))
) # Lets use XML!
except (HTTPError, IOError):
raise GrumbleError("THE INTERNET IS F*****G BROKEN. Please try again later.")
results = ET.fromstring(req)
if len(results) <= 0:
phenny.reply("Huh. rule34.xxx is missing {0}".format(q))
return
try:
link = "https:" + (choice(results).attrib["file_url"])
except AttributeError:
raise GrumbleError("THE INTERNET IS BROKEN. Please try again later.")
response = "!!NSFW!! -> {0} <- !!NSFW!!".format(link)
phenny.reply(response)
def relations(self, flavor, data, options={}):
"""
Extracts the relations for text, a URL or HTML.
For an overview, please refer to: http://www.alchemyapi.com/products/features/relation-extraction/
For the docs, please refer to: http://www.alchemyapi.com/api/relation-extraction/
INPUT:
flavor -> which version of the call, i.e. text, url or html.
data -> the data to analyze, either the text, the url or html code.
options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.
Available Options:
sentiment -> 0: disabled (default), 1: enabled. Requires one additional API transaction if enabled.
keywords -> extract keywords from the subject and object. 0: disabled (default), 1: enabled. Requires one additional API transaction if enabled.
entities -> extract entities from the subject and object. 0: disabled (default), 1: enabled. Requires one additional API transaction if enabled.
requireEntities -> only extract relations that have entities. 0: disabled (default), 1: enabled.
sentimentExcludeEntities -> exclude full entity name in sentiment analysis. 0: disabled, 1: enabled (default)
disambiguate -> disambiguate entities (i.e. Apple the company vs. apple the fruit). 0: disabled, 1: enabled (default)
linkedData -> include linked data with disambiguated entities. 0: disabled, 1: enabled (default).
coreference -> resolve entity coreferences. 0: disabled, 1: enabled (default)
showSourceText -> 0: disabled (default), 1: enabled.
maxRetrieve -> the maximum number of relations to extract (default: 50, max: 100)
OUTPUT:
The response, already converted from JSON to a Python object.
"""
#Make sure this request supports this flavor
if flavor not in AlchemyAPI.ENDPOINTS['relations']:
return { 'status':'ERROR', 'statusInfo':'relation extraction for ' + flavor + ' not available' }
#add the URL encoded data to the options and analyze
options[flavor] = urlquote(data)
return self.__analyze(AlchemyAPI.ENDPOINTS['relations'][flavor], options)
def feeds(self, flavor, data, options={}):
"""
Detects the RSS/ATOM feeds for a URL or HTML.
For an overview, please refer to: http://www.alchemyapi.com/products/features/feed-detection/
For the docs, please refer to: http://www.alchemyapi.com/api/feed-detection/
INPUT:
flavor -> which version of the call, i.e. url or html.
data -> the data to analyze, either the the url or html code.
options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.
Available Options:
none
OUTPUT:
The response, already converted from JSON to a Python object.
"""
#Make sure this request supports this flavor
if flavor not in AlchemyAPI.ENDPOINTS['feeds']:
return { 'status':'ERROR', 'statusInfo':'feed detection for ' + flavor + ' not available' }
#add the URL encoded data to the options and analyze
options[flavor] = urlquote(data)
return self.__analyze(AlchemyAPI.ENDPOINTS['feeds'][flavor], options)
def text_title(self, flavor, data, options={}):
"""
Extracts the title for a URL or HTML.
For an overview, please refer to: http://www.alchemyapi.com/products/features/text-extraction/
For the docs, please refer to: http://www.alchemyapi.com/api/text-extraction/
INPUT:
flavor -> which version of the call, i.e. text, url or html.
data -> the data to analyze, either the text, the url or html code.
options -> various parameters that can be used to adjust how the API works, see below for more info on the available options.
Available Options:
useMetadata -> utilize title info embedded in meta data, 0: disabled, 1: enabled (default)
OUTPUT:
The response, already converted from JSON to a Python object.
"""
#Make sure this request supports this flavor
if flavor not in AlchemyAPI.ENDPOINTS['text_title']:
return { 'status':'ERROR', 'statusInfo':'title extraction for ' + flavor + ' not available' }
#add the URL encoded data to the options and analyze
options[flavor] = urlquote(data)
return self.__analyze(AlchemyAPI.ENDPOINTS['text_title'][flavor], options)
ELASTICSEARCH_CONNECTIONS = {
'default': {
'hosts': [
{
'host': elasticsearch_url.hostname,
'port': elasticsearch_url.port,
},
],
'timeout': 60,
'max_retries': 1,
}
}
if elasticsearch_url.username is not None and elasticsearch_url.password is not None:
ELASTICSEARCH_CONNECTIONS['default']['http_auth'] = (
elasticsearch_url.username + ':' + urlquote(elasticsearch_url.password))
if elasticsearch_url.scheme == 'https':
ELASTICSEARCH_CONNECTIONS['default']['hosts'][0]['use_ssl'] = True
try:
from local_essarch_settings import ELASTICSEARCH_TEST_URL
except ImportError:
ELASTICSEARCH_TEST_URL = os.environ.get('ELASTICSEARCH_TEST_URL', 'http://localhost:19200')
elasticsearch_test_url = urlparse(ELASTICSEARCH_TEST_URL)
ELASTICSEARCH_TEST_CONNECTIONS = {
'default': {
'hosts': [
{
'host': elasticsearch_test_url.hostname,
'port': elasticsearch_test_url.port,
def _get_storage_entities(task_factory, storage_settings):
# type: (dict, settings.TaskFactoryStorageSettings) -> TaskSettings
"""Generate a task given a config
:param dict task_factory: task factory object
:param settings.TaskFactoryStorageSettings storage_settings:
storage settings
:rtype: FileInfo
:return: file info
"""
if not storage_settings.is_file_share:
# create blob client
blob_client = azureblob.BlockBlobService(
account_name=storage_settings.storage_settings.account,
account_key=storage_settings.storage_settings.account_key,
endpoint_suffix=storage_settings.storage_settings.endpoint)
# list blobs in container with include/exclude
blobs = blob_client.list_blobs(
container_name=storage_settings.container)
for blob in blobs:
if not _inclusion_check(blob.name, storage_settings.include,
storage_settings.exclude):
continue
file_path_with_container = '{}/{}'.format(
storage_settings.container, blob.name)
file_name = blob.name.split('/')[-1]
file_name_no_extension = file_name.split('.')[0]
if task_factory['file']['task_filepath'] == 'file_path':
task_filepath = blob.name
elif (task_factory['file']['task_filepath'] ==
'file_path_with_container'):
task_filepath = file_path_with_container
elif task_factory['file']['task_filepath'] == 'file_name':
task_filepath = file_name
elif (task_factory['file']['task_filepath'] ==
'file_name_no_extension'):
task_filepath = file_name_no_extension
else:
raise ValueError(
'invalid task_filepath specification: {}'.format(
task_factory['file']['task_filepath']))
# create blob url
url = 'https://{}.blob.{}/{}/{}'.format(
storage_settings.storage_settings.account,
storage_settings.storage_settings.endpoint,
storage_settings.container, urlquote(blob.name))
# create blob sas
sas = blob_client.generate_blob_shared_access_signature(
storage_settings.container,
blob.name,
permission=azureblob.BlobPermissions.READ,
expiry=datetime.datetime.utcnow() +
datetime.timedelta(days=_DEFAULT_SAS_EXPIRY_DAYS))
yield FileInfo(
is_blob=True,
url=url,
sas=sas,
file_path=blob.name,
file_path_with_container=file_path_with_container,
file_name=file_name,
file_name_no_extension=file_name_no_extension,
task_filepath=task_filepath,
)
else:
# create file share client
file_client = azurefile.FileService(
account_name=storage_settings.storage_settings.account,
account_key=storage_settings.storage_settings.account_key,
endpoint_suffix=storage_settings.storage_settings.endpoint)
# list files in share with include/exclude
for file in _list_all_files_in_fileshare(file_client,
storage_settings.container):
if not _inclusion_check(file, storage_settings.include,
storage_settings.exclude):
continue
file_path_with_container = '{}/{}'.format(
storage_settings.container, file)
file_name = file.split('/')[-1]
file_name_no_extension = file_name.split('.')[0]
if task_factory['file']['task_filepath'] == 'file_path':
task_filepath = file
elif (task_factory['file']['task_filepath'] ==
'file_path_with_container'):
task_filepath = file_path_with_container
elif task_factory['file']['task_filepath'] == 'file_name':
task_filepath = file_name
elif (task_factory['file']['task_filepath'] ==
'file_name_no_extension'):
task_filepath = file_name_no_extension
else:
raise ValueError(
'invalid task_filepath specification: {}'.format(
task_factory['file']['task_filepath']))
yield FileInfo(
is_blob=False,
url=None,
sas=None,
file_path=file,
file_path_with_container=file_path_with_container,
file_name=file_name,
file_name_no_extension=file_name_no_extension,
task_filepath=task_filepath,
)
def image_request_response(self, path):
"""Parse image request and create response."""
# Parse the request in path
if (len(path) > 1024):
raise IIIFError(code=414,
text="URI Too Long: Max 1024 chars, got %d\n" %
len(path))
# print "GET " + path
try:
self.iiif.identifier = self.identifier
self.iiif.parse_url(path)
except IIIFRequestPathError as e:
# Reraise as IIIFError with code=404 because we can't tell
# whether there was an encoded slash in the identifier or
# whether there was a bad number of path segments.
raise IIIFError(code=404, text=e.text)
except IIIFRequestBaseURI as e:
info_uri = self.server_and_prefix + '/' + \
urlquote(self.iiif.identifier) + '/info.json'
raise IIIFError(code=303, headers={'Location': info_uri})
except IIIFError as e:
# Pass through
raise e
except Exception as e:
# Something completely unexpected => 500
raise IIIFError(
code=500,
text=
"Internal Server Error: unexpected exception parsing request ("
+ str(e) + ")")
dr = degraded_request(self.identifier)
if (dr):
self.logger.info("image_request: degraded %s -> %s" %
(self.identifier, dr))
self.degraded = self.identifier
self.identifier = dr
self.iiif.quality = 'gray'
else:
# Parsed request OK, attempt to fulfill
self.logger.info("image_request: %s" % (self.identifier))
file = self.file
self.manipulator.srcfile = file
self.manipulator.do_first()
if (self.api_version < '2.0' and self.iiif.format is None
and 'Accept' in request.headers):
# In 1.0 and 1.1 conneg was specified as an alternative to format, see:
# http://iiif.io/api/image/1.0/#format
# http://iiif.io/api/image/1.1/#parameters-format
formats = {
'image/jpeg': 'jpg',
'image/tiff': 'tif',
'image/png': 'png',
'image/gif': 'gif',
'image/jp2': 'jps',
'application/pdf': 'pdf'
}
accept = do_conneg(request.headers['Accept'], list(formats.keys()))
# Ignore Accept header if not recognized, should this be an error
# instead?
if (accept in formats):
self.iiif.format = formats[accept]
(outfile, mime_type) = self.manipulator.derive(file, self.iiif)
# FIXME - find efficient way to serve file with headers
self.add_compliance_header()
return send_file(outfile, mimetype=mime_type)
def main(argv):
parser = argparse.ArgumentParser()
sub_parser = parser.add_subparsers(title='actions')
# common options
parser.add_argument('-c', '--config', default='api.conf', help='Specify which contains api key (format json)')
parser.add_argument('-K', '--insecure', action='store_true', default=False, help='Do not check hostname when using https')
parser.add_argument('-k', '--api-key', default=None, help='Specify the api key')
parser.add_argument('--url', default=None, help='specify url')
parser.add_argument('-l', '--admin', default=None, help='Specify the login of admin user')
parser.add_argument('--admin_password', default=None, help='Specify the admin\'s password')
parser.add_argument('-P', '--pwprompt', action='store_true', default=False, help='Prompt for password')
# sub parser for create_config command
parser_cf = sub_parser.add_parser('create_config', aliases=['cf'], help='Create a config file')
parser_cf.set_defaults(action='create_config')
# sub parser for create_user command
parser_cu = sub_parser.add_parser('create_user', aliases=['cu'], help='Create a user')
parser_cu.add_argument('--homedirectory', default='/var/www/nextcloud/data/<login>/files/', help='specify a homedirectory ("<login>" will be replaced by login)')
parser_cu.add_argument('--pool_template', help='specify a pool template')
parser_cu.add_argument('--vtl_id', help='specify vtl id')
parser_cu.add_argument('login', help='specify login of new user')
parser_cu.add_argument('password', nargs='?', default=None, help='specify password of new user')
parser_cu.set_defaults(action='create_user')
# sub parser for enable_user command
parser_eu = sub_parser.add_parser('enable_user', aliases=['eu'], help='enable a user')
parser_eu.add_argument('key', help='specify key')
parser_eu.set_defaults(action='enable_user')
# sub parser for disable_user command
parser_du = sub_parser.add_parser('disable_user', aliases=['du'], help='disable a user')
parser_du.add_argument('--key', help='specify key')
parser_du.add_argument('login', help='specify login of user')
parser_du.set_defaults(action='disable_user')
# sub parser for list_pool command
parser_lp = sub_parser.add_parser('list_pool', aliases=['lp'], help='List pools')
parser_lp.set_defaults(action='list_pool')
# sub parser for list_user command
parser_lu = sub_parser.add_parser('list_user', aliases=['lu'], help='List users')
parser_lu.set_defaults(action='list_user')
args = parser.parse_args(argv)
# check default options
base_url = None
if args.url is not None:
base_url = args.url
api_key = None
if args.api_key is not None:
api_key = args.api_key
admin_user = None
if args.admin is not None:
admin_user = args.admin
admin_password = None
if args.admin_password is not None:
admin_password = args.admin_password
# open config file if needed
if None in [base_url, api_key, admin_user, admin_password]:
try:
fd = open(args.config)
config = json.load(fd)
fd.close()
if 'api key' in config:
api_key = config['api key']
if 'base url' in config:
base_url = config['base url']
if 'login' in config:
admin_user = config['login']
if 'password' in config:
admin_password = config['password']
except OSError as err:
print('Error occured while reading file "%s" because %s' % (args.config, err), file=sys.stderr)
sys.exit(1)
if admin_password is None and args.pwprompt:
try:
admin_password = getpass('Admin\'s password: '******'Error, you should specify an api key', file=sys.stderr)
if base_url is None:
print('Error, you should specify an url', file=sys.stderr)
if admin_user is None:
print('Error, you should specify a login for admin user', file=sys.stderr)
if admin_password is None:
print('Error, you should specify a password for admin userl', file=sys.stderr)
if None in [base_url, api_key, admin_user, admin_password]:
sys.exit(1)
def authentication(info):
print('Login to "%s"... ' % base_url, end='', flush=True)
connection = newHttpConnection(info)
headers = {"Content-type": "application/json"}
params = json.dumps({'login': admin_user, 'password': admin_password, 'apikey': api_key})
connection.request('POST', base_url + '/auth/', params, headers)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
if response.status == 201:
print('connected')
return (connection, {'Cookie': response.getheader('Set-Cookie').split(';')[0]})
else:
print('connection failure because %s' % message)
connection.close()
sys.exit(2)
def newHttpConnection(info):
try:
if info.scheme == 'http':
return http.client.HTTPConnection(info.hostname, info.port)
elif info.scheme == 'https' and args.insecure:
ssl_context = ssl.SSLContext() # older python (< v3.5.3) need ssl.PROTOCOL_TLSv1 as parameter
ssl_context.verify_mode = False
ssl_context.check_hostname = False
return http.client.HTTPSConnection(info.hostname, info.port, context=ssl_context)
elif info.scheme == 'https':
return http.client.HTTPSConnection(info.hostname, info.port)
except Exception as err:
print('Error occured while creating http(s) connection because %s' % (err), file=sys.stderr)
sys.exit(1)
pp = pprint.PrettyPrinter(indent=4)
if args.action == 'create_config':
try:
fd = open(args.config, 'w')
json.dump({'api key': api_key, 'base url': base_url, 'login': admin_user, 'password': admin_password}, fd, sort_keys=True)
fd.close()
except OSError as err:
print('Error occured while writing info file "%s" because %s' % (args.config, err), file=sys.stderr)
sys.exit(1)
elif args.action == 'create_user':
user_password = args.password
if user_password is None and args.pwprompt:
try:
user_password = getpass('User\'s password: '******'Error, you should specify a password for user or use option --pwprompt', file=sys.stderr)
sys.exit(1)
url_info = urlparse(base_url)
(connection, cookie) = authentication(url_info)
# search user
print('Search user "%s"... ' % args.login, end='', flush=True)
connection.request('GET', '%s/user/search/?login=%s' % (base_url, args.login), headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
if response.status == 404:
print('not found')
else:
print('found, user id: %d' % message['users'][0])
# search archive format
print('Search archive format "Storiq One (TAR)"... ', end='', flush=True)
connection.request('GET', '%s/archiveformat/?name=%s' % (base_url, urlquote('Storiq One (TAR)')), headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
archive_format_id = None
if response.status == 200:
archive_format_id = message['archiveformat id']
print('found, id: %d' % archive_format_id)
else:
print('not found')
sys.exit(1)
vtl_id = args.vtl_id
if vtl_id is None:
# search vtl
print('Search vtl "%s"... ' % args.login, end='', flush=True)
connection.request('GET', '%s/vtl/' % (base_url), headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
if message['total_rows'] == 1:
vtl_id = int(message['vtls'][0]['vtlid'])
print('found, select vtl (id: %d)' % vtl_id)
elif message['total_rows'] > 1:
print('found, there is more than one vtl, you should specify vtl_id')
connection.close()
sys.exit(2)
else:
print('no vtl found')
connection.close()
sys.exit(2)
# get vtl information
print('Get vtl information (id: %d)... ' % vtl_id, end='', flush=True)
connection.request('GET', '%s/vtl/?id=%d' % (base_url, vtl_id), headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
mediaformat_id = None
if response.status == 200:
mediaformat_id = message['vtl']['mediaformat']
print('ok, will use mediaformat(%d)' % mediaformat_id)
else:
print('failed, no vtl found with id = %d' % vtl_id)
connection.close()
sys.exit(2)
# find blank media
print('Find blank media (mediaformat: %d)... ' % mediaformat_id, end='', flush=True)
connection.request('GET', '%s/media/search/?status=new&mediaformat=%d&order_by=id' % (base_url, mediaformat_id), headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
media_id = None
if response.status == 200:
if message['total_rows'] > 0:
media_id = message['medias'][0]
print('found %d media(s), will use media (id: %d)' % (message['total_rows'], media_id))
else:
print('no medias found')
connection.close()
sys.exit(2)
else:
print('error while finding blank media because %s' % message)
connection.close()
sys.exit(2)
# creating pool
pool = {
'name': 'pool_' + args.login,
'archiveformat': archive_format_id,
'mediaformat': mediaformat_id
}
if args.pool_template is not None:
pool['pooltemplate'] = args.pool_template
pool_header = { 'Content-type': 'application/json' }
pool_header.update(cookie)
print('Create new pool (name: %s)... ' % pool['name'], end='', flush=True)
connection.request('POST', '%s/pool/' % (base_url), json.dumps(pool), headers=pool_header)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
pool_id = None
if response.status == 201:
pool_id = message['pool_id']
print('created, new id: %d' % pool_id)
else:
print('failed because (%d, %s)' % (response.status, message['message']))
connection.close()
sys.exit(2)
# creating poolgroup
poolgroup = {
'name': 'poolgroup_' + args.login,
'pools': [pool_id]
}
print('Create new pool group (name: %s)... ' % poolgroup['name'], end='', flush=True)
connection.request('POST', '%s/poolgroup/' % (base_url), json.dumps(poolgroup), headers=pool_header)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
poolgroup_id = None
if response.status == 201:
poolgroup_id = message['poolgroup_id']
print('created, new id: %d' % poolgroup_id)
else:
print('failed because (%d, %s)' % (response.status, message['message']))
connection.close()
sys.exit(2)
# creating user
homedirectory = args.homedirectory
if homedirectory.find('<login>') > -1:
homedirectory = homedirectory.replace('<login>', args.login)
user = {
'login': args.login,
'password': user_password,
'fullname': args.login,
'email': args.login,
'homedirectory': homedirectory,
'isadmin': False,
'canarchive': True,
'canrestore': True,
'poolgroup': poolgroup_id,
'disabled': False
}
print('Create new user (name: %s)... ' % args.login, end='', flush=True)
connection.request('POST', '%s/user/' % (base_url), json.dumps(user), headers=pool_header)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
user_id = None
if response.status == 201:
user_id = message['user_id']
print('created, new id: %d' % user_id)
else:
print('failed because (%d, %s)' % (response.status, message['message']))
connection.close()
sys.exit(2)
# format media
task_info = {
'media': media_id,
'pool': pool_id
}
print('Create formatting task (media: %d, pool: %d)... ' % (media_id, pool_id), end='', flush=True)
connection.request('POST', '%s/media/format/' % (base_url), json.dumps(task_info), headers=pool_header)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
task_id = None
if response.status == 201:
task_id = message['job_id']
print('created, new task: %d' % task_id)
connection.close()
sys.exit(0)
else:
print('failed because (%d, %s)' % (response.status, message['message']))
connection.close()
sys.exit(2)
elif args.action == 'list_pool':
url_info = urlparse(base_url)
(connection, cookie) = authentication(url_info)
print('Getting pool list... ', end='', flush=True)
connection.request('GET', base_url + '/pool/', headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
print(message['total_rows'])
pools = message['pools']
for pool_id in pools:
connection.request('GET', '%s/pool/?id=%d' % (base_url, pool_id), headers=cookie)
response = connection.getresponse()
sub_message = json.loads(response.read().decode("utf-8"))
pool = sub_message['pool']
pp.pprint(pool)
connection.close()
elif args.action == 'list_user':
url_info = urlparse(base_url)
(connection, cookie) = authentication(url_info)
print('Getting user list... ', end='', flush=True)
connection.request('GET', base_url + '/user/', headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
print(message['total_rows'])
users = message['users']
for user_id in users:
connection.request('GET', '%s/user/?id=%d' % (base_url, user_id), headers=cookie)
response = connection.getresponse()
sub_message = json.loads(response.read().decode("utf-8"))
user = sub_message['user']
pp.pprint(user)
connection.close()
elif args.action == 'enable_user':
key = args.key
print('enable user : %s' % key)
url_info = urlparse(base_url)
(connection, cookie) = authentication(url_info)
print('Activating user... ', end='', flush=True)
connection.request('GET', base_url + '/user/update/?action=activate&key=' + key, headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
if response.status == 200:
print('activated')
connection.close()
sys.exit(0)
else:
print('not activated')
connection.close()
sys.exit(1)
elif args.action == 'disable_user':
print('disable user : %s' % args.login)
url_info = urlparse(base_url)
(connection, cookie) = authentication(url_info)
print('Deactivating user... ', end='', flush=True)
connection.request('GET', base_url + '/user/update/?action=deactivate&login='******'deactivated')
else:
print('not deactivated')
if args.key is not None:
url_info = urlparse(base_url)
(connection, cookie) = authentication(url_info)
print('Adding key... ', end='', flush=True)
connection.request('GET', base_url + '/user/update/?action=key&login='******'&key=' + args.key, headers=cookie)
response = connection.getresponse()
message = json.loads(response.read().decode("utf-8"))
if response.status == 200:
print('added')
else:
print('not added')
connection.close()
else:
print('Error, you should specify one action from ("create_config", "create_user")', file=sys.stderr)
sys.exit(1)
MYSQL_DATABASE = environ.get("MYSQL_DATABASE", MYSQL_DATABASE)
MYSQL_USER = environ.get("MYSQL_USER", MYSQL_USER)
MYSQL_PASSWORD = environ.get("MYSQL_PASSWORD", MYSQL_PASSWORD)
DHCP_DDNS_IP = environ.get("DHCP_DDNS_IP", DHCP_DDNS_IP)
DHCP_DDNS_PORT = int(environ.get("DHCP_DDNS_PORT", DHCP_DDNS_PORT))
CUSTOMER = environ.get("CUSTOMER", CUSTOMER)
APPS = environ.get("APPS", APPS)
if type(APPS) == str:
APPS = APPS.split(",")
MYSQL_URL = "mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8mb4".format(
MYSQL_USER,
urlquote(MYSQL_PASSWORD),
MYSQL_HOST,
MYSQL_PORT,
MYSQL_DATABASE,
)
MYSQL_URL = environ.get("MYSQL_URL", MYSQL_URL)
SCOPES = None
AUDIT_MODULES = None
def set_scopes(scopes):
global SCOPES
SCOPES = scopes
def load(self, bundle_id, bundle_version=None):
'''
Loads a bundle by downloading an index file, looking up the bundle location, and
then downloading the bundle
'''
import requests
self._setup_index()
binfo = self._index.get(bundle_id)
if not binfo:
raise LoadFailed(bundle_id, self, 'Bundle is not in the index')
if not isinstance(binfo, dict):
raise LoadFailed(bundle_id, self,
'Unexpected type of bundle info in the index')
if bundle_version is None:
max_vn = 0
for k in binfo.keys():
try:
val = int(k)
except ValueError:
L.warning(
"Got unexpected non-version-number key '%s' in bundle index info",
k)
else:
if max_vn < val:
max_vn = val
if not max_vn:
raise LoadFailed(bundle_id, self, 'No releases found')
bundle_version = max_vn
versioned_binfo = binfo.get(str(bundle_version))
if not versioned_binfo or not isinstance(versioned_binfo, dict):
raise LoadFailed(bundle_id, self,
f'No bundle info for version {bundle_version}')
bundle_url = versioned_binfo.get('url')
if not self._bundle_url_is_ok(bundle_url):
raise LoadFailed(
bundle_id, self, 'Did not find a valid URL for "%s" at'
' version %s' % (bundle_id, bundle_version))
hashes = versioned_binfo.get('hashes')
if not isinstance(hashes, dict) or not hashes:
raise LoadFailed(bundle_id, self,
f'No hash info for version {bundle_version}')
for hash_name in self.hash_preference:
bundle_hash = hashes.get(hash_name)
if bundle_hash:
break
else: # no break
for hash_name, bundle_hash in hashes.items():
if hash_name in hashlib.algorithms_available:
break
else: # no break
raise LoadFailed(
bundle_id, self,
f'No supported hash for version {bundle_version}')
try:
hsh = hashlib.new(hash_name)
except ValueError:
L.warning(
'Hash in hashlib.algorithms_available unsupported in hashlib.new'
)
raise LoadFailed(
bundle_id, self,
f'Unsupported hash {hash_name} for version {bundle_version}')
response = requests.get(bundle_url, stream=True)
if self.cachedir is not None:
bfn = urlquote(bundle_id)
with open(p(self.cachedir, bfn), 'wb') as f:
for chunk in response.iter_content(chunk_size=1024):
hsh.update(chunk)
f.write(chunk)
if bundle_hash != hsh.hexdigest():
raise LoadFailed(
bundle_id, self,
f'Failed to verify {hash_name} hash for version {bundle_version}'
)
with open(p(self.cachedir, bfn), 'rb') as f:
Unarchiver().unpack(f, self.base_directory)
else:
bio = io.BytesIO()
bundle_bytes = response.raw.read()
hsh.update(bundle_bytes)
if bundle_hash != hsh.hexdigest():
raise LoadFailed(
bundle_id, self,
f'Failed to verify {hash_name} hash for version {bundle_version}'
)
bio.write(bundle_bytes)
bio.seek(0)
Unarchiver().unpack(bio, self.base_directory)
def tasteometer(phenny, input):
input1 = input.group(2)
if not input1 or len(input1) == 0:
phenny.say("tasteometer: compares two users' musical taste")
phenny.say("syntax: .taste user1 user2")
return
input2 = input.group(3)
user1 = resolve_username(input1)
if not user1:
user1 = input1
user2 = resolve_username(input2)
if not user2:
user2 = input2
if not user2 or len(user2) == 0:
user2 = resolve_username(input.nick)
if not user2:
user2 = input.nick
try:
req = urlopen(
"%smethod=tasteometer.compare&type1=user&type2=user&value1=%s&value2=%s"
% (APIURL, urlquote(user1), urlquote(user2)))
except (HTTPError, http.client.BadStatusLine) as e:
if e.code == 400:
phenny.say(
"uhoh, someone doesn't exist on last.fm, perhaps they need to set user"
)
return
else:
phenny.say("uhoh. try again later, mmkay?")
return
doc = etree.parse(req)
root = doc.getroot()
score = root.xpath('comparison/result/score')
if len(score) == 0:
phenny.say("something isn't right. have those users scrobbled?")
return
score = float(score[0].text)
rating = ""
if score >= 0.9:
rating = "Super"
elif score >= 0.7:
rating = "Very High"
elif score >= 0.5:
rating = "High"
elif score >= 0.3:
rating = "Medium"
elif score >= 0.1:
rating = "Low"
else:
rating = "Very Low"
artists = root.xpath("comparison/result/artists/artist/name")
common_artists = ""
names = []
if len(artists) == 0:
common_artists = ". they don't have any artists in common."
else:
list(map(lambda a: names.append(a.text), artists))
common_artists = "and music they have in common includes: %s" % ", ".join(
names)
phenny.say("%s's and %s's musical compatibility rating is %s %s" %
(user1, user2, rating, common_artists))
def _trackerBranch(self, branch):
return "refs/buildbot/{}/{}".format(urlquote(self.repourl, ''),
self._removeHeads(branch))
def gcodeFileCommand(filename, target):
if target not in [FileDestinations.LOCAL, FileDestinations.SDCARD]:
abort(404)
if not _validate(target, filename):
abort(404)
# valid file commands, dict mapping command name to mandatory parameters
valid_commands = {
"select": [],
"unselect": [],
"slice": [],
"analyse": [],
"copy": ["destination"],
"move": ["destination"],
}
command, data, response = get_json_command_from_request(
request, valid_commands)
if response is not None:
return response
user = current_user.get_name()
if command == "select":
with Permissions.FILES_SELECT.require(403):
if not _verifyFileExists(target, filename):
abort(404)
# selects/loads a file
if not octoprint.filemanager.valid_file_type(filename,
type="machinecode"):
abort(
415,
description=
"Cannot select file for printing, not a machinecode file",
)
if not printer.is_ready():
abort(
409,
description=
"Printer is already printing, cannot select a new file",
)
printAfterLoading = False
if "print" in data and data["print"] in valid_boolean_trues:
with Permissions.PRINT.require(403):
if not printer.is_operational():
abort(
409,
description=
"Printer is not operational, cannot directly start printing",
)
printAfterLoading = True
sd = False
if target == FileDestinations.SDCARD:
filenameToSelect = filename
sd = True
else:
filenameToSelect = fileManager.path_on_disk(target, filename)
printer.select_file(filenameToSelect, sd, printAfterLoading, user)
elif command == "unselect":
with Permissions.FILES_SELECT.require(403):
if not printer.is_ready():
return make_response(
"Printer is already printing, cannot unselect current file",
409)
_, currentFilename = _getCurrentFile()
if currentFilename is None:
return make_response(
"Cannot unselect current file when there is no file selected",
409)
if filename != currentFilename and filename != "current":
return make_response(
"Only the currently selected file can be unselected", 400)
printer.unselect_file()
elif command == "slice":
with Permissions.SLICE.require(403):
if not _verifyFileExists(target, filename):
abort(404)
try:
if "slicer" in data:
slicer = data["slicer"]
del data["slicer"]
slicer_instance = slicingManager.get_slicer(slicer)
elif "cura" in slicingManager.registered_slicers:
slicer = "cura"
slicer_instance = slicingManager.get_slicer("cura")
else:
abort(415,
description="Cannot slice file, no slicer available")
except octoprint.slicing.UnknownSlicer:
abort(404)
if not any([
octoprint.filemanager.valid_file_type(
filename, type=source_file_type)
for source_file_type in slicer_instance.
get_slicer_properties().get("source_file_types", ["model"])
]):
abort(415, description="Cannot slice file, not a model file")
cores = psutil.cpu_count()
if (slicer_instance.get_slicer_properties().get(
"same_device", True)
and (printer.is_printing() or printer.is_paused())
and (cores is None or cores < 2)):
# slicer runs on same device as OctoPrint, slicing while printing is hence disabled
abort(
409,
description=
"Cannot slice on this slicer while printing on single core systems or systems of unknown core count due to performance reasons",
)
if "destination" in data and data["destination"]:
destination = data["destination"]
del data["destination"]
elif "gcode" in data and data["gcode"]:
destination = data["gcode"]
del data["gcode"]
else:
import os
name, _ = os.path.splitext(filename)
destination = (
name + "." + slicer_instance.get_slicer_properties().get(
"destination_extensions", ["gco", "gcode", "g"])[0])
full_path = destination
if "path" in data and data["path"]:
full_path = fileManager.join_path(target, data["path"],
destination)
else:
path, _ = fileManager.split_path(target, filename)
if path:
full_path = fileManager.join_path(target, path,
destination)
canon_path, canon_name = fileManager.canonicalize(
target, full_path)
sanitized_name = fileManager.sanitize_name(target, canon_name)
if canon_path:
full_path = fileManager.join_path(target, canon_path,
sanitized_name)
else:
full_path = sanitized_name
# prohibit overwriting the file that is currently being printed
currentOrigin, currentFilename = _getCurrentFile()
if (currentFilename == full_path and currentOrigin == target
and (printer.is_printing() or printer.is_paused())):
abort(
409,
description=
"Trying to slice into file that is currently being printed",
)
if "profile" in data and data["profile"]:
profile = data["profile"]
del data["profile"]
else:
profile = None
if "printerProfile" in data and data["printerProfile"]:
printerProfile = data["printerProfile"]
del data["printerProfile"]
else:
printerProfile = None
if ("position" in data and data["position"]
and isinstance(data["position"], dict)
and "x" in data["position"] and "y" in data["position"]):
position = data["position"]
del data["position"]
else:
position = None
select_after_slicing = False
if "select" in data and data["select"] in valid_boolean_trues:
if not printer.is_operational():
abort(
409,
description=
"Printer is not operational, cannot directly select for printing",
)
select_after_slicing = True
print_after_slicing = False
if "print" in data and data["print"] in valid_boolean_trues:
if not printer.is_operational():
abort(
409,
description=
"Printer is not operational, cannot directly start printing",
)
select_after_slicing = print_after_slicing = True
override_keys = [
k for k in data
if k.startswith("profile.") and data[k] is not None
]
overrides = {}
for key in override_keys:
overrides[key[len("profile."):]] = data[key]
def slicing_done(target, path, select_after_slicing,
print_after_slicing):
if select_after_slicing or print_after_slicing:
sd = False
if target == FileDestinations.SDCARD:
filenameToSelect = path
sd = True
else:
filenameToSelect = fileManager.path_on_disk(
target, path)
printer.select_file(filenameToSelect, sd,
print_after_slicing, user)
try:
fileManager.slice(
slicer,
target,
filename,
target,
full_path,
profile=profile,
printer_profile_id=printerProfile,
position=position,
overrides=overrides,
display=canon_name,
callback=slicing_done,
callback_args=(
target,
full_path,
select_after_slicing,
print_after_slicing,
),
)
except octoprint.slicing.UnknownProfile:
abort(404, description="Unknown profile")
location = url_for(
".readGcodeFile",
target=target,
filename=full_path,
_external=True,
)
result = {
"name": destination,
"path": full_path,
"display": canon_name,
"origin": FileDestinations.LOCAL,
"refs": {
"resource":
location,
"download":
url_for("index", _external=True) + "downloads/files/" +
target + "/" + urlquote(full_path),
},
}
r = make_response(jsonify(result), 202)
r.headers["Location"] = location
return r
elif command == "analyse":
with Permissions.FILES_UPLOAD.require(403):
if not _verifyFileExists(target, filename):
abort(404)
printer_profile = None
if "printerProfile" in data and data["printerProfile"]:
printer_profile = data["printerProfile"]
if not fileManager.analyse(
target, filename, printer_profile_id=printer_profile):
abort(400, description="No analysis possible")
elif command == "copy" or command == "move":
with Permissions.FILES_UPLOAD.require(403):
# Copy and move are only possible on local storage
if target not in [FileDestinations.LOCAL]:
abort(400,
description="Unsupported target for {}".format(command))
if not _verifyFileExists(target,
filename) and not _verifyFolderExists(
target, filename):
abort(404)
path, name = fileManager.split_path(target, filename)
destination = data["destination"]
dst_path, dst_name = fileManager.split_path(target, destination)
sanitized_destination = fileManager.join_path(
target, dst_path, fileManager.sanitize_name(target, dst_name))
# Check for exception thrown by _verifyFolderExists, if outside the root directory
try:
if (_verifyFolderExists(target, destination)
and sanitized_destination != filename):
# destination is an existing folder and not ourselves (= display rename), we'll assume we are supposed
# to move filename to this folder under the same name
destination = fileManager.join_path(
target, destination, name)
if _verifyFileExists(target,
destination) or _verifyFolderExists(
target, destination):
abort(409, description="File or folder does already exist")
except Exception:
abort(409,
description=
"Exception thrown by storage, bad folder/file name?")
is_file = fileManager.file_exists(target, filename)
is_folder = fileManager.folder_exists(target, filename)
if not (is_file or is_folder):
abort(400,
description="Neither file nor folder, can't {}".format(
command))
if command == "copy":
# destination already there? error...
if _verifyFileExists(target,
destination) or _verifyFolderExists(
target, destination):
abort(409, description="File or folder does already exist")
if is_file:
fileManager.copy_file(target, filename, destination)
else:
fileManager.copy_folder(target, filename, destination)
elif command == "move":
with Permissions.FILES_DELETE.require(403):
if _isBusy(target, filename):
abort(
409,
description=
"Trying to move a file or folder that is currently in use",
)
# destination already there AND not ourselves (= display rename)? error...
if (_verifyFileExists(target, destination)
or _verifyFolderExists(target, destination)
) and sanitized_destination != filename:
abort(409,
description="File or folder does already exist")
# deselect the file if it's currently selected
currentOrigin, currentFilename = _getCurrentFile()
if currentFilename is not None and filename == currentFilename:
printer.unselect_file()
if is_file:
fileManager.move_file(target, filename, destination)
else:
fileManager.move_folder(target, filename, destination)
location = url_for(
".readGcodeFile",
target=target,
filename=destination,
_external=True,
)
result = {
"name": name,
"path": destination,
"origin": FileDestinations.LOCAL,
"refs": {
"resource": location
},
}
if is_file:
result["refs"]["download"] = (
url_for("index", _external=True) + "downloads/files/" +
target + "/" + urlquote(destination))
r = make_response(jsonify(result), 201)
r.headers["Location"] = location
return r
return NO_CONTENT
def uploadGcodeFile(target):
input_name = "file"
input_upload_name = (input_name + "." +
settings().get(["server", "uploads", "nameSuffix"]))
input_upload_path = (input_name + "." +
settings().get(["server", "uploads", "pathSuffix"]))
if input_upload_name in request.values and input_upload_path in request.values:
if target not in [FileDestinations.LOCAL, FileDestinations.SDCARD]:
abort(404)
upload = octoprint.filemanager.util.DiskFileWrapper(
request.values[input_upload_name],
request.values[input_upload_path])
# Store any additional user data the caller may have passed.
userdata = None
if "userdata" in request.values:
import json
try:
userdata = json.loads(request.values["userdata"])
except Exception:
abort(400, description="userdata contains invalid JSON")
if target == FileDestinations.SDCARD and not settings().getBoolean(
["feature", "sdSupport"]):
abort(404)
sd = target == FileDestinations.SDCARD
selectAfterUpload = ("select" in request.values and
request.values["select"] in valid_boolean_trues
and Permissions.FILES_SELECT.can())
printAfterSelect = ("print" in request.values
and request.values["print"] in valid_boolean_trues
and Permissions.PRINT.can())
if sd:
# validate that all preconditions for SD upload are met before attempting it
if not (printer.is_operational()
and not (printer.is_printing() or printer.is_paused())):
abort(
409,
description=
"Can not upload to SD card, printer is either not operational or already busy",
)
if not printer.is_sd_ready():
abort(
409,
description="Can not upload to SD card, not yet initialized"
)
# determine future filename of file to be uploaded, abort if it can't be uploaded
try:
# FileDestinations.LOCAL = should normally be target, but can't because SDCard handling isn't implemented yet
canonPath, canonFilename = fileManager.canonicalize(
FileDestinations.LOCAL, upload.filename)
if request.values.get("path"):
canonPath = request.values.get("path")
if request.values.get("filename"):
canonFilename = request.values.get("filename")
futurePath = fileManager.sanitize_path(FileDestinations.LOCAL,
canonPath)
futureFilename = fileManager.sanitize_name(FileDestinations.LOCAL,
canonFilename)
except Exception:
canonFilename = None
futurePath = None
futureFilename = None
if futureFilename is None:
abort(415, description="Can not upload file, wrong format?")
# prohibit overwriting currently selected file while it's being printed
futureFullPath = fileManager.join_path(FileDestinations.LOCAL,
futurePath, futureFilename)
futureFullPathInStorage = fileManager.path_in_storage(
FileDestinations.LOCAL, futureFullPath)
if not printer.can_modify_file(futureFullPathInStorage, sd):
abort(
409,
description=
"Trying to overwrite file that is currently being printed",
)
if (fileManager.file_exists(FileDestinations.LOCAL,
futureFullPathInStorage)
and request.values.get("noOverwrite") in valid_boolean_trues):
abort(409,
description="File already exists and noOverwrite was set")
reselect = printer.is_current_file(futureFullPathInStorage, sd)
user = current_user.get_name()
def fileProcessingFinished(filename, absFilename, destination):
"""
Callback for when the file processing (upload, optional slicing, addition to analysis queue) has
finished.
Depending on the file's destination triggers either streaming to SD card or directly calls selectAndOrPrint.
"""
if (destination == FileDestinations.SDCARD
and octoprint.filemanager.valid_file_type(
filename, "machinecode")):
return filename, printer.add_sd_file(
filename,
absFilename,
on_success=selectAndOrPrint,
tags={"source:api", "api:files.sd"},
)
else:
selectAndOrPrint(filename, absFilename, destination)
return filename
def selectAndOrPrint(filename, absFilename, destination):
"""
Callback for when the file is ready to be selected and optionally printed. For SD file uploads this is only
the case after they have finished streaming to the printer, which is why this callback is also used
for the corresponding call to addSdFile.
Selects the just uploaded file if either selectAfterUpload or printAfterSelect are True, or if the
exact file is already selected, such reloading it.
"""
if octoprint.filemanager.valid_file_type(
added_file, "gcode") and (selectAfterUpload
or printAfterSelect or reselect):
printer.select_file(
absFilename,
destination == FileDestinations.SDCARD,
printAfterSelect,
user,
)
try:
added_file = fileManager.add_file(
FileDestinations.LOCAL,
futureFullPathInStorage,
upload,
allow_overwrite=True,
display=canonFilename,
)
except octoprint.filemanager.storage.StorageError as e:
if e.code == octoprint.filemanager.storage.StorageError.INVALID_FILE:
abort(400, description="Could not upload file, invalid type")
else:
abort(500, description="Could not upload file")
if octoprint.filemanager.valid_file_type(added_file, "stl"):
filename = added_file
done = True
else:
filename = fileProcessingFinished(
added_file,
fileManager.path_on_disk(FileDestinations.LOCAL, added_file),
target,
)
done = not sd
if userdata is not None:
# upload included userdata, add this now to the metadata
fileManager.set_additional_metadata(FileDestinations.LOCAL,
added_file, "userdata",
userdata)
sdFilename = None
if isinstance(filename, tuple):
filename, sdFilename = filename
payload = {
"name": futureFilename,
"path": filename,
"target": target,
"select": selectAfterUpload,
"print": printAfterSelect,
}
if userdata is not None:
payload["userdata"] = userdata
eventManager.fire(Events.UPLOAD, payload)
files = {}
location = url_for(
".readGcodeFile",
target=FileDestinations.LOCAL,
filename=filename,
_external=True,
)
files.update({
FileDestinations.LOCAL: {
"name": futureFilename,
"path": filename,
"origin": FileDestinations.LOCAL,
"refs": {
"resource":
location,
"download":
url_for("index", _external=True) + "downloads/files/" +
FileDestinations.LOCAL + "/" + urlquote(filename),
},
}
})
if sd and sdFilename:
location = url_for(
".readGcodeFile",
target=FileDestinations.SDCARD,
filename=sdFilename,
_external=True,
)
files.update({
FileDestinations.SDCARD: {
"name": sdFilename,
"path": sdFilename,
"origin": FileDestinations.SDCARD,
"refs": {
"resource": location
},
}
})
r = make_response(jsonify(files=files, done=done), 201)
r.headers["Location"] = location
return r
elif "foldername" in request.values:
foldername = request.values["foldername"]
if target not in [FileDestinations.LOCAL]:
abort(400, description="target is invalid")
canonPath, canonName = fileManager.canonicalize(target, foldername)
futurePath = fileManager.sanitize_path(target, canonPath)
futureName = fileManager.sanitize_name(target, canonName)
if not futureName or not futurePath:
abort(400, description="folder name is empty")
if "path" in request.values and request.values["path"]:
futurePath = fileManager.sanitize_path(FileDestinations.LOCAL,
request.values["path"])
futureFullPath = fileManager.join_path(target, futurePath, futureName)
if octoprint.filemanager.valid_file_type(futureName):
abort(409,
description="Can't create folder, please try another name")
try:
added_folder = fileManager.add_folder(target,
futureFullPath,
display=canonName)
except octoprint.filemanager.storage.StorageError as e:
if e.code == octoprint.filemanager.storage.StorageError.INVALID_DIRECTORY:
abort(400,
description="Could not create folder, invalid directory")
else:
abort(500, description="Could not create folder")
location = url_for(
".readGcodeFile",
target=FileDestinations.LOCAL,
filename=added_folder,
_external=True,
)
folder = {
"name": futureName,
"path": added_folder,
"origin": target,
"refs": {
"resource": location
},
}
r = make_response(jsonify(folder=folder, done=True), 201)
r.headers["Location"] = location
return r
else:
abort(400, description="No file to upload and no folder to create")
def analyse_recursively(files, path=None):
if path is None:
path = ""
result = []
for file_or_folder in files:
# make a shallow copy in order to not accidentally modify the cached data
file_or_folder = dict(file_or_folder)
file_or_folder["origin"] = FileDestinations.LOCAL
if file_or_folder["type"] == "folder":
if "children" in file_or_folder:
file_or_folder["children"] = analyse_recursively(
file_or_folder["children"].values(),
path + file_or_folder["name"] + "/",
)
file_or_folder["refs"] = {
"resource":
url_for(
".readGcodeFile",
target=FileDestinations.LOCAL,
filename=path + file_or_folder["name"],
_external=True,
)
}
else:
if ("analysis" in file_or_folder
and octoprint.filemanager.valid_file_type(
file_or_folder["name"], type="gcode")):
file_or_folder["gcodeAnalysis"] = file_or_folder[
"analysis"]
del file_or_folder["analysis"]
if ("history" in file_or_folder
and octoprint.filemanager.valid_file_type(
file_or_folder["name"], type="gcode")):
# convert print log
history = file_or_folder["history"]
del file_or_folder["history"]
success = 0
failure = 0
last = None
for entry in history:
success += 1 if "success" in entry and entry[
"success"] else 0
failure += (1 if "success" in entry
and not entry["success"] else 0)
if not last or ("timestamp" in entry
and "timestamp" in last
and entry["timestamp"] >
last["timestamp"]):
last = entry
if last:
prints = {
"success": success,
"failure": failure,
"last": {
"success": last["success"],
"date": last["timestamp"],
},
}
if "printTime" in last:
prints["last"]["printTime"] = last["printTime"]
file_or_folder["prints"] = prints
file_or_folder["refs"] = {
"resource":
url_for(
".readGcodeFile",
target=FileDestinations.LOCAL,
filename=file_or_folder["path"],
_external=True,
),
"download":
url_for("index", _external=True) + "downloads/files/" +
FileDestinations.LOCAL + "/" +
urlquote(file_or_folder["path"]),
}
result.append(file_or_folder)
return result
def _trackerBranch(self, branch):
# manually quote tilde for Python 3.7
url = urlquote(self.repourl, '').replace('~', '%7E')
return "refs/buildbot/{}/{}".format(url, self._removeHeads(branch))
def build_url(self, local_path, **kwargs):
# Make the path relative.
local_path = local_path.strip('/')
# We complain when we see non-normalized paths, as it is a good
# indicator that unsanitized data may be getting through.
# Mutating the scheme syntax to match is a little gross, but it works
# for today.
norm_path = os.path.normpath(local_path)
if local_path.replace(
'://', ':/') != norm_path or norm_path.startswith('../'):
raise ValueError('path is not normalized')
external = kwargs.pop('external', None) or kwargs.pop(
'_external', None)
scheme = kwargs.pop('scheme', None)
if scheme and not external:
raise ValueError('cannot specify scheme without external=True')
if kwargs.get('_anchor'):
raise ValueError('images have no _anchor')
if kwargs.get('_method'):
raise ValueError('images have no _method')
# Remote URLs are encoded into the query.
parsed = urlparse(local_path)
if parsed.scheme or parsed.netloc:
if parsed.scheme not in ALLOWED_SCHEMES:
raise ValueError('scheme %r is not allowed' % parsed.scheme)
kwargs['url'] = local_path
local_path = '_' # Must be something.
# Local ones are not.
else:
abs_path = self.find_img(local_path)
if abs_path:
kwargs['version'] = encode_int(int(os.path.getmtime(abs_path)))
# Prep the cache flag, which defaults to True.
cache = kwargs.pop('cache', True)
if not cache:
kwargs['cache'] = ''
# Prep the enlarge flag, which defaults to False.
enlarge = kwargs.pop('enlarge', False)
if enlarge:
kwargs['enlarge'] = '1'
# Prep the transform, which is a set of delimited strings.
transform = kwargs.get('transform')
if transform:
if isinstance(transform, string_types):
transform = re.split(r'[,;:_ ]', transform)
# We replace delimiters with underscores, and percent with p, since
# these won't need escaping.
kwargs['transform'] = '_'.join(
str(x).replace('%', 'p') for x in transform)
# Sign the query.
# Collapse to a dict first so that if we accidentally have two of the
# same kwarg (e.g. used `hidpi_sharpen` and `usm` which both turn into `usm`).
public_kwargs = {
LONG_TO_SHORT.get(k, k): v
for k, v in iteritems(kwargs)
if v is not None and not k.startswith('_')
}
query = urlencode(sorted(iteritems(public_kwargs)), True)
signer = Signer(current_app.secret_key)
sig = signer.get_signature('%s?%s' % (local_path, query))
url = '%s/%s?%s&s=%s' % (
current_app.config['IMAGES_URL'],
urlquote(local_path, "/$-_.+!*'(),"),
query,
sig,
)
if external:
url = '%s://%s%s/%s' % (scheme or request.scheme, request.host,
request.script_root, url.lstrip('/'))
return url
async def _perform_request(self, req: QueuedRequest):
def _put_back(_delta):
"""
Put request back into the queue after delta seconds
Resets the request expiration
"""
req.time_added = time.perf_counter()
return self.loop.call_later(_delta, self.queue.put, req)
req.tries += 1
route = req.route
delta = await self._ratelimits.get_delta(route)
if delta > 0:
return _put_back(delta)
options = req.options
options["headers"] = headers = {
"User-Agent": self.user_agent,
"X-Ratelimit-Precision": "millisecond",
"Authorization": f"Bot {self.token}",
}
if "json" in options:
headers["Content-Type"] = "application/json"
options["data"] = orjson.dumps(options.pop("json")).decode("utf-8")
if "reason" in options:
headers["X-Audit-Log-Reason"] = urlquote(options.pop("reason"), safe="/ ")
async with self.session.request(
method=route.method, url=route.url, raise_for_status=False, **options
) as resp:
data = await json_or_text(resp)
rl_bucket = resp.headers.get("X-Ratelimit-Remaining")
if rl_bucket is not None:
await self._ratelimits.set_bucket(route, rl_bucket)
rl_remaining = resp.headers.get("X-Ratelimit-Remaining")
if rl_remaining == 0:
reset = datetime.fromtimestamp(
resp.headers["X-Ratelimit-Reset"], timezone.utc
)
delta = (reset - datetime.utcnow()).total_seconds()
await self._ratelimits.set_delta(route, delta)
if 300 > resp.status >= 200:
return req.future.set_result(data)
if resp.status == 429 and resp.headers.get("Via"):
retry_after = data["retry_after"] / 1000.0
is_global = data.get("global", False)
if is_global:
await self._ratelimits.set_global(retry_after)
else:
await self._ratelimits.set_delta(route, retry_after)
return _put_back(retry_after)
if resp.status in {500, 502} and req.tries <= 5:
return _put_back(req.tries * 2)
resp.raise_for_status()
def request(
self,
route: Route,
session: Session,
*,
payload: Optional[Dict[str, Any]] = None,
multipart: Optional[List[Dict[str, Any]]] = None,
files: Optional[List[File]] = None,
reason: Optional[str] = None,
auth_token: Optional[str] = None,
params: Optional[Dict[str, Any]] = None,
) -> Any:
headers: Dict[str, str] = {}
files = files or []
to_send: Optional[Union[str, Dict[str, Any]]] = None
bucket = (route.webhook_id, route.webhook_token)
try:
lock = self._locks[bucket]
except KeyError:
self._locks[bucket] = lock = threading.Lock()
if payload is not None:
headers['Content-Type'] = 'application/json'
to_send = utils._to_json(payload)
if auth_token is not None:
headers['Authorization'] = f'Bot {auth_token}'
if reason is not None:
headers['X-Audit-Log-Reason'] = urlquote(reason, safe='/ ')
response: Optional[Response] = None
data: Optional[Union[Dict[str, Any], str]] = None
file_data: Optional[Dict[str, Any]] = None
method = route.method
url = route.url
webhook_id = route.webhook_id
with DeferredLock(lock) as lock:
for attempt in range(5):
for file in files:
file.reset(seek=attempt)
if multipart:
file_data = {}
for p in multipart:
name = p['name']
if name == 'payload_json':
to_send = {'payload_json': p['value']}
else:
file_data[name] = (p['filename'], p['value'],
p['content_type'])
try:
with session.request(method,
url,
data=to_send,
files=file_data,
headers=headers,
params=params) as response:
_log.debug(
'Webhook ID %s with %s %s has returned status code %s',
webhook_id,
method,
url,
response.status_code,
)
response.encoding = 'utf-8'
# Compatibility with aiohttp
response.status = response.status_code # type: ignore
data = response.text or None
if data and response.headers[
'Content-Type'] == 'application/json':
data = json.loads(data)
remaining = response.headers.get(
'X-Ratelimit-Remaining')
if remaining == '0' and response.status_code != 429:
delta = utils._parse_ratelimit_header(response)
_log.debug(
'Webhook ID %s has been pre-emptively rate limited, waiting %.2f seconds',
webhook_id, delta)
lock.delay_by(delta)
if 300 > response.status_code >= 200:
return data
if response.status_code == 429:
if not response.headers.get('Via'):
raise HTTPException(response, data)
retry_after: float = data[
'retry_after'] # type: ignore
_log.warning(
'Webhook ID %s is rate limited. Retrying in %.2f seconds',
webhook_id, retry_after)
time.sleep(retry_after)
continue
if response.status_code >= 500:
time.sleep(1 + attempt * 2)
continue
if response.status_code == 403:
raise Forbidden(response, data)
elif response.status_code == 404:
raise NotFound(response, data)
else:
raise HTTPException(response, data)
except OSError as e:
if attempt < 4 and e.errno in (54, 10054):
time.sleep(1 + attempt * 2)
continue
raise
if response:
if response.status_code >= 500:
raise DiscordServerError(response, data)
raise HTTPException(response, data)
raise RuntimeError('Unreachable code in HTTP handling.')
def scm_link(package, operation):
parts = (operation, package.pkgbase)
linkbase = ("https://github.com/z3ntu/archhurd_packages/%s/master/%s")
return linkbase % tuple(urlquote(part.encode('utf-8')) for part in parts)
def now_playing(phenny, input):
nick = input.nick
user = ""
arg = input.group(2)
if not arg or len(arg.strip()) == 0:
user = resolve_username(nick) # use the sender
if not user: #nick didnt resolve
user = nick
else: # use the argument
user = resolve_username(arg.strip())
if not user: # user didnt resolve
user = arg
user = user.strip()
try:
req = urlopen("%smethod=user.getrecenttracks&user=%s" %
(APIURL, urlquote(user)))
except HTTPError as e:
if e.code == 400:
phenny.say(
"%s doesn't exist on last.fm, perhaps they need to set user" %
(user))
return
else:
phenny.say("uhoh. try again later, mmkay?")
return
except http.client.BadStatusLine:
phenny.say("uhoh. try again later, mmkay?")
return
doc = etree.parse(req)
root = doc.getroot()
recenttracks = list(root)
if len(recenttracks) == 0:
phenny.say(
"%s hasn't played anything recently. this isn't you? try lastfm-set"
% (user))
return
tracks = list(recenttracks[0])
#print etree.tostring(recenttracks[0])
if len(tracks) == 0:
phenny.say(
"%s hasn't played anything recently. this isn't you? try lastfm-set"
% (user))
return
first = tracks[0]
now = True if first.get("nowplaying") == "true" else False
tags = {}
for e in first.getiterator():
tags[e.tag] = e
track = tags['name'].text.strip()
artist = tags['artist'].text.strip()
album = "unknown"
if tags['album'].text:
album = tags['album'].text
date = None
stamp = None
if not now:
date = tags['date'].get("uts")
stamp = int(date)
if now:
present = get_verb(nick)[1]
phenny.say("%s %s \"%s\" by %s on %s" %
(user.strip(), present.strip(), track, artist, album))
return
else:
past = get_verb(nick)[0]
phenny.say("%s %s \"%s\" by %s on %s %s" %
(user.strip(), past.strip(), track, artist, album,
pretty_date(stamp)))
def quote(uri):
return urlquote(uri, "!$&'()*+,-./:=@_~", encoding="latin-1")
async def request(
self,
route: Route,
session: aiohttp.ClientSession,
*,
payload: Optional[Dict[str, Any]] = None,
multipart: Optional[List[Dict[str, Any]]] = None,
files: Optional[List[File]] = None,
reason: Optional[str] = None,
auth_token: Optional[str] = None,
params: Optional[Dict[str, Any]] = None,
) -> Any:
headers: Dict[str, str] = {}
files = files or []
to_send: Optional[Union[str, aiohttp.FormData]] = None
bucket = (route.webhook_id, route.webhook_token)
try:
lock = self._locks[bucket]
except KeyError:
self._locks[bucket] = lock = asyncio.Lock()
if payload is not None:
headers['Content-Type'] = 'application/json'
to_send = utils.to_json(payload)
if auth_token is not None:
headers['Authorization'] = auth_token # Removed bot authorization here [OVERRIDEN BY BRNI]
if reason is not None:
headers['X-Audit-Log-Reason'] = urlquote(reason, safe='/ ')
response: Optional[aiohttp.ClientResponse] = None
data: Optional[Union[Dict[str, Any], str]] = None
method = route.method
url = route.url
webhook_id = route.webhook_id
async with AsyncDeferredLock(lock) as lock:
for attempt in range(5):
for file in files:
file.reset(seek=attempt)
if multipart:
form_data = aiohttp.FormData()
for p in multipart:
form_data.add_field(**p)
to_send = form_data
try:
async with session.request(method, url, data=to_send, headers=headers, params=params) as response:
log.debug(
'Webhook ID %s with %s %s has returned status code %s',
webhook_id,
method,
url,
response.status,
)
data = (await response.text(encoding='utf-8')) or None
if data and response.headers['Content-Type'] == 'application/json':
data = json.loads(data)
remaining = response.headers.get('X-Ratelimit-Remaining')
if remaining == '0' and response.status != 429:
delta = utils._parse_ratelimit_header(response)
log.debug(
'Webhook ID %s has been pre-emptively rate limited, waiting %.2f seconds', webhook_id, delta
)
lock.delay_by(delta)
if 300 > response.status >= 200:
return data
if response.status == 429:
if not response.headers.get('Via'):
raise HTTPException(response, data)
retry_after: float = data['retry_after'] # type: ignore
log.warning('Webhook ID %s is rate limited. Retrying in %.2f seconds', webhook_id, retry_after)
await asyncio.sleep(retry_after)
continue
if response.status >= 500:
await asyncio.sleep(1 + attempt * 2)
continue
if response.status == 403:
raise Forbidden(response, data)
elif response.status == 404:
raise NotFound(response, data)
else:
raise HTTPException(response, data)
except OSError as e:
if attempt < 4 and e.errno in (54, 10054):
await asyncio.sleep(1 + attempt * 2)
continue
raise
if response:
if response.status >= 500:
raise DiscordServerError(response, data)
raise HTTPException(response, data)
raise RuntimeError('Unreachable code in HTTP handling.')
def get_digikey_part_html_tree(dist,
pn,
extra_search_terms='',
url=None,
descend=2,
local_part_html=None):
'''Find the Digikey HTML page for a part number and return the URL and parse tree.'''
def merge_price_tiers(main_tree, alt_tree):
'''Merge the price tiers from the alternate-packaging tree into the main tree.'''
try:
insertion_point = main_tree.find('table',
id='product-dollars').find('tr')
for tr in alt_tree.find('table',
id='product-dollars').find_all('tr'):
insertion_point.insert_after(tr)
except AttributeError:
logger.log(
DEBUG_OBSESSIVE,
'Problem merging price tiers for Digikey part {} with alternate packaging!'
.format(pn))
pass
def merge_qty_avail(main_tree, alt_tree):
'''Merge the quantities from the alternate-packaging tree into the main tree.'''
try:
main_qty = get_digikey_qty_avail(main_tree)
alt_qty = get_digikey_qty_avail(alt_tree)
if main_qty is None:
merged_qty = alt_qty
elif alt_qty is None:
merged_qty = main_qty
else:
merged_qty = max(main_qty, alt_qty)
if merged_qty is not None:
insertion_point = main_tree.find(
'td', id='quantityAvailable').find('span', id='dkQty')
insertion_point.string = '{}'.format(merged_qty)
except AttributeError:
logger.log(
DEBUG_OBSESSIVE,
'Problem merging available quantities for Digikey part {} with alternate packaging!'
.format(pn))
pass
# Use the part number to lookup the part using the site search function, unless a starting url was given.
if url is None:
url = 'http://www.digikey.com/scripts/DkSearch/dksus.dll?WT.z_header=search_go&lang=en&keywords=' + urlquote(
pn + ' ' + extra_search_terms, safe='')
#url = 'http://www.digikey.com/product-search/en?KeyWords=' + urlquote(pn,safe='') + '&WT.z_header=search_go'
elif url[0] == '/':
url = 'http://www.digikey.com' + url
# Open the URL, read the HTML from it, and parse it into a tree structure.
req = FakeBrowser(url)
for _ in range(HTML_RESPONSE_RETRIES):
try:
response = urlopen(req)
html = response.read()
break
except WEB_SCRAPE_EXCEPTIONS:
logger.log(
DEBUG_DETAILED,
'Exception while web-scraping {} from {}'.format(pn, dist))
pass
else: # Couldn't get a good read from the website.
logger.log(DEBUG_OBSESSIVE,
'No HTML page for {} from {}'.format(pn, dist))
raise PartHtmlError
# Abort if the part number isn't in the HTML somewhere.
# (Only use the numbers and letters to compare PN to HTML.)
if re.sub('[\W_]', '',
str.lower(pn)) not in re.sub('[\W_]', '', str.lower(str(html))):
logger.log(DEBUG_OBSESSIVE,
'No part number {} in HTML page from {}'.format(pn, dist))
raise PartHtmlError
# Use the following code if Javascript challenge pages are used to block scrapers.
# try:
# ghst = Ghost()
# sess = ghst.start(plugins_enabled=False, download_images=False, show_scrollbars=False, javascript_enabled=False)
# html, resources = sess.open(url)
# print('type of HTML is {}'.format(type(html.content)))
# html = html.content
# except Exception as e:
# print('Exception reading with Ghost: {}'.format(e))
try:
tree = BeautifulSoup(html, 'lxml')
except Exception:
logger.log(DEBUG_OBSESSIVE,
'No HTML tree for {} from {}'.format(pn, dist))
raise PartHtmlError
# If the tree contains the tag for a product page, then return it.
if tree.find('div', class_='product-top-section') is not None:
# Digikey separates cut-tape and reel packaging, so we need to examine more pages
# to get all the pricing info. But don't descend any further if limit has been reached.
if descend > 0:
try:
# Find all the URLs to alternate-packaging pages for this part.
ap_urls = [
ap.find('li',
class_='lnkAltPack').find_all('a')[-1].get('href')
for ap in tree.find(
'div', class_='bota', id='additionalPackaging').
find_all('ul', class_='more-expander-item')
]
logger.log(
DEBUG_OBSESSIVE,
'Found {} alternate packagings for {} from {}'.format(
len(ap_urls), pn, dist))
try:
ap_trees_and_urls = [
get_digikey_part_html_tree(dist,
pn,
extra_search_terms,
ap_url,
descend=0)
for ap_url in ap_urls
]
except Exception:
logger.log(
DEBUG_OBSESSIVE,
'Failed to find alternate packagings for {} from {}'.
format(pn, dist))
# Put the main tree on the list as well and then look through
# the entire list for one that's non-reeled. Use this as the
# main page for the part.
ap_trees_and_urls.append((tree, url))
if digikey_part_is_reeled(tree):
for ap_tree, ap_url in ap_trees_and_urls:
if not digikey_part_is_reeled(ap_tree):
# Found a non-reeled part, so use it as the main page.
tree = ap_tree
url = ap_url
break # Done looking.
# Now go through the other pages, merging their pricing and quantity
# info into the main page.
for ap_tree, ap_url in ap_trees_and_urls:
if ap_tree is tree:
continue # Skip examining the main tree. It already contains its info.
try:
# Merge the pricing info from that into the main parse tree to make
# a single, unified set of price tiers...
merge_price_tiers(tree, ap_tree)
# and merge available quantity, using the maximum found.
merge_qty_avail(tree, ap_tree)
except AttributeError:
logger.log(
DEBUG_OBSESSIVE,
'Problem merging price/qty for {} from {}'.format(
pn, dist))
continue
except AttributeError:
logger.log(
DEBUG_OBSESSIVE,
'Problem parsing URLs from product page for {} from {}'.
format(pn, dist))
pass
return tree, url # Return the parse tree and the URL where it came from.
# If the tree is for a list of products, then examine the links to try to find the part number.
if tree.find('table', id='productTable') is not None:
logger.log(DEBUG_OBSESSIVE,
'Found product table for {} from {}'.format(pn, dist))
if descend <= 0:
logger.log(DEBUG_OBSESSIVE,
'Passed descent limit for {} from {}'.format(pn, dist))
raise PartHtmlError
else:
# Look for the table of products.
products = tree.find(
'table', id='productTable').find('tbody').find_all('tr')
# Extract the product links for the part numbers from the table.
# Extract links for both manufacturer and catalog numbers.
product_links = [
p.find('td', class_='tr-mfgPartNumber').a for p in products
]
product_links.extend(
[p.find('td', class_='tr-dkPartNumber').a for p in products])
# Extract all the part numbers from the text portion of the links.
part_numbers = [l.text for l in product_links]
# Look for the part number in the list that most closely matches the requested part number.
match = difflib.get_close_matches(pn, part_numbers, 1, 0.0)[0]
# Now look for the link that goes with the closest matching part number.
for l in product_links:
if l.text == match:
# Get the tree for the linked-to page and return that.
logger.log(
DEBUG_OBSESSIVE,
'Selecting {} from product table for {} from {}'.
format(l.text, pn, dist))
return get_digikey_part_html_tree(dist,
pn,
extra_search_terms,
url=l['href'],
descend=descend - 1)
# If the HTML contains a list of part categories, then give up.
if tree.find('form', id='keywordSearchForm') is not None:
logger.log(
DEBUG_OBSESSIVE,
'Found high-level part categories for {} from {}'.format(pn, dist))
raise PartHtmlError
# I don't know what happened here, so give up.
logger.log(DEBUG_OBSESSIVE,
'Unknown error for {} from {}'.format(pn, dist))
raise PartHtmlError
def encode_path(*components):
'Encode the path specified as a list of path components using URL encoding'
return '/' + '/'.join(
urlquote(x.encode('utf-8'), '').decode('ascii') for x in components)
def file_predict(contents, filename):
'''
Signature file_predict(contens=None, filename=None)
Docstring return a dash table and plot fig
Parameters contents: upload file
filename: string, file name
'''
content_type, content_string = contents.split(',')
decoded = base64.b64decode(content_string)
try:
if 'csv' in filename:
# Assume that the user uploaded a CSV file
df = pd.read_csv(io.StringIO(decoded.decode('utf-8')))
elif 'xls' in filename:
# Assume that the user uploaded an excel file
df = pd.read_excel(io.BytesIO(decoded))
# get predictions
start = time.time()
pred = get_pred(df, nlp, tokenizer)
# append predictions
df_pred = append_pred(df, pred)
# create download link
csv_str = df_pred.to_csv(index=False, encoding='utf-8')
csv_str = 'data:text/csv;charset=utf-8, ' + urlquote(csv_str)
# plot precision score if label is included in the upload file
if (df.columns == 'label').sum() > 0:
fig = plot_dash_precision(df, pred)
else:
fig = {}
# convert df to dash table
dash_tbl = html.Div([
dash_table.DataTable(data=df_pred.to_dict('records'),
columns=[{
'name': i,
'id': i
} for i in df_pred.columns],
page_size=20,
sort_action='native',
style_table={'overflowy': 'auto'},
style_header=css_tbl_header,
style_cell=css_tbl_cell,
style_data_conditional=css_tbl_condl)
],
style={
'font-family': 'Verdana',
'font-size': '1.4em'
})
end = time.time()
total_time = str(datetime.timedelta(seconds=end - start))
return dash_tbl, fig, csv_str, total_time
except Exception as e:
print(e)
err_msg = html.Div(
['There was an error processing this file.'],
style={
'font-size': '1.4em',
'font-weight': 'bold',
'color': '#CC4848 ',
'text-align': 'center'
})
return err_msg, {}, None
def get_mouser_part_html_tree(dist,
pn,
extra_search_terms='',
url=None,
descend=2,
local_part_html=None):
'''Find the Mouser HTML page for a part number and return the URL and parse tree.'''
# Use the part number to lookup the part using the site search function, unless a starting url was given.
if url is None:
url = 'http://www.mouser.com/Search/Refine.aspx?Keyword=' + urlquote(
pn + ' ' + extra_search_terms, safe='')
elif url[0] == '/':
url = 'http://www.mouser.com' + url
elif url.startswith('..'):
url = 'http://www.mouser.com/Search/' + url
# Open the URL, read the HTML from it, and parse it into a tree structure.
req = FakeBrowser(url)
req.add_header('Cookie', 'preferences=ps=www2&pl=en-US&pc_www2=USDe')
for _ in range(HTML_RESPONSE_RETRIES):
try:
response = urlopen(req)
html = response.read()
break
except WEB_SCRAPE_EXCEPTIONS:
logger.log(
DEBUG_DETAILED,
'Exception while web-scraping {} from {}'.format(pn, dist))
pass
else: # Couldn't get a good read from the website.
logger.log(DEBUG_OBSESSIVE,
'No HTML page for {} from {}'.format(pn, dist))
raise PartHtmlError
# Abort if the part number isn't in the HTML somewhere.
# (Only use the numbers and letters to compare PN to HTML.)
if re.sub('[\W_]', '',
str.lower(pn)) not in re.sub('[\W_]', '', str.lower(str(html))):
logger.log(DEBUG_OBSESSIVE,
'No part number {} in HTML page from {}'.format(pn, dist))
raise PartHtmlError
try:
tree = BeautifulSoup(html, 'lxml')
except Exception:
logger.log(DEBUG_OBSESSIVE,
'No HTML tree for {} from {}'.format(pn, dist))
raise PartHtmlError
# If the tree contains the tag for a product page, then just return it.
if tree.find('div', id='product-details') is not None:
return tree, url
# If the tree is for a list of products, then examine the links to try to find the part number.
if tree.find('table', class_='SearchResultsTable') is not None:
logger.log(DEBUG_OBSESSIVE,
'Found product table for {} from {}'.format(pn, dist))
if descend <= 0:
logger.log(DEBUG_OBSESSIVE,
'Passed descent limit for {} from {}'.format(pn, dist))
raise PartHtmlError
else:
# Look for the table of products.
products = tree.find('table',
class_='SearchResultsTable').find_all(
'tr',
class_=('SearchResultsRowOdd',
'SearchResultsRowEven'))
# Extract the product links for the part numbers from the table.
product_links = [
p.find('div', class_='mfrDiv').a for p in products
]
# Extract all the part numbers from the text portion of the links.
part_numbers = [l.text for l in product_links]
# Look for the part number in the list that most closely matches the requested part number.
match = difflib.get_close_matches(pn, part_numbers, 1, 0.0)[0]
# Now look for the link that goes with the closest matching part number.
for l in product_links:
if l.text == match:
# Get the tree for the linked-to page and return that.
logger.log(
DEBUG_OBSESSIVE,
'Selecting {} from product table for {} from {}'.
format(l.text, pn, dist))
return get_mouser_part_html_tree(dist,
pn,
extra_search_terms,
url=l['href'],
descend=descend - 1)
# I don't know what happened here, so give up.
logger.log(DEBUG_OBSESSIVE,
'Unknown error for {} from {}'.format(pn, dist))
raise PartHtmlError
def file_download_link(filename):
"""Create a Plotly Dash 'A' element that downloads a file from the app."""
location = "/download/{}".format(urlquote(filename))
return html.Div([html.H6(filename), html.A("download", href=location)])
def _build_raw_url(self, path, revision):
url = self.raw_file_url
url = url.replace("<revision>", revision)
url = url.replace("<filename>", urlquote(path))
return url
def data_url(self, content):
"""Encodes the content of an ignition file using RFC 2397."""
data = "," + urlquote(content)
return "data:" + data
if args.headercell is not None:
print('Reading from headercell: ' + args.headercell.name)
header = nf.read(args.headercell, nf.NO_CONVERT)
plain['cells'].insert(0, *header['cells'])
args.headercell.close()
for c in markdownlist:
line = c['source'].split('\n')
if 'Prerequisites' in line[0]:
colour = 'green'
symbol = 'star'
title = line[0].lstrip('#')
body = '\n'.join(line[1:])
safetitle = title.replace(' ', '-')
safetitle = safetitle.replace('`', '')
index = urlquote(safetitle, safe='?!$\\') + '%0A'
elif 'Overview' in line[0]:
colour = 'green'
symbol = 'file-o'
title = line[0].lstrip('#')
body = '\n'.join(line[1:])
safetitle = title.replace(' ', '-')
safetitle = safetitle.replace('`', '')
index = urlquote(safetitle, safe='?!$\\') + '%0A'
elif 'Info' in line[0]:
colour = 'blue'
symbol = 'info-circle'
subtitle = line[0].split(':')
title = ':'.join(subtitle[1:])
body = '\n'.join(line[1:])
safetitle = title.replace(' ', '-')
def escape_url_path(url):
urlparts = urlsplit(url)
path = urlquote(urlunquote(urlparts.path), '/')
query = urlquote(urlunquote(urlparts.query))
fragment = urlquote(urlunquote(urlparts.fragment))
return urlunsplit((urlparts.scheme, urlparts.netloc, path, query, fragment))
def __init__(
self,
*,
port=None,
user=None,
password=None,
host=None,
database=None,
pool_size=5,
overflow=10,
conn_str=None,
):
if conn_str is None:
if ((port is None) or (user is None) or (password is None)
or (host is None) or (database is None)):
raise ValueError(
"If conn_str is not given then all of the arguments port, user, password, host, database must be provided."
)
else:
if ((port is not None) or (user is not None)
or (password is not None) or (host is not None)
or (database is not None)):
raise ValueError(
"If conn_str is given, none of the arguments port, user, password, host, database are allowed."
)
if conn_str is None:
conn_str = "postgresql://{user}:{password}@{host}:{port}/{database}".format(
user=user,
password=urlquote(password),
host=host,
port=port,
database=database,
)
else:
pass
app_name = "flowmachine"
try:
app_name = "-".join((app_name, os.getlogin()))
except (FileNotFoundError, OSError):
logger.info(
"Couldn't get username for application name, using 'flowmachine'"
)
connect_args = {"application_name": app_name}
self.engine = sqlalchemy.create_engine(
conn_str,
echo=False,
strategy="threadlocal",
pool_size=pool_size,
max_overflow=overflow,
pool_timeout=None,
connect_args=connect_args,
)
self.inspector = sqlalchemy.inspect(self.engine)
self.max_connections = pool_size + overflow
if self.max_connections > os.cpu_count():
warnings.warn(
"Maximum number of connections (pool size + overflow = {}) is "
.format(self.max_connections) +
"greater than the available cpu cores ({}).".format(
os.cpu_count()))
self.__check_flowdb_version()
def open_db(self, dbname, **kwargs):
endpoint = '/'.join(['db', urlquote(dbname, safe='')])
return self._call('POST', endpoint, **kwargs)
