def parse_irc_uri(uri):
if not uri:
return {}
uri = uri.replace('#', '%23')
parsed = urlparse(uri)
if sys.version_info < (2, 7) and '?' in parsed.path:
query = parsed.path[parsed.path.find('?')+1:]
path = parsed.path[:parsed.path.find('?')]
else:
query = parsed.query
path = parsed.path
result = {}
if parsed.hostname:
result['server'] = parsed.hostname
if parsed.scheme:
result['ssl'] = '+ssl' in parsed.scheme
if path and len(path) > 1:
result['channel'] = unquote(path[1:])
if parsed.username:
result['nickname'] = unquote(parsed.username)
if parsed.password:
result['password'] = unquote(parsed.password)
if parsed.port:
result['port'] = int(parsed.port)
if query:
for keyword, value_list in parse_qs(query).items():
value = value_list[0]
result[keyword] = PARAMS.get(keyword, text_type)(value)
return result
def download_request(self, request, spider):
p = urlparse_cached(request)
scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = '{0!s}://{1!s}.s3.amazonaws.com{2!s}'.format(scheme, bucket, path)
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url='{0!s}://s3.amazonaws.com/{1!s}{2!s}'.format(scheme, bucket, path),
headers=request.headers.to_unicode_dict(),
data=request.body)
self._signer.add_auth(awsrequest)
request = request.replace(
url=url, headers=awsrequest.headers.items())
else:
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body)
request = request.replace(url=url, headers=signed_headers)
return self._download_http(request, spider)
def get_runs_by_run_metadata_key(run_metadata_key, value):
run_metadata_key = parse.unquote(run_metadata_key)
value = parse.unquote(value)
start_date = _parse_datetimes(flask.request.args.get('start_date', None))
stop_date = _parse_datetimes(flask.request.args.get('stop_date', None))
datetime_resolution = flask.request.args.get('datetime_resolution', 'day')
if datetime_resolution not in ['sec', 'min', 'hour', 'day']:
message = ('Datetime resolution: %s, is not a valid'
' choice' % datetime_resolution)
status_code = 400
return abort(make_response(message, status_code))
with session_scope() as session:
runs = (api.get_time_series_runs_by_key_value(run_metadata_key,
value,
start_date,
stop_date,
session))
# Groups runs by metadata
group_by = "build_name"
runs_by_build_name = _group_runs_by_key(runs, group_by)
# Group runs by the chosen data_range.
# That does not apply when you choose 'sec' since runs are already
# grouped by it.
aggregated_runs = \
RunAggregator(runs_by_build_name).aggregate(datetime_resolution)
return jsonify(_aggregate_runs(aggregated_runs))
def testGETListIncludeCertainFields(self):
modelList = self._getOrCreateModelList()
modelsByUrl = {self._getAbsoluteDetailURI(model): model for model in modelList}
includeFields = self._getIncludeFields()
fieldsQueryParamValue = includeFields.buildQueryParamValue()
modelCount = len(modelList)
queryParams = {self._PAGE_SIZE_FIELD_NAME: modelCount, self._QUERY_PARAM_FIELDS: fieldsQueryParamValue}
wildcardedParentLookups = self._getWildcardedParentLookups(self._getModelClass())
resp = self._doGETList(self._getModelClass(), queryParams, wildcardedParentLookups)
self.assertEqual(200, resp.status_code, resp.content)
stateAttrs, linkAttrs, embeddedAttrs = self._splitContent(resp.data)
self.assertEqual(stateAttrs[self._COUNT_FIELD_NAME], modelCount)
self.assertEqual(stateAttrs[self._PAGE_SIZE_FIELD_NAME], modelCount)
selfUrl = unquote(linkAttrs[self._SELF_FIELD_NAME])
self.assertTrue(
selfUrl.startswith(unquote(self._getAbsoluteListURI(self._getModelClass(), wildcardedParentLookups)))
)
self.assertTrue("{}={}".format(self._QUERY_PARAM_FIELDS, fieldsQueryParamValue) in selfUrl)
self.assertTrue("{}={}".format(self._PAGE_SIZE_FIELD_NAME, modelCount) in selfUrl)
self.assertEqual(modelCount, len(embeddedAttrs))
for embeddedObjectAttrs in embeddedAttrs:
modelObj = modelsByUrl[embeddedObjectAttrs[LINKS_FIELD_NAME][self._SELF_FIELD_NAME]]
self.assertIsNotNone(modelObj)
self.__assertIncludeFieldsContentEqual(includeFields, modelObj, embeddedObjectAttrs)
def login(self):
"""
登录账户
"""
# 登陆前清空 cookie, 能够防止再次登陆时因携带 cookie 可能提示有未进行教学评估的课程导致接口不可用
self.cookies.clear_session_cookies()
if self.campus == HF:
login_data = {'IDToken1': self.account, 'IDToken2': self.password}
login_url = 'http://ids1.hfut.edu.cn/amserver/UI/Login'
super(StudentSession, self).request('post', login_url, data=login_data)
method = 'get'
url = 'StuIndex.asp'
data = None
else:
method = 'post'
url = 'pass.asp'
data = {"user": self.account, "password": self.password, "UserStyle": 'student'}
# 使用重载的 request 会造成递归调用
response = super(StudentSession, self).request(method, url, data=data, allow_redirects=False)
logged_in = response.status_code == 302
if not logged_in:
msg = '登陆失败, 请检查你的账号和密码'
raise SystemLoginFailed(msg)
escaped_name = self.cookies.get('xsxm')
if six.PY3:
self.name = parse.unquote(escaped_name, ENV['SITE_ENCODING'])
else:
name = parse.unquote(escaped_name)
self.name = name.decode(ENV['SITE_ENCODING'])
def render_PUT(self, request):
parameters = http.parse_qs(request.content.read(), 1)
if 'name' not in parameters or not parameters['name'] or not parameters['name'][0]:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": "channel name cannot be empty"})
if 'description' not in parameters or not parameters['description']:
description = u''
else:
description = unquote(parameters['description'][0]).decode('utf-8')
my_key = self.session.trustchain_keypair
my_channel_pk = my_key.pub().key_to_bin()
# Do not allow to add a channel twice
if self.session.lm.mds.get_my_channel():
request.setResponseCode(http.CONFLICT)
return json.dumps({"error": "channel already exists"})
title = unquote(parameters['name'][0]).decode('utf-8')
self.session.lm.mds.ChannelMetadata.create_channel(title, description)
return json.dumps({
"added": hexlify(str(my_channel_pk)),
})
def get_request_data(self):
data = {}
if self.request.body:
items = self.request.body.decode('utf-8').split('&')
for item in items:
if '=' in item:
key, value = item.split('=')
else:
key, value = 'item', item
if key in data:
if not isinstance(data[key], (tuple, list)):
old = data[key]
data[key] = []
data[key].append(old)
data[key].append(unquote(value))
else:
data[key] = unquote(value)
else:
for arg in list(self.request.arguments.keys()):
data[arg] = self.get_argument(arg)
if data[arg] == '': # Tornado 3.0+ compatibility... Hard to test...
data[arg] = None
return data
def _get_parameters_from_request(self, request, exception=False):
"""Get parameters to log in OPERATION_LOG."""
user = request.user
referer_url = None
try:
referer_dic = urlparse.urlsplit(
urlparse.unquote(request.META.get('HTTP_REFERER')))
referer_url = referer_dic[2]
if referer_dic[3]:
referer_url += "?" + referer_dic[3]
if isinstance(referer_url, str):
referer_url = referer_url.decode('utf-8')
except Exception:
pass
return {
'domain_name': getattr(user, 'domain_name', None),
'domain_id': getattr(user, 'domain_id', None),
'project_name': getattr(user, 'project_name', None),
'project_id': getattr(user, 'project_id', None),
'user_name': getattr(user, 'username', None),
'user_id': request.session.get('user_id', None),
'request_scheme': request.scheme,
'referer_url': referer_url,
'request_url': urlparse.unquote(request.path),
'method': request.method if not exception else None,
'param': self._get_request_param(request),
}
def on_PUT(self, request, room_id, user_id):
requester = yield self.auth.get_user_by_req(request)
room_id = urlparse.unquote(room_id)
target_user = UserID.from_string(urlparse.unquote(user_id))
content = parse_json_object_from_request(request)
yield self.presence_handler.bump_presence_active_time(requester.user)
# Limit timeout to stop people from setting silly typing timeouts.
timeout = min(content.get("timeout", 30000), 120000)
if content["typing"]:
yield self.typing_handler.started_typing(
target_user=target_user,
auth_user=requester.user,
room_id=room_id,
timeout=timeout,
)
else:
yield self.typing_handler.stopped_typing(
target_user=target_user,
auth_user=requester.user,
room_id=room_id,
)
defer.returnValue((200, {}))
def getSongs(self):
format = "%Y-%m-%d %H:%M:%S"
for trackid,attributes in self.il['Tracks'].items():
s = Song()
s.name = attributes.get('Name')
s.artist = attributes.get('Artist')
s.album_artist = attributes.get('Album Artist')
s.composer = attributes.get('Composer')
s.album = attributes.get('Album')
s.genre = attributes.get('Genre')
s.kind = attributes.get('Kind')
if attributes.get('Size'):
s.size = int(attributes.get('Size'))
s.total_time = attributes.get('Total Time')
s.track_number = attributes.get('Track Number')
if attributes.get('Track Count'):
s.track_count = int(attributes.get('Track Count'))
if attributes.get('Disc Number'):
s.disc_number = int(attributes.get('Disc Number'))
if attributes.get('Disc Count'):
s.disc_count = int(attributes.get('Disc Count'))
if attributes.get('Year'):
s.year = int(attributes.get('Year'))
if attributes.get('Date Modified'):
s.date_modified = time.strptime(str(attributes.get('Date Modified')),format)
if attributes.get('Date Added'):
s.date_added = time.strptime(str(attributes.get('Date Added')),format)
if attributes.get('Bit Rate'):
s.bit_rate = int(attributes.get('Bit Rate'))
if attributes.get('Sample Rate'):
s.sample_rate = int(attributes.get('Sample Rate'))
s.comments = attributes.get("Comments ")
if attributes.get('Rating'):
s.rating = int(attributes.get('Rating'))
if attributes.get('Play Count'):
s.play_count = int(attributes.get('Play Count'))
if attributes.get('Location'):
if ( self.musicPathXML is None or self.musicPathSystem is None ):
# s.location = text_type(urlparse.unquote(urlparse.urlparse(attributes.get('Location')).path[1:]),"utf8")
s.location = text_type(urlparse.unquote(urlparse.urlparse(attributes.get('Location')).path[1:]))
else:
# s.location = text_type(urlparse.unquote(urlparse.urlparse(attributes.get('Location')).path[1:]).replace(self.musicPathXML,self.musicPathSystem),"utf8")
s.location = text_type(urlparse.unquote(urlparse.urlparse(attributes.get('Location')).path[1:]).replace(self.musicPathXML,self.musicPathSystem))
s.compilation = 'Compilation' in attributes
if attributes.get('Play Date UTC'):
s.lastplayed = time.strptime(str(attributes.get('Play Date UTC')),format)
if attributes.get('Total Time'):
s.length = int(attributes.get('Total Time'))
if attributes.get('Grouping'):
s.grouping = attributes.get('Grouping')
if self.filesOnly==True and attributes.get('Track Type') == 'File':
if self.legacymode:
self.songs.append(s)
else:
self.songs[int(trackid)] = s
elif self.filesOnly==False:
if self.legacymode:
self.songs.append(s)
else:
self.songs[int(trackid)] = s
def parse_header_string(data):
if not isinstance(data, (six.text_type, six.binary_type)):
data = str(data)
if six.PY2:
if isinstance(data, six.text_type):
# Under Python2 requests only returns binary_type, but if we get
# some stray text_type input, this should prevent unquote from
# interpreting %-encoded data as raw code-points.
data = data.encode('utf8')
try:
unquoted = unquote(data).decode('utf8')
except UnicodeDecodeError:
try:
return data.decode('utf8')
except UnicodeDecodeError:
return quote(data).decode('utf8')
else:
if isinstance(data, six.binary_type):
# Under Python3 requests only returns text_type and tosses (!) the
# rest of the headers. If that ever changes, this should be a sane
# approach.
try:
data = data.decode('ascii')
except UnicodeDecodeError:
data = quote(data)
try:
unquoted = unquote(data, errors='strict')
except UnicodeDecodeError:
return data
return unquoted
def object_request(self, req, api_version, account, container, obj,
allow_versioned_writes):
container_name = unquote(container)
object_name = unquote(obj)
orig_container = get_unversioned_container(container_name)
if orig_container != container_name:
orig_object, version = \
swift3_split_object_name_version(object_name)
req.environ['oio.query'] = {'version': version}
req.environ['PATH_INFO'] = '/%s/%s/%s/%s' % (api_version,
account,
quote(orig_container),
quote(orig_object))
elif req.method == 'DELETE':
ver_mode = req.headers.get('X-Backend-Versioning-Mode-Override',
'history')
if ver_mode == 'stack':
# Do not create a delete marker, delete the latest version
obj_inf = get_object_info(req.environ, self.app,
swift_source='VW')
req.environ['oio.query'] = {
'version': obj_inf.get('sysmeta', {}).get('version-id')
}
resp = req.get_response(self.app)
if req.method == 'HEAD':
close_if_possible(resp.app_iter)
return resp
def decode(encoded_str):
"""Decode an encrypted HTTP basic authentication string. Returns a tuple of
the form (username, password), and raises a DecodeError exception if
nothing could be decoded.
"""
split = encoded_str.strip().split(' ')
# If split is only one element, try to decode the username and password
# directly.
if len(split) == 1:
try:
username, password = b64decode(split[0]).decode().split(':', 1)
except:
raise DecodeError
# If there are only two elements, check the first and ensure it says
# 'basic' so that we know we're about to decode the right thing. If not,
# bail out.
elif len(split) == 2:
if split[0].strip().lower() == 'basic':
try:
username, password = b64decode(split[1]).decode().split(':', 1)
except:
raise DecodeError
else:
raise DecodeError
# If there are more than 2 elements, something crazy must be happening.
# Bail.
else:
raise DecodeError
return unquote(username), unquote(password)
def object_request(self, req, version, account, container, obj,
allow_versioned_writes):
account_name = unquote(account)
container_name = unquote(container)
object_name = unquote(obj)
container_info = None
resp = None
is_enabled = config_true_value(allow_versioned_writes)
if req.method in ('PUT', 'DELETE'):
container_info = get_container_info(
req.environ, self.app)
elif req.method == 'COPY' and 'Destination' in req.headers:
if 'Destination-Account' in req.headers:
account_name = req.headers.get('Destination-Account')
account_name = check_account_format(req, account_name)
container_name, object_name = check_destination_header(req)
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
version, account_name, container_name, object_name)
container_info = get_container_info(
req.environ, self.app)
if not container_info:
return self.app
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
object_versions = container_info.get(
'sysmeta', {}).get('versions-location')
if object_versions and isinstance(object_versions, six.text_type):
object_versions = object_versions.encode('utf-8')
elif not object_versions:
object_versions = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and object_versions:
is_enabled = True
if is_enabled and object_versions:
object_versions = unquote(object_versions)
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method in ('PUT', 'COPY'):
policy_idx = req.headers.get(
'X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
resp = vw_ctx.handle_obj_versions_put(
req, object_versions, object_name, policy_idx)
else: # handle DELETE
resp = vw_ctx.handle_obj_versions_delete(
req, object_versions, account_name,
container_name, object_name)
if resp:
return resp
else:
return self.app
def __call__(self, env, start_response):
request = Request(env)
if not request.path.startswith(self.endpoints_path):
return self.app(env, start_response)
if request.method != 'GET':
return HTTPMethodNotAllowed(
req=request, headers={"Allow": "GET"})(env, start_response)
try:
version, account, container, obj = self._parse_path(request)
except ValueError as err:
return HTTPBadRequest(str(err))(env, start_response)
if account is not None:
account = unquote(account)
if container is not None:
container = unquote(container)
if obj is not None:
obj = unquote(obj)
storage_policy_index = None
if obj is not None:
container_info = get_container_info(
{'PATH_INFO': '/v1/%s/%s' % (account, container)},
self.app, swift_source='LE')
storage_policy_index = container_info['storage_policy']
obj_ring = self.get_object_ring(storage_policy_index)
partition, nodes = obj_ring.get_nodes(
account, container, obj)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}/{obj}'
elif container is not None:
partition, nodes = self.container_ring.get_nodes(
account, container)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}'
else:
partition, nodes = self.account_ring.get_nodes(
account)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}'
endpoints = []
for node in nodes:
endpoint = endpoint_template.format(
ip=node['ip'],
port=node['port'],
device=node['device'],
partition=partition,
account=quote(account),
container=quote(container or ''),
obj=quote(obj or ''))
endpoints.append(endpoint)
resp = self.response_map[version](
request, endpoints=endpoints,
storage_policy_index=storage_policy_index)
return resp(env, start_response)
def url_unquote(v, name='(Unknown name)', md={}):
if six.PY2 and isinstance(v, unicode):
# unquote does not handle unicode. Encoding to a "safe"
# intermediate encoding before quoting, then unencoding the result.
return unquote(v.encode('utf-8')).decode('utf-8')
elif six.PY3 and isinstance(v, bytes):
return unquote(v.decode('utf-8')).encode('utf-8')
return unquote(str(v))
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
user_pass = '******' % (unquote(user), unquote(password))
creds = base64.b64encode(user_pass).strip()
else:
creds = None
return creds, proxy_url
def get_recent_failed_runs_rss(run_metadata_key, value):
run_metadata_key = parse.unquote(run_metadata_key)
value = parse.unquote(value)
url = request.url
if run_metadata_key not in feeds:
feeds[run_metadata_key] = {value: _gen_feed(url,
run_metadata_key,
value)}
feeds["last runs"][run_metadata_key] = {value: None}
elif value not in feeds[run_metadata_key]:
feeds[run_metadata_key][value] = _gen_feed(url,
run_metadata_key,
value)
feeds["last runs"][run_metadata_key][value] = None
fg = feeds[run_metadata_key][value]
with session_scope() as session:
failed_runs = api.get_recent_failed_runs_by_run_metadata(
run_metadata_key, value,
start_date=feeds["last runs"][run_metadata_key][value],
session=session)
if failed_runs:
last_run = sorted([x.run_at for x in failed_runs])[-1]
if feeds["last runs"][run_metadata_key][value] == last_run:
return feeds[run_metadata_key][value].rss_str()
feeds["last runs"][run_metadata_key][value] = last_run
else:
count = api.get_runs_counts_by_run_metadata(
run_metadata_key, value, session=session)
if count == 0:
msg = 'No matching runs found with %s=%s' % (
run_metadata_key, value)
return abort(make_response(msg, 404))
for run in failed_runs:
meta = api.get_run_metadata(run.uuid, session=session)
uuid = [x.value for x in meta if x.key == 'build_uuid'][0]
build_name = [x.value for x in meta if x.key == 'build_name'][0]
entry = fg.add_entry()
entry.id(uuid)
entry.title('Failed Run %s/%s' % (build_name, uuid[:7]))
entry.published(pytz.utc.localize(run.run_at))
entry.link({'href': run.artifacts, 'rel': 'alternate'})
metadata_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'g/%s/%s' % (run_metadata_key, value))
job_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'job/%s' % build_name)
content = '<ul>'
content += '<li><a href="%s">Metadata page</a></li>\n' % (
metadata_url)
content += '<li><a href="%s">Job Page</a></li>' % (job_url)
content += '</ul>'
entry.description(content)
response = make_response(feeds[run_metadata_key][value].rss_str())
response.headers['Content-Type'] = 'application/xml; charset=utf-8'
return response
def process_request(self, request):
super(SandstormMiddleware, self).process_request(request)
sid = request.META.get('HTTP_X_SANDSTORM_USER_ID')
if sid:
name = unquote(request.META.get('HTTP_X_SANDSTORM_USERNAME'))
handle = unquote(request.META.get('HTTP_X_SANDSTORM_PREFERRED_HANDLE'))
u = User(sandstorm_id=sid, name=name, handle=handle)
try:
u.save()
except:
pass
if hasattr(request, 'user'):
request.user = u
def test_hijack_urls(self):
self.assertEqual("/hijack/disable-hijack-warning/", reverse("disable_hijack_warning"))
self.assertEqual("/hijack/release-hijack/", reverse("release_hijack"))
self.assertEqual("/hijack/1/", reverse("login_with_id", args=[1]))
self.assertEqual("/hijack/2/", reverse("login_with_id", kwargs={"user_id": 2}))
self.assertEqual("/hijack/username/bob/", reverse("login_with_username", args=["bob"]))
self.assertEqual("/hijack/username/bob_too/", reverse("login_with_username", kwargs={"username": "******"}))
self.assertEqual(
"/hijack/email/[email protected]/", unquote(reverse("login_with_email", args=["*****@*****.**"]))
)
self.assertEqual(
"/hijack/email/[email protected]/",
unquote(reverse("login_with_email", kwargs={"email": "*****@*****.**"})),
)
def _get_proxy(self, url):
#>>> _parse_proxy('http://*****:*****@proxy.example.com/')
#('http', 'joe', 'password', 'proxy.example.com')
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or "http", hostport, '', '', '', ''))
#如果有用户生成证书用于连接
if user:
user_pass = to_bytes(
'%s:%s' % (unquote(user), unquote(password)),
encoding="utf-8")
creds = base64.b64encode(user_pass).strip()
else:
creds = None
return creds, proxy_url
def process_secure_data(text, request):
definitions = text.split('&')
cache_manager = VariableValueCacheManager(request['workspace'], request['user'])
for definition in definitions:
params = definition.split(',')
if len(params) == 1 and params[0].strip() == '':
continue
options = {}
for pair in params:
tokens = pair.split('=')
option_name = unquote(tokens[0].strip())
options[option_name] = unquote(tokens[1].strip())
action = options.get('action', 'data')
if action == 'data':
substr = options.get('substr', '')
var_ref = options.get('var_ref', '')
check_empty_params(substr=substr, var_ref=var_ref)
value = get_variable_value_by_ref(var_ref, cache_manager)
check_invalid_refs(var_ref=value)
encoding = options.get('encoding', 'none')
substr = substr.encode('utf8')
if encoding == 'url':
value = urlquote(value).encode('utf8')
elif encoding == 'base64':
value = base64.b64encode(value.encode('utf8'))[:-1]
else:
value = value.encode('utf8')
new_body = request['data'].read().replace(substr, value)
request['headers']['content-length'] = "%s" % len(new_body)
request['data'] = BytesIO(new_body)
elif action == 'basic_auth':
user_ref = options.get('user_ref', '')
password_ref = options.get('pass_ref', '')
check_empty_params(user_ref=user_ref, password_ref=password_ref)
user_value = get_variable_value_by_ref(user_ref, cache_manager)
password_value = get_variable_value_by_ref(password_ref, cache_manager)
check_invalid_refs(user_ref=user_value, password_ref=password_value)
token = base64.b64encode((user_value + ':' + password_value).encode('utf8'))[:-1]
request['headers']['Authorization'] = 'Basic ' + token.decode('ascii')
else:
raise ValidationError('Unsupported action: %s' % action)
def download_request(self, request, spider):
p = urlparse_cached(request)
scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body)
httpreq = request.replace(url=url, headers=signed_headers)
return self._download_http(httpreq, spider)
def object_request(self, req, api_version, account, container, obj,
allow_versioned_writes):
account_name = unquote(account)
container_name = unquote(container)
object_name = unquote(obj)
resp = None
is_enabled = config_true_value(allow_versioned_writes)
container_info = get_container_info(
req.environ, self.app)
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
versions_cont = container_info.get(
'sysmeta', {}).get('versions-location')
versioning_mode = container_info.get(
'sysmeta', {}).get('versions-mode', 'stack')
if not versions_cont:
versions_cont = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and versions_cont:
is_enabled = True
if is_enabled and versions_cont:
versions_cont = unquote(versions_cont).split('/')[0]
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method == 'PUT':
resp = vw_ctx.handle_obj_versions_put(
req, versions_cont, api_version, account_name,
object_name)
# handle DELETE
elif versioning_mode == 'history':
resp = vw_ctx.handle_obj_versions_delete_push(
req, versions_cont, api_version, account_name,
container_name, object_name)
else:
resp = vw_ctx.handle_obj_versions_delete_pop(
req, versions_cont, api_version, account_name,
container_name, object_name)
if resp:
return resp
else:
return self.app
def _handle_saml2_tokens(self, auth_context, project_id, domain_id):
user_id = auth_context['user_id']
group_ids = auth_context['group_ids']
idp = auth_context[federation.IDENTITY_PROVIDER]
protocol = auth_context[federation.PROTOCOL]
token_data = {
'user': {
'id': user_id,
'name': parse.unquote(user_id),
federation.FEDERATION: {
'identity_provider': {'id': idp},
'protocol': {'id': protocol}
}
}
}
if project_id or domain_id:
roles = self.v3_token_data_helper._populate_roles_for_groups(
group_ids, project_id, domain_id, user_id)
token_data.update({'roles': roles})
else:
token_data['user'][federation.FEDERATION].update({
'groups': [{'id': x} for x in group_ids]
})
return token_data
def _handle_mapped_tokens(self, auth_context, project_id, domain_id):
def get_federated_domain():
return CONF.federation.federated_domain_name or federation_constants.FEDERATED_DOMAIN_KEYWORD
federated_domain = get_federated_domain()
user_id = auth_context["user_id"]
group_ids = auth_context["group_ids"]
idp = auth_context[federation_constants.IDENTITY_PROVIDER]
protocol = auth_context[federation_constants.PROTOCOL]
token_data = {
"user": {
"id": user_id,
"name": parse.unquote(user_id),
federation_constants.FEDERATION: {
"groups": [{"id": x} for x in group_ids],
"identity_provider": {"id": idp},
"protocol": {"id": protocol},
},
"domain": {"id": federated_domain, "name": federated_domain},
}
}
if project_id or domain_id:
self.v3_token_data_helper.populate_roles_for_groups(token_data, group_ids, project_id, domain_id, user_id)
return token_data
def _handle_mapped_tokens(self, auth_context, project_id, domain_id):
user_id = auth_context['user_id']
group_ids = auth_context['group_ids']
idp = auth_context[federation_constants.IDENTITY_PROVIDER]
protocol = auth_context[federation_constants.PROTOCOL]
user_dict = self.identity_api.get_user(user_id)
user_name = user_dict['name']
token_data = {
'user': {
'id': user_id,
'name': parse.unquote(user_name),
federation_constants.FEDERATION: {
'groups': [{'id': x} for x in group_ids],
'identity_provider': {'id': idp},
'protocol': {'id': protocol}
},
'domain': {
'id': CONF.federation.federated_domain_name,
'name': CONF.federation.federated_domain_name
}
}
}
if project_id or domain_id:
self.v3_token_data_helper.populate_roles_for_federated_user(
token_data, group_ids, project_id, domain_id, user_id)
return token_data
def edit_start_date_view(request):
if request.method == "POST":
form = general_forms.StartDateForm(request.POST)
if form.is_valid():
new_start_date = form.cleaned_data["date"]
set_start_date(request, new_start_date)
messages.success(request, "Successfully changed start date")
redirect_destination = "eighth_admin_dashboard"
if "next_page" in request.GET:
redirect_destination = unquote(request.GET["next_page"])
return redirect(redirect_destination)
else:
messages.error(request, "Error changing start date.")
else:
initial_data = {
"date": get_start_date(request)
}
form = general_forms.StartDateForm(initial=initial_data)
context = {
"form": form,
"admin_page_title": "Change Start Date"
}
return render(request, "eighth/admin/edit_start_date.html", context)
def _handle_mapped_tokens(self, auth_context, project_id, domain_id):
def get_federated_domain():
return (CONF.federation.federated_domain_name or
federation_constants.FEDERATED_DOMAIN_KEYWORD)
federated_domain = get_federated_domain()
user_id = auth_context['user_id']
group_ids = auth_context['group_ids']
idp = auth_context[federation_constants.IDENTITY_PROVIDER]
protocol = auth_context[federation_constants.PROTOCOL]
token_data = {
'user': {
'id': user_id,
'name': parse.unquote(user_id),
federation_constants.FEDERATION: {
'groups': [{'id': x} for x in group_ids],
'identity_provider': {'id': idp},
'protocol': {'id': protocol}
},
'domain': {
'id': federated_domain,
'name': federated_domain
}
}
}
if project_id or domain_id:
self.v3_token_data_helper.populate_roles_for_groups(
token_data, group_ids, project_id, domain_id, user_id)
return token_data
def _load_backend(self, backend_uri, context):
"""
Return the instantiated backend object identified by the given
`backend_uri`.
The entry point that is used to create the backend object is determined
by the protocol part of the given URI.
"""
parsed = parse.urlparse(backend_uri)
options = dict(parse.parse_qsl(parsed.query))
try:
backend = self._entry_points[self.BACKENDS_ENTRY_POINT][parsed.scheme].load()
except KeyError:
raise BackendNotFoundError(
"The requested backend `%s` could not be found in the "
"registered entry points. Perhaps you forgot to install the "
"corresponding backend package?" % parsed.scheme
)
password = (parse.unquote(parsed.password)
if parsed.password
else parsed.password)
return backend(
username=parsed.username, password=password,
hostname=parsed.hostname, port=parsed.port,
path=parsed.path, options=options, context=context,
)
def clean_key_name(key_name):
if six.PY2:
return unquote(key_name.encode('utf-8')).decode('utf-8')
return unquote(key_name)
def getPicon(sname):
pp = PICON_PATH
if pp is not None:
# remove URL part
if ("://" in sname) or ("%3a//" in sname) or ("%3A//" in sname):
cname = unquote(sname.split(":")[-1])
sname = unquote(sname)
# sname = ":".join(sname.split(":")[:10]) -> old way
sname = ":".join(sname.split("://")[:1])
sname = GetWithAlternative(sname)
if PY3:
cname = unicodedata.normalize('NFKD', cname)
else:
cname = unicodedata.normalize('NFKD', six.text_type(cname, 'utf_8', errors='ignore')).encode('ASCII', 'ignore')
cname = re.sub('[^a-z0-9]', '', cname.replace('&', 'and').replace('+', 'plus').replace('*', 'star').replace(':', '').lower())
# picon by channel name for URL
if len(cname) > 0 and fileExists(pp + cname + ".png"):
return "/picon/" + cname + ".png"
if len(cname) > 2 and cname.endswith('hd') and fileExists(pp + cname[:-2] + ".png"):
return "/picon/" + cname[:-2] + ".png"
if len(cname) > 5:
series = re.sub(r's[0-9]*e[0-9]*$', '', cname)
if fileExists(pp + series + ".png"):
return "/picon/" + series + ".png"
sname = GetWithAlternative(sname)
if sname is not None:
pos = sname.rfind(':')
else:
return "/images/default_picon.png"
cname = None
if pos != -1:
cname = ServiceReference(sname[:pos].rstrip(':')).getServiceName()
sname = sname[:pos].rstrip(':').replace(':', '_') + ".png"
filename = pp + sname
if fileExists(filename):
return "/picon/" + sname
fields = sname.split('_', 8)
if len(fields) > 7 and not fields[6].endswith("0000"):
# remove "sub-network" from namespace
fields[6] = fields[6][:-4] + "0000"
sname = '_'.join(fields)
filename = pp + sname
if fileExists(filename):
return "/picon/" + sname
if len(fields) > 1 and fields[0] != '1':
# fallback to 1 for other reftypes
fields[0] = '1'
sname = '_'.join(fields)
filename = pp + sname
if fileExists(filename):
return "/picon/" + sname
if len(fields) > 3 and fields[2] != '1':
# fallback to 1 for tv services with nonstandard servicetypes
fields[2] = '1'
sname = '_'.join(fields)
filename = pp + sname
if fileExists(filename):
return "/picon/" + sname
if cname is not None: # picon by channel name
cname1 = filterName(cname).replace('/', '_')
if not PY3:
cname1 = cname1.encode('utf-8', 'ignore')
if fileExists(pp + cname1 + ".png"):
return "/picon/" + cname1 + ".png"
if PY3:
cname = unicodedata.normalize('NFKD', cname)
else:
cname = unicodedata.normalize('NFKD', six.text_type(cname, 'utf_8', errors='ignore')).encode('ASCII', 'ignore')
cname = re.sub('[^a-z0-9]', '', cname.replace('&', 'and').replace('+', 'plus').replace('*', 'star').lower())
if len(cname) > 0:
filename = pp + cname + ".png"
if fileExists(filename):
return "/picon/" + cname + ".png"
if len(cname) > 2 and cname.endswith('hd') and fileExists(pp + cname[:-2] + ".png"):
return "/picon/" + cname[:-2] + ".png"
return "/images/default_picon.png"
def parse(cls, conf, url=None, aliases=None):
"""Parse a URL as defined by :py:class:`TransportURL` and return a
TransportURL object.
Assuming a URL takes the form of::
transport://user:pass@host:port[,userN:passN@hostN:portN]/virtual_host?query
then parse the URL and return a TransportURL object.
Netloc is parsed following the sequence bellow:
* It is first split by ',' in order to support multiple hosts
* All hosts should be specified with username/password or not
at the same time. In case of lack of specification, username and
password will be omitted::
user:pass@host1:port1,host2:port2
[
{"username": "******", "password": "******", "host": "host1:port1"},
{"host": "host2:port2"}
]
If the url is not provided conf.transport_url is parsed instead.
:param conf: a ConfigOpts instance
:type conf: oslo.config.cfg.ConfigOpts
:param url: The URL to parse
:type url: str
:param aliases: A map of transport alias to transport name
:type aliases: dict
:returns: A TransportURL
"""
if not url:
conf.register_opts(
_transport_opts) #transport_url、rpc_backend、control_exchange
url = url or conf.transport_url
if not url:
return cls(conf) if aliases is None else cls(conf, aliases=aliases)
if not isinstance(url, six.string_types):
raise InvalidTransportURL(url, 'Wrong URL type')
#'http://blog.csdn.net/?ref=toolbar'----> (scheme='http', netloc='blog.csdn.net', path='/', params='', query='ref=toolbar', fragment='')
url = parse.urlparse(url)
if not url.scheme:
raise InvalidTransportURL(url.geturl(), 'No scheme specified')
transport = url.scheme
query = {}
if url.query:
for key, values in parse.parse_qs(url.query).items():
query[key] = ','.join(values)
virtual_host = None
if url.path.startswith('/'):
virtual_host = parse.unquote(url.path[1:])
hosts_with_credentials = []
hosts_without_credentials = []
hosts = []
for host in url.netloc.split(','):
if not host:
continue
hostname = host
username = password = port = None
if '@' in host:
username, hostname = host.rsplit('@', 1)
if ':' in username:
username, password = username.split(':', 1)
password = parse.unquote(password)
username = parse.unquote(username)
if not hostname:
hostname = None
elif hostname.startswith('['):
# Find the closing ']' and extract the hostname
host_end = hostname.find(']')
if host_end < 0:
# NOTE(Vek): Identical to what Python 2.7's
# urlparse.urlparse() raises in this case
raise ValueError('Invalid IPv6 URL')
port_text = hostname[host_end:]
hostname = hostname[1:host_end]
# Now we need the port; this is compliant with how urlparse
# parses the port data
port = None
if ':' in port_text:
port = port_text.split(':', 1)[1]
elif ':' in hostname:
hostname, port = hostname.split(':', 1)
if port == "":
port = None
if port is not None:
port = int(port)
if username is None or password is None:
hosts_without_credentials.append(hostname)
else:
hosts_with_credentials.append(hostname)
hosts.append(
TransportHost(hostname=hostname,
port=port,
username=username,
password=password))
if (len(hosts_with_credentials) > 0
and len(hosts_without_credentials) > 0):
LOG.warning(
_LW("All hosts must be set with username/password or "
"not at the same time. Hosts with credentials "
"are: %(hosts_with_credentials)s. Hosts without "
"credentials are %(hosts_without_credentials)s."), {
'hosts_with_credentials': hosts_with_credentials,
'hosts_without_credentials': hosts_without_credentials
})
if aliases is None:
return cls(conf, transport, virtual_host, hosts, query=query)
else:
return cls(conf, transport, virtual_host, hosts, aliases, query)
def _check_url(browser, port, url):
if not url.startswith("http"):
url = _formgrade_url(port, url)
url_matches = lambda browser: unquote(browser.current_url).rstrip("/"
) == url
WebDriverWait(browser, 10).until(url_matches)
def http(request_string, handle_errors=True):
"""Execute an HTTP request string via the publisher
This is used for HTTP doc tests.
"""
from six.moves.urllib.parse import unquote
from ZPublisher.HTTPRequest import WSGIRequest as Request
from ZPublisher.HTTPResponse import WSGIResponse
from ZPublisher.WSGIPublisher import publish_module
# Commit work done by previous python code.
transaction.commit()
# Discard leading white space to make call layout simpler
request_string = request_string.lstrip()
# Split off and parse the command line
l = request_string.find('\n')
command_line = request_string[:l].rstrip()
request_string = request_string[l + 1:]
method, path, protocol = command_line.split()
path = unquote(path)
env = {
'HTTP_HOST': 'localhost',
'HTTP_REFERER': 'localhost',
'REQUEST_METHOD': method,
'SERVER_PROTOCOL': protocol,
}
p = path.split('?', 1)
if len(p) == 1:
env['PATH_INFO'] = p[0]
elif len(p) == 2:
[env['PATH_INFO'], env['QUERY_STRING']] = p
else:
raise TypeError('')
header_output = HTTPHeaderOutput(
protocol, ('x-content-type-warning', 'x-powered-by'))
# With a HeaderParser the payload is always a string, Parser would create a
# list of messages for multipart messages.
parser = email.parser.HeaderParser()
msg = parser.parsestr(request_string)
headers = msg.items()
body = msg.get_payload()
if isinstance(body, text_type):
body = body.encode('utf-8')
# Store request body without headers
instream = BytesIO(body)
for name, value in headers:
name = ('_'.join(name.upper().split('-')))
if name not in ('CONTENT_TYPE', 'CONTENT_LENGTH'):
name = 'HTTP_' + name
env[name] = value.rstrip()
if 'HTTP_AUTHORIZATION' in env:
env['HTTP_AUTHORIZATION'] = auth_header(env['HTTP_AUTHORIZATION'])
if not handle_errors:
# Tell the publisher to skip exception views
env['x-wsgiorg.throw_errors'] = True
outstream = BytesIO()
response = WSGIResponse(stdout=outstream, stderr=sys.stderr)
request = Request(instream, env, response)
env['wsgi.input'] = instream
wsgi_headers = BytesIO()
def start_response(status, headers):
wsgi_headers.write(b'HTTP/1.1 ' + status.encode('ascii') + b'\r\n')
headers = b'\r\n'.join([(k + ': ' + v).encode('ascii')
for k, v in headers])
wsgi_headers.write(headers)
wsgi_headers.write(b'\r\n\r\n')
publish = partial(publish_module, _request=request, _response=response)
if handle_errors:
publish = HTTPExceptionHandler(publish)
wsgi_result = publish(env, start_response)
header_output.setResponseStatus(response.getStatus(), response.errmsg)
header_output.setResponseHeaders(response.headers)
header_output.headersl.extend(response._cookie_list())
header_output.appendResponseHeaders(response.accumulated_headers)
sync()
return DocResponseWrapper(response, outstream, path, header_output,
wsgi_result, wsgi_headers)
def _basic_auth_header(self, username, password):
user_pass = to_bytes('%s:%s' % (unquote(username), unquote(password)),
encoding=self.auth_encoding)
return base64.b64encode(user_pass).strip()
def entry_point(parameters):
item_type = None
action = parameters.get('action', None)
video_type = parameters.get('video_type', None)
title = unquote(parameters.get('title', ''))
year = parameters.get('year', '')
episode = parameters.get('episode', '')
season = parameters.get('season', '')
imdb_id = parameters.get('imdb_id', '')
if video_type == 'show' or video_type == 'season' or video_type == 'episode':
item_type = 'Series'
elif video_type == 'movie':
item_type = 'Movie'
if not item_type:
return
match = get_match(item_type, title, year, imdb_id)
if not match:
title_search_word = ''
title_words = title.split(' ')
for word in title_words:
if len(word) > len(title_search_word):
title_search_word = word
title_search_word = title_search_word.replace(':', '')
if title_search_word:
match = get_match(item_type, title_search_word, year, imdb_id)
str_season = str(season)
if len(str_season) == 1:
str_season = '0' + str_season
str_episode = str(episode)
if len(str_episode) == 1:
str_episode = '0' + str_episode
if action == 'play':
play_item_id = None
if video_type == 'movie':
if match:
play_item_id = match.get('ItemId')
if not play_item_id:
not_found('{title} ({year})'.format(title=title, year=year))
elif video_type == 'episode':
if not season or not episode:
return
if match:
item_id = match.get('ItemId')
season_id = get_season_id(item_id, season)
if season_id:
episode_id = get_episode_id(season_id, episode)
if episode_id:
play_item_id = episode_id
if not play_item_id:
not_found('{title} ({year}) - S{season}E{episode}'.format(title=title, year=year, season=str_season, episode=str_episode))
if play_item_id:
if video_type == 'episode':
playback_starting('{title} ({year}) - S{season}E{episode}'.format(title=title, year=year, season=str_season, episode=str_episode))
else:
playback_starting('{title} ({year})'.format(title=title, year=year))
xbmc.executebuiltin('RunPlugin(plugin://plugin.video.embycon/?mode=PLAY&item_id={item_id})'.format(item_id=play_item_id))
elif action == 'open':
url = media_type = None
if video_type == 'show':
if match:
item_id = match.get('ItemId')
media_type = 'series'
url = ('{server}/Shows/' + item_id +
'/Seasons'
'?userId={userid}' +
'&Fields=' + details_string +
'&format=json')
if not url:
not_found('{title} ({year})'.format(title=title, year=year))
elif video_type == 'season':
if not season:
return
if match:
item_id = match.get('ItemId')
season_id = get_season_id(item_id, season)
if season_id:
media_type = 'episodes'
url = ('{server}/Users/{userid}/items' +
'?ParentId=' + season_id +
'&IsVirtualUnAired=false' +
'&IsMissing=false' +
'&Fields=' + details_string +
'&format=json')
if not url:
not_found('{title} ({year}) - S{season}'.format(title=title, year=year, season=str_season))
if url and media_type:
xbmc.executebuiltin('ActivateWindow(Videos, plugin://plugin.video.embycon/?mode=GET_CONTENT&url={url}&media_type={media_type})'.format(url=quote(url), media_type=media_type))
def log_request(self, req, status_int, bytes_received, bytes_sent,
start_time, end_time, resp_headers=None):
"""
Log a request.
:param req: swob.Request object for the request
:param status_int: integer code for the response status
:param bytes_received: bytes successfully read from the request body
:param bytes_sent: bytes yielded to the WSGI server
:param start_time: timestamp request started
:param end_time: timestamp request completed
:param resp_headers: dict of the response headers
"""
resp_headers = resp_headers or {}
req_path = get_valid_utf8_str(req.path)
the_request = quote(unquote(req_path), QUOTE_SAFE)
if req.query_string:
the_request = the_request + '?' + req.query_string
logged_headers = None
if self.log_hdrs:
if self.log_hdrs_only:
logged_headers = '\n'.join('%s: %s' % (k, v)
for k, v in req.headers.items()
if k in self.log_hdrs_only)
else:
logged_headers = '\n'.join('%s: %s' % (k, v)
for k, v in req.headers.items())
method = self.method_from_req(req)
end_gmtime_str = time.strftime('%d/%b/%Y/%H/%M/%S',
time.gmtime(end_time))
duration_time_str = "%.4f" % (end_time - start_time)
start_time_str = "%.9f" % start_time
end_time_str = "%.9f" % end_time
policy_index = get_policy_index(req.headers, resp_headers)
self.access_logger.info(' '.join(
quote(str(x) if x else '-', QUOTE_SAFE)
for x in (
get_remote_client(req),
req.remote_addr,
end_gmtime_str,
method,
the_request,
req.environ.get('SERVER_PROTOCOL'),
status_int,
req.referer,
req.user_agent,
self.obscure_sensitive(req.headers.get('x-auth-token')),
bytes_received,
bytes_sent,
req.headers.get('etag', None),
req.environ.get('swift.trans_id'),
logged_headers,
duration_time_str,
req.environ.get('swift.source'),
','.join(req.environ.get('swift.log_info') or ''),
start_time_str,
end_time_str,
policy_index
)))
# Log timing and bytes-transferred data to StatsD
metric_name = self.statsd_metric_name(req, status_int, method)
metric_name_policy = self.statsd_metric_name_policy(req, status_int,
method,
policy_index)
# Only log data for valid controllers (or SOS) to keep the metric count
# down (egregious errors will get logged by the proxy server itself).
if metric_name:
self.access_logger.timing(metric_name + '.timing',
(end_time - start_time) * 1000)
self.access_logger.update_stats(metric_name + '.xfer',
bytes_received + bytes_sent)
if metric_name_policy:
self.access_logger.timing(metric_name_policy + '.timing',
(end_time - start_time) * 1000)
self.access_logger.update_stats(metric_name_policy + '.xfer',
bytes_received + bytes_sent)
def unquote_raw(value):
return urlparse.unquote(value)
def username(self):
"""Returns the username to use for fetching messages."""
return unquote(self._protocol_info.username)
def getChannelEpg(ref, begintime=-1, endtime=-1, encode=True):
ret = []
ev = {}
use_empty_ev = False
if ref:
ref = unquote(ref)
# When quering EPG we dont need URL, also getPicon doesn't like URL
if "://" in ref:
_ref = ":".join(ref.split(":")[:10]) + "::" + ref.split(":")[-1]
else:
_ref = ref
picon = getPicon(_ref)
epgcache = eEPGCache.getInstance()
events = epgcache.lookupEvent(['IBDTSENCW', (_ref, 0, begintime, endtime)])
if events is not None:
for event in events:
ev = {}
ev['picon'] = picon
ev['id'] = event[0]
if event[1]:
ev['date'] = "%s %s" % (tstrings[("day_" + strftime("%w", (localtime(event[1]))))], strftime("%d.%m.%Y", (localtime(event[1]))))
ev['begin'] = strftime("%H:%M", (localtime(event[1])))
ev['begin_timestamp'] = event[1]
ev['duration'] = int(event[2] / 60)
ev['duration_sec'] = event[2]
ev['end'] = strftime("%H:%M", (localtime(event[1] + event[2])))
ev['title'] = filterName(event[3], encode)
ev['shortdesc'] = convertDesc(event[4], encode)
ev['longdesc'] = convertDesc(event[5], encode)
ev['sref'] = ref
ev['sname'] = filterName(event[6], encode)
ev['tleft'] = int(((event[1] + event[2]) - event[7]) / 60)
if ev['duration_sec'] == 0:
ev['progress'] = 0
else:
ev['progress'] = int(((event[7] - event[1]) * 100 / event[2]) * 4)
ev['now_timestamp'] = event[7]
ev['genre'], ev['genreid'] = convertGenre(event[8])
ret.append(ev)
else:
use_empty_ev = True
ev['sref'] = ref
else:
use_empty_ev = True
ev['sref'] = ""
if use_empty_ev:
ev['date'] = 0
ev['begin'] = 0
ev['begin_timestamp'] = 0
ev['duration'] = 0
ev['duration_sec'] = 0
ev['end'] = 0
ev['title'] = "N/A"
ev['shortdesc'] = ""
ev['sname'] = ""
ev['longdesc'] = ""
ev['tleft'] = 0
ev['progress'] = 0
ev['now_timestamp'] = 0
ev['genre'] = ""
ev['genreid'] = 0
ret.append(ev)
return {"events": ret, "result": True}
def place_order(self,
instrument,
quantity=1,
bid_price=0.0,
transaction=None,
trigger='immediate',
order='market',
time_in_force='gfd'):
"""Place an order with Robinhood
Notes:
OMFG TEST THIS PLEASE!
Just realized this won't work since if type is LIMIT you need to use "price" and if
a STOP you need to use "stop_price". Oops.
Reference: https://github.com/sanko/Robinhood/blob/master/Order.md#place-an-order
Args:
instrument (dict): the RH URL and symbol in dict for the instrument to be traded
quantity (int): quantity of stocks in order
bid_price (float): price for order
transaction (:enum:`Transaction`): BUY or SELL enum
trigger (:enum:`Trigger`): IMMEDIATE or STOP enum
order (:enum:`Order`): MARKET or LIMIT
time_in_force (:enum:`TIME_IN_FORCE`): GFD or GTC (day or until cancelled)
Returns:
(:obj:`requests.request`): result from `orders` put command
"""
if isinstance(transaction, str):
transaction = Transaction(transaction)
if not bid_price:
bid_price = self.quote_data(instrument['symbol'])['bid_price']
payload = {
'account': self.get_account()['url'],
'instrument': unquote(instrument['url']),
'price': float(bid_price),
'quantity': quantity,
'side': transaction.name.lower(),
'symbol': instrument['symbol'],
'time_in_force': time_in_force.lower(),
'trigger': trigger,
'type': order.lower()
}
#data = 'account=%s&instrument=%s&price=%f&quantity=%d&side=%s&symbol=%s#&time_in_force=gfd&trigger=immediate&type=market' % (
# self.get_account()['url'],
# urllib.parse.unquote(instrument['url']),
# float(bid_price),
# quantity,
# transaction,
# instrument['symbol']
#)
res = self.session.post(endpoints.orders(), data=payload, timeout=15)
res.raise_for_status()
return res
def check_uri():
# type: () -> Tuple[unicode, unicode, int]
# split off anchor
if '#' in uri:
req_url, anchor = uri.split('#', 1)
for rex in self.anchors_ignore:
if rex.match(anchor):
anchor = None
break
else:
req_url = uri
anchor = None
# handle non-ASCII URIs
try:
req_url.encode('ascii')
except UnicodeError:
req_url = encode_uri(req_url)
try:
if anchor and self.app.config.linkcheck_anchors:
# Read the whole document and see if #anchor exists
response = requests.get(req_url,
stream=True,
config=self.app.config,
**kwargs)
found = check_anchor(response, unquote(anchor))
if not found:
raise Exception(__("Anchor '%s' not found") % anchor)
else:
try:
# try a HEAD request first, which should be easier on
# the server and the network
response = requests.head(req_url,
config=self.app.config,
**kwargs)
response.raise_for_status()
except HTTPError as err:
# retry with GET request if that fails, some servers
# don't like HEAD requests.
response = requests.get(req_url,
stream=True,
config=self.app.config,
**kwargs)
response.raise_for_status()
except HTTPError as err:
if err.response.status_code == 401:
# We'll take "Unauthorized" as working.
return 'working', ' - unauthorized', 0
else:
return 'broken', str(err), 0
except Exception as err:
if is_ssl_error(err):
return 'ignored', str(err), 0
else:
return 'broken', str(err), 0
if response.url.rstrip('/') == req_url.rstrip('/'):
return 'working', '', 0
else:
new_url = response.url
if anchor:
new_url += '#' + anchor
# history contains any redirects, get last
if response.history:
code = response.history[-1].status_code
return 'redirected', new_url, code
else:
return 'redirected', new_url, 0
def send_notification_for_subscriber(notif, bucket_name, object_path,
version_id, api_method, action,
event_name):
bucket_name = normalize_bucket_name(bucket_name)
if not event_type_matches(notif['Event'], action, api_method) or \
not filter_rules_match(notif.get('Filter'), object_path):
return
key = urlparse.unquote(object_path.replace('//', '/'))[1:]
s3_client = aws_stack.connect_to_service('s3')
try:
object_size = s3_client.head_object(Bucket=bucket_name,
Key=key).get('ContentLength', 0)
except botocore.exceptions.ClientError:
object_size = 0
# build event message
message = get_event_message(event_name=event_name,
bucket_name=bucket_name,
file_name=key,
file_size=object_size,
version_id=version_id)
message = json.dumps(message)
if notif.get('Queue'):
sqs_client = aws_stack.connect_to_service('sqs')
try:
queue_url = aws_stack.sqs_queue_url_for_arn(notif['Queue'])
sqs_client.send_message(QueueUrl=queue_url, MessageBody=message)
except Exception as e:
LOGGER.warning(
'Unable to send notification for S3 bucket "%s" to SQS queue "%s": %s'
% (bucket_name, notif['Queue'], e))
if notif.get('Topic'):
sns_client = aws_stack.connect_to_service('sns')
try:
sns_client.publish(TopicArn=notif['Topic'],
Message=message,
Subject='Amazon S3 Notification')
except Exception:
LOGGER.warning(
'Unable to send notification for S3 bucket "%s" to SNS topic "%s".'
% (bucket_name, notif['Topic']))
# CloudFunction and LambdaFunction are semantically identical
lambda_function_config = notif.get('CloudFunction') or notif.get(
'LambdaFunction')
if lambda_function_config:
# make sure we don't run into a socket timeout
connection_config = botocore.config.Config(read_timeout=300)
lambda_client = aws_stack.connect_to_service('lambda',
config=connection_config)
try:
lambda_client.invoke(FunctionName=lambda_function_config,
InvocationType='Event',
Payload=message)
except Exception:
LOGGER.warning(
'Unable to send notification for S3 bucket "%s" to Lambda function "%s".'
% (bucket_name, lambda_function_config))
if not filter(lambda x: notif.get(x), NOTIFICATION_DESTINATION_TYPES):
LOGGER.warning('Neither of %s defined for S3 notification.' %
'/'.join(NOTIFICATION_DESTINATION_TYPES))
def legacy_parse_uri(uri, to_quote):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
swift://account:user:[email protected]/container/obj
If to_quoted is True, the uri is assumed to have credentials that
have not been quoted, and the resulting uri will contain quoted
credentials.
If to_quoted is False, the uri is assumed to have credentials that
have been quoted, and the resulting uri will contain credentials
that have not been quoted.
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = _("URI cannot contain more than one occurrence of a scheme."
"If you have specified a URI like "
"swift://*****:*****@http://authurl.com/v1/container/obj"
", you need to change it to use the swift+http:// scheme, "
"like so: "
"swift+http://user:[email protected]/v1/container/obj")
raise exception.BadStoreUri(message=reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https')
scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
if netloc != '':
# > Python 2.6.1
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
else:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
netloc = path[0:path.find('/')].strip('/')
path = path[path.find('/'):].strip('/')
if creds:
cred_parts = creds.split(':')
# User can be account:user, in which case cred_parts[0:2] will be
# the account and user. Combine them into a single username of
# account:user
if to_quote:
if len(cred_parts) == 1:
reason = (_("Badly formed credentials '%(creds)s' in Swift "
"URI") % {
'creds': creds
})
raise exception.BadStoreUri(message=reason)
elif len(cred_parts) == 3:
user = '******'.join(cred_parts[0:2])
else:
user = cred_parts[0]
key = cred_parts[-1]
user = user
key = key
else:
if len(cred_parts) != 2:
reason = (_("Badly formed credentials in Swift URI."))
raise exception.BadStoreUri(message=reason)
user, key = cred_parts
user = urlparse.unquote(user)
key = urlparse.unquote(key)
else:
user = None
key = None
path_parts = path.split('/')
try:
obj = path_parts.pop()
container = path_parts.pop()
if not netloc.startswith('http'):
# push hostname back into the remaining to build full authurl
path_parts.insert(0, netloc)
auth_or_store_url = '/'.join(path_parts)
except IndexError:
reason = _("Badly formed S3 URI: %(uri)s") % {'uri': uri}
raise exception.BadStoreUri(message=reason)
if auth_or_store_url.startswith('http://'):
auth_or_store_url = auth_or_store_url[len('http://'):]
elif auth_or_store_url.startswith('https://'):
auth_or_store_url = auth_or_store_url[len('https://'):]
credstring = ''
if user and key:
if to_quote:
quote_user = urlparse.quote(user)
quote_key = urlparse.quote(key)
else:
quote_user = user
quote_key = key
credstring = '%s:%s@' % (quote_user, quote_key)
auth_or_store_url = auth_or_store_url.strip('/')
container = container.strip('/')
obj = obj.strip('/')
return '%s://%s%s/%s/%s' % (scheme, credstring, auth_or_store_url,
container, obj)
def authenticate_presign_url_signv4(method, path, headers, data, url, query_params, request_dict):
is_presign_valid = False
for port in PORT_REPLACEMENT:
match = re.match(HOST_COMBINATION_REGEX, urlparse.urlparse(request_dict["url"]).netloc)
if match and match.group(2):
request_dict["url"] = request_dict["url"].replace("%s" % match.group(2), "%s" % port)
else:
request_dict["url"] = "%s:%s" % (request_dict["url"], port)
# Calculating Signature
aws_request = create_request_object(request_dict)
ReadOnlyCredentials = namedtuple(
"ReadOnlyCredentials", ["access_key", "secret_key", "token"]
)
credentials = ReadOnlyCredentials(
TEST_AWS_ACCESS_KEY_ID,
TEST_AWS_SECRET_ACCESS_KEY,
query_params.get("X-Amz-Security-Token", None),
)
region = query_params["X-Amz-Credential"][0].split("/")[2]
signer = S3SigV4QueryAuth(
credentials, "s3", region, expires=int(query_params["X-Amz-Expires"][0])
)
signature = signer.add_auth(aws_request, query_params["X-Amz-Date"][0])
expiration_time = datetime.datetime.strptime(
query_params["X-Amz-Date"][0], "%Y%m%dT%H%M%SZ"
) + datetime.timedelta(seconds=int(query_params["X-Amz-Expires"][0]))
expiration_time = expiration_time.replace(tzinfo=datetime.timezone.utc)
# Comparing the signature in url with signature we calculated
query_sig = urlparse.unquote(query_params["X-Amz-Signature"][0])
if query_sig == signature:
is_presign_valid = True
break
# Comparing the signature in url with signature we calculated
if config.S3_SKIP_SIGNATURE_VALIDATION:
if not is_presign_valid:
LOGGER.warning(
"Signatures do not match, but not raising an error, as S3_SKIP_SIGNATURE_VALIDATION=1"
)
signature = query_sig
is_presign_valid = True
if not is_presign_valid:
return requests_error_response_xml_signature_calculation(
code=403,
code_string="SignatureDoesNotMatch",
aws_access_token=TEST_AWS_ACCESS_KEY_ID,
signature=signature,
message="The request signature we calculated does not match the signature you provided. \
Check your key and signing method.",
)
# Checking whether the url is expired or not
if is_expired(expiration_time):
return requests_error_response_xml_signature_calculation(
code=403,
code_string="AccessDenied",
message="Request has expired",
expires=query_params["X-Amz-Expires"][0],
)
def import_new_content(self, data): # noqa: C901
portal_workflow = api.portal.get_tool("portal_workflow")
added = []
if getattr(data, "len", None):
logger.info(u"Importing {} items".format(len(data)))
else:
logger.info(u"Importing data")
for index, item in enumerate(data, start=1):
if self.limit and len(added) >= self.limit:
break
uuid = item["UID"]
if uuid in self.DROP_UIDS:
continue
skip = False
for drop in self.DROP_PATHS:
if drop in item["@id"]:
skip = True
logger.info(u"Skipping {}".format(item['@id']))
if skip:
continue
if not index % 100:
logger.info("Imported {} items...".format(index))
new_id = unquote(item["@id"]).split("/")[-1]
if new_id != item["id"]:
logger.info(
u"Conflicting ids in url ({}) and id ({}). Using {}".
format(new_id, item["id"], new_id))
item["id"] = new_id
self.safe_portal_type = fix_portal_type(item["@type"])
item = self.handle_broken(item)
if not item:
continue
item = self.handle_dropped(item)
if not item:
continue
item = self.global_dict_hook(item)
if not item:
continue
# portal_type might change during a hook
self.safe_portal_type = fix_portal_type(item["@type"])
item = self.custom_dict_hook(item)
if not item:
continue
self.safe_portal_type = fix_portal_type(item["@type"])
container = self.handle_container(item)
if not container:
logger.info(
u"No container (parent was {}) found for {} {}".format(
item["parent"]["@type"], item["@type"], item["@id"]))
continue
factory_kwargs = item.get("factory_kwargs", {})
# Handle existing content
self.update_existing = False
if new_id in container:
if self.handle_existing_content == 0:
# Skip
logger.info(u"{} ({}) already exists. Skipping it.".format(
new_id, item["@id"]))
continue
elif self.handle_existing_content == 1:
# Replace content before creating it new
logger.info(
u"{} ({}) already exists. Replacing it.".format(
new_id, item["@id"]))
api.content.delete(container[new_id],
check_linkintegrity=False)
elif self.handle_existing_content == 2:
# Update existing item
logger.info(u"{} ({}) already exists. Updating it.".format(
new_id, item["@id"]))
self.update_existing = True
new = container[new_id]
else:
# Create with new id. Speed up by using random id.
duplicate = new_id
new_id = "{}-{}".format(new_id, random.randint(1000, 9999))
item["id"] = new_id
logger.info(
u"{} ({}) already exists. Created as {}".format(
duplicate, item["@id"], new_id))
if not self.update_existing:
# create without checking constrains and permissions
new = _createObjectByType(item["@type"], container, item["id"],
**factory_kwargs)
new, item = self.global_obj_hook_before_deserializing(new, item)
# import using plone.restapi deserializers
deserializer = getMultiAdapter((new, self.request),
IDeserializeFromJson)
try:
new = deserializer(validate_all=False, data=item)
except Exception as error:
logger.warning("cannot deserialize {}: {}".format(
item["@id"], repr(error)))
continue
# Blobs can be exported as only a path in the blob storage.
# It seems difficult to dynamically use a different deserializer,
# based on whether or not there is a blob_path somewhere in the item.
# So handle this case with a separate method.
self.import_blob_paths(new, item)
self.import_constrains(new, item)
self.global_obj_hook(new, item)
self.custom_obj_hook(new, item)
uuid = self.set_uuid(item, new)
if uuid != item["UID"]:
item["UID"] = uuid
if item["review_state"] and item["review_state"] != "private":
if portal_workflow.getChainFor(new):
try:
api.content.transition(to_state=item["review_state"],
obj=new)
except InvalidParameterError as e:
logger.info(e)
# Import workflow_history last to drop entries created during import
self.import_workflow_history(new, item)
# Set modification and creation-date as a custom attribute as last step.
# These are reused and dropped in ResetModifiedAndCreatedDate
modified = item.get("modified", item.get("modification_date",
None))
if modified:
modification_date = DateTime(dateutil.parser.parse(modified))
new.modification_date = modification_date
new.aq_base.modification_date_migrated = modification_date
created = item.get("created", item.get("creation_date", None))
if created:
creation_date = DateTime(dateutil.parser.parse(created))
new.creation_date = creation_date
new.aq_base.creation_date_migrated = creation_date
logger.info("Created item #{}: {} {}".format(
index, item["@type"], new.absolute_url()))
added.append(new.absolute_url())
if self.commit and not len(added) % self.commit:
self.commit_hook(added, index)
return added
def doMerge(outputio,
files,
authoropts=[],
titleopt=None,
descopt=None,
tags=[],
languages=['en'],
titlenavpoints=True,
originalnavpoints=True,
flattentoc=False,
printtimes=False,
coverjpgpath=None,
keepmetadatafiles=False,
source=None,
notify_progress=lambda x:x):
'''
outputio = output file name or BytesIO.
files = list of input file names or BytesIOs.
authoropts = list of authors to use, otherwise add from all input
titleopt = title, otherwise '<first title> Anthology'
descopt = description, otherwise '<title> by <author>' list for all input
tags = dc:subject tags to include, otherwise none.
languages = dc:language tags to include
titlenavpoints if true, put in a new TOC entry for each epub, nesting each epub's chapters under it
originalnavpoints if true, include the original TOCs from each epub
flattentoc if true, flatten TOC down to one level only.
coverjpgpath, Path to a jpg to use as cover image.
'''
notify_progress(0.0) # sets overall progress to 50%
printt = partial(cond_print,printtimes)
## Python 2.5 ZipFile is rather more primative than later
## versions. It can operate on a file, or on a BytesIO, but
## not on an open stream. OTOH, I suspect we would have had
## problems with closing and opening again to change the
## compression type anyway.
filecount=0
t = time()
## Write mimetype file, must be first and uncompressed.
## Older versions of python(2.4/5) don't allow you to specify
## compression by individual file.
## Overwrite if existing output file.
outputepub = ZipFile(outputio, "w", compression=ZIP_STORED, allowZip64=True)
outputepub.debug = 3
outputepub.writestr("mimetype", "application/epub+zip")
outputepub.close()
## Re-open file for content.
outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED, allowZip64=True)
outputepub.debug = 3
## Create META-INF/container.xml file. The only thing it does is
## point to content.opf
containerdom = getDOMImplementation().createDocument(None, "container", None)
containertop = containerdom.documentElement
containertop.setAttribute("version","1.0")
containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")
rootfiles = containerdom.createElement("rootfiles")
containertop.appendChild(rootfiles)
rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf",
"media-type":"application/oebps-package+xml"}))
outputepub.writestr("META-INF/container.xml",containerdom.toprettyxml(indent=' ',encoding='utf-8'))
## Process input epubs.
items = [] # list of (id, href, type) tuples(all strings) -- From .opfs' manifests
items.append(("ncx","toc.ncx","application/x-dtbncx+xml")) ## we'll generate the toc.ncx file,
## but it needs to be in the items manifest.
itemrefs = [] # list of strings -- idrefs from .opfs' spines
navmaps = [] # list of navMap DOM elements -- TOC data for each from toc.ncx files
is_ffdl_epub = [] # list of t/f
itemhrefs = {} # hash of item[id]s to itemref[href]s -- to find true start of book(s).
firstitemhrefs = []
booktitles = [] # list of strings -- Each book's title
allauthors = [] # list of lists of strings -- Each book's list of authors.
filelist = []
printt("prep output:%s"%(time()-t))
t = time()
booknum=1
firstmetadom = None
for file in files:
if file == None : continue
book = "%d" % booknum
bookdir = "%d/" % booknum
bookid = "a%d" % booknum
epub = ZipFile(file, 'r')
## Find the .opf file.
container = epub.read("META-INF/container.xml")
containerdom = parseString(container)
rootfilenodelist = containerdom.getElementsByTagNameNS("*","rootfile")
rootfilename = rootfilenodelist[0].getAttribute("full-path")
## Save the path to the .opf file--hrefs inside it are relative to it.
relpath = get_path_part(rootfilename)
metadom = parseString(epub.read(rootfilename))
# logger.debug("metadom:%s"%epub.read(rootfilename))
if booknum==1 and not source:
try:
firstmetadom = metadom.getElementsByTagNameNS("*","metadata")[0]
source=unicode(firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data)
except:
source=""
is_ffdl_epub.append(False)
## looking for any of:
## <dc:contributor id="id-2">FanFicFare [https://github.com/JimmXinu/FanFicFare]</dc:contributor>
## <dc:identifier opf:scheme="FANFICFARE-UID">test1.com-u98765-s68</dc:identifier>
## <dc:identifier id="fanficfare-uid">fanficfare-uid:test1.com-u98765-s68</dc:identifier>
## FFF writes dc:contributor and dc:identifier
## Sigil changes the unique-identifier, but leaves dc:contributor
## Calibre epub3->epub2 convert changes dc:contributor and modifies dc:identifier
for c in metadom.getElementsByTagName("dc:contributor") + metadom.getElementsByTagName("dc:identifier"):
# logger.debug("dc:contributor/identifier:%s"%getText(c.childNodes))
# logger.debug("dc:contributor/identifier:%s / %s"%(c.getAttribute('opf:scheme'),c.getAttribute('id')))
if ( getText(c.childNodes) in ["fanficdownloader [http://fanficdownloader.googlecode.com]",
"FanFicFare [https://github.com/JimmXinu/FanFicFare]"]
or 'fanficfare-uid' in c.getAttribute('opf:scheme').lower()
or 'fanficfare-uid' in c.getAttribute('id').lower() ):
# logger.debug("------------> is_ffdl_epub <-----------------")
is_ffdl_epub[-1] = True # set last.
break;
## Save indiv book title
try:
booktitles.append(metadom.getElementsByTagName("dc:title")[0].firstChild.data)
except:
booktitles.append("(Title Missing)")
## Save authors.
authors=[]
for creator in metadom.getElementsByTagName("dc:creator"):
try:
if( creator.getAttribute("opf:role") == "aut" or not creator.hasAttribute("opf:role") and creator.firstChild != None):
authors.append(creator.firstChild.data)
except:
pass
if len(authors) == 0:
authors.append("(Author Missing)")
allauthors.append(authors)
if keepmetadatafiles:
itemid=bookid+"rootfile"
itemhref = rootfilename
href=bookdir+itemhref
logger.debug("write rootfile %s to %s"%(itemhref,href))
outputepub.writestr(href,
epub.read(itemhref))
items.append((itemid,href,"origrootfile/xml"))
# spin through the manifest--only place there are item tags.
# Correction--only place there *should* be item tags. But
# somebody found one that did.
manifesttag=metadom.getElementsByTagNameNS("*","manifest")[0]
for item in manifesttag.getElementsByTagNameNS("*","item"):
itemid=bookid+item.getAttribute("id")
itemhref = normpath(unquote(item.getAttribute("href"))) # remove %20, etc.
href=bookdir+relpath+itemhref
# if item.getAttribute("properties") == "nav":
# # epub3 TOC file is only one with this type--as far as I know.
# # grab the whole navmap, deal with it later.
# el
if item.getAttribute("media-type") == "application/x-dtbncx+xml":
# epub2 TOC file is only one with this type--as far as I know.
# grab the whole navmap, deal with it later.
tocdom = parseString(epub.read(normpath(relpath+item.getAttribute("href"))))
# update all navpoint ids with bookid for uniqueness.
for navpoint in tocdom.getElementsByTagNameNS("*","navPoint"):
navpoint.setAttribute("id",bookid+navpoint.getAttribute("id"))
# update all content paths with bookdir for uniqueness.
for content in tocdom.getElementsByTagNameNS("*","content"):
content.setAttribute("src",normpath(bookdir+relpath+content.getAttribute("src")))
navmaps.append(tocdom.getElementsByTagNameNS("*","navMap")[0])
if keepmetadatafiles:
logger.debug("write toc.ncx %s to %s"%(relpath+itemhref,href))
outputepub.writestr(href,
epub.read(normpath(relpath+itemhref)))
items.append((itemid,href,"origtocncx/xml"))
else:
#href=href.encode('utf8')
# logger.debug("item id: %s -> %s:"%(itemid,href))
itemhrefs[itemid] = href
if href not in filelist:
try:
outputepub.writestr(href,
epub.read(normpath(relpath+itemhref)))
if re.match(r'.*/(file|chapter)\d+\.x?html',href):
filecount+=1
items.append((itemid,href,item.getAttribute("media-type")))
filelist.append(href)
except KeyError as ke: # Skip missing files.
logger.info("Skipping missing file %s (%s)"%(href,relpath+itemhref))
del itemhrefs[itemid]
itemreflist = metadom.getElementsByTagNameNS("*","itemref")
# logger.debug("itemhrefs:%s"%itemhrefs)
logger.debug("bookid:%s"%bookid)
logger.debug("itemreflist[0].getAttribute(idref):%s"%itemreflist[0].getAttribute("idref"))
# Looking for the first item in itemreflist that wasn't
# discarded due to missing files.
for itemref in itemreflist:
idref = bookid+itemref.getAttribute("idref")
if idref in itemhrefs:
firstitemhrefs.append(itemhrefs[idref])
break
for itemref in itemreflist:
itemrefs.append(bookid+itemref.getAttribute("idref"))
# logger.debug("adding to itemrefs:%s"%itemref.toprettyxml())
notify_progress(float(booknum-1)/len(files))
booknum=booknum+1;
printt("after file loop:%s"%(time()-t))
t = time()
## create content.opf file.
uniqueid="epubmerge-uid-%d" % time() # real sophisticated uid scheme.
contentdom = getDOMImplementation().createDocument(None, "package", None)
package = contentdom.documentElement
package.setAttribute("version","2.0")
package.setAttribute("xmlns","http://www.idpf.org/2007/opf")
package.setAttribute("unique-identifier","epubmerge-id")
metadata=newTag(contentdom,"metadata",
attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/",
"xmlns:opf":"http://www.idpf.org/2007/opf"})
metadata.appendChild(newTag(contentdom,"dc:identifier",text=uniqueid,attrs={"id":"epubmerge-id"}))
if( titleopt is None ):
titleopt = booktitles[0]+" Anthology"
metadata.appendChild(newTag(contentdom,"dc:title",text=titleopt))
# If cmdline authors, use those instead of those collected from the epubs
# (allauthors kept for TOC & description gen below.
if( len(authoropts) > 1 ):
useauthors=[authoropts]
else:
useauthors=allauthors
usedauthors=dict()
for authorlist in useauthors:
for author in authorlist:
if( author not in usedauthors ):
usedauthors[author]=author
metadata.appendChild(newTag(contentdom,"dc:creator",
attrs={"opf:role":"aut"},
text=author))
metadata.appendChild(newTag(contentdom,"dc:contributor",text="epubmerge"))
metadata.appendChild(newTag(contentdom,"dc:rights",text="Copyrights as per source stories"))
for l in languages:
metadata.appendChild(newTag(contentdom,"dc:language",text=l))
if not descopt:
# created now, but not filled in until TOC generation to save loops.
description = newTag(contentdom,"dc:description",text="Anthology containing:\n")
else:
description = newTag(contentdom,"dc:description",text=descopt)
metadata.appendChild(description)
if source:
metadata.appendChild(newTag(contentdom,"dc:identifier",
attrs={"opf:scheme":"URL"},
text=source))
metadata.appendChild(newTag(contentdom,"dc:source",
text=source))
for tag in tags:
metadata.appendChild(newTag(contentdom,"dc:subject",text=tag))
package.appendChild(metadata)
manifest = contentdom.createElement("manifest")
package.appendChild(manifest)
spine = newTag(contentdom,"spine",attrs={"toc":"ncx"})
package.appendChild(spine)
if coverjpgpath:
# in case coverjpg isn't a jpg:
coverext = 'jpg'
covertype = 'image/jpeg'
try:
coverext = coverjpgpath.split('.')[-1].lower()
covertype = imagetypes.get(coverext,covertype)
except:
pass
logger.debug("coverjpgpath:%s coverext:%s covertype:%s"%(coverjpgpath,coverext,covertype))
# <meta name="cover" content="cover.jpg"/>
metadata.appendChild(newTag(contentdom,"meta",{"name":"cover",
"content":"coverimageid"}))
guide = newTag(contentdom,"guide")
guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
"title":"Cover",
"href":"cover.xhtml"}))
package.appendChild(guide)
manifest.appendChild(newTag(contentdom,"item",
attrs={'id':"coverimageid",
'href':"cover."+coverext,
'media-type':covertype}))
# Note that the id of the cover xhmtl *must* be 'cover'
# for it to work on Nook.
manifest.appendChild(newTag(contentdom,"item",
attrs={'id':"cover",
'href':"cover.xhtml",
'media-type':"application/xhtml+xml"}))
spine.appendChild(newTag(contentdom,"itemref",
attrs={"idref":"cover",
"linear":"yes"}))
for item in items:
# logger.debug("new item:%s %s %s"%item)
(id,href,type)=item
manifest.appendChild(newTag(contentdom,"item",
attrs={'id':id,
'href':href,
'media-type':type}))
for itemref in itemrefs:
# logger.debug("itemref:%s"%itemref)
spine.appendChild(newTag(contentdom,"itemref",
attrs={"idref":itemref,
"linear":"yes"}))
## create toc.ncx file
tocncxdom = getDOMImplementation().createDocument(None, "ncx", None)
ncx = tocncxdom.documentElement
ncx.setAttribute("version","2005-1")
ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/")
head = tocncxdom.createElement("head")
ncx.appendChild(head)
head.appendChild(newTag(tocncxdom,"meta",
attrs={"name":"dtb:uid", "content":uniqueid}))
depthnode = newTag(tocncxdom,"meta",
attrs={"name":"dtb:depth", "content":"4"})
head.appendChild(depthnode)
head.appendChild(newTag(tocncxdom,"meta",
attrs={"name":"dtb:totalPageCount", "content":"0"}))
head.appendChild(newTag(tocncxdom,"meta",
attrs={"name":"dtb:maxPageNumber", "content":"0"}))
docTitle = tocncxdom.createElement("docTitle")
docTitle.appendChild(newTag(tocncxdom,"text",text=titleopt))
ncx.appendChild(docTitle)
tocnavMap = tocncxdom.createElement("navMap")
ncx.appendChild(tocnavMap)
booknum=0
printt("wrote initial metadata:%s"%(time()-t))
t = time()
for navmap in navmaps:
depthnavpoints = navmap.getElementsByTagNameNS("*","navPoint") # for checking more than one TOC entry
# logger.debug( [ x.toprettyxml() for x in navmap.childNodes ] )
## only gets top level TOC entries. sub entries carried inside.
navpoints = [ x for x in navmap.childNodes if isinstance(x,Element) and x.tagName=="navPoint" ]
# logger.debug("len(navpoints):%s"%len(navpoints))
# logger.debug( [ x.toprettyxml() for x in navpoints ] )
newnav = None
if titlenavpoints:
newnav = newTag(tocncxdom,"navPoint",{"id":"book%03d"%booknum})
navlabel = newTag(tocncxdom,"navLabel")
newnav.appendChild(navlabel)
# For purposes of TOC titling & desc, use first book author. Skip adding author if only one.
if len(usedauthors) > 1:
title = booktitles[booknum]+" by "+allauthors[booknum][0]
else:
title = booktitles[booknum]
navlabel.appendChild(newTag(tocncxdom,"text",text=title))
# Find the first 'spine' item's content for the title navpoint.
# Many epubs have the first chapter as first navpoint, so we can't just
# copy that anymore.
newnav.appendChild(newTag(tocncxdom,"content",
{"src":firstitemhrefs[booknum]}))
# logger.debug("newnav:%s"%newnav.toprettyxml())
tocnavMap.appendChild(newnav)
# logger.debug("tocnavMap:%s"%tocnavMap.toprettyxml())
else:
newnav = tocnavMap
if not descopt and len(allauthors[booknum]) > 0:
description.appendChild(contentdom.createTextNode(booktitles[booknum]+" by "+allauthors[booknum][0]+"\n"))
# If only one TOC point(total, not top level), or if not
# including title nav point, include sub book TOC entries.
if originalnavpoints and (len(depthnavpoints) > 1 or not titlenavpoints):
for navpoint in navpoints:
# logger.debug("navpoint:%s"%navpoint.toprettyxml())
newnav.appendChild(navpoint)
navpoint.is_ffdl_epub = is_ffdl_epub[booknum]
booknum=booknum+1;
# end of navmaps loop.
maxdepth = 0
contentsrcs = {}
removednodes = []
## Force strict ordering of playOrder, stripping out some.
playorder=0
# logger.debug("tocncxdom:%s"%tocncxdom.toprettyxml())
for navpoint in tocncxdom.getElementsByTagNameNS("*","navPoint"):
# logger.debug("navpoint:%s"%navpoint.toprettyxml())
if navpoint in removednodes:
continue
# need content[src] to compare for dups. epub wants dup srcs to have same playOrder.
contentsrc = None
for n in navpoint.childNodes:
if isinstance(n,Element) and n.tagName == "content":
contentsrc = n.getAttribute("src")
# logger.debug("contentsrc: %s"%contentsrc)
break
if( contentsrc not in contentsrcs ):
parent = navpoint.parentNode
try:
# if the epub was ever edited with Sigil, it changed
# the id, but the file name is the same.
if navpoint.is_ffdl_epub and \
( navpoint.getAttribute("id").endswith('log_page') \
or contentsrc.endswith("log_page.xhtml") ):
logger.debug("Doing sibs 'filter' 1")
sibs = [ x for x in parent.childNodes if isinstance(x,Element) and x.tagName=="navPoint" ]
# if only logpage and one chapter, remove them from TOC and just show story.
if len(sibs) == 2:
parent.removeChild(navpoint)
logger.debug("Removing %s:"% sibs[0].getAttribute("playOrder"))
parent.removeChild(sibs[1])
removednodes.append(sibs[1])
except:
pass
# New src, new number.
contentsrcs[contentsrc] = navpoint.getAttribute("id")
playorder += 1
navpoint.setAttribute("playOrder","%d" % playorder)
# logger.debug("playorder:%d:"%playorder)
# need to know depth of deepest navpoint for <meta name="dtb:depth" content="2"/>
npdepth = 1
dp = navpoint.parentNode
while dp and dp.tagName != "navMap":
npdepth += 1
dp = dp.parentNode
if npdepth > maxdepth:
maxdepth = npdepth
else:
# same content, look for ffdl and title_page and/or single chapter.
# easier to just set it now, even if the node gets removed later.
navpoint.setAttribute("playOrder","%d" % playorder)
logger.debug("playorder:%d:"%playorder)
parent = navpoint.parentNode
try:
# if the epub was ever edited with Sigil, it changed
# the id, but the file name is the same.
if navpoint.is_ffdl_epub and \
( navpoint.getAttribute("id").endswith('title_page') \
or contentsrc.endswith("title_page.xhtml") ):
parent.removeChild(navpoint)
logger.debug("Doing sibs 'filter' 2")
sibs = [ x for x in parent.childNodes if isinstance(x,Element) and x.tagName=="navPoint" ]
# if only one chapter after removing title_page, remove it too.
if len(sibs) == 1:
logger.debug("Removing %s:"% sibs[0].getAttribute("playOrder"))
parent.removeChild(sibs[0])
removednodes.append(sibs[0])
except:
pass
if flattentoc:
maxdepth = 1
# already have play order and pesky dup/single chapters
# removed, just need to flatten.
flattocnavMap = tocncxdom.createElement("navMap")
for n in tocnavMap.getElementsByTagNameNS("*","navPoint"):
flattocnavMap.appendChild(n)
ncx.replaceChild(flattocnavMap,tocnavMap)
printt("navmap/toc maddess:%s"%(time()-t))
t = time()
depthnode.setAttribute("content","%d"%maxdepth)
## content.opf written now due to description being filled in
## during TOC generation to save loops.
contentxml = contentdom.toprettyxml(indent=' ',encoding='utf-8')
# tweak for brain damaged Nook STR. Nook insists on name before content.
contentxml = contentxml.replace(ensure_binary('<meta content="coverimageid" name="cover"/>'),
ensure_binary('<meta name="cover" content="coverimageid"/>'))
outputepub.writestr("content.opf",contentxml)
outputepub.writestr("toc.ncx",tocncxdom.toprettyxml(indent=' ',encoding='utf-8'))
printt("wrote opf/ncx files:%s"%(time()-t))
t = time()
if coverjpgpath:
# write, not write string. Pulling from file.
outputepub.write(coverjpgpath,"cover."+coverext)
outputepub.writestr("cover.xhtml",'''
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head><title>Cover</title><style type="text/css" title="override_css">
@page {padding: 0pt; margin:0pt}
body { text-align: center; padding:0pt; margin: 0pt; }
div { margin: 0pt; padding: 0pt; }
</style></head><body><div>
<img src="cover.'''+coverext+'''" alt="cover"/>
</div></body></html>
''')
# declares all the files created by Windows. otherwise, when
# it runs in appengine, windows unzips the files as 000 perms.
for zf in outputepub.filelist:
zf.create_system = 0
outputepub.close()
printt("closed outputepub:%s"%(time()-t))
t = time()
return (source,filecount)
def is_local_repository(self):
url = self.get_remote_url()
return url.startswith(self.file_protocol) and \
os.path.exists(unquote(url[len(self.file_protocol):]))
def parse_data_uri(uri):
"""
Parse a data: URI, returning a 3-tuple of media type, dictionary of media
type parameters, and data.
"""
if not isinstance(uri, bytes):
uri = safe_url_string(uri).encode('ascii')
try:
scheme, uri = uri.split(b':', 1)
except ValueError:
raise ValueError("invalid URI")
if scheme.lower() != b'data':
raise ValueError("not a data URI")
# RFC 3986 section 2.1 allows percent encoding to escape characters that
# would be interpreted as delimiters, implying that actual delimiters
# should not be percent-encoded.
# Decoding before parsing will allow malformed URIs with percent-encoded
# delimiters, but it makes parsing easier and should not affect
# well-formed URIs, as the delimiters used in this URI scheme are not
# allowed, percent-encoded or not, in tokens.
if six.PY2:
uri = unquote(uri)
else:
uri = unquote_to_bytes(uri)
media_type = "text/plain"
media_type_params = {}
m = _mediatype_pattern.match(uri)
if m:
media_type = m.group().decode()
uri = uri[m.end():]
else:
media_type_params['charset'] = "US-ASCII"
while True:
m = _mediatype_parameter_pattern.match(uri)
if m:
attribute, value, value_quoted = m.groups()
if value_quoted:
value = re.sub(br'\\(.)', r'\1', value_quoted)
media_type_params[attribute.decode()] = value.decode()
uri = uri[m.end():]
else:
break
try:
is_base64, data = uri.split(b',', 1)
except ValueError:
raise ValueError("invalid data URI")
if is_base64:
if is_base64 != b";base64":
raise ValueError("invalid data URI")
data = base64.b64decode(data)
return _ParseDataURIResult(media_type, media_type_params, data)
def doUnMerge(inputio,outdir=None):
epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
outputios = []
## Find the .opf file.
container = epub.read("META-INF/container.xml")
containerdom = parseString(container)
rootfilenodelist = containerdom.getElementsByTagName("rootfile")
rootfilename = rootfilenodelist[0].getAttribute("full-path")
contentdom = parseString(epub.read(rootfilename))
## Save the path to the .opf file--hrefs inside it are relative to it.
relpath = get_path_part(rootfilename)
logger.debug("relpath:%s"%relpath)
# spin through the manifest--only place there are item tags.
# Correction--only place there *should* be item tags. But
# somebody found one that did.
manifesttag=contentdom.getElementsByTagNameNS("*","manifest")[0]
for item in manifesttag.getElementsByTagNameNS("*","item"):
# look for our fake media-type for original rootfiles.
if( item.getAttribute("media-type") == "origrootfile/xml" ):
# found one, assume the dir containing it is a complete
# original epub, do initial setup of epub.
itemhref = normpath(relpath+unquote(item.getAttribute("href")))
logger.debug("Found origrootfile:%s"%itemhref)
curepubpath = re.sub(r'([^\d/]+/)+$','',get_path_part(itemhref))
savehref = itemhref[len(curepubpath):]
logger.debug("curepubpath:%s"%curepubpath)
outputio = BytesIO()
outputepub = ZipFile(outputio, "w", compression=ZIP_STORED, allowZip64=True)
outputepub.debug = 3
outputepub.writestr("mimetype", "application/epub+zip")
outputepub.close()
## Re-open file for content.
outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED, allowZip64=True)
outputepub.debug = 3
## Create META-INF/container.xml file. The only thing it does is
## point to content.opf
containerdom = getDOMImplementation().createDocument(None, "container", None)
containertop = containerdom.documentElement
containertop.setAttribute("version","1.0")
containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")
rootfiles = containerdom.createElement("rootfiles")
containertop.appendChild(rootfiles)
rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":savehref,
"media-type":"application/oebps-package+xml"}))
outputepub.writestr("META-INF/container.xml",containerdom.toprettyxml(indent=' ',encoding='utf-8'))
outputepub.writestr(savehref,epub.read(itemhref))
for item2 in contentdom.getElementsByTagName("item"):
item2href = normpath(relpath+unquote(item2.getAttribute("href")))
if item2href.startswith(curepubpath) and item2href != itemhref:
save2href = item2href[len(curepubpath):]
logger.debug("Found %s -> %s"%(item2href,save2href))
outputepub.writestr(save2href,epub.read(item2href))
# declares all the files created by Windows. otherwise, when
# it runs in appengine, windows unzips the files as 000 perms.
for zf in outputepub.filelist:
zf.create_system = 0
outputepub.close()
outputios.append(outputio)
if outdir:
outfilenames=[]
for count,epubIO in enumerate(outputios):
filename="%s/%d.epub"%(outdir,count)
logger.debug("write %s"%filename)
outstream = open(filename,"wb")
outstream.write(epubIO.getvalue())
outstream.close()
outfilenames.append(filename)
return outfilenames
else:
return outputios
def _unquote(s):
return urlparse.unquote(s)
def _setUp(self):
super(GnocchiDriver, self)._setUp()
try:
shutil.copy(self.find_config_file("gnocchi/api-paste.ini"),
self.tempdir)
except RuntimeError:
pass
try:
shutil.copy(self.find_config_file("gnocchi/policy.json"),
self.tempdir)
except RuntimeError:
pass
if self.indexer_url is None:
pg = self.useFixture(
postgresql.PostgreSQLDriver(port=self.indexer_port))
self.indexer_url = pg.url
if self.storage_url is None:
self.storage_url = "file://%s" % self.tempdir
conffile = os.path.join(self.tempdir, "gnocchi.conf")
storage_parsed = urlparse.urlparse(self.storage_url)
storage_driver = storage_parsed.scheme
if storage_driver == "s3":
storage_config = {
"s3_access_key_id": (urlparse.unquote(storage_parsed.username
or "gnocchi")),
"s3_secret_access_key":
(urlparse.unquote(storage_parsed.password or "whatever")),
"s3_endpoint_url":
"http://%s:%s/%s" % (
storage_parsed.hostname,
storage_parsed.port,
storage_parsed.path,
)
}
elif storage_driver == "swift":
storage_config = {
"swift_authurl":
"http://%s:%s%s" % (
storage_parsed.hostname,
storage_parsed.port,
storage_parsed.path,
),
"swift_user": (urlparse.unquote(storage_parsed.username
or "admin:admin")),
"swift_key": (urlparse.unquote(storage_parsed.password
or "admin")),
}
elif storage_driver == "ceph":
storage_config = {
"ceph_conffile": storage_parsed.path,
}
elif storage_driver == "redis":
storage_config = {
"redis_url": self.storage_url,
}
elif storage_driver == "file":
storage_config = {
"file_basepath": (storage_parsed.path or self.tempdir),
}
else:
raise RuntimeError("Storage driver %s is not supported" %
storage_driver)
if self.coordination_driver == "redis":
r = self.useFixture(redis.RedisDriver(port=self.coordination_port))
storage_config["coordination_url"] = r.url
storage_config_string = "\n".join("%s = %s" % (k, v)
for k, v in storage_config.items())
statsd_resource_id = str(uuid.uuid4())
with open(conffile, "w") as f:
f.write("""[DEFAULT]
debug = %s
verbose = True
[api]
host = localhost
port = %s
[storage]
driver = %s
%s
[metricd]
metric_processing_delay = 1
metric_cleanup_delay = 1
[statsd]
resource_id = %s
creator = admin
user_id = admin
project_id = admin
[indexer]
url = %s""" % (self.debug, self.port, storage_driver, storage_config_string,
statsd_resource_id, self.indexer_url))
self._exec(["gnocchi-upgrade", "--config-file=%s" % conffile])
c, _ = self._exec(["gnocchi-metricd",
"--config-file=%s" % conffile],
wait_for_line="metrics wait to be processed")
c, _ = self._exec(
["gnocchi-statsd", "--config-file=%s" % conffile],
wait_for_line=("(Resource .* already exists"
"|Created resource )"))
_, v = self._exec(["gnocchi-api", "--", "--version"], stdout=True)
v = version.LooseVersion(drivers.fsdecode(v).strip())
if v < version.LooseVersion("4.1.0"):
LOG.debug("gnocchi version: %s, running uwsgi manually for api", v)
args = [
"uwsgi",
"--http",
"localhost:%d" % self.port,
"--wsgi-file",
spawn.find_executable("gnocchi-api"),
"--master",
"--die-on-term",
"--lazy-apps",
"--processes",
"4",
"--no-orphans",
"--enable-threads",
"--chdir",
self.tempdir,
"--add-header",
"Connection: close",
"--pyargv",
"--config-file=%s" % conffile,
]
virtual_env = os.getenv("VIRTUAL_ENV")
if virtual_env is not None:
args.extend(["-H", virtual_env])
else:
LOG.debug("gnocchi version: %s, running gnocchi-api", v)
args = ["gnocchi-api", "--config-file=%s" % conffile]
c, _ = self._exec(args,
wait_for_line="WSGI app 0 \(mountpoint=''\) ready")
self.addCleanup(self._kill, c)
self.http_url = "http://localhost:%d" % self.port
self.putenv("GNOCCHI_PORT", str(self.port))
self.putenv("URL", "gnocchi://localhost:%d" % self.port)
self.putenv("GNOCCHI_HTTP_URL", self.http_url)
self.putenv("GNOCCHI_ENDPOINT", self.http_url, True)
self.putenv("OS_AUTH_TYPE", "gnocchi-basic", True)
self.putenv("GNOCCHI_STATSD_RESOURCE_ID", statsd_resource_id, True)
self.putenv("GNOCCHI_USER", "admin", True)
def _lookup(self, recipient, *remainder):
recipient = unquote(recipient)
return GrantController(self.neighborhood, self.award,
recipient), remainder
def _lookup(self, filename, *args):
if args:
filename = unquote(filename)
else:
filename = unquote(request.path.rsplit(str('/'), 1)[-1])
return ScreenshotController(filename), args
def FillE2TimerList(xmlstring, sreference=None):
E2TimerList = []
try:
root = xml.etree.cElementTree.fromstring(xmlstring)
except:
return E2TimerList
if sreference is None:
sreference = None
else:
sreference = getServiceRef(sreference)
for timer in root.findall("e2timer"):
go = False
state = 0
try:
state = int(timer.findtext("e2state", 0))
except:
state = 0
disabled = 0
try:
disabled = int(timer.findtext("e2disabled", 0))
except:
disabled = 0
servicereference = str(
timer.findtext("e2servicereference",
'').decode("utf-8").encode("utf-8", 'ignore'))
if sreference is None:
go = True
else:
servicereference_str = ':'.join(
str(servicereference).split(':')[:11])
sreference_str = ':'.join(str(sreference).split(':')[:11])
if sreference_str.upper() == servicereference_str.upper(
) and state != TimerEntry.StateEnded and not disabled:
go = True
if go:
timebegin = 0
timeend = 0
duration = 0
startprepare = 0
repeated = 0
justplay = 0
afterevent = 3
eventId = -1
try:
timebegin = int(timer.findtext("e2timebegin", 0))
except:
timebegin = 0
try:
timeend = int(timer.findtext("e2timeend", 0))
except:
timeend = 0
try:
duration = int(timer.findtext("e2duration", 0))
except:
duration = 0
try:
startprepare = int(timer.findtext("e2startprepare", 0))
except:
startprepare = 0
try:
repeated = int(timer.findtext("e2repeated", 0))
except:
repeated = 0
try:
justplay = int(timer.findtext("e2justplay", 0))
except:
justplay = 0
try:
afterevent = int(timer.findtext("e2afterevent", 3))
except:
afterevent = 3
try:
eventId = int(timer.findtext("e2eit", -1))
except:
eventId = -1
E2TimerList.append(
E2Timer(servicereference=servicereference,
servicename=unquote(
str(
timer.findtext("e2servicename",
'n/a').decode("utf-8").encode(
"utf-8", 'ignore'))),
name=str(
timer.findtext("e2name",
'').decode("utf-8").encode(
"utf-8", 'ignore')),
disabled=disabled,
timebegin=timebegin,
timeend=timeend,
duration=duration,
startprepare=startprepare,
state=state,
repeated=repeated,
justplay=justplay,
eventId=eventId,
afterevent=afterevent,
dirname=str(
timer.findtext("e2location",
'').decode("utf-8").encode(
"utf-8", 'ignore')),
description=unquote(
str(
timer.findtext("e2description",
'').decode("utf-8").encode(
"utf-8", 'ignore'))),
type=0))
return E2TimerList
def check_uri():
# split off anchor
if '#' in uri:
req_url, anchor = uri.split('#', 1)
else:
req_url = uri
anchor = None
# handle non-ASCII URIs
try:
req_url.encode('ascii')
except UnicodeError:
req_url = encode_uri(req_url)
try:
if anchor and self.app.config.linkcheck_anchors and \
not anchor.startswith('!'):
# Read the whole document and see if #anchor exists
# (Anchors starting with ! are ignored since they are
# commonly used for dynamic pages)
req = Request(req_url)
f = opener.open(req, **kwargs)
encoding = 'utf-8'
if hasattr(f.headers, 'get_content_charset'):
encoding = f.headers.get_content_charset() or encoding
else:
encoding = get_content_charset(f) or encoding
found = check_anchor(TextIOWrapper(f, encoding),
unquote(anchor))
f.close()
if not found:
raise Exception("Anchor '%s' not found" % anchor)
else:
try:
# try a HEAD request, which should be easier on
# the server and the network
req = HeadRequest(req_url)
f = opener.open(req, **kwargs)
f.close()
except HTTPError as err:
if err.code != 405:
raise
# retry with GET if that fails, some servers
# don't like HEAD requests and reply with 405
req = Request(req_url)
f = opener.open(req, **kwargs)
f.close()
except HTTPError as err:
if err.code == 401:
# We'll take "Unauthorized" as working.
return 'working', ' - unauthorized', 0
else:
return 'broken', str(err), 0
except Exception as err:
return 'broken', str(err), 0
if f.url.rstrip('/') == req_url.rstrip('/'):
return 'working', '', 0
else:
new_url = f.url
if anchor:
new_url += '#' + anchor
code = getattr(req, 'redirect_code', 0)
return 'redirected', new_url, code
def __init__(self, app, account_name, **kwargs):
super(AccountController, self).__init__(app)
self.account_name = unquote(account_name)
if not self.app.allow_account_management:
self.allowed_methods.remove('PUT')
self.allowed_methods.remove('DELETE')
def _doi_uri_to_doi(uri):
parsed = urlparse(uri)
doi = parsed.path.split("/")[1]
# the doi from a url needs to be decoded
doi = unquote(doi)
return doi
def password(self):
"""Returns the password to use for fetching messages."""
return unquote(self._protocol_info.password)