def extract_script(service, converter=None):
"""
Calculate the script name from a services dictionary.
Attempts to get the script by trying each of these in order:
* The ``SCRIPT_NAME`` variable in ``environ``
The ``converter`` argument is an optional converter which is used to
decode and validate the script once it has been converted. If ``None``,
the ``decode_query`` converter is used.
"""
if converter is None:
converter = decode_script
environ = service and hasattr(service, 'environ') and \
service.environ or None
if environ:
script = environ.get('SCRIPT_NAME')
if script is None:
return None
if script.startswith('/'):
script = script[1:]
if script.endswith('/'):
raise URLConvertError(
"The SCRIPT_NAME %r cannot end with a '/' character" %
(script))
#script = script[:-1]
return Conversion(script).perform(converter).result
def extract_port(service, converter=None):
"""\
Calculate the port from a services dictionary. Returns the port as a
unicode string.
Attempts to get the host by trying each of these in order:
* The ``.config.server.display_port`` attribute of the ``service`` object if
such an attribute exists
* The ``X_PORT_FORWARDED_FOR`` key in the environ (totally unofficial)
* The ``SERVER_PORT`` key in the environ (can be trusted, but might not be the port
the user should use)
The ``converter`` argument is an optional converter which is used to
decode and validate the port once it has been converted. If ``None``,
the ``decode_port`` converter is used.
"""
if converter is None:
converter = decode_port
environ = service and hasattr(service,
'environ') and service.environ or None
if environ:
port = str(
environ.get('X_PORT_FORWARDED_FOR', environ.get('SERVER_PORT')))
if not port:
raise Exception('No port could be calculated from the service')
return Conversion(port).perform(converter).result
def extract_path(service, converter=None):
"""
Calculate the path from a services dictionary. The returned path does not
include the first ``/``.
Attempts to get the path by trying each of these in order:
* The ``PATH_INFO`` variable in ``environ`` which is expected to start
with a ``/`` character.
The ``converter`` argument is an optional converter which is used to
decode and validate the path once it has been converted. If ``None``,
the ``decode_path`` converter is used.
"""
if converter is None:
converter = decode_path
environ = service and hasattr(service,
'environ') and service.environ or None
if environ:
path = environ.get('PATH_INFO')
if path is None:
return None
if not path.startswith('/'):
raise URLConvertError(
"Expected the PATH_INFO %r to start with '/'" % (path))
path = path[1:]
return Conversion(path).perform(converter).result
def generate(self, routing_vars, default_url_parts=None):
log.debug('Converting routing variables %r to URL parts', routing_vars)
if default_url_parts is None:
value = dict(vars=routing_vars, current=dict())
else:
value = dict(vars=routing_vars, current=default_url_parts)
return Conversion(value).perform(self.to_url_converter)
def addExtras_post_converter(conversion, service=None):
if conversion.successful:
for k, v in add.items():
if conversion.value.has_key(k):
raise URLConvertError(
"The 'add' dictionary cannot contain the same key %r "
"as a routing variable defined in the rule" % k)
conversion.result[k] = v
conversion.children[k] = Conversion(v).perform(_no_conversion)
def update_config(bag, name, config):
from configconvert import handle_option_error, handle_section_error
if not bag.app.option.has_key(name):
raise handle_section_error(
bag,
name,
(
"'%s.plugin' (the name of the database plugin to use"%(name)
)
)
from stringconvert import unicodeToUnicode, unicodeToInteger,\
unicodeToBoolean
from recordconvert import toRecord
from configconvert import stringToObject
from conversionkit import Conversion, chainConverters
# Re-use the converters
unicode_to_integer = unicodeToInteger()
null = unicodeToUnicode()
database_config = toRecord(
missing_defaults=dict(
creator=connect,
fetch_converter=None,
execute_converter=None,
on_connect_sql=None,
),
missing_or_empty_errors = dict(
dsn="The required option '%s.dsn' is missing"%(name,),
unicode_results="The required option '%s.uniocde_results' is missing"%(name,),
),
converters=dict(
dsn = null,
unicode_results = unicodeToBoolean(),
on_connect_sql = unicodeToUnicode(),
creator = stringToObject(),
fetch_converter = stringToObject(),
execute_converter = stringToObject(),
),
)
import base64
if config.has_key('odsn'):
odsn = config['odsn']
parts = odsn.split('|')
s = str(parts[1])
p = base64.urlsafe_b64decode((s[:-4] + '=' * (4 - (len(s[4:])-4) % 4))[4:])
config['dsn'] = parts[0] + p + parts[2]
del config['odsn']
conversion = Conversion(config).perform(database_config)
if not conversion.successful:
handle_option_error(conversion, name)
else:
config = conversion.result
return config
def match_url(self, url, script=None):
url_parts = Conversion(url).perform(_to_parts).result
if script is not None:
if script.endswith('/'):
raise URLConvertError(
"According to the WSGI spec SCRIPT_NAME cannot end with "
"a '/' character, so neither should the 'script' argument")
url_parts['script'] = script
url_parts['path'] = url_parts['path'][len(script) + 1:]
result = self.match(url_parts)
return result
def generate_url(self, routing_vars, default_url_parts):
d = default_url_parts.copy()
rv = dict([(unicode(k), unicode(v)) for k, v in routing_vars.items()])
conversion = self.generate(rv, default_url_parts)
if conversion.successful:
res = conversion.result
d.update(res.url_parts)
# @@@ Should really use a post converter
result = Conversion(
AttributeDict(url=build_url(**d),
extra=res.extra)).perform(noConversion())
return result
raise Exception('No matching generation rule could be found')
def config(flow, name):
from configconvert import handle_option_error, handle_section_error
if not flow.config.option_group.has_key(name):
raise handle_section_error(
flow, name, "'%s.sendmail' or '%s.smtp.host'" % (name, name))
from nestedrecord import decodeNestedRecord
from conversionkit import chainConverters, chainPostConverters, Conversion
from stringconvert import unicodeToInteger, unicodeToBoolean, unicodeToUnicode
from stringconvert.email import listOfEmails
from recordconvert import toRecord
from configconvert import existingFile, existingDirectory
to_unicode = unicodeToUnicode()
smtp_converter = chainPostConverters(
toRecord(missing_errors=dict(
host="The required option '%s.smtp.host' is missing" %
(name, ), ),
empty_errors=dict(
host="The option '%s.smtp.host' cannot be empty" %
(name, ), ),
missing_defaults=dict(starttls=False),
converters=dict(
host=to_unicode,
username=to_unicode,
password=to_unicode,
port=unicodeToInteger(),
starttls=unicodeToBoolean(),
verbose=unicodeToBoolean(),
)),
requireIfPresent('username', ['password']),
)
mail_converter = chainConverters(
decodeNestedRecord(depth=1),
chainPostConverters(
toRecord(converters=dict(
sendmail=existingFile(),
smtp=smtp_converter,
debug_folder=existingDirectory(),
to_email_override=listOfEmails(split_name=False),
), ),
exacltyOneFieldFrom('sendmail', 'smtp'),
))
conversion = Conversion(
flow.config.option_group[name]).perform(mail_converter)
if not conversion.successful:
handle_option_error(conversion, name)
else:
flow.config[name] = conversion.result
return flow.config[name]
def ruleToParts_converter(conversion, service=None):
child_conversion = Conversion(conversion.value).perform(_to_parts)
if not child_conversion.successful:
conversion.error = child_conversion.error
return
parts = child_conversion.result
path_parts = parts['path'].split(u'|')
if len(path_parts) > 2:
raise URLConvertError("A rule can only contain one '|' character")
elif len(path_parts) == 2:
parts['path'] = path_parts[1]
parts['script'] = path_parts[0]
else:
parts['script'] = u'{*}'
if not parts.has_key('query'):
parts['query'] = u'{*}'
conversion.result = parts
def psycopg2_update_config(flow, name, config):
from configconvert import handle_option_error, handle_section_error
from stringconvert import unicodeToUnicode, unicodeToInteger,\
unicodeToBoolean
from recordconvert import toRecord
from configconvert import stringToObject
from conversionkit import Conversion, chainConverters
# Re-use the converters
unicode_to_integer = unicodeToInteger()
null = unicodeToUnicode()
database_config = chainConverters(
dbport,
toRecord(
missing_defaults=dict(
creator=psycopg2.connect,
fetch_converter=psycopg2_utf8_fetch,
execute_converter=None,
),
missing_or_empty_errors = dict(
database="The required option '%s.database' is missing"%(name,),
),
converters=dict(
database = null,
user = null,
# Could use formencode.validators.URL() for host?
host = null,
password = null,
port = unicode_to_integer,
creator = stringToObject(),
fetch_converter = stringToObject(),
execute_converter = stringToObject(),
),
# Keep the pool options as they are
filter_extra_fields = False
),
)
conversion = Conversion(config).perform(database_config)
if not conversion.successful:
handle_option_error(conversion, name)
else:
config = conversion.result
return config
def update_config(bag, name, config):
if config.get('pool', False):
raise Exception('The %s.pool option must be False for SQLite3' % name)
from configconvert import handle_option_error, handle_section_error
if not bag.app.option.has_key(name):
raise handle_section_error(
bag, name,
("'%s.database' and '%s.plugin' (the module module and "
"function to use to create a connection)" % (name, name)))
from stringconvert import unicodeToUnicode, unicodeToInteger,\
unicodeToBoolean
from recordconvert import toRecord
from configconvert import stringToObject
from conversionkit import Conversion, chainConverters
# Re-use the converters
unicode_to_integer = unicodeToInteger()
null = unicodeToUnicode()
database_config = toRecord(
missing_defaults=dict(
creator=connect,
fetch_converter=sqlite3_utf8_fetch,
execute_converter=None,
),
missing_or_empty_errors=dict(
database="The required option '%s.database' is missing" %
(name, ), ),
converters=dict(
database=null,
creator=stringToObject(),
fetch_converter=stringToObject(),
execute_converter=stringToObject(),
),
)
conversion = Conversion(bag.app.option[name]).perform(database_config)
if not conversion.successful:
handle_option_error(conversion, name)
else:
config = conversion.result
return config
def extract_query(service, converter=None):
"""
Calculate the query string from a services dictionary.
Attempts to get the query by trying each of these in order:
* The ``QUERY_STRING`` variable in ``environ``
The ``converter`` argument is an optional converter which is used to
decode and validate the query once it has been converted. If ``None``,
the ``decode_query`` converter is used.
"""
if converter is None:
converter = decode_query
environ = service and hasattr(service, 'environ') and \
service.environ or None
if environ:
query = environ.get('QUERY_STRING')
if query is None:
return None
return Conversion(query).perform(converter).result
def user_create(
service,
username,
password,
group=None,
name='user',
encrypt=_nothing,
terms=False,
email='',
):
"""\
Create a new user with the username, password and group name specified.
"""
conversion = Conversion(username).perform(valid_new_username, service)
if not conversion.successful:
raise Exception(conversion.error)
username = conversion.result
if group is not None and not group_exists(service, group, name=name):
raise NoSuchGroupError(
"There is no such group %r"%group
)
return user_create_store(service, username, password, group, name, encrypt, terms, email)
def extract_scheme(service, converter=None):
"""\
Calculate the scheme from a services dictionary.
Attempts to get the scheme by trying each of these in order:
* The ``.config.server.display_scheme`` attribute of the ``service`` object if
such an attribute exists
* The ``wsgi.url_scheme`` key in the environ
* Based on the port (with a call to ``extract_port()``)
The ``converter`` argument is an optional converter which is used to
decode and validate the scheme once it has been converted. If ``None``,
the ``decode_scheme`` converter is used.
"""
if converter is None:
converter = decode_scheme
scheme = None
if hasattr(service, 'config') and hasattr(service.config, 'server') and \
hasattr(service.config.server, 'display_scheme'):
scheme = service.config.app.display_scheme
else:
environ = service and hasattr(service,
'environ') and service.environ or None
if environ:
scheme = environ.get('wsgi.url_scheme')
if not scheme:
port = extract_port(service)
if port:
if port == '80':
scheme = 'http'
elif port == '443':
scheme = 'https'
if not scheme:
raise Exception('No scheme could be calculated from the service')
return Conversion(scheme).perform(converter).result
def extract_host(service, converter=None):
"""\
Calculate the host from a services dictionary.
Attempts to get the host by trying each of these in order:
* The ``.config.server.display_host`` attribute of the ``service`` object if
such an attribute exists
* The first part of an the ``X_FORWARDED_FOR`` key in the environ (can't be trusted)
* The ``HTTP_HOST`` key in the environ (can't be trusted)
* The ``SERVER_NAME`` key in the environ (can be trusted, but might not be the
host the user should use)
The ``converter`` argument is an optional converter which is used to
decode and validate the host once it has been converted. If ``None``,
the ``decode_host`` converter is used.
"""
if converter is None:
converter = decode_host
host = None
if hasattr(service, 'config') and hasattr(service.config, 'server') and \
hasattr(service.config.server, 'display_host'):
host = service.config.server.display_host
else:
environ = service and hasattr(service,
'environ') and service.environ or None
if environ:
if environ.has_key('X_FORWARDED_FOR'):
host = environ['X_FORWARDED_FOR'].split(',')[0].strip()
if not host:
host = environ.get('HTTP_HOST', environ.get('SERVER_NAME'))
if not host:
raise Exception('No host could be calculated from the service')
else:
host = host.split(':')[0]
return Conversion(host).perform(converter).result
def mailService_constructor(service, name, *k, **p):
from mail.helper import send_smtp, send_sendmail, prepare, plain
# Imports
from configconvert import handle_option_error, handle_section_error
from nestedrecord import decodeNestedRecord
from conversionkit import chainConverters, chainPostConverters, Conversion
from stringconvert import unicodeToInteger, unicodeToBoolean, unicodeToUnicode
from stringconvert.email import listOfEmails
from recordconvert import toRecord
from configconvert import existingFile, existingDirectory
# Config parsing
if not service.app.option.has_key(name):
raise handle_section_error(
service, name,
"'%s.sendmail' or '%s.smtp.host'" % (name, name))
to_unicode = unicodeToUnicode()
smtp_converter = chainPostConverters(
toRecord(missing_errors=dict(
host="The required option '%s.smtp.host' is missing" %
(name, ), ),
empty_errors=dict(
host="The option '%s.smtp.host' cannot be empty" %
(name, ), ),
missing_defaults=dict(starttls=False),
converters=dict(
host=to_unicode,
username=to_unicode,
password=to_unicode,
port=unicodeToInteger(),
starttls=unicodeToBoolean(),
verbose=unicodeToBoolean(),
)),
requireIfPresent('username', ['password']),
)
mail_converter = chainConverters(
decodeNestedRecord(depth=1),
chainPostConverters(
toRecord(converters=dict(
sendmail=existingFile(),
smtp=smtp_converter,
debug_folder=existingDirectory(),
to_email_override=listOfEmails(split_name=False),
), ),
exacltyOneFieldFrom('sendmail', 'smtp'),
))
conversion = Conversion(
service.app.option[name]).perform(mail_converter)
if not conversion.successful:
handle_option_error(conversion, name)
else:
service.app.config[name] = conversion.result
if service.app.config[name].get('debug_folder') and not \
os.path.exists(service.app.config[name]['debug_folder']):
os.mkdir(service.app.config[name]['debug_folder'])
def send(
message,
to,
from_email,
from_name,
subject=None,
type='plain',
charset=None,
):
if service.app.config[name].get('to_email_override'):
log.warning(
'Replacing the email %s with %s',
to,
service.app.config[name].get('to_email_override'),
)
to = service.app.config[name].get('to_email_override')
if to and isinstance(to, list) and isinstance(to[0], dict):
to = ['%s <%s>' % (x.name, x.email) for x in to]
subject = subject or service.app.config[name]['subject']
message = prepare(
plain(message, type=type, charset=charset),
from_name=from_name,
from_email=from_email,
to=to,
subject=subject,
)
debug_folder = service.app.config[name].get('debug_folder')
if debug_folder:
log.warning('Writing message to the debug folder, not sending '
'it directly')
fp = open(
os.path.join(debug_folder, '%s - %s.txt' % (subject, to)),
'wb')
fp.write(str(message))
fp.close()
else:
sendmail = service.app.config[name].get('sendmail')
if sendmail:
return send_sendmail(
message,
sendmail,
)
smtp_args = service.app.config[name]['smtp']
return send_smtp(message, **str_dict(smtp_args))
def start(service):
service[name] = AttributeDict(send=send)
def leave(service, error=False):
pass
return AttributeDict(start=start, enter=start, leave=leave)
def rule(rule, add=None, extra=None):
if not isinstance(rule, unicode):
raise URLConvertError(
'Expected the rule to be specified as a Unicode string')
if add is not None:
for k in parse_vars(rule):
# This gets checked later too, but better to do it now.
if k in add:
raise URLConvertError(
"The 'add' dictionary cannot contain the same key %r "
"as a routing variable defined in the rule" % k)
for k, v in add.items():
if not isinstance(k, unicode):
raise URLConvertError(
'Expected the key %r to be a Unicode string' % k)
if not isinstance(v, unicode):
raise URLConvertError(
'Expected the value %r to be a Unicode string' % v)
add = add.copy()
# Start making the chain
chain = []
# Split up the rule
parts = Conversion(rule).perform(_rule_to_parts).result
# Now create a match and generate converter for each part.
match = {}
generate = {}
# Start with the easy ones which either don't exist, take a static part,
# or have a single dynamic part
for key in ['scheme', 'query', 'port']:
if parts[key] not in [u'{}', u'{*}']:
if parts[key].startswith('{'):
if not parts[key].endswith('}'):
raise URLConvertError(
"Expected the %s part to end with '}' since it "
"can only contain one routing variable" % key)
var = parts[key][1:-1]
if '{' in var or '}' in var:
raise URLConvertError(
'The %s part of the rule is invalid' % key)
match[key] = matchDynamic(part=key, dynamic=var)
generate[key] = generateStaticOrDynamic(part=key, dynamic=var)
elif '{' in parts[key] or '}' in parts[key]:
raise URLConvertError('The %s part of the rule is invalid' %
key)
else:
match[key] = matchStatic(key, parts[key])
generate[key] = generateStaticOrDynamic(
part=key,
expected_value=parts[key],
)
if parts['host'] != u'{*}':
if u'{*}' in parts['host']:
raise URLConvertError(
"The host part %r is invalid, you cannot use {*} as well "
"as text or named variables." % parts['host'])
if not re.match(
'[A-Za-z0-9](([A-Za-z0-9\-])+.)*',
parts['host'][:].replace('{', 'a').replace('}', 'a'),
):
raise URLConvertError('The host name is invalid')
if not '{' in parts['host']:
# Static text. Check the characters.
match['host'] = matchStatic('host', parts['host'])
generate['host'] = generateStaticOrDynamic(
part='host',
expected_value=parts['host'],
)
else:
match['host'] = matchDynamicDomain(parts['host'])
generate['host'] = generateDynamicHost(parts['host'])
# The path and script
for key in ['path', 'script']:
if parts[key] not in [u'{*}']:
#if u'{}' in parts[key]:
# raise URLConvertError(
# "The %s part %r is invalid, you cannot use {} as well as "
# "named variables, only instead of them"%(key, parts[key])
# )
if not re.match('([A-Za-z0-9\-\%\/\:\"@&=+\$\,_\.\!\~\*\'\(\)])*',
parts[key][:].replace('{', 'a').replace('}', 'a')):
raise URLConvertError('The %s contains invalid characters' %
key)
if not '{' in parts['path']:
# Static text. Check the characters.
match[key] = matchStatic(key, parts[key])
generate[key] = generateStaticOrDynamic(
part=key,
expected_value=parts[key],
)
else:
match[key] = matchDynamicPath(key, parts[key])
generate[key] = generateDynamicPath(key, parts[key])
# Finally, create a converter capable of parsing all this data
post_converters = [
toDictionary(match),
mergeParts(),
]
if add is not None:
post_converters.append(addExtras(add))
post_converters.append(extraVariables(extra))
to_vars = chainConverters(missingKey(match.keys(), 'match'),
chainPostConverters(*post_converters))
to_url = chainPostConverters(
chainConverters(
removeExtras(add or {}),
# This is expecting a dictionary, we have a tuple, and want each
# of the generate converters called for each tuple.
# ie: We need a for_each with a key for each.
tryEach(generate.values(),
stop_on_first_result=False,
stop_on_first_error=True,
children_keys=generate.keys()),
),
joinVars(add or {}),
extraVariables(extra, 'url_parts'),
)
return AttributeDict(dict(to_vars=to_vars, to_url=to_url))
def match(self, url_parts):
log.debug('Converting URL parts %r to routing variables', url_parts)
return Conversion(url_parts).perform(self.to_vars_converter)