def make_token_lock(request, token, session_id):
return make_sha256(
bytes_join(
'-',
[to_unicode(request.user_agent or ''),
to_unicode(request.ip_address),
to_unicode(token),
to_unicode(session_id)]))
def clean_unicode(value, replace_case=None, mapping_dict=None):
value = to_unicode(value)
get_letter = (mapping_dict or MAPPING).get
if replace_case is not None:
replace_case = to_unicode(replace_case)
return unicode_join('', (get_letter(letter, replace_case) for letter in value))
else:
return unicode_join('', (get_letter(letter, letter) for letter in value))
def run_monitor(self):
try:
self.validate_daemons()
immediate_jobs = set(
to_unicode(k)
for k in self.config.cache.get_values(JOBS_IMMEDIATE_KEY, expire=None))
for apijob in list(JOBS):
run_job = False
if apijob.name in immediate_jobs:
run_job = True
immediate_jobs.remove(apijob.name)
elif apijob.will_run() and apijob.next_date <= NOW():
run_job = True
if run_job:
try:
daemon = start_system_thread(
'job_%s' % apijob.name,
apijob,
sleep_method=False)
except KeyError:
pass
else:
RUNNING_JOBS.append((apijob, daemon))
self.config.cache.replace_values(JOBS_IMMEDIATE_KEY, immediate_jobs, expire=None)
except Exception as error:
self.system_session().logging.log_critical('jobs_undefined_error', str(error))
return 5
else:
return 0.5
def ip_address(self):
value = self.environ.get('HTTP_X_FORWARDED_FOR') or self.environ.get('REMOTE_ADDR')
if value:
return to_unicode(value)
message = u('Missing IP Address')
raise Error('ip_address', message)
def send_immediately(self, *args, **kwargs):
fail_silently = kwargs.pop('fail_silently', False)
mime_message = self.create_message(*args, **kwargs)
try:
self.api_session_manager.mailer.send_immediately(mime_message, fail_silently=fail_silently)
except Exception as error:
if isinstance(error, SMTPRecipientsRefused):
for email, error_message in error.args[0].items():
code = error_message[0]
if code == 554:
raise Error('email', u('Invalid email "%s"') % to_unicode(email))
elif code == 450:
raise Error('email', u('Invalid email domain "%s"') % to_unicode(email))
raise
else:
return True
def _render(value, system):
value = to_unicode(value)
request = system.get('request')
if request is not None:
response = request.response
ct = response.content_type
if ct == response.default_content_type:
response.content_type = 'text/html'
return value
def log(self, code, message, level='INFO', extra=None):
level = level.upper()
header = ('-' * 30) + ' ' + level + ' ' + ('-' * 30)
print_(header)
arguments = [
('Application', self.request.application_name),
('Code', code),
('URL', self.request.url),
('Method', self.request.method),
('Date', NOW()),
('Language', self.request.locale_name),
('IP address', self.request.ip_address)]
if extra:
arguments.extend(extra.items())
if self.request.authenticated:
arguments.extend([
('Session type', self.request.authenticated.session_type),
('Session id', self.request.authenticated.session_id)])
else:
arguments.append(('Session', 'Without autentication'))
bigger = max(len(k) for k, v in arguments)
for key, value in arguments:
print_(key, ' ' * (bigger - len(key)), ':', to_unicode(value))
if level == 'CRITICAL':
sys_exc_info = sys.exc_info() # type, value, traceback
if sys_exc_info[2] is not None:
print_()
print_(string_join('', format_exception(*sys_exc_info)))
print_()
try:
message = to_unicode(message)
for line in message.split(linesep):
print_(' %s' % line)
except UnicodeEncodeError:
pass
print_('-' * len(header))
def format_email(email, encoding=None, force_development_email=None):
label = None
if isinstance(email, dict):
if 'label' in email:
label = email['label']
email = email['email']
elif not isinstance(email, string_types):
label, email = email
return to_unicode(
formataddr((
maybe_string(label, encoding=encoding),
to_string(force_development_email or email, encoding=encoding))))
def file_unique_code(open_file, block_size=OPEN_BLOCK_SIZE):
current_position = open_file.tell()
h = sha256()
open_file.seek(0)
block = open_file.read(block_size)
convert_to_bytes = bool(not isinstance(open_file, BytesIO) and 'b' not in open_file.mode)
while block:
if convert_to_bytes:
block = to_bytes(block)
h.update(block)
block = open_file.read(block_size)
open_file.seek(current_position)
return to_unicode(h.hexdigest())
def __call__(self, node, value):
number = to_unicode(value).lower()
if len(number) != self.length:
raise Invalid(node, _('Need to have ${length} chars', mapping={'length': self.length}))
digit_sum = 0
second_digit = False
for letter in reversed(number):
digit = PORTUGUESE_CC_LETTER_MAPPING.get(letter)
if second_digit:
digit *= 2
if digit > 9:
digit -= 9
digit_sum += digit
second_digit = not second_digit
if digit_sum % 10:
raise Invalid(node, _('Invalid code'))
def save_file(
self,
binary,
application_code,
code_key=None,
type_key=None,
data=None,
filename=None,
title=None,
parent_id=None):
file_path = self.save_file_path(binary, filename)
session_id = session_type = None
if self.request.authenticated:
session_id = self.request.authenticated.session_id
session_type = self.request.authenticated.session_type
# Add applications file relation
new = File(
file_id=file_path.id,
parent_id=parent_id,
key=make_unique_hash(70),
application_code=to_unicode(application_code),
code_key=maybe_unicode(code_key),
type_key=maybe_unicode(type_key),
data=maybe_unicode(data),
filename=maybe_unicode(filename),
title=maybe_unicode(title),
session_type=session_type,
session_id=session_id)
self.session.add(new)
self.session.flush()
# Add some attributes
new.size = file_path.size
new.mimetype = file_path.mimetype
return new
def open_url(url, data=None, timeout=None, headers=None, method='get'):
if timeout:
timeout = abs(float(timeout))
url = to_string(url)
if ' ' in url:
url = url.replace(' ', '%20')
if data:
if isinstance(data, string_types):
data = to_string(data)
elif isinstance(data, (dict, MultiDict)):
data = list(data.items())
if isinstance(data, (dict, tuple, list)):
data = dict((to_string(k), to_string(v)) for k, v in data)
data = urlencode(data)
if method.lower() == 'get':
url += '?%s' % data
data = None
else:
data = to_bytes(data)
req = Request(url, data=data, headers=headers or {})
req.get_method = lambda: method.upper()
try:
response = URL_OPENER.open(req, timeout=timeout)
except Exception as error:
message = u('Could not open the url: %s') % to_unicode(url)
raise Error('url', message, exception=error)
if isinstance(response, inesHTTPError):
raise Error('url', response.message, exception=response)
else:
return response
def construct_structure(self, request, schema, schema_type, types, models, to_translate, parent_name=None):
if isinstance(schema.typ, Sequence):
child = schema.children[0]
if not schema.name:
schema = child
name = camelcase(schema.name)
details = {
"model": name,
"type": "sequence",
"title": schema.title,
"description": schema.description or EMPTY_STRING,
}
models[name].append(details)
if isinstance(schema.title, TranslationString):
to_translate["title"].append(details)
if isinstance(schema.description, TranslationString):
to_translate["description"].append(details)
# Find and add child
child_details = self.construct_structure(
request, child, schema_type, types, models, to_translate, parent_name=schema.name
)
if isinstance(details, dict):
if isinstance(child.typ, Mapping):
details["type"] = "model"
details.update(child_details)
else:
details["fields"] = [child_details]
else:
details["fields"] = child_details
return details
elif isinstance(schema.typ, Tuple):
raise NotImplementedError("Tuple type need to be implemented")
elif isinstance(schema.typ, Mapping):
fields = []
for child in schema.children:
fields.append(
self.construct_structure(
request, child, schema_type, types, models, to_translate, parent_name=schema.name
)
)
name = schema.name or parent_name
if not name:
return fields
name = camelcase(name)
details = {
"type": "model",
"title": schema.title,
"description": schema.description or EMPTY_STRING,
"fields": fields,
"model": name,
}
models[name].append(details)
if isinstance(schema.title, TranslationString):
to_translate["title"].append(details)
if isinstance(schema.description, TranslationString):
to_translate["description"].append(details)
return details
else:
name = camelcase(schema.name)
details = {"fieldType": name, "title": schema.title, "description": schema.description or EMPTY_STRING}
if isinstance(schema.title, TranslationString):
to_translate["title"].append(details)
if isinstance(schema.description, TranslationString):
to_translate["description"].append(details)
if hasattr(schema, "model_reference"):
model = schema.model_reference["model"]
model_key = schema.model_reference.get("key") or "key"
model_query = schema.model_reference.get("query") or "name"
model_application = schema.model_reference.get("application_name") or request.application_name
details["modelReference"] = {
"applicationName": model_application,
"schemaName": schema.model_reference["schema"],
"key": camelcase(model[model_key].name),
"model": camelcase(model.name),
"queryField": camelcase(model[model_query].name),
}
types[name].append(details)
if isinstance(schema.typ, FilterByType):
for cls in schema.typ.__class__.__mro__[1:]:
if cls is not FilterByType:
details["type"] = str(cls.__name__).lower()
break
details["filter"] = True
elif hasattr(schema, "schema_type_name"):
details["type"] = camelcase(schema.schema_type_name)
else:
details["type"] = get_colander_type_name(schema.typ)
request_validation = []
if schema.validator:
if isinstance(schema.validator, All):
validators = schema.validator.validators
elif not is_nonstr_iter(schema.validator):
validators = [schema.validator]
else:
validators = schema.validator
for validator in validators:
if isinstance(validator, OneOfWithDescription):
details["options"] = []
add_option = details["options"].append
save_to_translate = False
for choice, description in validator.choices_with_descripton:
add_option({"value": choice, "text": description})
save_to_translate = save_to_translate or isinstance(description, TranslationString)
if save_to_translate:
to_translate["options"].append(details)
elif isinstance(validator, OneOf):
details["options"] = []
add_option = details["options"].append
for choice in validator.choices:
add_option({"value": choice, "text": to_unicode(choice).replace(UNDERSCORE, SPACE).title()})
else:
if isinstance(validator, Length):
validation_option = {}
if validator.min is not None:
validation_option["min"] = validator.min
if validator.max is not None:
validation_option["max"] = validator.max
else:
validation_option = True
request_validation.append((validator, validation_option))
if hasattr(schema, "use_when"):
details["useWhen"] = dict((camelcase(k), v) for k, v in schema.use_when.items())
if schema_type == "request":
validation = {}
if schema.required:
validation["required"] = True
if request_validation:
for validator, validation_option in request_validation:
validation[get_colander_type_name(validator)] = validation_option
if validation:
details["validation"] = validation
default = schema.missing
else:
if schema.missing is drop:
details["maybeNotSent"] = True
default = schema.default
if default is not drop and default is not required and default is not null:
if isinstance(schema.typ, Number):
default = schema.typ.num(default)
elif isinstance(schema.typ, BaseBoolean):
default = asbool(default)
details["default"] = default
return details
def loads(self, bstruct):
return to_unicode(bstruct)
def _render(value, system):
request = system.get('request')
output_schema = None
if isinstance(value, dict):
output_schema = value['schema']
value = value['value']
delimiter = ';'
quote_char = '"'
line_terminator = '\r\n'
quoting = QUOTE_ALL
encoder = None
if request is not None:
response = request.response
ct = response.content_type
if ct == response.default_content_type:
response.content_type = 'text/csv'
output = request.registry.config.lookup_output_schema(
request.matched_route.name,
request_method=request.method)
if output:
output_schema = output[0].schema
if output_schema:
value = self.lookup_rows(request, output_schema.children[0], value)
output_filename = getattr(output_schema, 'filename', None)
if output_filename:
if callable(output_filename):
output_filename = output_filename()
response.content_disposition = 'attachment; filename="%s"' % output_filename
_get_param = request.params.get
get_param = lambda k: _get_param(k) or _get_param(camelcase(k))
csv_delimiter = get_param('csv_delimiter')
if csv_delimiter:
delimiter = to_string(csv_delimiter)
if delimiter == '\\t':
delimiter = '\t'
else:
delimiter = delimiter[0]
csv_quote_char = get_param('csv_quote_char')
if csv_quote_char:
quote_char = to_string(csv_quote_char)
csv_line_terminator = get_param('csv_line_terminator')
if csv_line_terminator:
if csv_line_terminator == u('\\n\\r'):
line_terminator = '\n\r'
elif csv_line_terminator == u('\\n'):
line_terminator = '\n'
elif csv_line_terminator == u('\\r'):
line_terminator = '\r'
csv_encoding = get_param('csv_encoding')
if csv_encoding:
try:
encoder = codecs.lookup(csv_encoding)
except LookupError:
raise Error('csv_encoding', _('Invalid CSV encoding'))
else:
if encoder.name != 'utf-8':
request.response.charset = encoder.name
yes_text = to_string(request.translate(_('Yes')))
no_text = to_string(request.translate(_('No')))
csv_quoting = get_param('csv_quoting')
if csv_quoting:
csv_quoting = to_string(csv_quoting).lower()
if csv_quoting == 'minimal':
quoting = QUOTE_MINIMAL
elif csv_quoting == 'non_numeric':
quoting = QUOTE_NONNUMERIC
elif csv_quoting == 'none':
quoting = QUOTE_NONE
else:
yes_text = to_string('Yes')
no_text = to_string('No')
if not value:
return ''
f = StringIO()
csvfile = csv_writer(
f,
delimiter=delimiter,
quotechar=quote_char,
lineterminator=line_terminator,
quoting=quoting)
for value_items in value:
row = []
for item in value_items:
if item is None:
item = ''
elif isinstance(item, bool):
item = item and yes_text or no_text
elif isinstance(item, (string_types, integer_types)):
item = to_string(item)
elif isinstance(item, (DATE, DATETIME)):
item = to_string(item.isoformat())
else:
item = to_string(item or '')
row.append(item)
csvfile.writerow(row)
f.seek(0)
response = f.read()
f.close()
if encoder:
if PY3:
response = encoder.decode(encoder.encode(response)[0])[0]
else:
response = encoder.decode(response)[0]
else:
response = to_unicode(response)
return response
def get_url_body(*args, **kwargs):
return to_unicode(get_url_file(*args, **kwargs))
def create_message(
self,
subject,
recipients,
body=None,
html=None,
sender=None,
cc=None,
bcc=None,
reply_to=None,
content_charset='utf-8',
attachments=None,
message_id=None):
force_development_email = None
if not self.request.is_production_environ:
force_development_email = self.settings.get('force_development_email') or None
if body:
body = to_unicode(body, encoding=content_charset)
if not html:
html = body.replace(NEW_LINE, HTML_NEW_LINE)
if html:
html = to_unicode(html, encoding=content_charset)
if not html.lower().startswith('<html'):
html = '<html><body>%s</body></html>' % html
options = {}
# FROM sender
if sender:
options['sender'] = format_email(
sender,
encoding=content_charset,
force_development_email=force_development_email)
# Envelope CC
if cc:
if isinstance(cc, dict):
cc = [cc]
options['cc'] = [
format_email(e, content_charset, force_development_email=force_development_email)
for e in maybe_list(cc)]
# Envelope BCC
if bcc:
if isinstance(bcc, dict):
bcc = [bcc]
options['bcc'] = [
format_email(e, content_charset, force_development_email=force_development_email)
for e in maybe_list(bcc)]
if not message_id:
domain = self.settings.get('message_domain') or self.api_session_manager.default_domain
message_id = make_msgid(make_unique_hash(10), domain)
extra_headers = {
'Date': formatdate(localtime=True),
'Message-ID': to_string(message_id)}
# Force reply to another email
if reply_to:
extra_headers['Reply-To'] = '<%s>' % to_string(reply_to)
mime_attachments = []
if attachments:
if isinstance(attachments, dict):
attachments = [attachments]
for attachment in maybe_list(attachments):
f = attachment.get('file') or attachment['fp']
filename = to_string(attachment.get('filename') or basename(f.name)).replace(' ', '')
mimetype = to_string(attachment.get('content_type') or find_mimetype(filename, f))
f.seek(0)
mime_attachments.append(self.api_session_manager.attachment_cls(
data=f,
filename=filename,
content_type=mimetype))
return self.api_session_manager.message_cls(
subject=to_unicode(subject, encoding=content_charset),
html=html,
body=body,
recipients=[
format_email(e, content_charset, force_development_email=force_development_email)
for e in maybe_list(recipients)],
attachments=mime_attachments,
extra_headers=extra_headers,
**options)
def clean_filename(filename):
value = to_unicode(filename, errors='ignore')
return clean_unicode(value, replace_case='_', mapping_dict=FILENAME_MAPPING)
def string_unique_code(value):
value = to_bytes(value)
return to_unicode(sha256(value).hexdigest())
# -*- coding: utf-8 -*-
import os
from string import digits as base_digits
from string import ascii_letters as base_letters
from six import u
from ines import IGNORE_FULL_NAME_WORDS
from ines.convert import maybe_unicode
from ines.convert import to_unicode
from ines.convert import unicode_join
STRING_TO_DICT = lambda s: dict((l, l) for l in to_unicode(s))
LOWER_MAPPING = (u('àáâãäåæÆßçèéêëƒìíîïñðòóôõöšýùúüûž'), u('aaaaaaaabceeeefiiiinoooooosyyuuuuz'))
UPPER_MAPPING = (LOWER_MAPPING[0].upper(), LOWER_MAPPING[1].upper())
MAPPING = dict(zip(*LOWER_MAPPING))
MAPPING.update(zip(*UPPER_MAPPING))
MAPPING.update({u('Ø'): u('O'), u('ø'): u('0'), u('þ'): u('b'), u(' '): u(' ')})
MAPPING.update(STRING_TO_DICT(base_letters + base_digits))
FILENAME_MAPPING = MAPPING.copy()
FILENAME_MAPPING.update(STRING_TO_DICT('.-_'))
PHONE_MAPPING = STRING_TO_DICT('x+' + base_digits)
def clean_unicode(value, replace_case=None, mapping_dict=None):
value = to_unicode(value)
def maybe_email(value):
if validate_email(value):
return to_unicode(value)
def make_uuid_hash():
return to_unicode(uuid4().hex)
def referer_path_url(self):
if self.referer:
url = urlparse(self.referer)._replace(query=None).geturl()
return to_unicode(url)
def __call__(self, node, value):
if not to_unicode(value).isnumeric():
raise Invalid(node, _('Invalid number'))
def maybe_phone_number(number):
if number:
number = clean_phone_number(to_unicode(number))
if 21 > len(number or '') > 3:
return number
def get_files(
self,
key=None,
attributes=None,
only_one=False,
**kwargs):
attributes = set(attributes or ['id'])
return_open_file = 'open_file' in attributes
if return_open_file:
attributes.remove('open_file')
attributes.add('id')
columns = []
relate_with_path = False
for attribute in attributes:
if attribute in File.__table__.c:
columns.append(File.__table__.c[attribute])
if attribute not in ('id', 'created_date') and attribute in FilePath.__table__.c:
columns.append(FilePath.__table__.c[attribute])
relate_with_path = True
query = self.session.query(*columns or [File.id])
file_id = kwargs.get('file_id')
if file_id:
query = query.filter(FilePath.id.in_(maybe_set(file_id)))
relate_with_path = True
if relate_with_path:
query = query.filter(File.file_id == FilePath.id)
if 'id' in kwargs:
query = query.filter(File.id.in_(maybe_set(kwargs['id'])))
if key:
query = query.filter(File.key.in_(maybe_set(key)))
if 'application_code' in kwargs:
application_code = kwargs['application_code']
if application_code is None:
query = query.filter(File.application_code.is_(None))
else:
query = query.filter(File.application_code == to_unicode(application_code))
if 'code_key' in kwargs:
code_key = kwargs['code_key']
if code_key is None:
query = query.filter(File.code_key.is_(None))
else:
query = query.filter(File.code_key == to_unicode(code_key))
if 'type_key' in kwargs:
type_key = kwargs['type_key']
if type_key is None:
query = query.filter(File.type_key.is_(None))
else:
query = query.filter(File.type_key == to_unicode(type_key))
if 'data' in kwargs:
data = kwargs['data']
if data is None:
query = query.filter(File.data.is_(None))
else:
query = query.filter(File.data == to_unicode(data))
if 'parent_id' in kwargs:
parent_id = kwargs['parent_id']
if parent_id is None:
query = query.filter(File.parent_id.is_(None))
else:
query = query.filter(File.parent_id == int(parent_id))
parent_key = kwargs.get('parent_key')
if parent_key:
parent = aliased(File)
query = query.filter(File.parent_id == parent.id).filter(parent.key == to_unicode(parent_key))
order_by = kwargs.get('order_by')
if order_by is not None:
query = query.order_by(order_by)
if only_one:
response = query.first()
else:
response = query.all()
if not return_open_file or not response:
return response
if only_one:
response = [response]
# Add items to SQLAlchemy tuple result
files_binary = self.get_files_binary(*(f.id for f in response))
new_namedtuple = new_lightweight_named_tuple(response[0], 'open_file')
response[:] = [
new_namedtuple(r + (files_binary[r.id], ))
for r in response]
if only_one:
return response[0]
else:
return response
def clean_phone_number(value):
value = to_unicode(value or '', errors='ignore').lower()
return clean_unicode(value, replace_case='', mapping_dict=PHONE_MAPPING)
