def make_token_lock(request, token, session_id): return make_sha256( bytes_join( '-', [to_unicode(request.user_agent or ''), to_unicode(request.ip_address), to_unicode(token), to_unicode(session_id)]))
def clean_unicode(value, replace_case=None, mapping_dict=None): value = to_unicode(value) get_letter = (mapping_dict or MAPPING).get if replace_case is not None: replace_case = to_unicode(replace_case) return unicode_join('', (get_letter(letter, replace_case) for letter in value)) else: return unicode_join('', (get_letter(letter, letter) for letter in value))
def run_monitor(self): try: self.validate_daemons() immediate_jobs = set( to_unicode(k) for k in self.config.cache.get_values(JOBS_IMMEDIATE_KEY, expire=None)) for apijob in list(JOBS): run_job = False if apijob.name in immediate_jobs: run_job = True immediate_jobs.remove(apijob.name) elif apijob.will_run() and apijob.next_date <= NOW(): run_job = True if run_job: try: daemon = start_system_thread( 'job_%s' % apijob.name, apijob, sleep_method=False) except KeyError: pass else: RUNNING_JOBS.append((apijob, daemon)) self.config.cache.replace_values(JOBS_IMMEDIATE_KEY, immediate_jobs, expire=None) except Exception as error: self.system_session().logging.log_critical('jobs_undefined_error', str(error)) return 5 else: return 0.5
def ip_address(self): value = self.environ.get('HTTP_X_FORWARDED_FOR') or self.environ.get('REMOTE_ADDR') if value: return to_unicode(value) message = u('Missing IP Address') raise Error('ip_address', message)
def send_immediately(self, *args, **kwargs): fail_silently = kwargs.pop('fail_silently', False) mime_message = self.create_message(*args, **kwargs) try: self.api_session_manager.mailer.send_immediately(mime_message, fail_silently=fail_silently) except Exception as error: if isinstance(error, SMTPRecipientsRefused): for email, error_message in error.args[0].items(): code = error_message[0] if code == 554: raise Error('email', u('Invalid email "%s"') % to_unicode(email)) elif code == 450: raise Error('email', u('Invalid email domain "%s"') % to_unicode(email)) raise else: return True
def _render(value, system): value = to_unicode(value) request = system.get('request') if request is not None: response = request.response ct = response.content_type if ct == response.default_content_type: response.content_type = 'text/html' return value
def log(self, code, message, level='INFO', extra=None): level = level.upper() header = ('-' * 30) + ' ' + level + ' ' + ('-' * 30) print_(header) arguments = [ ('Application', self.request.application_name), ('Code', code), ('URL', self.request.url), ('Method', self.request.method), ('Date', NOW()), ('Language', self.request.locale_name), ('IP address', self.request.ip_address)] if extra: arguments.extend(extra.items()) if self.request.authenticated: arguments.extend([ ('Session type', self.request.authenticated.session_type), ('Session id', self.request.authenticated.session_id)]) else: arguments.append(('Session', 'Without autentication')) bigger = max(len(k) for k, v in arguments) for key, value in arguments: print_(key, ' ' * (bigger - len(key)), ':', to_unicode(value)) if level == 'CRITICAL': sys_exc_info = sys.exc_info() # type, value, traceback if sys_exc_info[2] is not None: print_() print_(string_join('', format_exception(*sys_exc_info))) print_() try: message = to_unicode(message) for line in message.split(linesep): print_(' %s' % line) except UnicodeEncodeError: pass print_('-' * len(header))
def format_email(email, encoding=None, force_development_email=None): label = None if isinstance(email, dict): if 'label' in email: label = email['label'] email = email['email'] elif not isinstance(email, string_types): label, email = email return to_unicode( formataddr(( maybe_string(label, encoding=encoding), to_string(force_development_email or email, encoding=encoding))))
def file_unique_code(open_file, block_size=OPEN_BLOCK_SIZE): current_position = open_file.tell() h = sha256() open_file.seek(0) block = open_file.read(block_size) convert_to_bytes = bool(not isinstance(open_file, BytesIO) and 'b' not in open_file.mode) while block: if convert_to_bytes: block = to_bytes(block) h.update(block) block = open_file.read(block_size) open_file.seek(current_position) return to_unicode(h.hexdigest())
def __call__(self, node, value): number = to_unicode(value).lower() if len(number) != self.length: raise Invalid(node, _('Need to have ${length} chars', mapping={'length': self.length})) digit_sum = 0 second_digit = False for letter in reversed(number): digit = PORTUGUESE_CC_LETTER_MAPPING.get(letter) if second_digit: digit *= 2 if digit > 9: digit -= 9 digit_sum += digit second_digit = not second_digit if digit_sum % 10: raise Invalid(node, _('Invalid code'))
def save_file( self, binary, application_code, code_key=None, type_key=None, data=None, filename=None, title=None, parent_id=None): file_path = self.save_file_path(binary, filename) session_id = session_type = None if self.request.authenticated: session_id = self.request.authenticated.session_id session_type = self.request.authenticated.session_type # Add applications file relation new = File( file_id=file_path.id, parent_id=parent_id, key=make_unique_hash(70), application_code=to_unicode(application_code), code_key=maybe_unicode(code_key), type_key=maybe_unicode(type_key), data=maybe_unicode(data), filename=maybe_unicode(filename), title=maybe_unicode(title), session_type=session_type, session_id=session_id) self.session.add(new) self.session.flush() # Add some attributes new.size = file_path.size new.mimetype = file_path.mimetype return new
def open_url(url, data=None, timeout=None, headers=None, method='get'): if timeout: timeout = abs(float(timeout)) url = to_string(url) if ' ' in url: url = url.replace(' ', '%20') if data: if isinstance(data, string_types): data = to_string(data) elif isinstance(data, (dict, MultiDict)): data = list(data.items()) if isinstance(data, (dict, tuple, list)): data = dict((to_string(k), to_string(v)) for k, v in data) data = urlencode(data) if method.lower() == 'get': url += '?%s' % data data = None else: data = to_bytes(data) req = Request(url, data=data, headers=headers or {}) req.get_method = lambda: method.upper() try: response = URL_OPENER.open(req, timeout=timeout) except Exception as error: message = u('Could not open the url: %s') % to_unicode(url) raise Error('url', message, exception=error) if isinstance(response, inesHTTPError): raise Error('url', response.message, exception=response) else: return response
def construct_structure(self, request, schema, schema_type, types, models, to_translate, parent_name=None): if isinstance(schema.typ, Sequence): child = schema.children[0] if not schema.name: schema = child name = camelcase(schema.name) details = { "model": name, "type": "sequence", "title": schema.title, "description": schema.description or EMPTY_STRING, } models[name].append(details) if isinstance(schema.title, TranslationString): to_translate["title"].append(details) if isinstance(schema.description, TranslationString): to_translate["description"].append(details) # Find and add child child_details = self.construct_structure( request, child, schema_type, types, models, to_translate, parent_name=schema.name ) if isinstance(details, dict): if isinstance(child.typ, Mapping): details["type"] = "model" details.update(child_details) else: details["fields"] = [child_details] else: details["fields"] = child_details return details elif isinstance(schema.typ, Tuple): raise NotImplementedError("Tuple type need to be implemented") elif isinstance(schema.typ, Mapping): fields = [] for child in schema.children: fields.append( self.construct_structure( request, child, schema_type, types, models, to_translate, parent_name=schema.name ) ) name = schema.name or parent_name if not name: return fields name = camelcase(name) details = { "type": "model", "title": schema.title, "description": schema.description or EMPTY_STRING, "fields": fields, "model": name, } models[name].append(details) if isinstance(schema.title, TranslationString): to_translate["title"].append(details) if isinstance(schema.description, TranslationString): to_translate["description"].append(details) return details else: name = camelcase(schema.name) details = {"fieldType": name, "title": schema.title, "description": schema.description or EMPTY_STRING} if isinstance(schema.title, TranslationString): to_translate["title"].append(details) if isinstance(schema.description, TranslationString): to_translate["description"].append(details) if hasattr(schema, "model_reference"): model = schema.model_reference["model"] model_key = schema.model_reference.get("key") or "key" model_query = schema.model_reference.get("query") or "name" model_application = schema.model_reference.get("application_name") or request.application_name details["modelReference"] = { "applicationName": model_application, "schemaName": schema.model_reference["schema"], "key": camelcase(model[model_key].name), "model": camelcase(model.name), "queryField": camelcase(model[model_query].name), } types[name].append(details) if isinstance(schema.typ, FilterByType): for cls in schema.typ.__class__.__mro__[1:]: if cls is not FilterByType: details["type"] = str(cls.__name__).lower() break details["filter"] = True elif hasattr(schema, "schema_type_name"): details["type"] = camelcase(schema.schema_type_name) else: details["type"] = get_colander_type_name(schema.typ) request_validation = [] if schema.validator: if isinstance(schema.validator, All): validators = schema.validator.validators elif not is_nonstr_iter(schema.validator): validators = [schema.validator] else: validators = schema.validator for validator in validators: if isinstance(validator, OneOfWithDescription): details["options"] = [] add_option = details["options"].append save_to_translate = False for choice, description in validator.choices_with_descripton: add_option({"value": choice, "text": description}) save_to_translate = save_to_translate or isinstance(description, TranslationString) if save_to_translate: to_translate["options"].append(details) elif isinstance(validator, OneOf): details["options"] = [] add_option = details["options"].append for choice in validator.choices: add_option({"value": choice, "text": to_unicode(choice).replace(UNDERSCORE, SPACE).title()}) else: if isinstance(validator, Length): validation_option = {} if validator.min is not None: validation_option["min"] = validator.min if validator.max is not None: validation_option["max"] = validator.max else: validation_option = True request_validation.append((validator, validation_option)) if hasattr(schema, "use_when"): details["useWhen"] = dict((camelcase(k), v) for k, v in schema.use_when.items()) if schema_type == "request": validation = {} if schema.required: validation["required"] = True if request_validation: for validator, validation_option in request_validation: validation[get_colander_type_name(validator)] = validation_option if validation: details["validation"] = validation default = schema.missing else: if schema.missing is drop: details["maybeNotSent"] = True default = schema.default if default is not drop and default is not required and default is not null: if isinstance(schema.typ, Number): default = schema.typ.num(default) elif isinstance(schema.typ, BaseBoolean): default = asbool(default) details["default"] = default return details
def loads(self, bstruct): return to_unicode(bstruct)
def _render(value, system): request = system.get('request') output_schema = None if isinstance(value, dict): output_schema = value['schema'] value = value['value'] delimiter = ';' quote_char = '"' line_terminator = '\r\n' quoting = QUOTE_ALL encoder = None if request is not None: response = request.response ct = response.content_type if ct == response.default_content_type: response.content_type = 'text/csv' output = request.registry.config.lookup_output_schema( request.matched_route.name, request_method=request.method) if output: output_schema = output[0].schema if output_schema: value = self.lookup_rows(request, output_schema.children[0], value) output_filename = getattr(output_schema, 'filename', None) if output_filename: if callable(output_filename): output_filename = output_filename() response.content_disposition = 'attachment; filename="%s"' % output_filename _get_param = request.params.get get_param = lambda k: _get_param(k) or _get_param(camelcase(k)) csv_delimiter = get_param('csv_delimiter') if csv_delimiter: delimiter = to_string(csv_delimiter) if delimiter == '\\t': delimiter = '\t' else: delimiter = delimiter[0] csv_quote_char = get_param('csv_quote_char') if csv_quote_char: quote_char = to_string(csv_quote_char) csv_line_terminator = get_param('csv_line_terminator') if csv_line_terminator: if csv_line_terminator == u('\\n\\r'): line_terminator = '\n\r' elif csv_line_terminator == u('\\n'): line_terminator = '\n' elif csv_line_terminator == u('\\r'): line_terminator = '\r' csv_encoding = get_param('csv_encoding') if csv_encoding: try: encoder = codecs.lookup(csv_encoding) except LookupError: raise Error('csv_encoding', _('Invalid CSV encoding')) else: if encoder.name != 'utf-8': request.response.charset = encoder.name yes_text = to_string(request.translate(_('Yes'))) no_text = to_string(request.translate(_('No'))) csv_quoting = get_param('csv_quoting') if csv_quoting: csv_quoting = to_string(csv_quoting).lower() if csv_quoting == 'minimal': quoting = QUOTE_MINIMAL elif csv_quoting == 'non_numeric': quoting = QUOTE_NONNUMERIC elif csv_quoting == 'none': quoting = QUOTE_NONE else: yes_text = to_string('Yes') no_text = to_string('No') if not value: return '' f = StringIO() csvfile = csv_writer( f, delimiter=delimiter, quotechar=quote_char, lineterminator=line_terminator, quoting=quoting) for value_items in value: row = [] for item in value_items: if item is None: item = '' elif isinstance(item, bool): item = item and yes_text or no_text elif isinstance(item, (string_types, integer_types)): item = to_string(item) elif isinstance(item, (DATE, DATETIME)): item = to_string(item.isoformat()) else: item = to_string(item or '') row.append(item) csvfile.writerow(row) f.seek(0) response = f.read() f.close() if encoder: if PY3: response = encoder.decode(encoder.encode(response)[0])[0] else: response = encoder.decode(response)[0] else: response = to_unicode(response) return response
def get_url_body(*args, **kwargs): return to_unicode(get_url_file(*args, **kwargs))
def create_message( self, subject, recipients, body=None, html=None, sender=None, cc=None, bcc=None, reply_to=None, content_charset='utf-8', attachments=None, message_id=None): force_development_email = None if not self.request.is_production_environ: force_development_email = self.settings.get('force_development_email') or None if body: body = to_unicode(body, encoding=content_charset) if not html: html = body.replace(NEW_LINE, HTML_NEW_LINE) if html: html = to_unicode(html, encoding=content_charset) if not html.lower().startswith('<html'): html = '<html><body>%s</body></html>' % html options = {} # FROM sender if sender: options['sender'] = format_email( sender, encoding=content_charset, force_development_email=force_development_email) # Envelope CC if cc: if isinstance(cc, dict): cc = [cc] options['cc'] = [ format_email(e, content_charset, force_development_email=force_development_email) for e in maybe_list(cc)] # Envelope BCC if bcc: if isinstance(bcc, dict): bcc = [bcc] options['bcc'] = [ format_email(e, content_charset, force_development_email=force_development_email) for e in maybe_list(bcc)] if not message_id: domain = self.settings.get('message_domain') or self.api_session_manager.default_domain message_id = make_msgid(make_unique_hash(10), domain) extra_headers = { 'Date': formatdate(localtime=True), 'Message-ID': to_string(message_id)} # Force reply to another email if reply_to: extra_headers['Reply-To'] = '<%s>' % to_string(reply_to) mime_attachments = [] if attachments: if isinstance(attachments, dict): attachments = [attachments] for attachment in maybe_list(attachments): f = attachment.get('file') or attachment['fp'] filename = to_string(attachment.get('filename') or basename(f.name)).replace(' ', '') mimetype = to_string(attachment.get('content_type') or find_mimetype(filename, f)) f.seek(0) mime_attachments.append(self.api_session_manager.attachment_cls( data=f, filename=filename, content_type=mimetype)) return self.api_session_manager.message_cls( subject=to_unicode(subject, encoding=content_charset), html=html, body=body, recipients=[ format_email(e, content_charset, force_development_email=force_development_email) for e in maybe_list(recipients)], attachments=mime_attachments, extra_headers=extra_headers, **options)
def clean_filename(filename): value = to_unicode(filename, errors='ignore') return clean_unicode(value, replace_case='_', mapping_dict=FILENAME_MAPPING)
def string_unique_code(value): value = to_bytes(value) return to_unicode(sha256(value).hexdigest())
# -*- coding: utf-8 -*- import os from string import digits as base_digits from string import ascii_letters as base_letters from six import u from ines import IGNORE_FULL_NAME_WORDS from ines.convert import maybe_unicode from ines.convert import to_unicode from ines.convert import unicode_join STRING_TO_DICT = lambda s: dict((l, l) for l in to_unicode(s)) LOWER_MAPPING = (u('àáâãäåæÆßçèéêëƒìíîïñðòóôõöšýùúüûž'), u('aaaaaaaabceeeefiiiinoooooosyyuuuuz')) UPPER_MAPPING = (LOWER_MAPPING[0].upper(), LOWER_MAPPING[1].upper()) MAPPING = dict(zip(*LOWER_MAPPING)) MAPPING.update(zip(*UPPER_MAPPING)) MAPPING.update({u('Ø'): u('O'), u('ø'): u('0'), u('þ'): u('b'), u(' '): u(' ')}) MAPPING.update(STRING_TO_DICT(base_letters + base_digits)) FILENAME_MAPPING = MAPPING.copy() FILENAME_MAPPING.update(STRING_TO_DICT('.-_')) PHONE_MAPPING = STRING_TO_DICT('x+' + base_digits) def clean_unicode(value, replace_case=None, mapping_dict=None): value = to_unicode(value)
def maybe_email(value): if validate_email(value): return to_unicode(value)
def make_uuid_hash(): return to_unicode(uuid4().hex)
def referer_path_url(self): if self.referer: url = urlparse(self.referer)._replace(query=None).geturl() return to_unicode(url)
def __call__(self, node, value): if not to_unicode(value).isnumeric(): raise Invalid(node, _('Invalid number'))
def maybe_phone_number(number): if number: number = clean_phone_number(to_unicode(number)) if 21 > len(number or '') > 3: return number
def get_files( self, key=None, attributes=None, only_one=False, **kwargs): attributes = set(attributes or ['id']) return_open_file = 'open_file' in attributes if return_open_file: attributes.remove('open_file') attributes.add('id') columns = [] relate_with_path = False for attribute in attributes: if attribute in File.__table__.c: columns.append(File.__table__.c[attribute]) if attribute not in ('id', 'created_date') and attribute in FilePath.__table__.c: columns.append(FilePath.__table__.c[attribute]) relate_with_path = True query = self.session.query(*columns or [File.id]) file_id = kwargs.get('file_id') if file_id: query = query.filter(FilePath.id.in_(maybe_set(file_id))) relate_with_path = True if relate_with_path: query = query.filter(File.file_id == FilePath.id) if 'id' in kwargs: query = query.filter(File.id.in_(maybe_set(kwargs['id']))) if key: query = query.filter(File.key.in_(maybe_set(key))) if 'application_code' in kwargs: application_code = kwargs['application_code'] if application_code is None: query = query.filter(File.application_code.is_(None)) else: query = query.filter(File.application_code == to_unicode(application_code)) if 'code_key' in kwargs: code_key = kwargs['code_key'] if code_key is None: query = query.filter(File.code_key.is_(None)) else: query = query.filter(File.code_key == to_unicode(code_key)) if 'type_key' in kwargs: type_key = kwargs['type_key'] if type_key is None: query = query.filter(File.type_key.is_(None)) else: query = query.filter(File.type_key == to_unicode(type_key)) if 'data' in kwargs: data = kwargs['data'] if data is None: query = query.filter(File.data.is_(None)) else: query = query.filter(File.data == to_unicode(data)) if 'parent_id' in kwargs: parent_id = kwargs['parent_id'] if parent_id is None: query = query.filter(File.parent_id.is_(None)) else: query = query.filter(File.parent_id == int(parent_id)) parent_key = kwargs.get('parent_key') if parent_key: parent = aliased(File) query = query.filter(File.parent_id == parent.id).filter(parent.key == to_unicode(parent_key)) order_by = kwargs.get('order_by') if order_by is not None: query = query.order_by(order_by) if only_one: response = query.first() else: response = query.all() if not return_open_file or not response: return response if only_one: response = [response] # Add items to SQLAlchemy tuple result files_binary = self.get_files_binary(*(f.id for f in response)) new_namedtuple = new_lightweight_named_tuple(response[0], 'open_file') response[:] = [ new_namedtuple(r + (files_binary[r.id], )) for r in response] if only_one: return response[0] else: return response
def clean_phone_number(value): value = to_unicode(value or '', errors='ignore').lower() return clean_unicode(value, replace_case='', mapping_dict=PHONE_MAPPING)