def ping_url(protocol, url):
url = to_string(url)
protocol = to_string(protocol).lower()
url_parts = url.split('/', 1)
host = url_parts[0]
if len(url_parts) == 1:
path = '/'
else:
path = '/%s' % url_parts[1]
if protocol == 'https':
connection = HTTPSConnection(host)
elif protocol == 'http':
connection = HTTPConnection(host)
else:
raise ValueError('url', u('Invalid protocol %s. Use only http or https.') % protocol)
valid_url = False
try:
connection.request('HEAD', path)
response = connection.getresponse()
except Exception:
pass
else:
if response.status != 404:
valid_url = True
finally:
connection.close()
return valid_url
def __call__(self, environ, start_response):
content_type = get_content_type(environ.get('CONTENT_TYPE'))
if content_type == 'application/json' and 'wsgi.input' in environ:
body = environ['wsgi.input'].read()
if body:
arguments = []
body = to_string(body)
try:
body_json = loads(body)
for key, values in dict(body_json).items():
key = to_string(key)
values = maybe_list(values)
for value in values:
if value is None:
arguments.append('%s=' % key)
else:
arguments.append('%s=%s' % (key, quote(dump_query_value(value))))
body = string_join('&', arguments)
except (ValueError, UnicodeEncodeError):
headers = [('Content-type', 'application/json')]
error = HTTPInvalidJSONPayload()
start_response(error.status, headers)
return format_error_response_to_json(error)
environ_add_POST(environ, body or '')
return self.application(environ, start_response)
def clean_string(value, replace_case=None, mapping_dict=None):
value = to_string(value)
get_letter = (mapping_dict or MAPPING).get
if replace_case is not None:
replace_case = to_string(replace_case)
return ''.join(get_letter(letter, replace_case) for letter in value)
else:
return ''.join(get_letter(letter, letter) for letter in value)
def get_token_file_path(self, token_256):
token_256 = to_string(token_256)
return normpath(
join_paths(
self.settings['token.path'],
token_256[0],
token_256))
def run_monitor(self):
try:
self.validate_daemons()
immediate_jobs = set(
to_string(k)
for k in self.config.cache.get_values(JOBS_IMMEDIATE_KEY, expire=None))
for apijob in list(JOBS):
run_job = False
if apijob.name in immediate_jobs:
run_job = True
immediate_jobs.remove(apijob.name)
elif apijob.will_run() and apijob.next_date <= NOW():
run_job = True
if run_job:
try:
daemon = start_system_thread(
'job_%s' % apijob.name,
apijob,
sleep_method=False)
except KeyError:
pass
else:
RUNNING_JOBS.append((apijob, daemon))
self.config.cache.replace_values(JOBS_IMMEDIATE_KEY, immediate_jobs, expire=None)
except Exception as error:
self.system_session().logging.log_critical('jobs_undefined_error', str(error))
return 5
else:
return 0.5
def send_immediately(self, *args, **kwargs):
fail_silently = kwargs.pop('fail_silently', False)
mime_message = self.create_message(*args, **kwargs)
try:
self.api_session_manager.mailer.send_immediately(mime_message, fail_silently=fail_silently)
except Exception as error:
if isinstance(error, SMTPRecipientsRefused):
for email, error_message in error.args[0].items():
code = error_message[0]
if code == 554:
raise Error('email', 'Invalid email "%s"' % to_string(email))
elif code == 450:
raise Error('email', 'Invalid email domain "%s"' % to_string(email))
raise
else:
return True
def parse_request_type(content_type):
if content_type is None:
return 'text/plain'
else:
content_type = to_string(content_type)
if ';' in content_type:
content_type = content_type.split(';', 1)[0]
return '/'.join(map(str.strip, content_type.lower().split('/')))
def parse_request_type(content_type):
if content_type is None:
return 'text/plain'
content_type = to_string(content_type)
if ';' in content_type:
content_type = content_type.split(';', 1)[0]
return string_join('/', (c.strip().lower() for c in content_type.split('/')))
def clean_junk_locks(self):
for path, dirnames, filenames in walk_on_path(self.path):
filenames = filenames or []
for dirname in dirnames:
folder_path = join_paths(path, dirname)
for filename in get_dir_filenames(folder_path):
if not filename.startswith('.'):
filenames.append(join_paths(dirname, filename))
for filename in filenames:
filename = to_string(filename)
if filename.startswith('.'):
continue
file_path = join_paths(path, filename)
if '.' in filename:
# Delete inactive positions locks
binary = get_file_binary(file_path, mode='r')
if binary:
info = binary.split()
if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]):
try:
getpgid(int(info[1]))
except OSError as error:
if error.errno is errno.ESRCH:
remove_file_quietly(
file_path,
retries=self.retries,
retry_errno=self.retry_errno)
else:
# Clean locks wait list
# Get last modified time, to check if file as been updated in the process
modified_time = file_modified_time(file_path)
if modified_time:
binary = get_file_binary(file_path, mode='r')
if binary:
# Find alive locks
keep_codes = binary.splitlines()
for i, line in enumerate(keep_codes):
info = line.split()
if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]):
try:
getpgid(int(info[1]))
except OSError as error:
if error.errno is errno.ESRCH:
# Add empty line to keep position number
keep_codes[i] = ''
# Check if file as been updated in the process
last_modified_time = file_modified_time(file_path)
if last_modified_time and modified_time == last_modified_time:
if not any(keep_codes):
remove_file_quietly(file_path)
else:
with open(file_path, 'w') as f:
f.write(NEW_LINE.join(keep_codes))
def _render(value, system):
value = to_string(value)
request = system.get('request')
if request is not None:
response = request.response
ct = response.content_type
if ct == response.default_content_type:
response.content_type = 'text/html'
return value
def __call__(self, environ, start_response):
try:
for chunk in self.application(environ, start_response):
yield chunk
except (BaseException, Exception) as error:
# Save / log error
request = make_request(self.config, environ)
try:
small_message = '%s: %s' % (error.__class__.__name__, to_string(error))
except (BaseException, Exception):
small_message = error
print_message = True
if request.api is not None:
api_manager = self.settings.get('api_manager')
if api_manager is not None:
logging_api_name = api_manager.__api_name__
else:
logging_api_name = 'logging'
try:
getattr(request.api, logging_api_name).log_critical(
'internal_server_error',
str(small_message))
except (BaseException, Exception):
pass
else:
print_message = False
if print_message:
print(''.join(format_exception(*sys.exc_info())))
if isinstance(request.registry.config, APIConfigurator):
headers = [('Content-type', 'application/json')]
internal_server_error = HTTPInternalServerError()
start_response(internal_server_error.status, headers)
response = format_error_response_to_json(internal_server_error, request=request)
yield response
else:
return_default = False
error_view = self.config.settings.get('errors.interface.global_error_view')
if error_view is None:
return_default = True
else:
try:
response = error_view(error, request)
except Exception:
return_default = True
else:
start_response(response.status, response.headerlist)
yield response.body
if return_default:
internal_server_error = HTTPInternalServerError()
for response in internal_server_error(environ, start_response):
yield response
def make_dir(path, mode=0o777, make_dir_recursively=False):
path = to_string(path)
try:
mkdir(path, mode)
except OSError as error:
if make_dir_recursively and error.errno is errno.ENOENT:
makedirs(path, mode)
elif error.errno is not errno.EEXIST:
raise
return path
def __call__(self, environ, start_response):
http_origin = environ.get('HTTP_ORIGIN')
if not self.allow_all_origins and http_origin not in self.allowed_origins:
return self.application(environ, start_response)
http_method = environ.get('REQUEST_METHOD')
if http_method not in self.allowed_methods:
method_not_allowed = HTTPMethodNotAllowed()
start_response(
method_not_allowed.status,
[('Content-type', 'application/json')])
return format_error_response_to_json(method_not_allowed)
cors_headers = []
if self.allow_all_origins:
cors_headers.append(('Access-Control-Allow-Origin', '*'))
else:
cors_headers.append(('Access-Control-Allow-Origin', to_string(http_origin)))
if http_method == 'OPTIONS':
methods = environ.get('HTTP_ACCESS_CONTROL_REQUEST_METHOD')
if methods:
cors_headers.append(('Access-Control-Allow-Methods', to_string(methods)))
http_headers = environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
if http_headers:
cors_headers.append(('Access-Control-Allow-Headers', to_string(http_headers)))
if self.max_age is not None:
cors_headers.append(('Access-Control-Max-Age', to_string(self.max_age)))
start_response(HTTPNoContent().status, cors_headers)
return []
else:
def start_response_decorator(status, headers, exc_info=None):
headers.extend(cors_headers)
return start_response(status, headers, exc_info)
return self.application(environ, start_response_decorator)
def validate_skype_username(username, validate_with_api=False):
if username:
username = to_string(username)
if SKYPE_USERNAME_REGEX.match(username):
if not validate_with_api:
return True
response = open_json_url(
'https://login.skype.com/json/validator',
data={'new_username': username},
method='get')
if response['data']['markup'].lower() == 'skype name not available':
return True
return False
def open_url(url, data=None, timeout=None, headers=None, method='get'):
if timeout:
timeout = abs(float(timeout))
url = to_string(url)
if ' ' in url:
url = url.replace(' ', '%20')
if data:
if isinstance(data, string_types):
data = to_string(data)
elif isinstance(data, (dict, MultiDict)):
data = list(data.items())
if isinstance(data, (dict, tuple, list)):
data = dict((to_string(k), to_string(v)) for k, v in data)
data = urlencode(data)
if method.lower() == 'get':
url += '?%s' % data
data = None
else:
data = to_bytes(data)
req = Request(url, data=data, headers=headers or {})
req.get_method = lambda: method.upper()
try:
response = URL_OPENER.open(req, timeout=timeout)
except Exception as error:
message = u('Could not open the url: %s') % to_unicode(url)
raise Error('url', message, exception=error)
if isinstance(response, inesHTTPError):
raise Error('url', response.message, exception=response)
else:
return response
def format_email(email, encoding=None, force_development_email=None):
label = None
if isinstance(email, dict):
if 'label' in email:
label = email['label']
email = email['email']
elif not isinstance(email, string_types):
label, email = email
return to_unicode(
formataddr((
maybe_string(label, encoding=encoding),
to_string(force_development_email or email, encoding=encoding))))
def format_email(email, encoding=None, force_development_email=None):
label = None
if isinstance(email, dict):
email = to_string(email['email'], encoding=encoding)
if 'label' in email:
label = to_string(email['label'], encoding=encoding)
elif not isinstance(email, str):
label = to_string(email[0], encoding=encoding)
email = to_string(email[1], encoding=encoding)
else:
email = to_string(email, encoding=encoding)
return formataddr((label, force_development_email and to_string(force_development_email) or email))
def __call__(self, node, value):
number = to_string(value).lower()
if len(number) != self.length:
raise Invalid(node, _('Need to have ${length} chars', mapping={'length': self.length}))
digit_sum = 0
second_digit = False
for letter in reversed(number):
digit = PORTUGUESE_CC_LETTER_MAPPING.get(letter)
if second_digit:
digit *= 2
if digit > 9:
digit -= 9
digit_sum += digit
second_digit = not second_digit
if digit_sum % 10:
raise Invalid(node, _('Invalid code'))
def save_file(
self,
binary,
application_code,
code_key=None,
type_key=None,
data=None,
filename=None,
title=None,
parent_id=None):
file_path = self.save_file_path(binary, filename)
session_id = session_type = None
if self.request.authenticated:
session_id = self.request.authenticated.session_id
session_type = self.request.authenticated.session_type
# Add applications file relation
new = File(
file_id=file_path.id,
parent_id=parent_id,
key=make_unique_hash(70),
application_code=to_string(application_code),
code_key=maybe_string(code_key),
type_key=maybe_string(type_key),
data=maybe_string(data),
filename=maybe_string(filename),
title=maybe_string(title),
session_type=session_type,
session_id=session_id)
self.session.add(new)
self.session.flush()
# Add some attributes
new.size = file_path.size
new.mimetype = file_path.mimetype
return new
def put_binary_on_file(
path,
binary,
mode='wb',
retries=3,
retry_errno=DEFAULT_RETRY_ERRNO,
make_dir_recursively=False):
if 'b' in mode:
binary = to_bytes(binary)
else:
binary = to_string(binary)
try:
with open(path, mode) as f:
f.write(binary)
except IOError as error:
if error.errno is errno.ENOENT:
# Missing folder, create and try again
make_dir(dirname(path), make_dir_recursively=make_dir_recursively)
elif error.errno not in retry_errno:
raise
else:
# Try again, or not!
retries -= 1
if not retries:
raise
return put_binary_on_file(
path,
binary,
mode=mode,
retries=retries,
retry_errno=retry_errno,
make_dir_recursively=make_dir_recursively)
else:
return True
def create_message(
self,
subject,
recipients,
body=None,
html=None,
sender=None,
cc=None,
bcc=None,
reply_to=None,
content_charset='utf-8',
attachments=None,
message_id=None):
force_development_email = None
if not self.request.is_production_environ:
force_development_email = self.settings.get('force_development_email') or None
if body:
body = to_string(body, encoding=content_charset)
if not html:
html = body.replace(NEW_LINE, HTML_NEW_LINE)
if html:
html = to_string(html, encoding=content_charset)
if not html.lower().startswith('<html'):
html = '<html><body>%s</body></html>' % html
options = {}
# FROM sender
if sender:
options['sender'] = format_email(
sender,
encoding=content_charset,
force_development_email=force_development_email)
# Envelope CC
if cc:
if isinstance(cc, dict):
cc = [cc]
options['cc'] = [
format_email(e, content_charset, force_development_email=force_development_email)
for e in maybe_list(cc)]
# Envelope BCC
if bcc:
if isinstance(bcc, dict):
bcc = [bcc]
options['bcc'] = [
format_email(e, content_charset, force_development_email=force_development_email)
for e in maybe_list(bcc)]
if not message_id:
domain = self.settings.get('message_domain') or self.api_session_manager.default_domain
message_id = make_msgid(make_unique_hash(10), domain)
extra_headers = {
'Date': formatdate(localtime=True),
'Message-ID': to_string(message_id)}
# Force reply to another email
if reply_to:
extra_headers['Reply-To'] = '<%s>' % to_string(reply_to)
mime_attachments = []
if attachments:
if isinstance(attachments, dict):
attachments = [attachments]
for attachment in maybe_list(attachments):
f = attachment.get('file') or attachment['fp']
filename = to_string(attachment.get('filename') or basename(f.name)).replace(' ', '')
mimetype = to_string(attachment.get('content_type') or find_mimetype(filename, f))
f.seek(0)
mime_attachments.append(self.api_session_manager.attachment_cls(
data=f,
filename=filename,
content_type=mimetype))
return self.api_session_manager.message_cls(
subject=to_string(subject, encoding=content_charset),
html=html,
body=body,
recipients=[
format_email(e, content_charset, force_development_email=force_development_email)
for e in maybe_list(recipients)],
attachments=mime_attachments,
extra_headers=extra_headers,
**options)
def validate_gps(value):
value = to_string(value)
return bool(GPS_REGEX.match(value))
def _render(value, system):
request = system.get('request')
output_schema = None
if isinstance(value, dict):
output_schema = value['schema']
value = value['value']
delimiter = ';'
quote_char = '"'
line_terminator = '\r\n'
quoting = QUOTE_ALL
encoder = None
if request is not None:
response = request.response
ct = response.content_type
if ct == response.default_content_type:
response.content_type = 'text/csv'
output = request.registry.config.lookup_output_schema(
request.matched_route.name,
request_method=request.method)
if output:
output_schema = output[0].schema
if output_schema:
value = self.lookup_rows(request, output_schema.children[0], value)
output_filename = getattr(output_schema, 'filename', None)
if output_filename:
if callable(output_filename):
output_filename = output_filename()
response.content_disposition = 'attachment; filename="%s"' % output_filename
_get_param = request.params.get
get_param = lambda k: _get_param(k) or _get_param(camelcase(k))
csv_delimiter = get_param('csv_delimiter')
if csv_delimiter:
delimiter = to_string(csv_delimiter)
if delimiter == '\\t':
delimiter = '\t'
else:
delimiter = delimiter[0]
csv_quote_char = get_param('csv_quote_char')
if csv_quote_char:
quote_char = to_string(csv_quote_char)
csv_line_terminator = get_param('csv_line_terminator')
if csv_line_terminator:
if csv_line_terminator == '\\n\\r':
line_terminator = '\n\r'
elif csv_line_terminator == '\\n':
line_terminator = '\n'
elif csv_line_terminator == '\\r':
line_terminator = '\r'
csv_encoding = get_param('csv_encoding')
if csv_encoding:
try:
encoder = codecs.lookup(csv_encoding)
except LookupError:
raise Error('csv_encoding', _('Invalid CSV encoding'))
else:
if encoder.name != 'utf-8':
request.response.charset = encoder.name
yes_text = request.translate(_('Yes'))
no_text = request.translate(_('No'))
csv_quoting = get_param('csv_quoting')
if csv_quoting:
csv_quoting = to_string(csv_quoting).lower()
if csv_quoting == 'minimal':
quoting = QUOTE_MINIMAL
elif csv_quoting == 'non_numeric':
quoting = QUOTE_NONNUMERIC
elif csv_quoting == 'none':
quoting = QUOTE_NONE
else:
yes_text = 'Yes'
no_text = 'No'
if not value:
return ''
f = StringIO()
csvfile = csv_writer(
f,
delimiter=delimiter,
quotechar=quote_char,
lineterminator=line_terminator,
quoting=quoting)
for value_items in value:
row = []
for item in value_items:
if item is None:
item = ''
elif not isinstance(item, str):
if isinstance(item, bool):
item = item and yes_text or no_text
elif isinstance(item, (float, int)):
item = str(item)
elif isinstance(item, (DATE, DATETIME)):
item = item.isoformat()
elif not isinstance(item, str):
item = to_string(item)
row.append(item)
csvfile.writerow(row)
f.seek(0)
response = f.read()
f.close()
if encoder:
response = encoder.decode(encoder.encode(response)[0])[0]
else:
response = to_string(response)
return response
def validate_email(value):
value = to_string(value)
return bool(EMAIL_REGEX.match(value))
def dump_query_value(value):
if isinstance(value, string_types):
return to_string(value)
else:
return to_string(compact_dump(value))
def get_files(
self,
key=None,
attributes=None,
only_one=False,
**kwargs):
attributes = set(attributes or ['id'])
return_open_file = 'open_file' in attributes
if return_open_file:
attributes.remove('open_file')
attributes.add('id')
columns = []
relate_with_path = False
for attribute in attributes:
if attribute in File.__table__.c:
columns.append(File.__table__.c[attribute])
if attribute not in ('id', 'created_date') and attribute in FilePath.__table__.c:
columns.append(FilePath.__table__.c[attribute])
relate_with_path = True
query = self.session.query(*columns or [File.id])
if relate_with_path:
query = query.filter(File.file_id == FilePath.id)
if 'id' in kwargs:
query = query.filter(File.id.in_(maybe_set(kwargs['id'])))
if key:
query = query.filter(File.key.in_(maybe_set(key)))
if 'file_id' in kwargs:
query = query.filter(File.file_id.in_(maybe_set(kwargs['file_id'])))
if 'application_code' in kwargs:
application_code = kwargs['application_code']
if application_code is None:
query = query.filter(File.application_code.is_(None))
else:
query = query.filter(File.application_code == to_string(application_code))
if 'code_key' in kwargs:
code_key = kwargs['code_key']
if code_key is None:
query = query.filter(File.code_key.is_(None))
else:
query = query.filter(File.code_key == to_string(code_key))
if 'type_key' in kwargs:
type_key = kwargs['type_key']
if type_key is None:
query = query.filter(File.type_key.is_(None))
else:
query = query.filter(File.type_key == to_string(type_key))
if 'data' in kwargs:
data = kwargs['data']
if data is None:
query = query.filter(File.data.is_(None))
else:
query = query.filter(File.data == to_string(data))
if 'parent_id' in kwargs:
parent_id = kwargs['parent_id']
if parent_id is None:
query = query.filter(File.parent_id.is_(None))
else:
query = query.filter(File.parent_id == int(parent_id))
parent_key = kwargs.get('parent_key')
if parent_key:
parent = aliased(File)
query = query.filter(File.parent_id == parent.id).filter(parent.key == to_string(parent_key))
order_by = kwargs.get('order_by')
if order_by is not None:
query = query.order_by(order_by)
if only_one:
response = query.first()
else:
response = query.all()
if not return_open_file or not response:
return response
if only_one:
response = [response]
# Add items to SQLAlchemy tuple result
files_binary = self.get_files_binary(*(f.id for f in response))
new_namedtuple = new_lightweight_named_tuple(response[0], 'open_file')
response[:] = [
new_namedtuple(r + (files_binary[r.id], ))
for r in response]
if only_one:
return response[0]
else:
return response
def maybe_email(value):
value = to_string(value)
if validate_email(value):
return value
def user_agent_is_mobile(user_agent):
if user_agent:
user_agent = to_string(user_agent)
return MOBILE_REGEX_B.search(user_agent) or MOBILE_REGEX_V.search(user_agent[0:4])
return False
def maybe_phone_number(number):
if number:
number = to_string(number)
number = clean_phone_number(number)
if 21 > len(number or '') > 3:
return number
def get_content_type(value):
if value:
return to_string(value).split(';', 1)[0].strip()