def refresh_webhook(request):
id_ = request.matchdict["id"]
source, _ = sources.check_id_key(id_=id_, key=request.matchdict["key"])
if source is None:
raise HTTPNotFound(f"Unknown id {id_}")
if source.get_type() != "git":
raise HTTPServerError(
f"Non GIT source {id_} cannot be refreshed by a webhook")
source_git = cast(sources.git.GitSource, source)
if request.headers.get("X-GitHub-Event") != "push":
LOG.info("Ignoring webhook notif for a non-push event on %s", id_)
return {"status": 200, "ignored": True, "reason": "Not a push"}
ref = request.json.get("ref")
if ref is None:
raise HTTPServerError(f"Webhook for {id_} is missing the ref")
if ref != "refs/heads/" + source_git.get_branch():
LOG.info("Ignoring webhook notif for non-matching branch %s on %s",
source_git.get_branch(), id_)
return {
"status": 200,
"ignored": True,
"reason": f"Not {source_git.get_branch()} branch"
}
return _refresh(request)
def refresh_all_webhook(request):
if not sources.MASTER_SOURCE:
raise HTTPServerError("Master source not initialized")
key = request.matchdict["key"]
sources.MASTER_SOURCE.validate_key(key)
if request.headers.get("X-GitHub-Event") != "push":
LOG.info("Ignoring webhook notif for a non-push event on %s")
return {"status": 200, "ignored": True, "reason": "Not a push"}
ref = request.json.get("ref")
if ref is None:
raise HTTPServerError("Webhook is missing the ref")
nb_refresh = 0
for id_, source in sources.get_sources().items():
if not source or source.get_type() != "git":
continue
source_git = cast(sources.git.GitSource, source)
if ref != "refs/heads/" + source_git.get_branch():
LOG.info(
"Ignoring webhook notif for non-matching branch %s!=refs/heads/%s on %s",
ref,
source_git.get_branch(),
id_,
)
continue
sources.refresh(id_=id_, key=key)
nb_refresh += 1
return {"status": 200, "nb_refresh": nb_refresh}
def check_celery(request):
'''
GET request that executes a task via celery and retrieve result to verify celery service
is functioning
:param request: Pyramid request object
'''
if pyramid.threadlocal.get_current_registry().settings is not None:
queue = pyramid.threadlocal.get_current_registry().settings.get(
'pdf.health_check.job.queue')
timeout = float(
pyramid.threadlocal.get_current_registry().settings.get(
'pdf.celery_timeout'))
else:
queue = 'health_check'
timeout = 10.0
try:
celery_response = health_check.apply_async(queue=queue)
heartbeat_message = celery_response.get(timeout=timeout)
except Exception as e:
logger.error("Heartbeat failed at celery. Check celery. %s", str(e))
return HTTPServerError()
if heartbeat_message[0:9] == 'heartbeat':
return HTTPOk()
else:
logger.error("Heartbeat failed at celery. Check celery.")
return HTTPServerError()
def heartbeat(request):
check_list = [check_datasource, check_celery]
results = [check_task(request) for check_task in check_list]
results = map(lambda x: isinstance(x,
HTTPServerError().__class__), results)
if True in results:
return HTTPServerError()
else:
return HTTPOk()
def create_shot(request):
"""runs when adding a new shot
"""
logged_in_user = get_logged_in_user(request)
name = request.params.get('name')
code = request.params.get('code')
status_id = request.params.get('status_id')
status = Status.query.filter_by(id=status_id).first()
project_id = request.params.get('project_id')
project = Project.query.filter_by(id=project_id).first()
logger.debug('project_id : %s' % project_id)
if name and code and status and project:
# get descriptions
description = request.params.get('description')
sequence_id = request.params['sequence_id']
sequence = Sequence.query.filter_by(id=sequence_id).first()
# get the status_list
status_list = StatusList.query.filter_by(
target_entity_type='Shot').first()
# there should be a status_list
# TODO: you should think about how much possible this is
if status_list is None:
return HTTPServerError(detail='No StatusList found')
new_shot = Shot(name=name,
code=code,
description=description,
sequence=sequence,
status_list=status_list,
status=status,
created_by=logged_in_user,
project=project)
DBSession.add(new_shot)
else:
logger.debug('there are missing parameters')
logger.debug('name : %s' % name)
logger.debug('code : %s' % code)
logger.debug('status : %s' % status)
logger.debug('project : %s' % project)
HTTPServerError()
return HTTPOk()
def upload_file(request):
"""
POSTing a file upload is very different than any other endpoint in assembl
API because all of the content will be passed in using a MULTIPART_HEADER,
with all of data as well as the file (along with its metadata)
"""
# Testing purposes on front-end
# raise Exception("Upload file exception occured!")
db = Document.default_db
ctx = request.context
user_id = authenticated_userid(request) or Everyone
discusison_id = ctx.get_discussion_id()
discussion = Discussion.get(discusison_id)
mime = request.POST['mime_type']
file_name = request.POST['name']
# Check if the file has previously existed, if so, change the name by appending "(n)"
# to it's name
try:
blob = File(discussion=discussion, mime_type=mime, title=file_name)
db.add(blob)
with request.POST['file'].file as f:
blob.add_file_data(f)
db.flush()
except Exception as e:
raise HTTPServerError(e)
view = 'default'
return blob.generic_json(view, user_id, ctx.get_permissions())
def get_extract_by_id(self):
time.sleep(2)
if self._request.matchdict.get('egrid') == 'CH1234':
with open('samples/extract.json') as f:
content = json.loads(f.read())
return HTTPOk(json_body=content)
return HTTPServerError('An error occurred.')
def format_exception(context, request):
"""
Catch all app exceptions and render them nicely
This will keep the status code, but will always return parseable json
Returns
-------
error : str
Identifying error key
message : str
Human-readable error message
stacktrace : str, optional
If pyramid.debug = true, also return the stacktrace to the client
"""
message = context.message if hasattr(context, "message") else str(context)
LOG.exception(message)
if not request.path.startswith("/api/") and not request.path.startswith(
"/admin/"):
if isinstance(context, HTTPException):
return context
else:
return HTTPServerError(message)
error = {"error": getattr(context, "error", "unknown"), "message": message}
if asbool(request.registry.settings.get("pyramid.debug", False)):
error["stacktrace"] = traceback.format_exc()
request.response.status_code = getattr(context, "status_code", 500)
return error
def update_sequence(request):
"""runs when adding a new sequence
"""
logged_in_user = get_logged_in_user(request)
sequence_id = request.params.get('sequence_id')
sequence = Sequence.query.filter_by(id=sequence_id).first()
name = request.params.get('name')
code = request.params.get('code')
status_id = request.params.get('status_id')
status = Status.query.filter_by(id=status_id).first()
if sequence and code and name and status:
# get descriptions
description = request.params.get('description')
#update the sequence
sequence.name = name
sequence.code = code
sequence.description = description
sequence.status = status
sequence.updated_by = logged_in_user
sequence.date_updated = datetime.datetime.now()
DBSession.add(sequence)
else:
logger.debug('there are missing parameters')
logger.debug('name : %s' % name)
logger.debug('status : %s' % status)
HTTPServerError()
return HTTPOk()
def upload_file(request):
cfg = get_cfg(request.registry.settings, 'naki.storage.')
root = cfg.get('root', None)
if not root:
raise HTTPServerError()
prefix = 'storage:'
filename = request.matchdict['filename']
print('File: ' + filename)
target_dir = request.GET.get('dir', 'upload')
while target_dir.startswith('/'):
target_dir = target_dir[1:]
print(target_dir)
# print(request.body)
path = os.path.join(root, target_dir)
print(root, path)
filepath = os.path.join(path, filename)
os.makedirs(path, exist_ok=True)
f = open(filepath, 'wb')
f.write(request.body)
f.close()
mime = guess_mime(magic.Magic(mime=True), filepath)
return APIResponse({
'name': filename,
'mime': mime,
'used': False,
'path': strip_path(root, filepath),
'metadata': load_metadata(filepath, mime)
})
def __init__(self, request):
if not isinstance(request.pyramid_oereb_processor, Processor):
raise HTTPServerError('Missing processor instance')
self._request = request
self._params = {k.upper(): v for k, v in request.params.items()}
self._real_estate_reader = request.pyramid_oereb_processor.real_estate_reader
self._municipality_reader = request.pyramid_oereb_processor.municipality_reader
def collection_get(self):
if not self._root:
raise HTTPServerError()
prefix = 'storage:'
used_paths = [x[0][len(prefix):] for x in DBSession.query(Link.uri).filter(Link.type == 'data').all() if
x[0].startswith(prefix)]
if self._request.GET.get('dirs', '0') == '1':
directories = {}
for root, dirs, files in os.walk(self._root):
d = directories
for dd in self._strip_path(root).split('/')[1:]:
d = d['dirs'][dd]
d['dirs'] = {dd: {} for dd in dirs}
d['file_count'] = len(files)
d['new_files'] = len([x for x in files if not self._strip_path(os.path.join(root, x)) in used_paths])
return APIResponse(directories)
print(self._request.matchdict['path'])
req_path = os.path.join(self._root, *self._request.matchdict['path'])
print(req_path)
r, d, f = next(os.walk(req_path))
res = {'files': [self._process_file(file, os.path.join(r, file), used_paths) for file in f if not file.endswith('.mods.xml')],
'path': self._strip_path(r),
'dirs': d}
print(res)
return APIResponse(res)
def persona_login(request):
if request.method != 'POST':
return HTTPMethodNotAllowed('Only POST is allowed')
assertion = request.POST.get('assertion', None)
if assertion is None:
return HTTPBadRequest('The assertion parameter is required')
if 'next_url' in request.params and request.params['next_url']:
request.session['next_url'] = request.params['next_url']
settings = request.registry.settings
data = {
'assertion': assertion,
'audience': get_audience(settings['public_url_root'])
}
response = requests.post(settings['persona_verifier_url'],
data=data,
verify=True)
if response.ok:
verification_data = response.json()
if verification_data['status'] == 'okay':
email = verification_data['email']
info = {'email': email}
user_id = hashlib.sha1(email.encode('utf-8')).hexdigest()
return register_or_update(request, 'persona', user_id, info,
request.route_path('home'))
else:
return HTTPForbidden(
'Mozilla Persona verifier can not verify your identity')
else:
return HTTPServerError(
'Mozilla Persona verifier is not working properly')
def test_HTTPException_server_error(self, LOGGER):
exc = HTTPServerError()
http_exc = exc_to_http_exc(exc)
self.assertIs(http_exc, exc)
self.assertEqual(http_exc.code, 500)
self.assertEqual(LOGGER.mock_calls, [
call.error(ANY, exc, ANY, 500, exc_info=True),
])
def heartbeat(request):
'''
service end point for heartbeat
'''
base_upload_path = request.registry.settings['hpz.frs.upload_base_path']
if check_database() and check_file_write(base_upload_path):
return HTTPOk()
return HTTPServerError()
def _proc_iter(proc: subprocess.Popen):
while True:
block = proc.stdout.read(4096) # type: ignore
if not block:
break
yield block
if proc.wait() != 0:
raise HTTPServerError("Error building the tarball")
def status(request):
try:
DBSession.execute("SET TIME ZONE 'UTC'")
alert_too_old = request.registry.settings['alert_too_old']
latest_dt = utcnow() - timedelta(hours=alert_too_old)
last_position = DBSession.query(Position.date_time).filter(
Position.date_time >= latest_dt).limit(1).scalar()
DBSession.commit()
if last_position is None:
raise HTTPServerError('Positions have not been received recently')
return {'version': __version__}
except DBAPIError as e:
DBSession.rollback()
LOGGER.warn(e)
raise e
except NoResultFound:
raise HTTPServerError('Positions have not been received recently')
class TestHTTPServerError(ExceptionViewTest):
view = error.http_server_error
exception = HTTPServerError("This is the error message")
response_status = 500
report_to_sentry = True
expected_result = {
"message": exception.args[0] # pylint:disable=unsubscriptable-object
}
def exc_to_http_exc(exc):
"""
Takes any :exc:`~exceptions.Exception` instance, returns a
:exc:`pyramid.httpexceptions.HTTPException` instance.
"""
if isinstance(exc, HTTPException):
code = getattr(exc, 'code', None)
if isinstance(code, (int, long)) and 200 <= code < 500:
LOGGER.debug(
'HTTPException: %r ("%s", code: %s)',
exc, ascii_str(exc), code)
else:
LOGGER.error(
'HTTPException: %r ("%s", code: %r)',
exc, ascii_str(exc), code,
exc_info=True)
http_exc = exc
elif isinstance(exc, AuthorizationError):
LOGGER.debug(
'Authorization not successful: %r (public message: "%s")',
exc, ascii_str(exc.public_message))
http_exc = HTTPForbidden(exc.public_message)
elif isinstance(exc, ParamCleaningError):
LOGGER.debug(
'Request parameters not valid: %r (public message: "%s")',
exc, ascii_str(exc.public_message))
http_exc = HTTPBadRequest(exc.public_message)
elif isinstance(exc, TooMuchDataError):
LOGGER.debug(
'Too much data requested: %r (public message: "%s")',
exc, ascii_str(exc.public_message))
http_exc = HTTPForbidden(exc.public_message)
else:
if isinstance(exc, DataAPIError):
if isinstance(exc, ResultCleaningError):
LOGGER.error(
'Result cleaning error: %r (public message: "%s")',
exc, ascii_str(exc.public_message),
exc_info=True)
else:
LOGGER.error(
'%r (public message: "%s")',
exc, ascii_str(exc.public_message),
exc_info=True)
public_message = (
None
if exc.public_message == DataAPIError.default_public_message
else exc.public_message)
else:
LOGGER.error(
'Non-HTTPException/DataAPIError exception: %r',
exc,
exc_info=True)
public_message = None
http_exc = HTTPServerError(public_message)
return http_exc
def subrequest_object(request, object_id):
subreq = make_subrequest(request, "/" + object_id)
subreq.headers['Accept'] = 'application/json'
# Tweens are suppressed here because this is an internal call and doesn't need things like HTML processing.
# -kmp 2-Feb-2021
response = request.invoke_subrequest(subreq, use_tweens=False)
if response.status_code >= 300: # alas, the response from a pyramid subrequest has no .raise_for_status()
raise HTTPServerError("Error obtaining object: %s" % object_id)
object_json = response.json
return object_json
def heartbeat(request):
'''
service end point for heartbeat
'''
try:
check_celery(request)
except Exception as e:
logger.error("TSB Heartbeat failed. Check TSB worker. %s", str(e))
return HTTPServerError()
return HTTPOk()
def refresh_all(request):
if not sources.MASTER_SOURCE:
raise HTTPServerError("Master source not initialized")
key = request.matchdict["key"]
sources.MASTER_SOURCE.validate_key(key)
nb_refresh = 0
for id_ in sources.get_sources().keys():
sources.refresh(id_=id_, key=key)
nb_refresh += 1
return {"status": 200, "nb_refresh": nb_refresh}
def saml_metadata_view(request):
complete_url = request.route_url('social.complete', backend="saml")
saml_backend = load_backend(
load_strategy(request),
"saml",
redirect_uri=complete_url,
)
metadata, errors = saml_backend.generate_metadata_xml()
if errors:
raise HTTPServerError("SAML issue:" + errors)
return Response(metadata, content_type='text/xml', charset="utf-8")
def test_server_error(self, monkeypatch):
monkeypatch.setattr(pyramid.logger, "exception", mock.Mock())
request = testing.DummyRequest()
exc = HTTPServerError(detail="detail", explanation="explanation")
response = pyramid.pyramid_handler(exc, request)
assert request.response.status == "500 Internal Server Error"
assert response == {"message": "explanation", "debug_message": "detail", "code": None}
assert pyramid.logger.exception.call_args == mock.call("explanation")
def confirm_email_sent(request):
localizer = request.localizer
# TODO: How to make this not become a spambot?
email = request.matchdict.get('email')
if not email:
raise HTTPNotFound()
if '@' not in email:
raise HTTPBadRequest("Not an email")
email = EmailString.normalize_email_case(email)
email_objects = AbstractAgentAccount.default_db.query(
AbstractAgentAccount).filter_by(email_ci=email)
verified_emails = [e for e in email_objects if e.verified]
unverified_emails = [e for e in email_objects if not e.verified]
if len(verified_emails) > 1:
# TODO!: Merge accounts.
raise HTTPServerError("Multiple verified emails")
elif len(verified_emails):
if len(unverified_emails):
# TODO!: Send an email, mention duplicates, and...
# offer to merge accounts?
# Send an email to other emails of the duplicate? Sigh!
pass
return HTTPFound(location=maybe_contextual_route(
request,
'login',
_query=dict(identifer=email,
error=localizer.translate(
_("This email is already confirmed.")))))
else:
if len(unverified_emails):
# Normal case: Send an email. May be spamming
for email_account in unverified_emails:
send_confirmation_email(request, email_account)
slug = request.matchdict.get('discussion_slug', None)
slug_prefix = "/" + slug if slug else ""
return dict(
get_default_context(request),
action="%s/confirm_email_sent/%s" % (slug_prefix, email),
email=email,
title=localizer.translate(_('Confirmation requested')),
description=localizer.translate(
_('A confirmation e-mail has been sent to your account and should be in your inbox in a few minutes. '
'It contains a confirmation link, please click on it in order to confirm your e-mail address. '
'If you did not receive any confirmation e-mail (check your spams), click here.'
)))
else:
# We do not have an email to this name.
return HTTPFound(location=maybe_contextual_route(
request,
'register',
email=email,
_query=dict(error=localizer.translate(
_("We do not know about this email.")))))
def process(self, session, request):
response = request.response
response.content_type = 'text/plain; charset=UTF-8'
response.headerlist.extend(cors_headers(request))
response.headerlist.append(
('Cache-Control',
'no-store, no-cache, must-revalidate, max-age=0'))
data = request.body_file.read()
if not data:
return HTTPServerError("Payload expected.")
try:
messages = decode(data)
except:
return HTTPServerError("Broken JSON encoding.")
for msg in messages:
session.message(msg)
response.status = 204
def delete(self):
contact_id = self.request.swagger_data["contact_id"]
contact = ContactObject(self.user.user_id, contact_id=contact_id)
try:
contact.delete()
except Exception as exc:
if isinstance(exc, ForbiddenAction):
raise HTTPForbidden(exc)
else:
raise HTTPServerError(exc)
return Response(None, 204)
def callback(self, request, order):
token = request.POST.get('stripeToken')
if not token:
return HTTPBadRequest('missing stripeToken')
months = int(order.time.days / 30)
username = order.user.username
try:
charge = stripe.Charge.create(
amount=int(order.amount * 100),
currency='eur',
card=token,
description='%d months for %s' % (months, username),
)
order.payment['charge_id'] = charge['id']
if charge['refunded'] or not charge['paid']:
request.message.error('Error: The order has not been payed or '
'has been refunded')
loc = request.route_url('order_view', hexid=hex(order.id)[2:])
return HTTPSeeOther(location=loc)
order.paid_amount = float(charge['amount']) / 100.0
order.paid = True
order.user.add_paid_time(order.time)
loc = request.route_url('order_view', hexid=hex(order.id)[2:])
return HTTPSeeOther(location=loc)
except stripe.error.CardError as e:
message = e.json_body['error']['message']
request.message.error('Card error: ' + message)
order.payment['error'] = message
loc = request.route_url('order_view', hexid=hex(order.id)[2:])
return HTTPSeeOther(location=loc)
except stripe.error.AuthenticationError:
log.critical('Authentication with Stripe failed')
return HTTPServerError()
except stripe.error.StripeError as e:
log.critical('Stripe error: ' + str(e))
return HTTPServerError()
def site_gallery_image_update(request):
site = Site.get_from_key_and_user_id(request.matchdict['site_key'], request.authenticated_userid)
gallery = Gallery.get_from_site_id_and_gallery_id(site.id, request.matchdict['gallery_id'])
if not gallery:
return HTTPNotFound()
image = Image.get_from_gallery_id_and_image_id(gallery.id, request.matchdict['image_id'])
if not image:
return HTTPNotFound()
if 'title' in request.POST:
image.title = request.POST['title']
if 'description' in request.POST:
image.description = request.POST['description']
request.session.flash("Image title and description was updated!")
if 'return_page_section_id' in request.GET:
section = PageSection.get_from_id(request.GET['return_page_section_id'])
if section and section.page.site_id == gallery.site_id:
url = request.current_route_url(_route_name='site_page', page_id=section.page_id)+ '#page-section-' + str(section.id)
return HTTPFound(location=url)
return HTTPFound(location=request.current_route_url(_route_name='site_gallery'))
if 'delete_image' in request.POST:
image.delete(request.imbo)
return HTTPFound(location=request.current_route_url(_route_name='site_gallery'))
if 'file' in request.POST and getattr(request.POST['file'], 'file') and not image.parent_image_id:
result = request.imbo.add_image_from_string(request.POST['file'].file)
if not result or 'imageIdentifier' not in result:
return HTTPServerError()
new_image = Image(parent_image=image, imbo_id=result['imageIdentifier'], width=result['width'],
height=result['height'], site=site, order=int(time.time()))
DBSession.add(new_image)
DBSession.flush()
return new_image
return HTTPBadRequest()
def delete(self):
message_id = self.request.swagger_data["message_id"]
message = ObjectMessage(self.user.user_id, message_id=message_id)
try:
message.get_db()
message.get_index()
except NotFound:
raise ResourceNotFound
try:
message.delete_db()
message.delete_index()
except Exception as exc:
raise HTTPServerError(exc)
return Response(None, 204)