def __init__(self, location=None, name=None, description=None, **kw):
HTTPUnprocessableEntity.__init__(self)
if location is None:
assert name is None and description is None and not kw
self.detail = None
else:
self.detail = dict(location=location, name=name,
description=description, **kw)
def users_update(request):
"""
Updates user object
"""
user = User.by_id(request.matchdict.get('user_id'))
if not user:
return HTTPNotFound()
post_data = request.safe_json_body or {}
if request.method == 'PATCH':
form = forms.UserUpdateForm(MultiDict(post_data),
csrf_context=request)
if form.validate():
form.populate_obj(user, ignore_none=True)
if form.user_password.data:
user.set_password(user.user_password)
if form.status.data:
user.status = 1
else:
user.status = 0
else:
return HTTPUnprocessableEntity(body=form.errors_json)
return user.get_dict(exclude_keys=['security_code_date', 'notes',
'security_code', 'user_password'])
def api_frontpage_(request):
form = request.web_input(since=None, count=0)
since = None
try:
if form.since:
since = d.parse_iso8601(form.since)
count = int(form.count)
except ValueError:
raise HTTPUnprocessableEntity(json=_ERROR_UNEXPECTED)
else:
count = min(count or 100, 100)
submissions = index.filter_submissions(request.userid, index.recent_submissions())
ret = []
for e, sub in enumerate(submissions, start=1):
if (since is not None and since >= sub['unixtime']) or (count and e > count):
break
tidy_submission(sub)
ret.append(sub)
return ret
def put_label_comment(
request: Request, name: CommentLabelOption, reason: str
) -> Response:
"""Add a label to a comment."""
comment = request.context
if not request.user.is_label_available(name):
raise HTTPUnprocessableEntity("That label is not available.")
savepoint = request.tm.savepoint()
weight = request.user.comment_label_weight
if weight is None:
weight = request.registry.settings["tildes.default_user_comment_label_weight"]
label = CommentLabel(comment, request.user, name, weight, reason)
request.db_session.add(label)
_mark_comment_read_from_interaction(request, comment)
try:
# manually flush before attempting to commit, to avoid having all objects
# detached from the session in case of an error
request.db_session.flush()
request.tm.commit()
except FlushError:
savepoint.rollback()
# re-query the comment to get complete data
comment = (
request.query(Comment)
.join_all_relationships()
.filter_by(comment_id=comment.comment_id)
.one()
)
return {"comment": comment}
def post_comment_reply(request: Request, markdown: str) -> dict:
"""Post a reply to a comment with Intercooler."""
parent_comment = request.context
wait_mins = _reply_wait_minutes(request, request.user, parent_comment.user)
if wait_mins:
incr_counter("comment_back_and_forth_warnings")
raise HTTPUnprocessableEntity(
f"You can't reply to this user yet. Please wait {wait_mins} minutes."
)
new_comment = Comment(
topic=parent_comment.topic,
author=request.user,
markdown=markdown,
parent_comment=parent_comment,
)
request.db_session.add(new_comment)
request.db_session.add(
LogComment(LogEventType.COMMENT_POST, request, new_comment))
if CommentNotification.should_create_reply_notification(new_comment):
notification = CommentNotification(
parent_comment.user, new_comment,
CommentNotificationType.COMMENT_REPLY)
request.db_session.add(notification)
_mark_comment_read_from_interaction(request, parent_comment)
# commit and then re-query the new comment to get complete data
request.tm.commit()
new_comment = (request.query(Comment).join_all_relationships().filter_by(
comment_id=new_comment.comment_id).one())
return {"comment": new_comment}
def alert_channels_POST(request):
"""
Creates a new email alert channel for user, sends a validation email
"""
user = request.user
form = forms.EmailChannelCreateForm(MultiDict(request.unsafe_json_body),
csrf_context=request)
if not form.validate():
return HTTPUnprocessableEntity(body=form.errors_json)
email = form.email.data.strip()
channel = EmailAlertChannel()
channel.channel_name = 'email'
channel.channel_value = email
security_code = generate_random_string(10)
channel.channel_json_conf = {'security_code': security_code}
user.alert_channels.append(channel)
email_vars = {
'user': user,
'email': email,
'request': request,
'security_code': security_code,
'email_title': "AppEnlight :: "
"Please authorize your email"
}
UserService.send_email(request,
recipients=[email],
variables=email_vars,
template='/email_templates/authorize_email.jinja2')
request.session.flash(_('Your alert channel was ' 'added to the system.'))
request.session.flash(
_('You need to authorize your email channel, a message was '
'sent containing necessary information.'), 'warning')
DBSession.flush()
channel.get_dict()
def run_workflow(context, request):
input_json = request.json
# set env_name for awsem runner in tibanna
env = request.registry.settings.get('env.name')
# for testing
if not env:
env = ENV_WEBDEV
input_json['output_bucket'] = _wfoutput_bucket_for_env(env)
input_json['env_name'] = env
# hand-off to tibanna for further processing
aws_lambda = boto3.client('lambda', region_name='us-east-1')
res = aws_lambda.invoke(FunctionName=TIBANNA_WORKFLOW_RUNNER_LAMBDA_FUNCTION,
Payload=json.dumps(input_json))
res_decode = res['Payload'].read().decode()
res_dict = json.loads(res_decode)
arn = res_dict['_tibanna']['response']['executionArn']
# just loop until we get proper status
for _ in range(2):
res = aws_lambda.invoke(FunctionName=TIBANNA_WORKFLOW_STATUS_LAMBDA_FUNCTION,
Payload=json.dumps({'executionArn': arn}))
res_decode = res['Payload'].read().decode()
res_dict = json.loads(res_decode)
if res_dict['status'] == 'RUNNING':
break
sleep(2)
if res_dict['status'] == 'FAILED':
# get error from execution and sent a 422 response
sfn = boto3.client('stepfunctions', region_name='us-east-1')
hist = sfn.get_execution_history(executionArn=res_dict['executionArn'], reverseOrder=True)
for event in hist['events']:
if event.get('type') == 'ExecutionFailed':
raise HTTPUnprocessableEntity(str(event['executionFailedEventDetails']))
return res_dict
def api_messages_submissions_(request):
form = request.web_input(count=0, backtime=0, nexttime=0)
try:
count = int(form.count)
backtime = int(form.backtime)
nexttime = int(form.nexttime)
except ValueError:
raise HTTPUnprocessableEntity(json=_ERROR_UNEXPECTED)
else:
count = min(count or 100, 100)
submissions = message.select_submissions(
request.userid, count + 1, include_tags=True, backtime=backtime, nexttime=nexttime)
backtime, nexttime = d.paginate(submissions, backtime, nexttime, count, 'unixtime')
ret = []
for sub in submissions:
tidy_submission(sub)
ret.append(sub)
return {
'backtime': backtime, 'nexttime': nexttime,
'submissions': ret,
}
def application_regenerate_key(request):
"""
Regenerates API keys for application
"""
resource = request.context.resource
form = forms.CheckPasswordForm(MultiDict(request.unsafe_json_body),
csrf_context=request)
form.password.user = request.user
if form.validate():
resource.api_key = resource.generate_api_key()
resource.public_key = resource.generate_api_key()
msg = 'API keys regenerated - please update your application config.'
request.session.flash(_(msg))
else:
return HTTPUnprocessableEntity(body=form.errors_json)
if request.has_permission('edit'):
include_sensitive_info = True
resource_dict = resource.get_dict(
include_perms=include_sensitive_info,
include_processing_rules=include_sensitive_info)
return resource_dict
def post_register(
request: Request,
username: str,
password: str,
password_confirm: str,
invite_code: str,
) -> HTTPFound:
"""Process a registration request."""
if not request.params.get("accepted_terms"):
raise HTTPUnprocessableEntity(
"Terms of Use and Privacy Policy must be accepted.")
if password != password_confirm:
raise HTTPUnprocessableEntity(
"Password and confirmation do not match.")
# attempt to fetch and lock the row for the specified invite code (lock prevents
# concurrent requests from using the same invite code)
lookup_code = UserInviteCode.prepare_code_for_lookup(invite_code)
code_row = (
request.query(UserInviteCode).filter(
UserInviteCode.code == lookup_code,
UserInviteCode.invitee_id == None, # noqa
).with_for_update(skip_locked=True).one_or_none())
if not code_row:
incr_counter("invite_code_failures")
raise HTTPUnprocessableEntity("Invalid invite code")
# create the user and set inviter to the owner of the invite code
user = User(username, password)
user.inviter_id = code_row.user_id
# flush the user insert to db, will fail if username is already taken
request.db_session.add(user)
try:
request.db_session.flush()
except IntegrityError:
raise HTTPUnprocessableEntity(
"That username has already been registered.")
# the flush above will generate the new user's ID, so use that to update the invite
# code with info about the user that registered with it
code_row.invitee_id = user.user_id
# subscribe the new user to all groups except ~test
all_groups = request.query(Group).all()
for group in all_groups:
if group.path == "test":
continue
request.db_session.add(GroupSubscription(user, group))
_send_welcome_message(user, request)
incr_counter("registrations")
# log the user in to the new account
remember(request, user.user_id)
# set request.user before logging so the user is associated with the event
request.user = user
request.db_session.add(Log(LogEventType.USER_REGISTER, request))
# redirect to the front page
raise HTTPFound(location="/")
def deploy_process_from_payload(payload, container):
# type: (JSON, AnyContainer) -> HTTPException
"""
Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload`` matching
:class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`.
:returns: HTTPOk if the process registration was successful
:raises HTTPException: otherwise
"""
_check_deploy(payload)
# use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload
payload_copy = deepcopy(payload)
# validate identifier naming for unsupported characters
process_description = payload.get("processDescription")
process_info = process_description.get("process", {})
process_href = process_description.pop("href", None)
# retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference)
deployment_profile_name = payload.get("deploymentProfileName", "").lower()
ows_context = process_info.pop("owsContext", None)
reference = None
package = None
if process_href:
reference = process_href # reference type handled downstream
elif isinstance(ows_context, dict):
offering = ows_context.get("offering")
if not isinstance(offering, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering'."
)
content = offering.get("content")
if not isinstance(content, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering.content'."
)
package = None
reference = content.get("href")
elif deployment_profile_name:
if not any(
deployment_profile_name.endswith(typ)
for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]):
raise HTTPBadRequest(
"Invalid value for parameter 'deploymentProfileName'.")
execution_units = payload.get("executionUnit")
if not isinstance(execution_units, list):
raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.")
for execution_unit in execution_units:
if not isinstance(execution_unit, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'executionUnit'.")
package = execution_unit.get("unit")
reference = execution_unit.get("href")
# stop on first package/reference found, simultaneous usage will raise during package retrieval
if package or reference:
break
else:
raise HTTPBadRequest(
"Missing one of required parameters [href, owsContext, deploymentProfileName]."
)
# obtain updated process information using WPS process offering, CWL/WPS reference or CWL package definition
process_info = _get_deploy_process_info(process_info, reference, package)
# validate process type against weaver configuration
settings = get_settings(container)
process_type = process_info["type"]
if process_type == PROCESS_WORKFLOW:
weaver_config = get_weaver_configuration(settings)
if weaver_config != WEAVER_CONFIGURATION_EMS:
raise HTTPBadRequest(
"Invalid [{0}] package deployment on [{1}].".format(
process_type, weaver_config))
restapi_url = get_wps_restapi_base_url(settings)
description_url = "/".join(
[restapi_url, "processes", process_info["identifier"]])
execute_endpoint = "/".join([description_url, "jobs"])
# ensure that required "processEndpointWPS1" in db is added,
# will be auto-fixed to localhost if not specified in body
process_info["processEndpointWPS1"] = process_description.get(
"processEndpointWPS1")
process_info["executeEndpoint"] = execute_endpoint
process_info["payload"] = payload_copy
process_info["jobControlOptions"] = process_description.get(
"jobControlOptions", [])
process_info["outputTransmission"] = process_description.get(
"outputTransmission", [])
process_info["processDescriptionURL"] = description_url
# insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext"
if "owsContext" not in process_info and reference:
process_info["owsContext"] = {
"offering": {
"content": {
"href": str(reference)
}
}
}
elif isinstance(ows_context, dict):
process_info["owsContext"] = ows_context
try:
store = get_db(container).get_store(StoreProcesses)
saved_process = store.save_process(Process(process_info),
overwrite=False)
except ProcessRegistrationError as ex:
raise HTTPConflict(detail=str(ex))
except ValueError as ex:
# raised on invalid process name
raise HTTPBadRequest(detail=str(ex))
json_response = {
"processSummary": saved_process.process_summary(),
"deploymentDone": True
}
return HTTPOk(
json=json_response
) # FIXME: should be 201 (created), update swagger accordingly
def deploy_process_from_payload(payload, container, overwrite=False):
# type: (JSON, AnyContainer, bool) -> HTTPException
"""
Deploy the process after resolution of all references and validation of the parameters from payload definition.
Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload``
matching :class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`.
:param payload: JSON payload that was specified during the process deployment request.
:param container: container to retrieve application settings.
:param overwrite: whether to allow override of an existing process definition if conflict occurs.
:returns: HTTPOk if the process registration was successful.
:raises HTTPException: for any invalid process deployment step.
"""
# use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload
payload_copy = deepcopy(payload)
payload = _check_deploy(payload)
# validate identifier naming for unsupported characters
process_description = payload.get("processDescription")
process_info = process_description.get("process", {})
process_href = process_description.pop("href", None)
# retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference)
deployment_profile_name = payload.get("deploymentProfileName", "").lower()
ows_context = process_info.pop("owsContext", None)
reference = None
package = None
if process_href:
reference = process_href # reference type handled downstream
elif isinstance(ows_context, dict):
offering = ows_context.get("offering")
if not isinstance(offering, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering'."
)
content = offering.get("content")
if not isinstance(content, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering.content'."
)
package = None
reference = content.get("href")
elif deployment_profile_name:
if not any(
deployment_profile_name.endswith(typ)
for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]):
raise HTTPBadRequest(
"Invalid value for parameter 'deploymentProfileName'.")
execution_units = payload.get("executionUnit")
if not isinstance(execution_units, list):
raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.")
for execution_unit in execution_units:
if not isinstance(execution_unit, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'executionUnit'.")
package = execution_unit.get("unit")
reference = execution_unit.get("href")
# stop on first package/reference found, simultaneous usage will raise during package retrieval
if package or reference:
break
else:
raise HTTPBadRequest(
"Missing one of required parameters [href, owsContext, deploymentProfileName]."
)
if process_info.get("type", "") == PROCESS_BUILTIN:
raise HTTPBadRequest(
"Invalid process type resolved from package: [{0}]. Deployment of {0} process is not allowed."
.format(PROCESS_BUILTIN))
# update and validate process information using WPS process offering, CWL/WPS reference or CWL package definition
settings = get_settings(container)
headers = getattr(
container, "headers", {}
) # container is any request (as when called from API Deploy request)
process_info = _validate_deploy_process_info(process_info, reference,
package, settings, headers)
restapi_url = get_wps_restapi_base_url(settings)
description_url = "/".join(
[restapi_url, "processes", process_info["identifier"]])
execute_endpoint = "/".join([description_url, "jobs"])
# ensure that required "processEndpointWPS1" in db is added,
# will be auto-fixed to localhost if not specified in body
process_info["processEndpointWPS1"] = process_description.get(
"processEndpointWPS1")
process_info["executeEndpoint"] = execute_endpoint
process_info["payload"] = payload_copy
process_info["jobControlOptions"] = process_description.get(
"jobControlOptions", [])
process_info["outputTransmission"] = process_description.get(
"outputTransmission", [])
process_info["processDescriptionURL"] = description_url
# insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext"
if "owsContext" not in process_info and reference:
process_info["owsContext"] = {
"offering": {
"content": {
"href": str(reference)
}
}
}
elif isinstance(ows_context, dict):
process_info["owsContext"] = ows_context
# bw-compat abstract/description (see: ProcessDeployment schema)
if "description" not in process_info or not process_info["description"]:
process_info["description"] = process_info.get("abstract", "")
# FIXME: handle colander invalid directly in tween (https://github.com/crim-ca/weaver/issues/112)
try:
store = get_db(container).get_store(StoreProcesses)
process = Process(process_info)
sd.ProcessSummary().deserialize(
process) # make if fail before save if invalid
store.save_process(process, overwrite=overwrite)
process_summary = process.summary()
except ProcessRegistrationError as ex:
raise HTTPConflict(detail=str(ex))
except (ValueError, colander.Invalid) as ex:
# raised on invalid process name
raise HTTPBadRequest(detail=str(ex))
return HTTPCreated(
json={
"description": sd.OkPostProcessesResponse.description,
"processSummary": process_summary,
"deploymentDone": True
})
def validate_csrf_token(self, field):
if field.current_token != field.data:
raise HTTPUnprocessableEntity()
def process_pedigree(context, request):
"""
Endpoint to handle creation of a family of individuals provided a pedigree
file. Uses a dcicutils.misc_utils.VirtualApp to handle POSTing and PATCHing
items. The request.json contains attachment information and file content.
Currently, only handles XML input formatted from the Proband app.
This endpoint takes the following options, provided through request params:
- config_uri: should be 'development.ini' for dev, else 'production.ini'
Response dict contains the newly created family, as well as the up-to-date
Cohort properties.
Args:
request (Request): the current request. Attachment data should be
given in the request JSON.
Returns:
dict: reponse, including 'status', and 'cohort' and 'family' on success
Raises:
HTTPUnprocessableEntity: on an error. Extra information may be logged
"""
cohort = str(context.uuid) # used in logging
# verify that attachment data in request.json has type and href
if not {'download', 'type', 'href'} <= set(request.json.keys()):
raise HTTPUnprocessableEntity('Cohort %s: Request JSON must include following'
' keys: download, type, href. Found: %s'
% (cohort, request.json.keys()))
# verification on the attachment. Currently only handle .pbxml
# pbxml uploads don't get `type` attribute from <input> element
if request.json['type'] != '' or not request.json['download'].endswith('.pbxml'):
raise HTTPUnprocessableEntity('Cohort %s: Bad pedigree file upload. Use .pbxml'
' file. Found: %s (file type), %s (file name)'
% (cohort, request.json['type'], request.json['download']))
config_uri = request.params.get('config_uri', 'production.ini')
# TODO: get pedigree timestamp dynamically, maybe from query_params
# ped_timestamp = request.params.get('timestamp')
ped_datetime = datetime.utcnow()
ped_timestamp = ped_datetime.isoformat() + '+00:00'
app = get_app(config_uri, 'app')
# get user email for VirtualApp authentication
email = get_trusted_email(request, context="Cohort %s" % cohort)
environ = {'HTTP_ACCEPT': 'application/json', 'REMOTE_USER': email}
testapp = VirtualApp(app, environ)
# parse XML and create family by two rounds of POSTing/PATCHing individuals
response = {'title': 'Pedigree Processing'}
refs = {}
try:
xml_data = etree_to_dict(fromstring(request.json['href']), refs, 'managedObjectID')
except Exception as exc:
response['status'] = 'failure'
response['detail'] = 'Error parsing pedigree XML: %s' % str(exc)
return response
# add "affected" metadata to refs for easy access
family_pheno_feats = []
for meta_key, meta_val in xml_data.get('meta', {}).items():
if meta_key.startswith('affected'):
refs[meta_key] = meta_val
if meta_val.get('id') and meta_val.get('ontology') == 'HPO':
family_pheno_feats.append(meta_val['id'])
# extra values that are used when creating the pedigree
cohort_props = context.upgrade_properties()
post_extra = {'project': cohort_props['project'],
'institution': cohort_props['institution']}
xml_extra = {'ped_datetime': ped_datetime}
family_uuids = create_family_proband(testapp, xml_data, refs, 'managedObjectID',
cohort, post_extra, xml_extra)
# create Document for input pedigree file
# pbxml files are not handled by default. Do some mimetype processing
mimetypes.add_type('application/proband+xml', '.pbxml')
use_type = 'application/proband+xml'
data_href = 'data:%s;base64,%s' % (use_type, b64encode(request.json['href'].encode()).decode('ascii'))
attach = {'attachment': {'download': request.json['download'],
'type': use_type, 'href': data_href}}
attach.update(post_extra)
try:
attach_res = testapp.post_json('/Document', attach)
assert attach_res.status_code == 201
except Exception as exc:
log.error('Failure to POST Document in process-pedigree! Exception: %s' % exc)
error_msg = ('Cohort %s: Error encountered on POST in process-pedigree.'
' Check logs. These items were already created: %s'
% (cohort, family_uuids['members']))
raise HTTPUnprocessableEntity(error_msg)
# add extra fields to the family object
attach_uuid = attach_res.json['@graph'][0]['uuid']
family_uuids['original_pedigree'] = attach_uuid
family_uuids['timestamp'] = ped_timestamp
if xml_data.get('meta', {}).get('notes'):
family_uuids['clinic_notes'] = xml_data['meta']['notes']
ped_src = 'Proband app'
if xml_data.get('meta', {}).get('version'):
ped_src += (' ' + xml_data['meta']['version'])
family_uuids['pedigree_source'] = ped_src
family_uuids['family_phenotypic_features'] = []
for hpo_id in family_pheno_feats:
try:
pheno_res = testapp.get('/phenotypes/' + hpo_id, status=200).json
except Exception as exc:
error_msg = ('Cohort %s: Cannot GET family feature %s. Error: %s'
% (cohort, hpo_id, str(exc)))
log.error(error_msg)
# HACKY. Skip raising this error if local
if config_uri == 'production.ini':
raise HTTPUnprocessableEntity(error_msg)
else:
family_uuids['family_phenotypic_features'].append(pheno_res['uuid'])
# PATCH the Cohort with new family
cohort_families = cohort_props.get('families', []) + [family_uuids]
cohort_patch = {'families': cohort_families}
try:
cohort_res = testapp.patch_json('/' + cohort, cohort_patch)
assert cohort_res.status_code == 200
except Exception as exc:
log.error('Failure to PATCH Cohort %s in process-pedigree with '
'data %s! Exception: %s' % (cohort, cohort_patch, exc))
error_msg = ('Cohort %s: Error encountered on PATCH in process-pedigree.'
' Check logs. These items were already created: %s'
% (cohort, family_uuids['members'] + [attach_uuid]))
raise HTTPUnprocessableEntity(error_msg)
# get the fully embedded cohort to put in response for front-end
response['context'] = testapp.get('/cohorts/' + cohort + '?frame=page&datastore=database', status=200).json
response['status'] = 'success'
return response
def create_family_proband(testapp, xml_data, refs, ref_field, cohort,
post_extra=None, xml_extra=None):
"""
Proband-specific object creation protocol. We can expand later on
General process (in development):
- POST individuals with required fields and attribution (`post_extra` kwarg)
- PATCH non-required fields
Can be easily extended by adding tuples to `to_convert` dict
Args:
testapp (dcicutils.misc_utils.VirtualApp): test application for posting/patching
xml_data (dict): parsed XMl data, probably from `etree_to_dict`
refs: (dict): reference-based parsed XML data
ref_field (str): name of reference field from the XML data
cohort (str): identifier of the cohort
post_extra (dict): keys/values given here are added to POST
xml_extra (dict): key/values given here are added to each XML object
processed using the PROBAND_MAPPING
Returns:
dict: family created, including members and proband with full context
"""
# key family members by uuid
family_members = {}
uuids_by_ref = {}
proband = None
errors = []
xml_type = 'people'
item_type = 'Individual'
for round in ['first', 'second']:
for xml_obj in xml_data.get(xml_type, []):
ref = xml_obj.get(ref_field)
if not ref: # element does not have a managed ID
continue
data = {}
if round == 'first' and post_extra is not None:
data.update(post_extra)
if xml_extra is not None:
xml_obj.update(xml_extra)
for xml_key in xml_obj:
converted = PROBAND_MAPPING[item_type].get(xml_key)
if converted is None:
log.info('Unknown field %s for %s in process-pedigree!' % (xml_key, item_type))
continue
# convert all conversions to lists, since some xml fields map
# to multiple metadata fields and this makes it simpler
if not isinstance(converted, list):
converted = [converted]
for converted_dict in converted:
if round == 'first':
if converted_dict.get('linked', False) is True:
continue
ref_val = converted_dict['value'](xml_obj)
if ref_val is not None:
data[converted_dict['corresponds_to']] = ref_val
elif round == 'second':
if converted_dict.get('linked', False) is False:
continue
ref_val = converted_dict['value'](xml_obj)
# more complex function based on xml refs needed
if ref_val is not None and 'xml_ref_fxn' in converted_dict:
# will update data in place
converted_dict['xml_ref_fxn'](testapp, ref_val, refs, data,
cohort, uuids_by_ref)
elif ref_val is not None:
data[converted_dict['corresponds_to']] = uuids_by_ref[ref_val]
# POST if first round
if round == 'first':
try:
post_res = testapp.post_json('/' + item_type, data)
assert post_res.status_code == 201
except Exception as exc:
log.error('Failure to POST %s in process-pedigree with '
'data %s! Exception: %s' % (item_type, data, exc))
error_msg = ('Cohort %s: Error encountered on POST in process-pedigree.'
' Check logs. These items were already created: %s'
% (cohort, list(uuids_by_ref.values())))
raise HTTPUnprocessableEntity(error_msg)
else:
idv_props = post_res.json['@graph'][0]
uuids_by_ref[ref] = idv_props['uuid']
# PATCH if second round, with adding uuid to the data
if round == 'second' and data:
try:
patch_res = testapp.patch_json('/' + uuids_by_ref[ref], data)
assert patch_res.status_code == 200
except Exception as exc:
log.error('Failure to PATCH %s in process-pedigree with '
'data %s! Exception: %s' % (uuids_by_ref[ref], data, exc))
error_msg = ('Cohort %s: Error encountered on PATCH in process-pedigree.'
' Check logs. These items were already created: %s'
% (cohort, list(uuids_by_ref.values())))
raise HTTPUnprocessableEntity(error_msg)
else:
idv_props = patch_res.json['@graph'][0]
# update members info on POST or PATCH
family_members[idv_props['uuid']] = idv_props
# update proband only on first round (not all items hit in second)
if round == 'first' and xml_obj.get('proband') == '1':
if proband and idv_props['uuid'] != proband:
log.error('Cohort %s: Multiple probands found! %s conflicts with %s'
% (idv_props['uuid'], proband))
else:
proband = idv_props['uuid']
# process into family structure, keeping only uuids of items
# invert uuids_by_ref to sort family members by managedObjectID (xml ref)
refs_by_uuid = {v: k for k, v in uuids_by_ref.items()}
family = {'members': sorted([m['uuid'] for m in family_members.values()],
key=lambda v: int(refs_by_uuid[v]))}
if proband and proband in family_members:
family['proband'] = family_members[proband]['uuid']
else:
log.error('Cohort %s: No proband found for family %s' % family)
return family
def get_queried_jobs(request):
"""
Retrieve the list of jobs which can be filtered, sorted, paged and categorized using query parameters.
"""
settings = get_settings(request)
service, process = validate_service_process(request)
params = dict(request.params)
LOGGER.debug("Job search queries (raw):\n%s", repr_json(params, indent=2))
for param_name in ["process", "processID", "provider", "service"]:
params.pop(param_name, None)
filters = {**params, "process": process, "provider": service}
if params.get("datetime", False):
# replace white space with '+' since request.params replaces '+' with whitespaces when parsing
filters["datetime"] = params["datetime"].replace(" ", "+")
try:
filters = sd.GetJobsQueries().deserialize(filters)
except Invalid as ex:
raise HTTPBadRequest(
json={
"code": "JobInvalidParameter",
"description": "Job query parameters failed validation.",
"error": Invalid.__name__,
"cause": str(ex),
"value": repr_json(ex.value or filters, force_string=False),
})
detail = filters.pop("detail", False)
groups = filters.pop("groups", "").split(",") if filters.get(
"groups", False) else filters.pop("groups", None)
filters["tags"] = list(
filter(
lambda s: s,
filters["tags"].split(",") if filters.get("tags", False) else ""))
filters["notification_email"] = (encrypt_email(
filters["notification_email"], settings) if filters.get(
"notification_email", False) else None)
filters["service"] = filters.pop("provider", None)
filters["min_duration"] = filters.pop("minDuration", None)
filters["max_duration"] = filters.pop("maxDuration", None)
filters["job_type"] = filters.pop("type", None)
dti = datetime_interval_parser(filters["datetime"]) if filters.get(
"datetime", False) else None
if dti and dti.get("before", False) and dti.get(
"after", False) and dti["after"] > dti["before"]:
raise HTTPUnprocessableEntity(
json={
"code":
"InvalidDateFormat",
"description":
"Datetime at the start of the interval must be less than the datetime at the end."
})
filters.pop("datetime", None)
filters["datetime_interval"] = dti
LOGGER.debug("Job search queries (processed):\n%s",
repr_json(filters, indent=2))
store = get_db(request).get_store(StoreJobs)
items, total = store.find_jobs(request=request, group_by=groups, **filters)
body = {"total": total}
def _job_list(jobs):
return [j.json(settings) if detail else j.id for j in jobs]
paging = {}
if groups:
for grouped_jobs in items:
grouped_jobs["jobs"] = _job_list(grouped_jobs["jobs"])
body.update({"groups": items})
else:
paging = {"page": filters["page"], "limit": filters["limit"]}
body.update({"jobs": _job_list(items), **paging})
try:
body.update({"links": get_job_list_links(total, filters, request)})
except IndexError as exc:
raise HTTPBadRequest(
json={
"code": "JobInvalidParameter",
"description": str(exc),
"cause": "Invalid paging parameters.",
"error": type(exc).__name__,
"value": repr_json(paging, force_string=False)
})
body = sd.GetQueriedJobsSchema().deserialize(body)
return HTTPOk(json=body)
def post_login(request: Request, username: str, password: str,
from_url: str) -> Response:
"""Process a log in request."""
incr_counter("logins")
# Look up the user for the supplied username
user = (request.query(User).undefer_all_columns().filter(
User.username == username).one_or_none())
# If the username doesn't exist, tell them so - usually this isn't considered a good
# practice, but it's completely trivial to check if a username exists on Tildes
# anyway (by visiting /user/<username>), so it's better to just let people know if
# they're trying to log in with the wrong username
if not user:
incr_counter("login_failures")
# log the failure - need to manually commit because of the exception
log_entry = Log(
LogEventType.USER_LOG_IN_FAIL,
request,
{
"username": username,
"reason": "Nonexistent username"
},
)
request.db_session.add(log_entry)
request.tm.commit()
raise HTTPUnprocessableEntity("That username does not exist")
if not user.is_correct_password(password):
incr_counter("login_failures")
# log the failure - need to manually commit because of the exception
log_entry = Log(
LogEventType.USER_LOG_IN_FAIL,
request,
{
"username": username,
"reason": "Incorrect password"
},
)
request.db_session.add(log_entry)
request.tm.commit()
raise HTTPUnprocessableEntity("Incorrect password")
# Don't allow banned users to log in
if user.is_banned:
if user.ban_expiry_time:
# add an hour to the ban's expiry time since the cronjob runs hourly
unban_time = user.ban_expiry_time + timedelta(hours=1)
unban_time = unban_time.strftime("%Y-%m-%d %H:%M (UTC)")
raise HTTPUnprocessableEntity(
"That account is temporarily banned. "
f"The ban will be lifted at {unban_time}")
raise HTTPUnprocessableEntity("That account has been banned")
# If 2FA is enabled, save username to session and make user enter code
if user.two_factor_enabled:
request.session["two_factor_username"] = username
return render_to_response(
"tildes:templates/intercooler/login_two_factor.jinja2",
{
"keep": request.params.get("keep"),
"from_url": from_url
},
request=request,
)
raise finish_login(request, user, from_url)
