def updated_followed_entity(user_id, params):
entities = params['entities']
followed_entities = []
for entity in entities:
entity_id = entity['entity_id']
entity_type = entity['entity_type']
following = entity['following']
followed_entity = db_session_users.query(UserFollowedEntity).filter_by(
entity_id=entity_id, entity_type=entity_type,
user_id=user_id).first()
if not followed_entity:
followed_entity = UserFollowedEntity({
'entity_id': entity_id,
'entity_type': entity_type,
'user_id': user_id
})
followed_entity.following = following
followed_entities.append(followed_entity)
db_session_users.add_all(followed_entities)
db_session_users.commit()
return {
"followed_entities": [
merge_two_dicts(e.to_dict(), get_entity_from_es(e))
for e in followed_entities
]
}
def add_all_users_to_email_list():
all_users = db_session_users.query(User).filter_by(enabled=True).all()
# for each user
users = []
for user in all_users:
# check to see if the email_updates in user.properties is a string (deprecated)
if user.properties and 'email_updates' in user.properties and isinstance(
user.properties['email_updates'], unicode):
# update to become a hash table (latest)
# also set whatever the user's actual preferences are
properties = {
'email_updates': {
'agency_weekly':
user.properties['email_updates'] == 'weekly',
'agency_daily':
user.properties['email_updates'] == 'daily',
'topics_weekly':
True, # all users are being set to recieve topics email by default
}
}
elif 'email_updates' not in user.properties: # catch those users who may have not signed up for email yet
properties = {
'email_updates': {
'agency_weekly': True,
'agency_daily': False,
'topics_weekly': True,
}
}
user.properties = merge_two_dicts(user.properties, properties)
users.append(user)
# after each user has been updated save it in the db
db_session_users.add_all(users)
db_session_users.commit()
def confirm_user(params):
def error_response(msg='Invalid email or token'):
response = jsonify({
'error': msg,
})
response.status_code = 400
return response
email = params.get('email')
token = params.get('token')
if email is None or token is None:
return error_response()
email = email.lower()
g.user_email = email
g.user_token = token
user = db_session_users.query(User).filter_by(email=email).first()
if user.enabled:
return error_response('User is already enabled')
if user.reset_token != token:
return error_response()
user.enabled = True
user.reset_token = None # once enabled, the token is done
user.properties = merge_two_dicts(user.properties, {'confirmed_date': datetime.datetime.utcnow().isoformat() })
db_session_users.add(user)
db_session_users.commit()
return jsonify({})
def queue_created_document(params, user_id):
created_doc = UserCreatedDocument(
merge_two_dicts({"user_id": user_id}, params))
db_session_users.add(created_doc)
db_session_users.commit()
db_session_users.refresh(created_doc)
return {'user_created_document': created_doc.to_dict()}
def send_confirm_email(params):
def error_response(msg='Error: Email is not in use.'):
response = jsonify({
'error': msg,
})
response.status_code = 400
return response
if 'email' not in params:
return error_response()
email = params.get('email').lower()
user = db_session_users.query(User).filter_by(email=email).first()
if user is None or user.password_hash is None:
return error_response()
user.properties = merge_two_dicts(user.properties, {'confirmation_resent_time': datetime.datetime.utcnow().isoformat() })
db_session_users.add(user)
db_session_users.commit()
try:
_send_activation_email('confirm', user)
except SMTPException:
return error_response('Could not send email', code=500)
return jsonify({'data': 'email sent'})
def node_dict(self, key):
if 'documents_' in key:
old_val = self.docs_map.get(key, {}) ## TODO ???
else:
old_val = self.named_clusters[key]
if 'date' in old_val:
if not old_val['date']:
old_val.pop('date', None)
old_val.pop('rule', None)
if 'times_cited' in old_val and old_val['times_cited'] > 0:
size = max(2, math.log(old_val['times_cited']))
else:
size = 2
size = NodeMagnificationRate * size
new_val = {
'key': key,
'size': size,
'strength': 100,
'x': random.randint(0, 1200),
'y': random.randint(0, 1200),
'color': node_color(key, old_val)
}
if key == 'acts_1099': print old_val
return merge_two_dicts(old_val, new_val)
def create_marketing_campaign(user_id, params):
marketing_campaign = MarketingCampaign(
merge_two_dicts(params, {'created_by_user_id': user_id}))
marketing_campaign.gen_token()
db_session_users.add(marketing_campaign)
db_session_users.commit()
db_session_users.refresh(marketing_campaign)
return {"marketing_campaign": marketing_campaign.to_dict()}
def pure_send_confirmation_email(user):
user = db_session_users().query(User).filter_by(id=user.id).first()
if user is not None:
try:
_send_activation_email('confirm', user)
user.properties = merge_two_dicts(user.properties, {'recent_confirmation_email_success': True})
except:
email_helper.send_email(
'*****@*****.**',
'*****@*****.**',
'error sending confirmation email during daily heroku script',
template='feedback-inline',
vars={'feedback': 'user (user_id='+str(user.id)+') ' + user.email,
'User_first_name': 'complaibot',
'User_last_name': 'complaibot', },
)
user.properties = merge_two_dicts(user.properties, {'recent_confirmation_email_success': False})
db_session_users.add(user)
db_session_users.commit()
def get_all_followed_entities(user_id, params):
if params:
entity_type = params['entity_type']
all_entities = db_session_users.query(UserFollowedEntity).filter_by(
user_id=user_id, entity_type=entity_type, following=True).all()
else:
all_entities = db_session_users.query(UserFollowedEntity).filter_by(
user_id=user_id, following=True).all()
return {
"followed_entities": [
merge_two_dicts(e.to_dict(), get_entity_from_es(e))
for e in all_entities
]
}
def link_secondary_signups(user):
if not hasattr(user, 'linkedin_user_info') and not hasattr(user, 'google_user_info'):
return
# user exists in db but is using linkedin for the first times.
# link accounts. account must have common email
if hasattr(user, 'linkedin_user_info'):
# linkedin is source of truth for name
user.first_name = user.linkedin_user_info['first_name']
user.last_name = user.linkedin_user_info['last_name']
user.linkedin = user.linkedin_user_info['linkedin_id']
user.industry = user.linkedin_user_info['industry']
user.company = user.linkedin_user_info['company']
if user.linkedin_user_info['is_contributor'] :
user.roles.append('contributor')
new_properties = {
'linkedin_data': user.linkedin_user_info['linkedin_data'],
'secondary_signup_dates': {'linkedin': datetime.datetime.utcnow().isoformat()}
}
if hasattr(user, 'google_user_info'):
user.google_id = user.google_user_info['google_id']
if not user.linkedin:
# linkedin is source of truth for name
user.first_name = user.google_user_info['first_name']
user.last_name = user.google_user_info['last_name']
if user.google_user_info['is_contributor']:
user.roles.append('contributor')
new_properties = {
'secondary_signup_dates': {'google': datetime.datetime.utcnow().isoformat()}
}
if not user.enabled:
user.enabled = True
new_properties['confirmed_date'] = datetime.datetime.utcnow().isoformat()
user.properties = merge_two_dicts(user.properties, new_properties)
db_session_users.add(user)
try:
db_session_users.commit()
except IntegrityError:
return error_response()
db_session_users.refresh(user)
def update_aggregated_annotation(aggregated_annotation_task_id, params):
# NB: probably shouldn't change doc_id, topic_id or annotation_task_group_id;
# these are set at crawler level when aggregated_annotations are created
original_agg_annotation = db_session_users.query(AggregatedAnnotations)\
.filter_by(id=aggregated_annotation_task_id)\
.first()
# perform updates
# TODO: add some checks against setting illegal foreign keys
if 'annotation_task_group_id' in params:
original_agg_annotation.annotation_task_group_id = params['annotation_task_group_id']
if 'doc_id' in params:
original_agg_annotation.doc_id = params['doc_id']
if 'topic_id' in params and params['topic_id'] in AggregatedAnnotations.topic_id_name_mapping:
original_agg_annotation.topic_id = params['topic_id']
if 'is_gold_standard' in params:
original_agg_annotation.is_gold_standard = params['is_gold_standard']
if 'gold_topic_annotation_id' in params:
original_agg_annotation.gold_topic_annotation_id = params['gold_topic_annotation_id']
if 'is_active_for_gold_annotation' in params:
original_agg_annotation.is_active_for_gold_annotation = params['is_active_for_gold_annotation']
if 'gold_difficulty' in params:
original_agg_annotation.gold_difficulty = params['gold_difficulty']
if 'arbitrary_tags' in params:
original_agg_annotation.arbitrary_tags = params['arbitrary_tags']
# NB: 'is_in_agreement' should depend on annotation agreement somewhere
if 'is_in_agreement' in params:
original_agg_annotation.is_in_agreement = params['is_in_agreement']
if 'notes' in params:
original_agg_annotation.notes = params['notes']
# for the dict 'details' do a merge that retains all existing keys;
# if same key "k1" appears in both dicts, older value is overwritten by new value in params['details']["k1"]
if 'details' in params:
original_agg_annotation.details = merge_two_dicts(original_agg_annotation.details, params['details'])
# commit updates to database
db_session_users.add(original_agg_annotation)
db_session_users.commit()
# return updated values
agg_task_dict = original_agg_annotation.to_dict()
return jsonify({"aggregated_annotation": agg_task_dict})
def decorate_documents(documents, user_id, user_docs=None, decorate_children=False, flagged_docs=None):
if not user_docs: user_docs = get_user_document_ids(user_id)
read_docs = set([d[0] for d in user_docs if d[1] == True])
bookmarked_docs = set([d[0] for d in user_docs if d[2] == True])
doc_tag_map = get_doc_tag_map(user_id, documents)
if flagged_docs is None:
flagged_docs = get_all_flagged_documents()
docs = [
merge_two_dicts(d, {
"read": doc_boolean(d, read_docs),
"bookmarked": doc_boolean(d, bookmarked_docs),
"tags": doc_tag_map[d['id']],
"flagged": getFlaggedData(d['id'], flagged_docs)
}
) for d in documents ]
if decorate_children:
for doc in docs:
doc['children'] = decorate_documents(doc['children'], user_id)
return docs
def update_user_details(email, params):
user = db_session_users.query(User).filter_by(email=email).first()
if g.admin_user:
if 'isQA' in params:
update_user_role('qa', params['isQA'], user)
if 'isContributor' in params:
update_user_role('contributor', params['isContributor'], user)
if params['isContributor']:
# deactivate current subscriptions and subscribe user to a contributor subscription
subscribe_users_to_plan([user.id], 'contributor_monthly_recur')
user.isSubscribedContributor = True
else:
# deactivate contributor subscription and subscription them to a 30 day free trial
subscribe_users_to_plan([user.id], 'free_trial_extension')
user.isSubscribedContributor = False
if 'suspended' in params:
user.suspended = params['suspended']
if 'suspended_reason' in params:
user.suspended_reason = params['suspended_reason']
# n.b. timestamping the action so we know when it happened
user.properties = merge_two_dicts(
user.properties,
{"suspended_time": dt.datetime.utcnow().isoformat()})
if 'enabled' in params:
user.enabled = params['enabled']
# n.b. timestamping the action so we know when it happened
user.properties = merge_two_dicts(
user.properties,
{"confirmed_date": dt.datetime.utcnow().isoformat()})
# prevent any actions by non-admin users on a different user row than logged in for
elif g.user_id != user.id:
return jsonify({"errors": "Not found"}), 404
if params.get('first_name', False): user.first_name = params['first_name']
if params.get('last_name', False): user.last_name = params['last_name']
if BASE_URL_FOR_EMAIL:
base_url = BASE_URL_FOR_EMAIL
else:
base_url = 'http://%s:%s' % (API_BOUND_HOST, API_PORT)
if 'email' in params and user.email.lower() != params['email'].lower():
new_user_email = params['email'].lower()
mandrill_recipient = [{
'email': user.email,
'name': user.first_name,
'type': 'to'
}, {
'email': new_user_email,
'name': user.first_name,
'type': 'to'
}]
email_helper.send_via_mandrill(
mandrill_recipient,
'*****@*****.**',
'Compliance.ai',
'Your Compliance.ai Email has Changed!',
template='confirm-change-inline',
vars={
'base_url': base_url,
'user_first_name': user.first_name,
'user_account_item': 'email',
'user_email': user.email,
'account_change_txt_begin': 'please contact us at',
'link_txt': "*****@*****.**",
'account_change_txt_end': 'immediately',
'url': 'mailto:[email protected]',
},
)
if params.get('email', False): user.email = params['email'].lower()
if params.get('properties', False):
user.properties = merge_two_dicts(user.properties,
params['properties'])
if params.get('company', False): user.company = params['company']
if 'team_id' in params: user.team_id = params['team_id']
if params.get('industry', False): user.industry = params['industry']
if params.get('discipline', False): user.discipline = params['discipline']
if params.get('level', False): user.level = params['level']
# update the password if the current and new passwords were provided,
# and the current password is correct
if params.get('current_password', False) and params.get(
'new_password', False):
if user.compare_password(params['current_password']):
user.update_password(params['new_password'])
mandrill_recipient = [{
'email': user.email,
'name': user.first_name,
'type': 'to'
}]
user.gen_reset_token()
reset_url = '%s/activate?&email=%s&token=%s' % (
base_url, urllib.quote_plus(
user.email), urllib.quote_plus(user.reset_token))
reset_url += '&reset=1'
email_helper.send_via_mandrill(
mandrill_recipient,
'*****@*****.**',
'Compliance.ai',
'Your Compliance.ai Password has Changed!',
template='confirm-change-inline',
vars={
'base_url': base_url,
'user_first_name': user.first_name,
'user_account_item': 'password',
'user_email': user.email,
'url': reset_url,
'account_change_txt_begin': "you'll need to click",
'link_txt': "here",
'account_change_txt_end': "and reset your password",
},
)
else:
return jsonify({"errors": {'field': 'password'}}), 400
db_session_users.add(user)
db_session_users.commit()
db_session_users.flush()
db_session_users.refresh(user)
if hasattr(user, 'isSubscribedContributor'):
return jsonify({
'user': user.to_dict(),
'isSubscribedContributor': user.isSubscribedContributor
})
return jsonify({'user': user.to_dict()})
def get_all_topic_annotations_in_group(annotation_task_group_id, params):
"""
Returns list of topic_annotations (possibly with augmented evaluation information for given
annotation_task_topic_group.
Optional flag is include_aggregated_annotation_info, which tells whether to query the aggregated_annotations
table for additional annotation info
Optional filters on user_id, tag, difficulty, is_gold_evaluation, accuracy.
Options sorting on tag, difficulty, accuracy.
:param annotation_task_group_id (int): id (primary key) of annotation_task_topic_group
:param (in url request): all top-level:
'user_id': filter on given user_id (int)
'is_gold_evaluation': filter on value of is_gold_evaluation (bool)
'user_arbitrary_tag': filter on arbitrary_tags from annotation_jobs table (text)
'user_difficulty': filter on self-reported user difficulty from annotation_jobs table
('easy', 'medium' or 'hard')
'is_correct_judgment': filter on accuracy of annotations (i.e. agreement between
annotation and gold annotation)
(bool - True means 'agree'/'accurate')
'sorting': sort on given parameter; available are 'accuracy', 'difficulty', 'tag'
'is_correct_judgment': accuracy of annotations (i.e. agreement
between annotation and gold annotation)
'user_difficulty': sort on self-reported user difficulty from
annotation_jobs table
'include_aggregated_annotation_info': whether to include info from aggregated_annotations table
(necessary if want to filter/sort on accuracy) (bool)
:return (list(dict)): potentially-sorted list of dicts: [dict_1, dict_2, ...]
Each dict is topic_annotation.to_dict() object augmented with additional keys:
{
'user_difficulty': 'easy'/'medium'/'hard' (self-reported user difficulty)
'user_tags': ['text_1', 'text_2', ... ] (user-chosen arbitrary tags)
'gold_judgment': bool (judgment of gold annotation for this annotation),
'is_correct_judgment': bool (whether this annotation agrees with gold judgment),
}
'gold_judgment' and 'is_correct_judgment' keys only present if 'include_aggregated_annotation_info' in params.
"""
###############################################################################
# make base query in topic_annotations table for this AnnotationTaskTopicGroup
###############################################################################
# get ids of AnnotationTasks in this group
# subquery for AnnotationTask.ids of AnnotationTasks that point to annotation_task_topic_group of interest
task_ids_subquery = db_session_users.query(AnnotationTask.id)\
.filter_by(annotation_task_topic_group_id=annotation_task_group_id)\
.subquery()
# query TopicAnnotation table for all topic_annotations contained in any of above annotation_tasks
# use subqueryload for annotation_job objects (to avoid another database query execution when accessing them later)
base_query = db_session_users.query(TopicAnnotation)\
.options(subqueryload(TopicAnnotation.annotation_job))\
.filter(TopicAnnotation.annotation_task_id.in_(task_ids_subquery))
#####################################
# filtering and additional queries
#####################################
# filtering on topic_annotation table (part of base query table, so done first before query execution)
if 'user_id' in params:
base_query = base_query.filter_by(user_id=params['user_id'])
if 'is_gold_evaluation' in params:
base_query = base_query.filter_by(
is_gold_evaluation=params['is_gold_evaluation'])
# create list of dictionaries to return with annotation_job-related fields #
list_of_topic_annotation_objects = base_query.all(
) # NB: one database query executed here
# create list of dicts to return
list_of_topic_annotation_dicts = []
for topic_annotation_object in list_of_topic_annotation_objects:
annotation_job_dict = {
'user_difficulty':
topic_annotation_object.annotation_job.user_difficulty,
'user_tags': topic_annotation_object.annotation_job.arbitrary_tags
}
topic_annotation_to_dict = topic_annotation_object.to_dict()
topic_annotation_dict = merge_two_dicts(annotation_job_dict,
topic_annotation_to_dict)
list_of_topic_annotation_dicts.append(topic_annotation_dict)
# add aggregated_annotation info if necessary (i.e. gold judgment and is_correct_judgment)
if 'include_aggregated_annotation_info' in params:
# if there is not aggregated_annotation info for a particular annotation, relevant fields left as None,
# so all topic_annotation_dicts will have 'gold_judgment' and 'is_correct_judgment' fields
# do one query to get all relevant gold topic_annotation judgments
set_of_doc_ids = set([
topic_annotation_dict['doc_id']
for topic_annotation_dict in list_of_topic_annotation_dicts
])
gold_judgment_ids_subquery = db_session_users.query(AggregatedAnnotations.gold_topic_annotation_id)\
.filter(AggregatedAnnotations.doc_id.in_(set_of_doc_ids))\
.filter_by(annotation_task_group_id=annotation_task_group_id)\
.subquery()
gold_judgment_objects = db_session_users.query(TopicAnnotation.doc_id,
TopicAnnotation.is_positive)\
.filter(TopicAnnotation.id.in_(gold_judgment_ids_subquery))\
.all() # NB: another database query executed here
# create lookup table (keyed by doc_id) from gold_judgment_objects
gold_judgment_dict = {
gold_judgment_object.doc_id: gold_judgment_object.is_positive
for gold_judgment_object in gold_judgment_objects
}
# run through list_of_topic_annotation_dicts, look up gold judgment (if exists) for each topic_annotation,
# and add to list_of_topic_annotation_dicts
for topic_annotation_dict in list_of_topic_annotation_dicts:
gold_judgment = None
is_correct_judgment = None
if topic_annotation_dict['doc_id'] in gold_judgment_dict:
gold_judgment = gold_judgment_dict[
topic_annotation_dict['doc_id']]
is_correct_judgment = gold_judgment == topic_annotation_dict[
'is_positive']
topic_annotation_dict.update({
'gold_judgment':
gold_judgment,
'is_correct_judgment':
is_correct_judgment
})
# filtering/sorting on annotation_job fields (this is done on list_of_topic_annotation_dicts list)
# NB: tags here are added by the annotators to annotation_jobs (not admin), so no guarantee they are accurate
###################
# more filtering
###################
if 'user_arbitrary_tag' in params:
list_of_topic_annotation_dicts = [
d for d in list_of_topic_annotation_dicts
if params['user_arbitrary_tag'] in d['user_tags']
]
if 'user_difficulty' in params:
list_of_topic_annotation_dicts = [
d for d in list_of_topic_annotation_dicts
if d['user_difficulty'] == params['user_difficulty']
]
if 'is_correct_judgment' in params and 'include_aggregated_annotation_info' in params:
# NB: str() call here is because booleans come through as strings in url parameters
list_of_topic_annotation_dicts = [
d for d in list_of_topic_annotation_dicts
if str(d['is_correct_judgment']) == params['is_correct_judgment']
]
###################
# sorting
###################
if 'sorting' in params:
# sorting by difficulty or by accuracy
if params['sorting'] == 'user_difficulty' or (
params['sorting'] == 'is_correct_judgment'
and 'include_aggregated_annotation_info' in params):
list_of_topic_annotation_dicts = sorted(
list_of_topic_annotation_dicts,
key=lambda dic: dic[params['sorting']])
#############################################
# return list of topic_annotation dicts
#############################################
return jsonify(list_of_topic_annotation_dicts)
def activate_user(params):
email = params.get('email')
token = params.get('token')
new_password = params.get('new_password')
first_name = params.get('first_name')
last_name = params.get('last_name')
is_contributor = params.get('is_contributor')
dry_run = params.get('dry_run', False) # validate token, email, enabled state only
linkedin_id = params.get('linkedin_id')
google_id = params.get('google_id')
enabled = params.get('enabled')
# params to go into json field in db
json_params = [
'agencies', 'state_agencies',
'other_agencies', 'other_state_agencies', 'other_topics',
'user_style'
]
def error_response(msg='Invalid email or token'):
response = jsonify({
'error': msg,
})
response.status_code = 400
return response
# confirmation_required variable tracks whether this is an activation sourced from a marketing campaign,
# a signup withouot a token, or from the invite -> activate flow.
# use confirmation_required to indicate we need to send a confirmation email later on
confirmation_required = False
marketing_campaign = db_session_users.query(MarketingCampaign).filter_by(token=token).first()
if marketing_campaign is not None or token is None:
confirmation_required = True
else:
if email is None:
return error_response()
else:
email = email.lower()
g.user_email = email
user = db_session_users.query(User).filter_by(email=email).scalar()
if user is None:
return error_response()
if dry_run and user.enabled:
return error_response('User is already enabled')
enabled_at_start = user.enabled
if not user.reset_token or user.reset_token != token:
# send an email to support, but only if the user is in the db to prevent spamming
if dry_run:
template_vars = {
'email': email,
}
email_helper.send_email(
'*****@*****.**',
'*****@*****.**',
'A user attempted to use an invalid token during activation',
template='activate-fail',
vars=template_vars,
)
return error_response()
if dry_run:
return jsonify({'marketing_campaign': marketing_campaign is not None})
if not new_password:
return error_response('Missing fields')
# for the marketing campaign approach, create an entry in the users table,
# for the invite-based registration approach, mark the user enabled
if confirmation_required:
email = email.lower()
g.user_email = email
# check if this user exists in the database (the invite use-case), so we can use the existing entry if so
# and create a new entry if not
user = db_session_users.query(User).filter_by(email=email).first()
# this is for when a user comes to our site without being invited through the admin tool
if user is None:
user = User({
'email': email,
'first_name': first_name,
'last_name': last_name,
'password': new_password,
'enabled': False,
})
# this is for when the user is instead invited to our site, but then instead of trying to enter via the
# invitation link, they use the regular user signup flow. they will now get the confirmation email
# and have to fully activate their account there
else:
# triple check to prevent any shenanigans for enabled users, or user accounts
# that somehow exist but were not invited, and also if the invite has already been skipped
# and we have successfully moved onto the confirmation step
# n.b. relying on hash values is a little funky here, but it seems to work
if user.enabled or "invited_by" not in user.properties or "invite_skipped" in user.properties:
return error_response()
user.properties["invite_skipped"] = True # n.b. record that the invite workflow was skipped
user.first_name = first_name
user.last_name = last_name
user.update_password(new_password)
if linkedin_id:
user.linkedin = linkedin_id
user.industry = params.get('industry')
user.company = params.get('company')
user.properties['linkedin_data'] = params.get('linkedin_data')
user.enabled = enabled
user.properties['confirmed_date'] = datetime.datetime.utcnow().isoformat()
if google_id:
user.google_id = google_id
user.enabled = enabled
user.properties['confirmed_date'] = datetime.datetime.utcnow().isoformat()
# mark internal users with the internal user flag so we can differentiate user types when
# calculating various stats
if email.endswith("@jurispect.com") or email.endswith("@compliance.ai"):
user.is_internal_user = True
if marketing_campaign is not None:
user.marketing_campaigns.append(marketing_campaign)
user.gen_reset_token()
enabled_at_start = False
try:
_send_activation_email('confirm', user)
except SMTPException:
db_session_users.rollback()
return error_response('Could not send email', code=500)
else:
user.enabled = True
user.update_password(new_password)
if first_name:
user.first_name = first_name
if last_name:
user.last_name = last_name
# only allow the token to be used once:
user.reset_token = None
new_props = {p: params[p] for p in json_params if params.get(p)}
# n.b. since this route is shared with password resets, we need to skip updating the activation time
# when it is a password reset action
if not enabled_at_start:
new_props['activation_time'] = datetime.datetime.utcnow().isoformat()
if not params.get('user_style') and email.endswith('@firstrepublic.com'):
new_props['user_style'] = 'first-republic'
if len(new_props) > 0:
user.properties = merge_two_dicts(user.properties, new_props)
if is_contributor:
user.roles = ['contributor']
# FIXME: this is needed for marketing-campaign sourced users but yields a double commit
# probably not super noticeable, but should fix if we have the time
db_session_users.add(user)
try:
db_session_users.commit()
except IntegrityError:
return error_response()
db_session_users.refresh(user)
topic_ids = []
topic_ids.extend(params.get('topics', AggregatedAnnotations.topic_id_name_mapping.keys()))
for topic_id in topic_ids:
userTopic = UserTopic({
'user_id': user.id,
'topic_id': topic_id,
'following': True
})
db_session_users.add(userTopic)
news_ids = [x['id'] for x in jsearch.query_records({'size': 1000}, doc_type='news_sources')]
for news_id in news_ids:
userFollowedEntity = UserFollowedEntity({
'user_id': user.id,
'entity_id': news_id,
'entity_type': 'news_sources',
'following': True
})
db_session_users.add(userFollowedEntity)
agency_ids = []
agency_ids.extend(params.get('agencies', []))
new_ids = []
# verify that the agency ids are correct
# using DefaultAgenciesToFollowAtSignup since users now skip onboarding
for agency_id in DefaultAgenciesToFollowAtSignup:
try:
agency = jsearch.get_record(agency_id, 'agencies')
new_ids.append(agency['id'])
except NotFoundError:
pass
for agency_id in new_ids:
user_agency = UserAgency({'user_id': user.id, 'agency_id': agency_id, 'following': True})
db_session_users.add(user_agency)
state_jurisdictions = []
state_jurisdictions.extend(params.get('state_agencies', []))
state_ids = []
# get selected state jurisdiction ids and add them to follow entity table
for state_jurisdiction in state_jurisdictions:
try:
state = get_state_by_short_name('jurisdictions', state_jurisdiction)
state_ids.append(state['id'])
except NotFoundError:
pass
updated_followed_entity(user.id, {'entities': [{ 'entity_id': state_id, 'entity_type': 'jurisdictions', 'following': True } for state_id in state_ids]})
# send a support mail if the user requests a new source
other_agencies = params.get('other_agencies', '')
other_state_agencies = params.get('other_state_agencies', '')
other_topics = params.get('other_topics', '')
if other_agencies or other_state_agencies or other_topics:
template_vars = {
'other_agencies': other_agencies,
'other_state_agencies': other_state_agencies,
'other_topics': other_topics,
'name': '%s %s' % (first_name, last_name),
'email': email,
}
email_helper.send_email(
'*****@*****.**',
'*****@*****.**',
'A new user has requested additional sources or topics',
template='additional-sources',
vars=template_vars,
)
try:
db_session_users.commit()
except IntegrityError:
return error_response()
# start free trials.
user = db_session_users.query(User).filter_by(email=email.lower()).first()
latest_subscription = db_session_users.query(Subscription).filter_by(user_id=user.id, latest=True).first()
if latest_subscription is None:
# new users with .edu email get a 120 month free trial.
if user.email.endswith('.edu'):
subscribe_users_to_plan([user.id],'free_trial_120months')
# all other users get a 1 month free trial
else:
start_free_trial(user.id)
create_folder(user.id, {'name': 'Read'})
create_folder(user.id, {'name': 'Bookmarked'})
if confirmation_required:
# special case login for unenabled marketing campaign users allow access for 7 days only
expiration_datetime = datetime.datetime.utcnow() + datetime.timedelta(days=7)
token = jwt.encode({'user_id': user.id, 'exp': expiration_datetime}, SECRET_JWT)
# Flag 'is_new' defines if user is returning or just registered. True means just registered user
return jsonify({"jwt_token": token, "is_new": True, 'email': email.lower()})
else:
# return empty if user not from marketing campaign
return jsonify({})
def docket_timeline(docket_id, params, user_id):
params = merge_two_dicts(params, {"docket_id": docket_id, 'skip_unused_fields': True})
docs, total = get_filtered_documents(params, user_id)
return docs
def get_filtered_documents(params, user_id):
## Get the query parameters:
query_list = safe_getlist('query', params)
more_like_doc_id = params.get('more_like_doc_id', None)
agency_list = safe_getlist('agency_id', params)
agency_skip_list = safe_getlist('skip_agency', params)
category_list = safe_getlist('category', params)
skip_category_list = safe_getlist('skip_category', params)
provenance_list = safe_getlist('provenance', params)
meta_table_list = safe_getlist('meta_table', params)
docket_list = safe_getlist('docket_id', params)
regulation_id_list = safe_getlist('regulation_id', params)
citation_id_list = safe_getlist('citation_id', params)
concept_id_list = safe_getlist('concept_id', params)
act_id_list = safe_getlist('act_id', params)
bank_id_list = safe_getlist('bank_id', params)
topic_id_list = safe_getlist('topic_id', params)
published_to = params.get('published_to', None)
published_from = params.get('published_from', None)
compliance_to = params.get('compliance_to', None)
compliance_from = params.get('compliance_from', None)
comments_close_to = params.get('comments_close_to', None)
comments_close_from = params.get('comments_close_from', None)
key_date_to = params.get('key_date_to', None)
key_date_from = params.get('key_date_from', None)
spider_name_list = safe_getlist('spider_name', params)
limit = params.get('limit', 20)
offset = params.get('offset', 0)
sort = params.get('sort', None)
order = params.get('order', None)
read = params.get('read', None)
bookmarked = params.get('bookmarked', None)
tag_id = params.get('tag_id', None)
folder_id = params.get('folder_id', None)
full_text = params.get('full_text', False)
skip_unused_fields = params.get('skip_unused_fields', False)
all_agencies = params.get('all_agencies', False)
all_topics = params.get('all_topics', False)
created_at = params.get('created_at', False)
flagged_status = params.get('flagged_status', None)
pdf_url = params.get('pdf_url', None)
get_count_only = params.get('get_count_only', None)
include_mentions_for_filter = params.get('include_mentions_for_filter', False)
published_at = params.get('published_at', False)
created_to = params.get('created_to', None)
created_from = params.get('created_from', None)
## Don't sort when not requested
## Update the sort attr to use the full path into jurasticsearch docs
## Use 'desc' as the default order when sort is requested
if sort:
if AttributeName.get(sort, False): sort = AttributeName[sort]
if not order: order = 'desc'
## Filter agencies in the following manner:
# 1. By list provided; else
# 2. By agencies followed; else
# 3. Using the global default for Financial Services
# Unless there is a query_list passed, in which case we don't want to constrain the search
# NB: Remove the default filter for easy testing (for now)
show_all_sources_timeline = False
include_docs_with_no_agency = False
if not agency_list and not topic_id_list:
show_all_sources_timeline = True
followed_agencies = get_followed_agency_ids(user_id)
if all_topics:
topic_id_list = get_all_topic_ids_for_search()
else:
topic_id_list = get_user_followed_topic_ids()
if query_list or all_agencies:
# use a blacklist to restrict the agencies that show up in our system for search as a unit
include_docs_with_no_agency = True
agency_list = get_all_agency_ids_for_search()
elif followed_agencies:
agency_list = followed_agencies
else:
agency_list = DefaultAgencies
if agency_skip_list:
skip_list_set = set([int(a) for a in agency_skip_list])
agency_list = [a for a in agency_list if a not in skip_list_set]
full_request_body = filtered_request_template()
query = full_request_body["query"]
query["bool"] = {"must": []}
## Add the search query, if there is one
if query_list:
query_text = " ".join(query_list)
search_queryreform(query_text, query)
full_request_body["highlight"] = default_highlight_parameters()
# i can't see a reason why we'd want to use a query string AND more_like_this doc id
# n.b. highlights don't work out of the box with more_like_this queries
elif more_like_doc_id:
query["bool"]["must"].append({
"more_like_this": {
"fields": ["title"],
"like": [
{
"_index" : ACTIVE_INDEX_NAME,
"_type" : "documents",
"_id" : str(more_like_doc_id)
}
],
# these values are from the examples at https://www.elastic.co/guide/en/elasticsearch/reference/2.3/query-dsl-mlt-query.html
# and their presence seems useful for providing reasonable results. they could certainly be tweaked though
"min_term_freq": 1,
"max_query_terms": 12
}
})
## Setup the response object containers (and tracking variables):
date_filter = None
date_filter_should = None # for OR type queries
user_docs = None
tagged_doc_ids = None
folder_doc_ids = None
clause_count = 0
flagged_docs = None
## Add filters:
filter_terms = []
# In the case of where mainstream news is sent in as a category
# (not for search results) query for followed_mainstream_news docs
# otherwise (in the case of all_agencies), return all mainstream news sources
if 'Mainstream News' in category_list and not all_agencies:
agency_list_filter = elastic_list_filter(agency_list, "agencies.id")
mainstream_news_should_clauses = [agency_list_filter]
followed_mainstream_news = get_all_followed_entities(user_id, {'entity_type': 'news_sources'})
if len(followed_mainstream_news['followed_entities']) > 0:
followed_mainstream_news_id_list = [e['entity_id'] for e in followed_mainstream_news['followed_entities']]
mainstream_news_should_clauses.append({
"terms": {"mainstream_news.news_source.id": followed_mainstream_news_id_list}
})
filter_terms.append({"bool": {"should": mainstream_news_should_clauses}})
elif show_all_sources_timeline and not include_docs_with_no_agency:
query['bool']['must'].append({
"bool": {
"should": [
topic_id_filter(topic_id_list),
{"terms": {"agencies.id": agency_list}}
]
}
})
# in some search results, we need to include document types like Whitepapers, which don't have an agency
# in this case, we use our agency whitelist with an OR on the value being missing (the whitepaper case)
elif show_all_sources_timeline and include_docs_with_no_agency:
agency_list_filter = elastic_list_filter(agency_list, "agencies.id")
no_agency_should_clauses = [agency_list_filter, {"bool": {"must_not": {"exists": {"field": "agencies.id"}}}}]
if len(topic_id_list) > 0:
no_agency_should_clauses.append(topic_id_filter(topic_id_list))
filter_terms.append({"bool": {"should": no_agency_should_clauses}})
else:
add_elastic_list_filter(agency_list, "agencies.id", filter_terms)
if len(topic_id_list) > 0:
query['bool']['must'].append(topic_id_filter(topic_id_list))
# if an explicit category filter was passed in, ignore the skip_category filter. they should be mutually exclusive
if len(category_list) > 0:
add_elastic_list_filter(category_list, "category", filter_terms)
elif len(skip_category_list) > 0:
if not is_mainstream_news_enabled():
skip_category_list.append("Mainstream News")
skip_filter = elastic_list_filter(skip_category_list, "category")
filter_terms.append({"bool": {"must_not": skip_filter}})
else:
if not is_mainstream_news_enabled():
skip_filter = elastic_list_filter(["Mainstream News"], "category")
filter_terms.append({"bool": {"must_not": skip_filter}})
add_elastic_list_filter(provenance_list, "provenance", filter_terms)
add_elastic_list_filter(meta_table_list, "meta_table", filter_terms)
add_elastic_list_filter(regulation_id_list, "cited_associations.named_regulation_ids", filter_terms)
add_elastic_list_filter(citation_id_list, "cited_associations.citation_ids", filter_terms)
add_elastic_list_filter(concept_id_list, "cited_associations.concept_ids", filter_terms)
add_elastic_list_filter(act_id_list, "cited_associations.act_ids", filter_terms)
add_elastic_list_filter(docket_list, "dockets.docket_id", filter_terms)
add_elastic_list_filter(spider_name_list, "spider_name", filter_terms)
add_elastic_list_filter(bank_id_list, "cited_associations.bank_ids", filter_terms)
if filter_terms: clause_count += 1
## Add date filters:
if published_to or published_from:
date_filter = elastic_date_filter("publication_date", published_from, published_to)
clause_count += 1
elif compliance_to or compliance_from:
date_filter = elastic_date_filter("rule.effective_on", compliance_from, compliance_to)
clause_count += 1
elif comments_close_to or comments_close_from:
date_filter = elastic_date_filter("rule.comments_close_on", comments_close_from, comments_close_to)
clause_count += 1
elif key_date_to or key_date_from:
date_filter_effective = {"range": elastic_date_filter("rule.effective_on", key_date_from, key_date_to)}
date_filter_comments = {"range": elastic_date_filter("rule.comments_close_on", key_date_from, key_date_to)}
date_filter_should = [date_filter_effective, date_filter_comments]
clause_count += 2
elif created_at:
date_filter = {"created_at":{"from": "now-%sh" % created_at}}
clause_count += 1
elif published_at:
date_filter = {"publication_date":{"from": "now-%sh" % published_at}}
clause_count += 1
elif created_to or created_from:
date_filter = elastic_date_filter("created_at", created_from, created_to)
clause_count += 1
if read or bookmarked:
user_docs = get_user_document_ids(user_id)
clause_count += 1
if tag_id:
tagged_doc_ids = get_doc_ids_for_tag(user_id, tag_id)
clause_count += 1
if folder_id:
folder_doc_ids = get_doc_ids_for_folder(folder_id)
clause_count += 1
if flagged_status:
flagged_docs = get_flagged_doc_id_info_map(user_id, flagged_status)
clause_count += 1
if pdf_url:
pdf_url_filter = es_filter(pdf_url, 'pdf_url')
clause_count += 1
## Build the final query to send to jurasticsearch:
if clause_count >= 1:
query["bool"]["must"].extend(filter_terms)
if date_filter_should:
query["bool"]["must"].append({
"bool": {
"should": date_filter_should
}
})
if date_filter:
query["bool"]["must"].append({"range": date_filter})
if read:
read_bool = str_to_bool(read)
if read_bool:
query["bool"]["must"].append(read_filter(user_docs))
elif not read_bool:
query["bool"]["must_not"] = read_filter(user_docs)
else:
print "WARNING: read_flag is not boolean: {}".format(read_bool)
if bookmarked:
bookmarked_bool = str_to_bool(bookmarked)
if bookmarked_bool:
query["bool"]["must"].append(bookmarked_filter(user_docs))
elif not bookmarked_bool:
query["bool"]["must_not"] = bookmarked_filter(user_docs)
else:
print "WARNING: bookmarked_flag is not boolean: {}".format(bookmarked_bool)
if tagged_doc_ids:
query["bool"]["must"].append({"terms": {"id": tagged_doc_ids}})
if folder_doc_ids is not None:
query["bool"]["must"].append({"terms": {"id": folder_doc_ids}})
if flagged_status:
ValidFlaggedStatuses = [UserFlaggedDocument.FLAGGED_STATUS, UserFlaggedDocument.HIDDEN_STATUS, UserFlaggedDocument.PROCESSED_STATUS]
if flagged_status == "exclude_all":
query["bool"]["must_not"] = flagged_filter(flagged_docs)
elif flagged_status == "include_all" or flagged_status in ValidFlaggedStatuses or flagged_status == "contributor_flagged":
query["bool"]["must"].append(flagged_filter(flagged_docs))
else:
print "WARNING: flagged_status must be one of: {}" + str(ValidFlaggedStatuses)
if pdf_url:
query["bool"]["filter"] = pdf_url_filter
# return just the document count if that is all that was requested
if get_count_only is not None:
result = jsearch.count_records(full_request_body)
count = result["hits"]["total"]
return [], count
# otherwise, send the query and return the source dict for each doc:
else:
# Start out by excluding a bunch of fields from the payload that are not currently used in any way by the
# front-end application, if the user has passed the skip_unused fields flag
excluded_fields = []
if skip_unused_fields:
excluded_fields += UNUSED_FIELDS_FOR_DOC_LIST
if not full_text:
excluded_fields.append("full_text")
if len(excluded_fields) > 0:
full_request_body["_source"] = {"exclude": excluded_fields}
if limit: full_request_body["size"] = limit
if sort and order: full_request_body["sort"] = {sort: {"order": order}} ## TODO: relevance support
if offset: full_request_body["from"] = offset
hits = jsearch.query_documents(full_request_body)['hits']
if query_list:
docs = [merge_two_dicts(x["_source"], search_attrs(x)) for x in hits['hits']]
else:
docs = [x['_source'] for x in hits['hits']]
docs = decorate_documents(docs, user_id, user_docs=user_docs, flagged_docs=flagged_docs)
if include_mentions_for_filter:
decorate_documents_with_filter_mentions(docs, act_id_list, regulation_id_list, concept_id_list,
bank_id_list, limit)
# TODO: Allow turning this off to return all topics with their probabilities by an optional flag
docs = apply_threshold_topics_in_document_response(docs)
return docs, hits['total']
def update_annotation_task(annotation_task_id, params):
original_annotation_task = db_session_users.query(
AnnotationTask).filter_by(id=annotation_task_id).first()
num_annotation_jobs = db_session_users.query(AnnotationJob).filter_by(
annotation_task_id=annotation_task_id).count()
# n.b. updating the topics hash means that we need to swap this task for a new one for
# consistency of which annotations were generated against which task, as it is valuable to know
# which topics were presented when annotations are created, likewise what the user and other config options
# note: there is no need for a new task until this task has jobs added for it, so when the job count is still 0,
# we can do a direct update to the existing task instead
# note: onboarding mode tasks (with is_training_task=True) also do not create a new task when they date updated
if num_annotation_jobs > 0 and not original_annotation_task.is_training_task and \
('topics' in params or
'user_ids' in params or
'config' in params or
'term_sampling_group_ids' in params or
'is_training_task' in params or
'include_gold_annotations' in params or
'annotation_task_topic_group_id' in params):
new_annotation_task_dict = original_annotation_task.__dict__
if 'topics' in params:
new_annotation_task_dict['topics'] = params['topics']
# must be done before setting is_training_task due to check in setting is_training_task
if 'annotation_task_topic_group_id' in params:
new_annotation_task_dict[
'annotation_task_topic_group_id'] = params[
'annotation_task_topic_group_id']
# update is_training_task to be False if no longer in annotation_task_topic_group
if new_annotation_task_dict[
'annotation_task_topic_group_id'] is None:
new_annotation_task_dict['is_training_task'] = False
if 'user_ids' in params:
new_annotation_task_dict['user_ids'] = params['user_ids']
# n.b. changing status between active/inactive in effect toggles this task on/off
if 'status' in params and params['status'] in VALID_TASK_STATUSES:
new_annotation_task_dict['status'] = params['status']
if 'config' in params:
new_annotation_task_dict['config'] = merge_two_dicts(
new_annotation_task_dict['config'], params['config'])
if 'num_touches' in new_annotation_task_dict[
'config'] and new_annotation_task_dict['config'][
'num_touches'] == '':
del new_annotation_task_dict['config']['num_touches']
if 'name' in params:
new_annotation_task_dict['name'] = params['name']
# only allow setting this to True if this annotation_task is in an annotation_task_topic_group
# TODO: currently possible to do illegal is_training_task update (with not effect) that creates new task
if 'is_training_task' in params and params['is_training_task'] is True:
if new_annotation_task_dict[
'annotation_task_topic_group_id'] is not None:
new_annotation_task_dict['is_training_task'] = params[
'is_training_task']
if 'include_gold_annotations' in params:
new_annotation_task_dict['include_gold_annotations'] = params[
'include_gold_annotations']
if 'is_contributor_task' in params:
new_annotation_task_dict['is_contributor_task'] = params[
'is_contributor_task']
new_annotation_task = AnnotationTask(new_annotation_task_dict)
db_session_users.add(new_annotation_task)
# n.b. connect the tasks together and mark the old one as inactive
original_annotation_task.active_task = new_annotation_task
original_annotation_task.status = AnnotationTask.INACTIVE_STATUS
db_session_users.add(original_annotation_task)
# also deal with any even older annotation tasks and make sure they get their active_task_id updated too
for older_annotation_task in \
db_session_users.query(AnnotationTask).filter_by(active_task_id=original_annotation_task.id):
older_annotation_task.active_task = new_annotation_task
db_session_users.add(older_annotation_task)
db_session_users.commit()
db_session_users.refresh(new_annotation_task)
# update any annotation_task_groups containing original_annotation_task to point to new task
all_task_groups = db_session_users.query(
AnnotationTaskTopicGroup).all() # get all task groups
for group in all_task_groups:
if annotation_task_id in group.annotation_task_ids:
# update task id list to point to new annotation task id
# list is a tuple, so cannot be mutated in-place
new_ids = [
id for id in group.annotation_task_ids
if id != annotation_task_id
]
new_ids.append(new_annotation_task.id)
group.annotation_task_ids = new_ids
db_session_users.add(group)
db_session_users.commit()
if 'term_sampling_group_ids' in params:
new_term_sampling_ids = params['term_sampling_group_ids']
else:
existing_term_sampling_group_ids = [
t[0] for t in db_session_users.query(
AnnotationTaskTermSamplingGroup.term_sampling_group_id).
filter_by(annotation_task_id=original_annotation_task.id)
]
new_term_sampling_ids = existing_term_sampling_group_ids
for term_sampling_group_id in new_term_sampling_ids:
attsg = AnnotationTaskTermSamplingGroup({
'annotation_task_id':
new_annotation_task.id,
'term_sampling_group_id':
term_sampling_group_id
})
db_session_users.add(attsg)
if len(new_term_sampling_ids) > 0:
db_session_users.commit()
task_to_return = new_annotation_task
# allow basic updates of status/config/name
else:
# n.b. changing status between active/inactive in effect toggles this task on/off
if 'status' in params and params['status'] in VALID_TASK_STATUSES:
original_annotation_task.status = params['status']
if 'name' in params:
original_annotation_task.name = params['name']
if 'is_contributor_task' in params:
original_annotation_task.is_contributor_task = params[
'is_contributor_task']
# must be done before setting is_training_task due to check in setting is_training_task
if 'annotation_task_topic_group_id' in params:
original_annotation_task.annotation_task_topic_group_id = params[
'annotation_task_topic_group_id']
# update is_training_task to be False if no longer in annotation_task_topic_group
if original_annotation_task.annotation_task_topic_group_id is None:
original_annotation_task.is_training_task = False
## n.b. these all can get updated here in the annotation jobs == 0 use case ##
# only allow setting this to True if this annotation_task is in an annotation_task_topic_group
if 'is_training_task' in params and params['is_training_task'] is True:
if original_annotation_task.annotation_task_topic_group_id is not None:
original_annotation_task.is_training_task = params[
'is_training_task']
if 'include_gold_annotations' in params:
original_annotation_task.include_gold_annotations = params[
'include_gold_annotations']
if 'topics' in params:
original_annotation_task.topics = params['topics']
if 'user_ids' in params:
original_annotation_task.user_ids = params['user_ids']
if 'config' in params:
original_annotation_task.config = merge_two_dicts(
original_annotation_task.config, params['config'])
if 'num_touches' in original_annotation_task.config and original_annotation_task.config[
'num_touches'] == '':
del original_annotation_task.config['num_touches']
if 'term_sampling_group_ids' in params:
new_term_sampling_ids = params['term_sampling_group_ids']
existing_tsgs = db_session_users.query(AnnotationTaskTermSamplingGroup)\
.filter_by(annotation_task_id=original_annotation_task.id).all()
existing_tsg_id_map = {
t.term_sampling_group_id: t
for t in existing_tsgs
}
removed_ids = [
item for item in existing_tsg_id_map.keys()
if item not in new_term_sampling_ids
]
for term_sampling_group_id in new_term_sampling_ids:
if term_sampling_group_id not in existing_tsg_id_map:
attsg = AnnotationTaskTermSamplingGroup({
'annotation_task_id':
original_annotation_task.id,
'term_sampling_group_id':
term_sampling_group_id
})
db_session_users.add(attsg)
for term_sampling_group_id in removed_ids:
db_session_users.query(
AnnotationTaskTermSamplingGroup).filter_by(
annotation_task_id=original_annotation_task.id,
term_sampling_group_id=term_sampling_group_id).delete(
)
db_session_users.add(original_annotation_task)
db_session_users.commit()
task_to_return = original_annotation_task
task_dict = task_to_return.to_dict()
task_dict["old_tasks"] = [
t.to_dict() for t in db_session_users.query(AnnotationTask).filter_by(
active_task_id=task_to_return.id)
]
term_sampling_group_ids = db_session_users.query(AnnotationTaskTermSamplingGroup.term_sampling_group_id) \
.filter_by(annotation_task_id=task_to_return.id)
task_dict["term_sampling_group_ids"] = [
t[0] for t in term_sampling_group_ids
]
return {"annotation_task": task_dict}
def get_all_folders(user_id):
# Gather all of the user's folders and shared folder ids
all_personal_user_folders = db_session_users.query(UserFolder).filter_by(
user_id=user_id).all()
all_user_shared_folder_ids = [
f[0]
for f in db_session_users.query(UserSharedFolder.folder_id).filter_by(
user_id=user_id).all()
]
# filter out personal folders using shared folders ids
personal_folders = []
for folder in all_personal_user_folders:
if folder.id not in all_user_shared_folder_ids:
personal_folders.append(folder.to_dict())
# Gather all of the user's shared folders
shared_folders = []
all_user_shared_folders = db_session_users.query(
UserSharedFolder).filter_by(user_id=user_id).all()
all_users_of_shared_folder = db_session_users.query(
UserSharedFolder).filter(
UserSharedFolder.folder_id.in_(
[f.folder_id for f in all_user_shared_folders])).all()
all_relevant_users = db_session_users.query(User).filter(
User.id.in_([f.user_id for f in all_users_of_shared_folder]))
relevant_user_lookup = {u.id: u for u in all_relevant_users}
folder_id_to_user_shared_folders = defaultdict(list)
for f in all_users_of_shared_folder:
folder_id_to_user_shared_folders[f.folder_id].append(f)
for folder in db_session_users.query(UserFolder).filter(
UserFolder.id.in_([f.folder_id for f in all_user_shared_folders])):
# Gather all of the users with whom the folder was shared
shared_folder_users = []
for shared_folder_user in folder_id_to_user_shared_folders[folder.id]:
# Finally gather the permissions assigned to each user in the shared folder
user_permission = {}
if (shared_folder_user.owner):
user_permission['user_permission_access'] = 'owner'
elif (shared_folder_user.editor):
user_permission['user_permission_access'] = 'editor'
else:
user_permission['user_permission_access'] = 'viewer'
shared_folder_users.append(
merge_two_dicts(
relevant_user_lookup[shared_folder_user.user_id].to_dict(),
user_permission))
folder_users_data = {}
folder_users_data['shared_folder_users'] = shared_folder_users
shared_folders.append(
merge_two_dicts(folder.to_dict(), folder_users_data))
return jsonify({
'personal_folders': personal_folders,
'shared_folders': shared_folders
})
def add_users_to_shared_folder(folder_id, params):
shared_folder = db_session_users.query(UserSharedFolder).filter_by(
folder_id=folder_id).first()
if not shared_folder:
return jsonify(
{'errors':
"No shared folder exists for id: " + str(folder_id)}), 404
shared_folder_owner_id = db_session_users.query(
UserFolder.user_id).filter_by(id=folder_id).first()
user_shared_folders = []
mandrill_recipients = []
for user in params['users']:
user_access_to_folder = db_session_users.query(
UserSharedFolder).filter_by(folder_id=folder_id,
user_id=user['id']).first()
if user_access_to_folder:
return jsonify({'errors': 'Folder already shared with user'}), 409
elif (shared_folder_owner_id != user['id']): # no need to notify owner
new_user_shared_folder = UserSharedFolder({
"user_id": user['id'],
"folder_id": folder_id,
"editor": user['editor'],
"viewer": user['viewer']
})
user_shared_folders.append(new_user_shared_folder)
# new sharees need to be notified and emailed
user_from_db = db_session_users.query(User).filter_by(
id=user['id']).first()
mandrill_recipients.append({
'email': user_from_db.email,
'name': user_from_db.first_name,
'type': 'to'
})
if 'notif_status' in user_from_db.properties:
folder_status = {"user_folders": {"viewed_status": False}}
new_notif_status = merge_two_dicts(
user_from_db.properties['notif_status'], folder_status)
else:
new_notif_status = {"user_folders": {"viewed_status": False}}
new_props = merge_two_dicts(user_from_db.properties,
{'notif_status': new_notif_status})
user_from_db.properties = merge_two_dicts(user_from_db.properties,
new_props)
db_session_users.add(user_from_db)
db_session_users.commit()
db_session_users.add_all(user_shared_folders)
db_session_users.commit()
from app import socket
for user in user_shared_folders:
socket.emit('foldersNotification', room=user.user_id)
if BASE_URL_FOR_EMAIL:
base_url = BASE_URL_FOR_EMAIL
else:
base_url = 'http://%s:%s' % (API_BOUND_HOST, API_PORT)
if 'user_msg' in params:
user_msg = params['user_msg']
else:
user_msg = ''
shared_folder_owner = db_session_users.query(User).filter_by(
id=shared_folder_owner_id).first()
if shared_folder_owner.first_name is not None:
owner_first_name = shared_folder_owner.first_name + ' has'
else:
owner_first_name = 'I've'
folder_url = '%s/content?folderTimelineView=true&no_skipping=true&folder_id=%s' % (
base_url,
folder_id,
)
#email sharees
email_helper.send_via_mandrill(
mandrill_recipients,
'*****@*****.**',
'Compliance.ai',
'A Compliance.ai folder has been shared with you!',
template='shared-folder-inline',
vars={
'url': folder_url,
'base_url': base_url,
'user_msg': user_msg,
'owner_first_name': owner_first_name,
},
)
return jsonify({'folder_shared_with_users': True})
def invite_user(params, user_id):
email = params.get('email')
resend = params.get('resend', False)
def error_response(msg, code=400):
response = jsonify({
'error': msg,
})
response.status_code = code
return response
current_user = db_session_users.query(User).filter_by(id=user_id).first()
if not 'admin' in current_user.roles:
return error_response('Only admin users can send invitations', code=403)
if email is None:
return error_response('Email must be specified')
else:
email = email.lower()
if not EmailHelper.validate_email(email):
return error_response('Domain is not allowed')
user = db_session_users.query(User).filter_by(email=email).scalar()
if user is None:
user = User({
'email': email,
'enabled': False,
'properties': {'invited_by': current_user.email},
})
# mark internal users with the internal user flag so we can differentiate user types when
# calculating various stats
if email.endswith("@jurispect.com") or email.endswith("@compliance.ai"):
user.is_internal_user = True
user.gen_reset_token()
elif resend is True and user is not None:
# Add resent email time to user properties to surface in the FE
user.properties = merge_two_dicts(user.properties, { 'resent_invite_time': datetime.datetime.utcnow().isoformat() })
elif resend is False:
return error_response('User already exists')
user.gen_reset_token()
try:
_send_activation_email('invite', user)
except SMTPException:
db_session_users.rollback()
return error_response('Could not send email', code=500)
response = jsonify({
'success': True,
})
db_session_users.add(user)
try:
db_session_users.commit()
except IntegrityError:
return error_response('User already exists')
return response
