def create_annotation_task(params): new_annotation_task = AnnotationTask(params) term_sampling_group_ids = params.get("term_sampling_group_ids", None) # FIXME: enforcing default values until the front-end has been updated to provide this explicitly if new_annotation_task.type is None: new_annotation_task.type = AnnotationTask.TOPIC_ANNOTATION_TYPE elif new_annotation_task.type == 'contributor': new_annotation_task.is_contributor_task = True db_session_users.add(new_annotation_task) db_session_users.commit() db_session_users.refresh(new_annotation_task) if term_sampling_group_ids is not None: for term_sampling_group_id in term_sampling_group_ids: attsg = AnnotationTaskTermSamplingGroup({ 'annotation_task_id': new_annotation_task.id, 'term_sampling_group_id': term_sampling_group_id }) db_session_users.add(attsg) db_session_users.commit() task_dict = new_annotation_task.to_dict() term_sampling_group_ids = db_session_users.query(AnnotationTaskTermSamplingGroup.term_sampling_group_id) \ .filter_by(annotation_task_id=new_annotation_task.id) task_dict["term_sampling_group_ids"] = [ t[0] for t in term_sampling_group_ids ] return {"annotation_task": task_dict}
def handle_request_invoice(user_id, params): plan = params.get('plan', None) plan_from_db = db_session_users.query(Plan).filter_by( stripe_id=plan).first() if not plan_from_db: return 'Invalid plan' plan_name = plan_from_db.name plan_id = plan_from_db.id current_user = db_session_users.query(User).filter_by(id=user_id).first() message = current_user.first_name + ' ' + current_user.last_name + ' is requesting to pay by invoice for the ' + plan_name + ' subscription. ' + current_user.first_name + ', the team at Compliance.ai will respond to your request soon!' try: email_helper.send_email( '*****@*****.**', current_user.email, 'Invoice request from ' + current_user.first_name, template='feedback-inline', vars={ 'feedback': message, 'User_first_name': current_user.first_name, 'User_last_name': current_user.last_name, }, ) except SMTPException as e: return error_response('Could not send invoice email.', code=500) invoice_for_db = { 'user_id': user_id, 'plan_id': plan_id, 'status': 'requested' } invoice_for_db = Invoice(invoice_for_db) db_session_users.add(invoice_for_db) db_session_users.commit() db_session_users.refresh(invoice_for_db) return {'invoice': 'invoice request sent'}
def update_saved_search(user_id, saved_search_id, params): name = params.get('name', None) name_conflict_user = db_session_users.query(UserSavedSearch).filter_by( user_id=user_id, name=name).first() if name_conflict_user: return jsonify({ 'errors': "Saved search name: " + name + " is already being used" }), 409 search_args = params.get('search_args', None) saved_search = db_session_users.query(UserSavedSearch).filter_by( id=saved_search_id, user_id=user_id).first() if not saved_search: return jsonify({"errors": "No saved search for this user and id"}), 404 if name: saved_search.name = name if search_args: saved_search.search_args = search_args db_session_users.add(saved_search) db_session_users.commit() db_session_users.refresh(saved_search) return jsonify({"saved_search": saved_search.to_dict()})
def queue_created_document(params, user_id): created_doc = UserCreatedDocument( merge_two_dicts({"user_id": user_id}, params)) db_session_users.add(created_doc) db_session_users.commit() db_session_users.refresh(created_doc) return {'user_created_document': created_doc.to_dict()}
def create_review_for_job(annotation_task_id, annotation_job_id, user_id, params): annotation_job = db_session_users.query(AnnotationJob).filter_by(id=annotation_job_id).first() annotation_job.status = AnnotationJob.COMPLETE_STATUS annotation_job.completed_at = datetime.datetime.now() db_session_users.add(annotation_job) db_session_users.commit() db_session_users.refresh(annotation_job) job = annotation_job.to_dict() doc = None if 'multiple_field' in params: multiple_field = params.get('multiple_field', None) flagged_doc = UserFlaggedDocument({ 'user_id': user_id, 'doc_id': annotation_job.doc_id, 'issue_type': UserFlaggedDocument.CONTRIBUTOR_ISSUE_TYPE, 'multiple_field': multiple_field }) db_session_users.add(flagged_doc) db_session_users.commit() db_session_users.refresh(flagged_doc) doc = flagged_doc.to_dict() return {"annotation_job": job, "flagged_doc": doc}
def update_subscription(subscription_id, params): original_subscription = db_session_users.query(Subscription).filter_by( id=subscription_id).first() new_subscription_dict = original_subscription.__dict__ today = datetime.datetime.utcnow() new_subscription_dict['latest'] = True new_subscription_dict['notes'] = None if 'expiration_date' in params: new_exiration_date = params['expiration_date'] new_exiration_date_obj = datetime.datetime.strptime( new_exiration_date, "%Y-%m-%d") new_subscription_dict['expiration_date'] = new_exiration_date_obj # update status of subscription depending on new expiration date if new_exiration_date_obj < today or new_exiration_date_obj.date( ) == today.date(): new_subscription_dict[ 'status_reason'] = Subscription.EXPIRED_STATUS_REASON new_subscription_dict['status'] = Subscription.INACTIVE_STATUS elif new_subscription_dict['status'] != Subscription.ACTIVE_STATUS: new_subscription_dict[ 'status_reason'] = Subscription.REACTIVATED_STATUS_REASON new_subscription_dict['status'] = Subscription.ACTIVE_STATUS if 'plan_id' in params: new_plan_id = params['plan_id'] plan = db_session_users().query(Plan).filter_by(id=new_plan_id).first() if plan: new_subscription_dict['plan_id'] = new_plan_id new_subscription_dict['stripe_id'] = plan.stripe_id new_subscription_dict['start_date'] = datetime.datetime.utcnow() new_subscription_dict[ 'status_reason'] = Subscription.REACTIVATED_STATUS_REASON new_subscription_dict['status'] = Subscription.ACTIVE_STATUS if plan.recurring: new_subscription_dict['expiration_date'] = None else: new_subscription_dict[ 'expiration_date'] = get_default_expiration_date(plan) else: return {'errors': "Plan is not found"} if 'payment_type' in params: new_subscription_dict['payment_type'] = params['payment_type'] if 'notes' in params: new_subscription_dict['notes'] = params['notes'] new_subscription_dict['modified_by_user_id'] = g.user_id new_subscription = Subscription(new_subscription_dict) db_session_users.add(new_subscription) # deactivate old subscription deactivate_subscriptions(new_subscription_dict['user_id']) db_session_users.commit() db_session_users.refresh(new_subscription) return {'new_subscription': new_subscription.to_dict()}
def create_marketing_campaign(user_id, params): marketing_campaign = MarketingCampaign( merge_two_dicts(params, {'created_by_user_id': user_id})) marketing_campaign.gen_token() db_session_users.add(marketing_campaign) db_session_users.commit() db_session_users.refresh(marketing_campaign) return {"marketing_campaign": marketing_campaign.to_dict()}
def handle_payment_event(event): payment_event_for_db = Payment_Event({ 'stripe_id': event.id, 'properties': serializeClass(event) }) db_session_users.add(payment_event_for_db) db_session_users.commit() db_session_users.refresh(payment_event_for_db) return {'event': 'event received'}
def create_annotation_task_group(params): # create annotation_task_group object and add to database new_annotation_task_group = AnnotationTaskTopicGroup(params) db_session_users.add(new_annotation_task_group) db_session_users.commit() db_session_users.refresh(new_annotation_task_group) #TODO: add check here about whether tasks ids in params actually exist task_group_dict = new_annotation_task_group.to_dict() return {"annotation_task_group": task_group_dict}
def deactivate_subscriptions(user_id): # a user should have only 1 current subscription at a time latest_subscriptions = db_session_users.query(Subscription).filter_by( user_id=user_id, latest=True).all() if latest_subscriptions is not None: for latest_subscription in latest_subscriptions: latest_subscription.latest = False latest_subscription.status = Subscription.INACTIVE_STATUS latest_subscription.end_date = datetime.datetime.utcnow() db_session_users.add(latest_subscription) db_session_users.commit() db_session_users.refresh(latest_subscription)
def update_annotation_task_group(annotation_task_group_id, params): # params is dict that can contain keys "name", "description", "annotation_task_ids" # and "arbitrary_tags" # get original annotation task group original_annotation_task_group = db_session_users.query(AnnotationTaskTopicGroup)\ .filter_by(id=annotation_task_group_id)\ .first() # check that task group exists if original_annotation_task_group is None: return jsonify({'errors': 'This annotation task group does not exist'}), 400 # apply easy updates to annotation task group (i.e. just overwriting name or description) if "name" in params: original_annotation_task_group.name = params["name"] if "description" in params: original_annotation_task_group.description = params["description"] # apply more difficult updates (i.e. changing arbitrary_tags or annotation_task_ids) # NB: for now these are simple overwrites - if needed can later update for more granular control if "annotation_task_ids" in params: original_annotation_task_group.annotation_task_ids = params[ "annotation_task_ids"] if "arbitrary_tags" in params: original_annotation_task_group.arbitrary_tags = params[ "arbitrary_tags"] # n.b. for case where wrong topic was chosen at outset if "topic_id" in params: original_annotation_task_group.topic_id = params['topic_id'] if "gold_annotator_user_ids" in params: original_annotation_task_group.gold_annotator_user_ids = params[ 'gold_annotator_user_ids'] if "active_topic_annotation_model_id" in params: original_annotation_task_group.active_topic_annotation_model_id = params[ 'active_topic_annotation_model_id'] # update database with new values db_session_users.add(original_annotation_task_group) db_session_users.commit() task_group_to_return = original_annotation_task_group # original task group has been updated # return updated annotation task group task_group_dict = task_group_to_return.to_dict() return jsonify({"annotation_task_group": task_group_dict})
def flag_document(user_id, doc_id, params): issue_severity = params.get('issue_severity', UserFlaggedDocument.REVIEW_SEVERITY) issue_type = params.get('issue_type', None) notes = params.get('notes', None) field = params.get('field', None) user_flagged_document_id = params.get('id', None) status = params.get('status', None) # for creating new user flagged documents if user_flagged_document_id is None: if not issue_type or issue_type not in ValidIssueTypes: return { 'errors': "Issue type must be one of: " + str(ValidIssueTypes) } if issue_severity not in ValidIssueSeverities: return { 'errors': "Issue severity must be one of: " + str(ValidIssueSeverities) } flagged_doc = UserFlaggedDocument({ 'user_id': user_id, 'doc_id': doc_id, 'issue_severity': issue_severity, 'issue_type': issue_type, 'notes': notes, 'field': field, }) # for updating an existing user flagged document (status is the only relevant use-case) else: flagged_doc = db_session_users.query(UserFlaggedDocument).filter_by( id=user_flagged_document_id).first() if status is not None: if status in ValidIssueStatuses: flagged_doc.status = status else: return { 'errors': "Issue status must be one of: " + str(ValidIssueStatuses) } db_session_users.add(flagged_doc) db_session_users.commit() db_session_users.refresh(flagged_doc) return flagged_doc.to_dict()
def start_free_trial(user_id): stripe_id = 'free_trial' free_trial = db_session_users.query(Plan).filter_by( stripe_id=stripe_id).first() subscription_for_db = { 'user_id': user_id, 'stripe_id': free_trial.stripe_id, 'plan_id': free_trial.id, 'latest': True, 'start_date': datetime.datetime.utcnow(), 'expiration_date': get_default_expiration_date(free_trial), 'status': 'active' } subscription_for_db = Subscription(subscription_for_db) deactivate_subscriptions(user_id) db_session_users.add(subscription_for_db) db_session_users.commit() db_session_users.refresh(subscription_for_db)
def subscribe_users_to_plan(user_ids, plan_stripe_id, payment_type=None, modified_by_user_id=None): if not (payment_type == 'invoice' or payment_type == 'stripe' or payment_type == None): return 'please use a valid payment_type' for user_id in user_ids: user = db_session_users.query(User).filter_by(id=user_id).first() if user is None: return "user doesn't exist" plan = db_session_users.query(Plan).filter_by( stripe_id=plan_stripe_id).first() if plan is None: return 'please use a valid plan' subscription_for_db = { 'user_id': user_id, 'stripe_id': plan.stripe_id, 'plan_id': plan.id, 'latest': True, 'start_date': datetime.datetime.utcnow(), 'status': 'active' } if modified_by_user_id is not None: subscription_for_db['modified_by_user_id'] = modified_by_user_id if payment_type == 'invoice' or payment_type == 'invoice': subscription_for_db['payment_type'] = payment_type else: # currently only free_trials have exiration dates. all other plans are recurring subscription_for_db[ 'expiration_date'] = get_default_expiration_date(plan) try: deactivate_subscriptions(user_id) except: return 'unable to deactivate existing subscriptions' try: subscription_for_db = Subscription(subscription_for_db) db_session_users.add(subscription_for_db) db_session_users.commit() except: return 'unable to post subscription to database'
def update_aggregated_annotation(aggregated_annotation_task_id, params): # NB: probably shouldn't change doc_id, topic_id or annotation_task_group_id; # these are set at crawler level when aggregated_annotations are created original_agg_annotation = db_session_users.query(AggregatedAnnotations)\ .filter_by(id=aggregated_annotation_task_id)\ .first() # perform updates # TODO: add some checks against setting illegal foreign keys if 'annotation_task_group_id' in params: original_agg_annotation.annotation_task_group_id = params['annotation_task_group_id'] if 'doc_id' in params: original_agg_annotation.doc_id = params['doc_id'] if 'topic_id' in params and params['topic_id'] in AggregatedAnnotations.topic_id_name_mapping: original_agg_annotation.topic_id = params['topic_id'] if 'is_gold_standard' in params: original_agg_annotation.is_gold_standard = params['is_gold_standard'] if 'gold_topic_annotation_id' in params: original_agg_annotation.gold_topic_annotation_id = params['gold_topic_annotation_id'] if 'is_active_for_gold_annotation' in params: original_agg_annotation.is_active_for_gold_annotation = params['is_active_for_gold_annotation'] if 'gold_difficulty' in params: original_agg_annotation.gold_difficulty = params['gold_difficulty'] if 'arbitrary_tags' in params: original_agg_annotation.arbitrary_tags = params['arbitrary_tags'] # NB: 'is_in_agreement' should depend on annotation agreement somewhere if 'is_in_agreement' in params: original_agg_annotation.is_in_agreement = params['is_in_agreement'] if 'notes' in params: original_agg_annotation.notes = params['notes'] # for the dict 'details' do a merge that retains all existing keys; # if same key "k1" appears in both dicts, older value is overwritten by new value in params['details']["k1"] if 'details' in params: original_agg_annotation.details = merge_two_dicts(original_agg_annotation.details, params['details']) # commit updates to database db_session_users.add(original_agg_annotation) db_session_users.commit() # return updated values agg_task_dict = original_agg_annotation.to_dict() return jsonify({"aggregated_annotation": agg_task_dict})
def create_rated_search(user_id, params): is_relevant = params.get('is_relevant', None) doc_id = params.get('doc_id', None) search_args = params.get('search_args', None) search_entry = find_or_return_new_search_query(search_args, save_and_refresh_if_new=True) # NOTE: Although data is recorded on the UserSearchResultRating table, this method is # also used to keep track of ratings for data outside of a user search (ie. topic buttons relevancy) user_search_result_rating = db_session_users.query(UserSearchResultRating)\ .filter_by(user_id=user_id, doc_id=doc_id, search_query_id=search_entry.id).first() if user_search_result_rating is None: user_search_result_rating = \ UserSearchResultRating({'user_id': user_id, 'search_query_id': search_entry.id, 'doc_id': doc_id}) user_search_result_rating.is_relevant = is_relevant db_session_users.add(user_search_result_rating) db_session_users.commit() db_session_users.refresh(user_search_result_rating) return {}
def create_saved_search(user_id, params): name = params.get('name', None) name_conflict_user = db_session_users.query(UserSavedSearch).filter_by( user_id=user_id, name=name).first() if name_conflict_user: return jsonify({ 'errors': "Saved search name: " + name + " is already being used" }), 409 search_args = params['search_args'] saved_search = UserSavedSearch({ 'user_id': user_id, 'search_args': search_args, 'name': name }) db_session_users.add(saved_search) db_session_users.commit() db_session_users.refresh(saved_search) return jsonify({"saved_search": saved_search.to_dict()})
def update_marketing_campaign(marketing_campaign_id, params): marketing_campaign = db_session_users.query(MarketingCampaign).filter_by( id=marketing_campaign_id).first() if 'start_date' in params: marketing_campaign.start_date = params['start_date'] if 'end_date' in params: marketing_campaign.end_date = params['end_date'] if 'name' in params: marketing_campaign.name = params['name'] if 'notes' in params: marketing_campaign.notes = params['notes'] # n.b. for regenerating the token if 'token' in params: marketing_campaign.gen_token() db_session_users.add(marketing_campaign) db_session_users.commit() db_session_users.refresh(marketing_campaign) return {"marketing_campaign": marketing_campaign.to_dict()}
def update_annotation_task(annotation_task_id, params): original_annotation_task = db_session_users.query( AnnotationTask).filter_by(id=annotation_task_id).first() num_annotation_jobs = db_session_users.query(AnnotationJob).filter_by( annotation_task_id=annotation_task_id).count() # n.b. updating the topics hash means that we need to swap this task for a new one for # consistency of which annotations were generated against which task, as it is valuable to know # which topics were presented when annotations are created, likewise what the user and other config options # note: there is no need for a new task until this task has jobs added for it, so when the job count is still 0, # we can do a direct update to the existing task instead # note: onboarding mode tasks (with is_training_task=True) also do not create a new task when they date updated if num_annotation_jobs > 0 and not original_annotation_task.is_training_task and \ ('topics' in params or 'user_ids' in params or 'config' in params or 'term_sampling_group_ids' in params or 'is_training_task' in params or 'include_gold_annotations' in params or 'annotation_task_topic_group_id' in params): new_annotation_task_dict = original_annotation_task.__dict__ if 'topics' in params: new_annotation_task_dict['topics'] = params['topics'] # must be done before setting is_training_task due to check in setting is_training_task if 'annotation_task_topic_group_id' in params: new_annotation_task_dict[ 'annotation_task_topic_group_id'] = params[ 'annotation_task_topic_group_id'] # update is_training_task to be False if no longer in annotation_task_topic_group if new_annotation_task_dict[ 'annotation_task_topic_group_id'] is None: new_annotation_task_dict['is_training_task'] = False if 'user_ids' in params: new_annotation_task_dict['user_ids'] = params['user_ids'] # n.b. changing status between active/inactive in effect toggles this task on/off if 'status' in params and params['status'] in VALID_TASK_STATUSES: new_annotation_task_dict['status'] = params['status'] if 'config' in params: new_annotation_task_dict['config'] = merge_two_dicts( new_annotation_task_dict['config'], params['config']) if 'num_touches' in new_annotation_task_dict[ 'config'] and new_annotation_task_dict['config'][ 'num_touches'] == '': del new_annotation_task_dict['config']['num_touches'] if 'name' in params: new_annotation_task_dict['name'] = params['name'] # only allow setting this to True if this annotation_task is in an annotation_task_topic_group # TODO: currently possible to do illegal is_training_task update (with not effect) that creates new task if 'is_training_task' in params and params['is_training_task'] is True: if new_annotation_task_dict[ 'annotation_task_topic_group_id'] is not None: new_annotation_task_dict['is_training_task'] = params[ 'is_training_task'] if 'include_gold_annotations' in params: new_annotation_task_dict['include_gold_annotations'] = params[ 'include_gold_annotations'] if 'is_contributor_task' in params: new_annotation_task_dict['is_contributor_task'] = params[ 'is_contributor_task'] new_annotation_task = AnnotationTask(new_annotation_task_dict) db_session_users.add(new_annotation_task) # n.b. connect the tasks together and mark the old one as inactive original_annotation_task.active_task = new_annotation_task original_annotation_task.status = AnnotationTask.INACTIVE_STATUS db_session_users.add(original_annotation_task) # also deal with any even older annotation tasks and make sure they get their active_task_id updated too for older_annotation_task in \ db_session_users.query(AnnotationTask).filter_by(active_task_id=original_annotation_task.id): older_annotation_task.active_task = new_annotation_task db_session_users.add(older_annotation_task) db_session_users.commit() db_session_users.refresh(new_annotation_task) # update any annotation_task_groups containing original_annotation_task to point to new task all_task_groups = db_session_users.query( AnnotationTaskTopicGroup).all() # get all task groups for group in all_task_groups: if annotation_task_id in group.annotation_task_ids: # update task id list to point to new annotation task id # list is a tuple, so cannot be mutated in-place new_ids = [ id for id in group.annotation_task_ids if id != annotation_task_id ] new_ids.append(new_annotation_task.id) group.annotation_task_ids = new_ids db_session_users.add(group) db_session_users.commit() if 'term_sampling_group_ids' in params: new_term_sampling_ids = params['term_sampling_group_ids'] else: existing_term_sampling_group_ids = [ t[0] for t in db_session_users.query( AnnotationTaskTermSamplingGroup.term_sampling_group_id). filter_by(annotation_task_id=original_annotation_task.id) ] new_term_sampling_ids = existing_term_sampling_group_ids for term_sampling_group_id in new_term_sampling_ids: attsg = AnnotationTaskTermSamplingGroup({ 'annotation_task_id': new_annotation_task.id, 'term_sampling_group_id': term_sampling_group_id }) db_session_users.add(attsg) if len(new_term_sampling_ids) > 0: db_session_users.commit() task_to_return = new_annotation_task # allow basic updates of status/config/name else: # n.b. changing status between active/inactive in effect toggles this task on/off if 'status' in params and params['status'] in VALID_TASK_STATUSES: original_annotation_task.status = params['status'] if 'name' in params: original_annotation_task.name = params['name'] if 'is_contributor_task' in params: original_annotation_task.is_contributor_task = params[ 'is_contributor_task'] # must be done before setting is_training_task due to check in setting is_training_task if 'annotation_task_topic_group_id' in params: original_annotation_task.annotation_task_topic_group_id = params[ 'annotation_task_topic_group_id'] # update is_training_task to be False if no longer in annotation_task_topic_group if original_annotation_task.annotation_task_topic_group_id is None: original_annotation_task.is_training_task = False ## n.b. these all can get updated here in the annotation jobs == 0 use case ## # only allow setting this to True if this annotation_task is in an annotation_task_topic_group if 'is_training_task' in params and params['is_training_task'] is True: if original_annotation_task.annotation_task_topic_group_id is not None: original_annotation_task.is_training_task = params[ 'is_training_task'] if 'include_gold_annotations' in params: original_annotation_task.include_gold_annotations = params[ 'include_gold_annotations'] if 'topics' in params: original_annotation_task.topics = params['topics'] if 'user_ids' in params: original_annotation_task.user_ids = params['user_ids'] if 'config' in params: original_annotation_task.config = merge_two_dicts( original_annotation_task.config, params['config']) if 'num_touches' in original_annotation_task.config and original_annotation_task.config[ 'num_touches'] == '': del original_annotation_task.config['num_touches'] if 'term_sampling_group_ids' in params: new_term_sampling_ids = params['term_sampling_group_ids'] existing_tsgs = db_session_users.query(AnnotationTaskTermSamplingGroup)\ .filter_by(annotation_task_id=original_annotation_task.id).all() existing_tsg_id_map = { t.term_sampling_group_id: t for t in existing_tsgs } removed_ids = [ item for item in existing_tsg_id_map.keys() if item not in new_term_sampling_ids ] for term_sampling_group_id in new_term_sampling_ids: if term_sampling_group_id not in existing_tsg_id_map: attsg = AnnotationTaskTermSamplingGroup({ 'annotation_task_id': original_annotation_task.id, 'term_sampling_group_id': term_sampling_group_id }) db_session_users.add(attsg) for term_sampling_group_id in removed_ids: db_session_users.query( AnnotationTaskTermSamplingGroup).filter_by( annotation_task_id=original_annotation_task.id, term_sampling_group_id=term_sampling_group_id).delete( ) db_session_users.add(original_annotation_task) db_session_users.commit() task_to_return = original_annotation_task task_dict = task_to_return.to_dict() task_dict["old_tasks"] = [ t.to_dict() for t in db_session_users.query(AnnotationTask).filter_by( active_task_id=task_to_return.id) ] term_sampling_group_ids = db_session_users.query(AnnotationTaskTermSamplingGroup.term_sampling_group_id) \ .filter_by(annotation_task_id=task_to_return.id) task_dict["term_sampling_group_ids"] = [ t[0] for t in term_sampling_group_ids ] return {"annotation_task": task_dict}
def pop_annotation_job_from_queue(annotation_task_id, user_id): time_now = datetime.datetime.now() # grabs queued annotation jobs for this task that are assigned to the user (or nobody), # ordered first by whether they are have a user assignment, next by highest priority, # and finally falling back on the oldest created annotation_job = db_session_users.query(AnnotationJob).filter_by(annotation_task_id=annotation_task_id)\ .filter_by(status=AnnotationJob.QUEUED_STATUS)\ .filter(or_(AnnotationJob.user_id == user_id, AnnotationJob.user_id == None))\ .order_by(AnnotationJob.user_id.nullslast(), AnnotationJob.priority.desc(), AnnotationJob.created_at.asc()).first() # if by chance, we are in the period of time between when a task was updated, but before the next queuing run # came around, we want to make sure to look up annotation jobs for older annotation tasks too if annotation_job is None: old_annotation_task_ids = db_session_users.query(AnnotationTask.id).filter_by(active_task_id=annotation_task_id).subquery() annotation_job = db_session_users.query(AnnotationJob)\ .filter(AnnotationJob.annotation_task_id.in_(old_annotation_task_ids)) \ .filter_by(status=AnnotationJob.QUEUED_STATUS) \ .filter(or_(AnnotationJob.user_id == user_id, AnnotationJob.user_id == None)) \ .order_by(AnnotationJob.user_id.nullslast(), AnnotationJob.priority.desc(), AnnotationJob.created_at.asc()).first() if annotation_job is None: return {"annotation_job": None} annotation_job.status = AnnotationJob.ASSIGNED_STATUS annotation_job.user_id = user_id annotation_job.assigned_at = time_now db_session_users.add(annotation_job) db_session_users.commit() db_session_users.refresh(annotation_job) # n.b. mitigation strategy for race condition would look like: while the assigned user_id is not me -> query again # change status to error status if document is not found in index try: doc_dict = jsearch.get_record(annotation_job.doc_id) except NotFoundError: annotation_job.status = AnnotationJob.ERROR_STATUS annotation_job.notes = "Document is not found" db_session_users.add(annotation_job) db_session_users.commit() db_session_users.refresh(annotation_job) return {"errors": "Document is not found. Doc ID: " + str(annotation_job.doc_id)} # if this is training job, return info about correct judgment if annotation_job.is_gold_evaluation: # get gold judgment info to return with annotation_job object topic_group_id_subquery = db_session_users.query(AnnotationTask.annotation_task_topic_group_id)\ .filter_by(id=annotation_job.annotation_task_id)\ .subquery() # should contain just one result gold_judgment_id_subquery = db_session_users.query(AggregatedAnnotations.gold_topic_annotation_id)\ .filter_by(doc_id=annotation_job.doc_id)\ .filter(AggregatedAnnotations.annotation_task_group_id.in_(topic_group_id_subquery))\ .subquery() gold_judgment_object = db_session_users.query(TopicAnnotation.is_positive, TopicAnnotation.admin_notes)\ .filter(TopicAnnotation.id.in_(gold_judgment_id_subquery))\ .first() # this query should return just one object anyway return {'annotation_job': annotation_job.to_dict(), 'document': doc_dict, 'correct_judgment': gold_judgment_object.is_positive, 'correct_judgment_notes': gold_judgment_object.admin_notes} return {'annotation_job': annotation_job.to_dict(), 'document': doc_dict}
def create_annotations_for_job(annotation_task_id, annotation_job_id, user_id, task_type, params): annotation_job = db_session_users.query(AnnotationJob).filter_by(id=annotation_job_id).first() # if this is onboarding job, notes are required and skipping is not allowed if annotation_job.is_gold_evaluation: if 'notes' not in params: return jsonify({'Error': 'Notes are required for onboarding jobs'}), 400 # 400 error means "bad request" if 'skip' in params: return jsonify({'Error': 'Onboarding jobs cannot be skipped'}), 400 # if we get a key named "complete_later", annotation should stay in the queue, # for example, user opens previous annotation # if we get a key named "error" in the params, this is being flagged as an error case # otherwise assume it is posting annotations and therefore complete if 'complete_later' in params: annotation_job.status = AnnotationJob.QUEUED_STATUS elif 'error' in params: annotation_job.status = AnnotationJob.ERROR_STATUS elif 'skip' in params: annotation_job.status = AnnotationJob.SKIPPED_STATUS annotation_job.was_skipped = True else: annotation_job.status = AnnotationJob.COMPLETE_STATUS annotation_job.completed_at = datetime.datetime.now() if 'notes' in params: annotation_job.notes = params['notes'] # user difficulty for annotation job # NB: this doesn't include ability to set difficulty for each individual topic_annotation, in case # that there is more than one topic_annotation per annotation job if 'user_difficulty' in params: annotation_job.user_difficulty = params['user_difficulty'] # allowed tags are defined in annotation_task_group for this annotation_job; # a list of these tags can be retrieved when job is popped from jobs queue if 'arbitrary_tags' in params: annotation_job.arbitrary_tags = params['arbitrary_tags'] # TopicAnnotation instances created here if task_type == AnnotationTask.TOPIC_ANNOTATION_TYPE and 'topic_annotations' in params: topic_annotations = [] for topic_annotation in params['topic_annotations']: # if this is update to existing TopicAnnotation object if 'topic_annotation_id' in topic_annotation: existing_annotation = db_session_users.query(TopicAnnotation)\ .filter_by(id=topic_annotation['topic_annotation_id'])\ .first() existing_annotation.is_positive = topic_annotation['is_positive'] if 'admin_notes' in topic_annotation: existing_annotation.admin_notes = topic_annotation['admin_notes'] # overwrite of previous notes topic_annotations.append(existing_annotation) # otherwise create new TopicAnnotation object else: new_topic_annotation_dict = topic_annotation.copy() # assumes no nested objects (otherwise deepcopy) new_topic_annotation_dict["annotation_job_id"] = annotation_job_id new_topic_annotation_dict["annotation_task_id"] = annotation_task_id new_topic_annotation_dict["user_id"] = user_id new_topic_annotation_dict["doc_id"] = annotation_job.doc_id # whether this is for annotator training/onboarding new_topic_annotation_dict["is_gold_evaluation"] = annotation_job.is_gold_evaluation new_topic_annotation_object = TopicAnnotation(new_topic_annotation_dict) topic_annotations.append(new_topic_annotation_object) # if there is topic_annotation_excerpt information, make topic_annotation_excerpt objects if 'topic_annotation_excerpts' in topic_annotation: topic_annotation_excerpts = [] for topic_annotation_excerpt in topic_annotation['topic_annotation_excerpts']: new_topic_annotation_excerpt_dict = topic_annotation_excerpt.copy() new_topic_annotation_excerpt_object = TopicAnnotationExcerpt(new_topic_annotation_excerpt_dict) topic_annotation_excerpts.append(new_topic_annotation_excerpt_object) # add all new topic_annotation_excerpts to topic_annotation object new_topic_annotation_object.topic_annotation_excerpts = topic_annotation_excerpts db_session_users.add_all(topic_annotations) elif task_type == AnnotationTask.SLOT_FILL_TYPE and 'selected_sentences' in params: selected_sentences = [] # TODO add support for annotation updates for selected_sentence in params['selected_sentences']: selected_sentence["annotation_job_id"] = annotation_job_id selected_sentence["annotation_task_id"] = annotation_task_id selected_sentence["user_id"] = user_id selected_sentence["doc_id"] = annotation_job.doc_id selected_sentence["index_build"] = ACTIVE_INDEX_NAME selected_sentences.append(SelectedSentence(selected_sentence)) db_session_users.add_all(selected_sentences) db_session_users.add(annotation_job) db_session_users.commit() # n.b. return a deliberately simple response here to avoid a large json response that isn't strictly needed return jsonify({"success": True})
def post_subscribe_customer(user_id, user_email, params): stripe_response = params.get('stripe_response', None) charge_token = stripe_response['id'] plan = params.get('plan', None) plan_from_db = db_session_users.query(Plan).filter_by( stripe_id=plan).first() payment_type = params.get('payment_type', None) mild_error = 'Your subscription purchase was not successful and your card was not charged. Please refresh the page and enter your payment details again or contact your bank. You can also reach out to us for help at [email protected].' severe_error = "Congrats! Your subscription has been paid and is in process. Please refresh the page, continue to use the app, and we'll update your subscription details soon." existing_subscription_error = 'You already have a subscription. Please reach out to us for help at [email protected].' if not plan_from_db: # invalid plan return jsonify({'errors': mild_error}), 409 # check for stripe customer and that user not subscribed to stripe. existing_stripe_customer = db_session_users.query(Customer).filter_by( user_id=user_id).first() existing_stripe_subscription = db_session_users.query( Subscription).filter_by(user_id=user_id, payment_type='stripe', latest=True).first() if existing_stripe_subscription is not None: if existing_stripe_customer is None: # This user already has a stripe subscription but is not a Stripe customer. return jsonify({'errors': existing_subscription_error}), 409 else: # This user already is a stripe customer and has a stripe subscription. return jsonify({'errors': existing_subscription_error}), 409 if existing_stripe_customer is None: # create stripe customer try: customer = stripe.Customer.create( description='compliance.ai customer', source=charge_token) stripe_customer_id = customer[ 'id'] #customer_id (stripe_id) should be unique customer_for_db = { 'user_id': user_id, 'stripe_id': stripe_customer_id, 'properties': serializeClass(customer) } except stripe.error.CardError as e: # Since it's a decline, stripe.error.CardError will be caught body = e.json_body err = body['error'] print "Status is: %s" % e.http_status print "Type is: %s" % err['type'] print "Code is: %s" % err['code'] # param is '' in this case print "Param is: %s" % err['param'] print "Message is: %s" % err['message'] return jsonify({ 'errors': "There is an error with your card: %s" % err['message'] }), 409 except Exception as e: print 'error creating stripe customer', e return jsonify({'errors': mild_error}), 409 # add customer to db try: class_for_db = Customer(customer_for_db) db_session_users.add(class_for_db) db_session_users.commit() except: # delete stripe customer - this also deletes any latest stripe subscriptions a customer may have try: cu = stripe.Customer.retrieve(stripe_customer_id) cu.delete() except Exception as e: print 'unable to delete stripe customer', e return jsonify({'errors': mild_error}), 409 else: stripe_customer_id = existing_stripe_customer.stripe_id # create stripe subscription try: subscription = stripe.Subscription.create(customer=stripe_customer_id, plan=plan) except stripe.error.CardError as e: # Since it's a decline, stripe.error.CardError will be caught body = e.json_body err = body['error'] print "Status is: %s" % e.http_status print "Type is: %s" % err['type'] print "Code is: %s" % err['code'] # param is '' in this case print "Param is: %s" % err['param'] print "Message is: %s" % err['message'] return jsonify({ 'errors': "There is an error with your card: %s" % err['message'] }), 409 except Exception as e: print 'error creating stripe subscription', e return jsonify({'errors': mild_error}), 409 try: # a user should have only 1 latest subscription at a time deactivate_subscriptions( user_id ) # TODO add this to subscription write to db to avoid edge case problems plan_id = plan_from_db.id subscription_for_db = { 'user_id': user_id, 'stripe_id': subscription['id'], 'plan_id': plan_id, 'latest': True, 'status': 'active', 'start_date': datetime.datetime.utcnow(), 'payment_type': payment_type, 'properties': serializeClass(subscription), 'period_count': 1 } db_session_users.add(Subscription(subscription_for_db)) db_session_users.commit() return jsonify({'subscription': 'Customer is subscribed'}) except Exception as e: print 'error writing subscriber to db. The following stripe subscription was made and should be dealt with manually on stripe:', subscription # is there a stripe method to immediately cancel subscription without charging customer. if so, put it here. stripe docs don't indicate this is possible???? try: email_helper.send_email( '*****@*****.**', '*****@*****.**', 'Urgent: Subscription Error', template='feedback-inline', vars={ 'feedback': 'A subscription was created for this user (user_id=' + str(user_id) + ') but the subscription failed to write to the database. Handle the subscription in the stripe dashboard. Here is the subscription from stripe:' + str(json.dumps(subscription, sort_keys=True, indent=2)) + '. Here is the error writing to the db: ' + str(e), 'User_first_name': 'complaibot', 'User_last_name': 'complaibot', }, ) except SMTPException as e: return error_response('Could not send error email.', code=500) return jsonify({'errors': severe_error}), 409
def update_document(user_id, doc_id, params): user_flagged_document_id = params.get('user_flagged_document_id', None) fix_contributor_notes = params.get('fix_contributor_notes', None) skip_contributor_notes = params.get('skip_contributor_notes', None) flagged_doc = None if user_flagged_document_id is not None: flagged_doc = db_session_users.query(UserFlaggedDocument).filter_by( id=user_flagged_document_id).first() # Change status without making updates in document # in case admin decides to not update document flagged by contributor if skip_contributor_notes: if flagged_doc is not None: flagged_doc.status = UserFlaggedDocument.PROCESSED_STATUS db_session_users.add(flagged_doc) db_session_users.commit() return {"status_updated": True} notes = params.get('notes', None) changes = {} category = params.get('category', None) if category: changes["category"] = category publication_date = params.get('publication_date', None) if publication_date: changes["publication_date"] = publication_date summary_text = params.get('summary_text', None) if summary_text: changes["summary_text"] = summary_text title = params.get('title', None) if title: changes["title"] = title topics_to_add = params.get('topics_to_add', None) if topics_to_add: changes["topics_to_add"] = topics_to_add topics_to_remove = params.get('topics_to_remove', None) if topics_to_remove: changes["topics_to_remove"] = topics_to_remove if not changes: return { 'errors': "changes submitted to update document must not be empty" } updated_doc = UserDocumentUpdate({ 'user_id': user_id, 'doc_id': doc_id, 'notes': notes, 'changes': changes, }) # Update status of flagged document if flagged_doc is not None: if fix_contributor_notes: flagged_doc.status = UserFlaggedDocument.FIXED_STATUS else: flagged_doc.status = UserFlaggedDocument.PROCESSED_STATUS db_session_users.add(flagged_doc) db_session_users.add(updated_doc) db_session_users.commit() db_session_users.refresh(updated_doc) return updated_doc.to_dict()
def update_research_mode_expanded_view(aggregated_annotation_id, params): """ updates to topic_annotation columns can include "is_positive", "admin_notes" and "user_id" updates to annotation_job can include "notes", "tags" and "difficulty" If "is_positive" is updated, params must also contain the "user_id" of the annotator who did the updating. In this case, the user_id field in both the topic_annotation and its corresponding annotation_job are updated If 'arbitrary_tags' or 'difficulty' is updated and the job is a gold standard, the tags/difficulty are copied to 'arbitrary_tags' or 'gold_difficulty' of the aggregated_annotation containing this task. aggregated_annotation_id: id of aggregated_annotation containing the topic_annotation/annotationjob to be updated params is of form { 'topic_annotation_id': topic_annotation_id (int) 'topic_annotation_updates': {column1_name: column1_value, ...}, 'annotation_job_updates': {column1_name: column1_value, ...} } For key-value pairs for updates: topic_annotation_updates: 'is_positive': True/False (bool) 'admin_notes': 'string' (string, arbitrary) 'user_id': user_id (int, foreign key from user table; required if 'is_positive' is present) annotation_job_updates: 'arbitrary_tags': ["tag1", "tag2", ... ] (list) 'user_difficulty': 'easy'/'medium'/'hard' (string) 'notes': 'bananas, almond milk. wait wrong notes.' (string) """ # get original topic_annotation and annotation_job objects topic_annotation = db_session_users.query(TopicAnnotation)\ .filter_by(id=params['topic_annotation_id'])\ .first() annotation_job = db_session_users.query(AnnotationJob)\ .filter_by(id=topic_annotation.annotation_job_id)\ .first() aggregated_annotation_object = None # only needed if arbitrary_tags or user_difficulty updates occur # do topic_annotation updates if 'topic_annotation_updates' in params: ta_update_dict = params['topic_annotation_updates'] if 'is_positive' in ta_update_dict: # in this case params must also contain the 'user_id' of person doing updating # record information about the previous user_id and their annotation_status # record this in a list of dicts that is under 'previous_annotators' key in details field if not topic_annotation.details: topic_annotation.details = {'previous_annotators': []} previous_annotator_dict = { 'user_id': topic_annotation.user_id, 'updated_at': topic_annotation.updated_at.utcnow().isoformat(), 'is_positive': topic_annotation.is_positive } topic_annotation.details['previous_annotators'].append(previous_annotator_dict) # update user_id in both topic_annotation and corresponding annotation_job topic_annotation.user_id = ta_update_dict['user_id'] annotation_job.user_id = ta_update_dict['user_id'] # do topic_annotation update topic_annotation.is_positive = ta_update_dict['is_positive'] if 'admin_notes' in ta_update_dict: # simple overwrite topic_annotation.admin_notes = ta_update_dict['admin_notes'] db_session_users.add(topic_annotation) # do annotation_job updates if 'annotation_job_updates' in params: job_update_dict = params['annotation_job_updates'] # get corresponding aggregated_annotation object in case potential updates to gold standard are needed if ('arbitrary_tags' in job_update_dict) or ('user_difficulty' in job_update_dict): aggregated_annotation_object = db_session_users.query(AggregatedAnnotations)\ .filter_by(id=aggregated_annotation_id)\ .first() if 'arbitrary_tags' in job_update_dict: # tags are required on frontend to come from annotation_task_groups.arbitrary_tags for relevant task group # assumes exactly one task group contains task that spawned this annotation - if no group contains # this task, then an update to arbitrary_tags is not allowed allowed_tags = [] # get current active task id for this task active_task_id = topic_annotation.annotation_task_id current_active_task_id = topic_annotation.annotation_task.active_task_id if current_active_task_id is not None: active_task_id = current_active_task_id # find the group that contains this task; if no group or more than one group, raises error (from .one()) task_group = db_session_users.query(AnnotationTaskTopicGroup)\ .filter(AnnotationTaskTopicGroup.annotation_task_ids.any(active_task_id))\ .one() if task_group: allowed_tags = task_group.arbitrary_tags # allowed tags pulled from group containing task updated_tags = [tag for tag in job_update_dict['arbitrary_tags'] if tag in allowed_tags] annotation_job.arbitrary_tags = updated_tags # if this job is a gold annotation, update aggregated_annotation.arbitrary_tags to be these tags if aggregated_annotation_object.gold_topic_annotation_id == params['topic_annotation_id']: aggregated_annotation_object.arbitrary_tags = updated_tags if 'user_difficulty' in job_update_dict: # difficulty must be 'easy', 'medium', or 'hard' - todo: handle error case more explicitly? new_difficulty = job_update_dict['user_difficulty'] if new_difficulty in ['easy', 'medium', 'hard']: annotation_job.user_difficulty = new_difficulty # if this job is a gold annotation, update aggregated_annotation.gold_difficulty to be this difficulty if aggregated_annotation_object.gold_topic_annotation_id == params['topic_annotation_id']: aggregated_annotation_object.gold_difficulty = new_difficulty if 'notes' in job_update_dict: annotation_job.notes = job_update_dict['notes'] db_session_users.add(annotation_job) if aggregated_annotation_object is not None: db_session_users.add(aggregated_annotation_object) # commit changes to database db_session_users.commit() # simple return statement return jsonify({"success": True})