def delete(self, id_): ''' Delete the category identified by `id`. **Example Response** ..sourcecode:: json { "message": "Category `main` deleted", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str message: the API response message :status 200: deleted :status 400: invalid request body :status 401: authentication required :status 404: category does not exist ''' # Get label. id_ = get_int_arg('id_', id_) category = g.db.query(Category).filter(Category.id == id_).first() if category is None: raise NotFound("Category '%s' does not exist." % id_) # Delete label g.db.delete(category) try: g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest( 'You must delete archived results that' 'use category "{}" before it can be deleted.'.format( category.name)) except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='category', message={ 'id': category.id, 'name': category.name, 'status': 'deleted', 'resource': None }) message = 'Category id "{}" deleted'.format(category.id) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete archive identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) archive = g.db.query(Archive).filter(Archive.id == id_).filter( Archive.user_id == g.user.id).first() if archive is None: raise NotFound("Archive '%s' does not exist." % id_) # Delete site try: g.db.delete(archive) g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest('Could not delete archive.') # Send redis notifications notify_mask_client(channel='archive', message={ 'id': archive.id, 'name': archive.username, 'status': 'deleted', 'resource': None }) message = 'Archive id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete site identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) site = g.db.query(Site).filter(Site.id == id_).first() if site is None: raise NotFound("Site '%s' does not exist." % id_) # Delete site try: g.db.delete(site) g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest('"{}" must be removed from all groups ' 'before deleting.'.format(site.name)) # Send redis notifications notify_mask_client(channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'deleted', 'resource': None }) message = 'Site id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete proxy identified by `id_`. ''' # Get proxy. id_ = get_int_arg('id_', id_) proxy = g.db.query(Proxy).filter(Proxy.id == id_).first() if proxy is None: raise NotFound("Proxy '%s' does not exist." % id_) # Delete proxy try: g.db.delete(proxy) g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest('Could not delete proxy.') # Send redis notifications notify_mask_client(channel='proxy', message={ 'proxy': proxy.as_dict(), 'status': 'deleted', 'resource': None }) message = 'Proxy id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def get(self, id_): ''' Get the application user identified by `id`. Normal users may request only their own ID. Admin users can request any ID. **Example Response** .. sourcecode:: json { "agency": "Department Of Justice", "created": "2015-05-05T14:30:09.676268", "email": "*****@*****.**", "id": 2029, "is_admin": true, "location": "Washington, DC", "modified": "2015-05-05T14:30:09.676294", "name": "Lt. John Doe", "phone": "+12025551234", "thumb": "iVBORw0KGgoAAAANS...", "url": "https://quickpin/api/user/2029" } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str agency: the name of the organization/agency that this person is affiliated with (default: null) :>json str created: record creation timestamp in ISO-8601 format :>json str email: e-mail address :>json bool is_admin: true if this user has admin privileges, false otherwise :>json str location: location name, e.g. city or state (default: null) :>json str modified: record modification timestamp in ISO-8601 format :>json str name: user's full name, optionally including title or other salutation information (default: null) :>json str phone: phone number :>json str phone_e164: phone number in E.164 format :>json str thumb: PNG thumbnail for this user, base64 encoded :>json str url: url to view data about this user :status 200: ok :status 401: authentication required :status 404: user does not exist ''' id_ = get_int_arg('id_', id_) user = g.db.query(User).filter(User.id == id_).first() if not g.user.is_admin and g.user.id != user.id: raise Forbidden('You may only view your own profile.') if user is None: raise NotFound("User '%d' does not exist." % id_) return jsonify(**self._user_dict(user))
def get(self, id_): ''' Get the application user identified by `id`. **Example Response** .. sourcecode:: json { "agency": "Department Of Justice", "created": "2015-05-05T14:30:09.676268", "email": "*****@*****.**", "id": 2029, "is_admin": true, "location": "Washington, DC", "modified": "2015-05-05T14:30:09.676294", "name": "Lt. John Doe", "phone": "+12025551234", "thumb": "iVBORw0KGgoAAAANS...", "url": "https://quickpin/api/user/2029" } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str agency: the name of the organization/agency that this person is affiliated with (default: null) :>json str created: record creation timestamp in ISO-8601 format :>json str email: e-mail address :>json bool is_admin: true if this user has admin privileges, false otherwise :>json str location: location name, e.g. city or state (default: null) :>json str modified: record modification timestamp in ISO-8601 format :>json str name: user's full name, optionally including title or other salutation information (default: null) :>json str phone: phone number :>json str phone_e164: phone number in E.164 format :>json str thumb: PNG thumbnail for this user, base64 encoded :>json str url: url to view data about this user :status 200: ok :status 401: authentication required :status 404: user does not exist ''' id_ = get_int_arg('id_', id_) user = g.db.query(User).filter(User.id == id_).first() if user is None: raise NotFound("User '%d' does not exist." % id_) return jsonify(**self._user_dict(user))
def get(self, id_): ''' Get the category identified by `id`. **Example Response** .. sourcecode: json { "id": 1, "name": "gender", "sites": [ { "id": 2, "name": "", "url": "". "status_code": "", "search_pattern": "", "category": "" }, ... ] } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json int id: unique identifier for category :>json str name: the category name :>json str url: URL url-for for retriving more data about this category :status 200: ok :status 400: invalid argument[s] :status 401: authentication required :status 404: category does not exist ''' # Get category. id_ = get_int_arg('id_', id_) category = g.db.query(Category).filter(Category.id == id_).first() if category is None: raise NotFound("Category '%s' does not exist." % id_) response = category.as_dict() response['url-for'] = url_for('CategoryView:get', id_=category.id) # Send response. return jsonify(**response)
def delete(self, id_): ''' Delete the note identified by `id`. **Example Response** .. sourcecode:: json { "message": "note `12` deleted", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str message: the API response message :status 202: deleted :status 400: invalid request body :status 401: authentication required :status 404: note does not exist ''' # Get note. redis = worker.get_redis() id_ = get_int_arg('id_', id_) note = g.db.query(ProfileNote).filter(ProfileNote.id == id_).first() if note is None: raise NotFound("Note `%s` does not exist." % id_) # Delete note g.db.delete(note) try: g.db.commit() except DBAPIError as e: raise BadRequest('Database error: {}'.format(e)) message = 'Note `{}` deleted'.format(note.id) redis.publish('profile_notes', json.dumps({ 'id': id_, 'status': 'deleted', })) response = jsonify(message=message) response.status_code = 202 return response
def delete(self, id_): """ Delete the proile identified by `id`. **Example Response** .. sourcecode:: json { "message": "Profile ID `1234` deleted", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str message: the API response message :status 202: deleted :status 400: invalid request body :status 401: authentication required :status 404: profile does not exist """ # Get profile. id_ = get_int_arg('id_', id_) profile = g.db.query(Profile).filter(Profile.id == id_).first() if profile is None: raise NotFound("Profile '%s' does not exist." % id_) # Delete profile g.db.delete(profile) try: g.db.commit() except DBAPIError as e: raise BadRequest('Database error: {}'.format(e)) # Queue jobs to delete profile and posts from index app.queue.schedule_delete_profile_from_index(id_) app.queue.schedule_delete_profile_posts_from_index(id_) message = 'Profile ID `{}` deleted'.format(profile.id) response = jsonify(message=message) response.status_code = 202 return response
def get(self, id_): ''' Get the note identified by `id`. **Example Response** .. sourcecode:: json { "id": 1, "category": "user annotation", "body": "This is a user annotation.", "profile_id": "10", "created_at": "2015-12-14T16:23:18.101558", "url": "https://quickpin/api/note/2", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json int id: unique identifier for note :>json str category: the user-defined category of this note :>json str body: the note :>json str profile_id: the unique id of the profile this note belongs to :>json str created_at: when the note was created as iso-formatted date string :>json str url: API endpoint URL for this note object :status 200: ok :status 400: invalid argument[s] :status 401: authentication required :status 404: note does not exist ''' # Get note. id_ = get_int_arg('id_', id_) note = g.db.query(ProfileNote).filter(ProfileNote.id == id_).first() if note is None: raise NotFound("Note '%s' does not exist." % id_) response = note.as_dict() response['url'] = url_for('ProfileNoteView:get', id_=note.id) # Send response. return jsonify(**response)
def delete(self, id_): """ Delete the label identified by `id`. **Example Response** .. sourcecode:: json { "message": "Label `gender` deleted", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str message: the API response message :status 202: deleted :status 400: invalid request body :status 401: authentication required :status 404: label does not exist """ # Get label. id_ = get_int_arg('id_', id_) label = g.db.query(Label).filter(Label.id == id_).first() if label is None: raise NotFound("Label '%s' does not exist." % id_) # Delete label g.db.delete(label) try: g.db.commit() except DBAPIError as e: raise BadRequest('Database error: {}'.format(e)) message = 'Label {} deleted'.format(label.name) response = jsonify(message=message) response.status_code = 202 return response
def delete(self, id_): ''' Delete the label identified by `id`. **Example Response** ..sourcecode:: json { "message": "Label `gender` deleted", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str message: the API response message :status 202: deleted :status 400: invalid request body :status 401: authentication required :status 404: label does not exist ''' # Get label. id_ = get_int_arg('id_', id_) label = g.db.query(Label).filter(Label.id == id_).first() if label is None: raise NotFound("Label '%s' does not exist." % id_) # Delete label g.db.delete(label) try: g.db.commit() except DBAPIError as e: raise BadRequest('Database error: {}'.format(e)) message = 'Label {} deleted'.format(label.name) response = jsonify(message=message) response.status_code = 202 return response
def get(self, id_): """ Get the label identified by `id`. **Example Response** .. sourcecode:: json { "id": 1, "name": "gender", "url": "https://quickpin/api/label/1", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json int id: unique identifier for label :>json str name: the label name :>json str url: URL endpoint for retriving more data about this label :status 200: ok :status 400: invalid argument[s] :status 401: authentication required :status 404: label does not exist """ # Get label. id_ = get_int_arg('id_', id_) label = g.db.query(Label).filter(Label.id == id_).first() if label is None: raise NotFound("Label '%s' does not exist." % id_) response = label.as_dict() response['url'] = url_for('LabelView:get', id_=label.id) # Send response. return jsonify(**response)
def get(self, id_): ''' Get the label identified by `id`. **Example Response** .. sourcecode: json { "id": 1, "name": "gender", "url": "https://quickpin/api/label/1", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json int id: unique identifier for label :>json str name: the label name :>json str url: URL endpoint for retriving more data about this label :status 200: ok :status 400: invalid argument[s] :status 401: authentication required :status 404: label does not exist ''' # Get label. id_ = get_int_arg('id_', id_) label = g.db.query(Label).filter(Label.id == id_).first() if label is None: raise NotFound("Label '%s' does not exist." % id_) response = label.as_dict() response['url'] = url_for('LabelView:get', id_=label.id) # Send response. return jsonify(**response)
def delete(self, id_): ''' Delete site identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) site = g.db.query(Site).filter(Site.id == id_).first() if site is None: raise NotFound("Site '%s' does not exist." % id_) try: # Remove site from categories categories = g.db.query(Category).filter( Category.sites.contains(site)).all() for category in categories: category.sites.remove(site) # Delete site g.db.delete(site) g.db.commit() except Exception as e: g.db.rollback() raise BadRequest(e) # Send redis notifications notify_mask_client(channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'deleted', 'resource': None }) message = 'Site id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete file identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) file_ = g.db.query(File).filter(File.id == id_).first() data_dir = app.config.get_path("data") if file_ is None: raise NotFound("File '%s' does not exist." % id_) # Restrict access to files according to their access_type and owner. # access_type can be Private ('p') or shared ('s'). if file_.access_type == 'p' and file_.user_id != g.user.id: raise Unauthorized('You are not authorized to view this file.') # Get filesystem path relpath = file_.relpath() file_object_path = os.path.join(data_dir, relpath) # Delete db file record try: g.db.delete(file_) g.db.commit() except Exception as e: g.db.rollback() raise BadRequest(e) # Delete file from filesystem if os.path.isfile(file_object_path): os.unlink(file_object_path) message = 'File id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete file identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) file_ = g.db.query(File).filter(File.id == id_).first() data_dir = app.config.get_path("data") if file_ is None: raise NotFound("File '%s' does not exist." % id_) # Get filesystem path relpath = file_.relpath() file_object_path = os.path.join(data_dir, relpath) # Delete db file record try: g.db.delete(file_) g.db.commit() except Exception as e: g.db.rollback() raise BadRequest(e) # Delete file from filesystem if os.path.isfile(file_object_path): os.unlink(file_object_path) message = 'File id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def put(self, id_): ''' Update the profile identified by `id` with submitted data. The following attribute are modifiable: * is_interesting * lables **Example Request** .. sourcecode:: json { "is_interesting": true, "labels": [ {"name": "male"}, {"name": "british"}, ... ], ... } **Example Response** .. sourcecode:: json { "avatar_url": "https://quickpin/api/file/1", "avatar_thumb_url": "https://quickpin/api/file/2", "description": "A human being.", "follower_count": 71, "friend_count": 28, "id": 1, "is_stub": false, "is_interesting": true, "join_date": "2012-01-30T15:11:35", "labels": [ { "id": 1, "name": "male" }, { "id": 2, "name": "british" }, ], "last_update": "2015-08-18T10:51:16", "location": "Washington, DC", "name": "John Doe", "post_count": 1666, "private": false, "site": "twitter", "site_name": "Twitter", "time_zone": "Central Time (US & Canada)", "upstream_id": "11009418", "url": "https://quickpin/api/profile/1", "username": "******", "usernames": [ { "end_date": "2012-06-30T15:00:00", "start_date": "2012-01-01T12:00:00", "username": "******" }, ... ] } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json bool is_interesting: whether profile is marked as interesting :>json list labels: whether profile is marked as interesting :>header Content-Type: application/json :>json str avatar_url: URL to the user's current avatar :>json str avatar_thumb_url: URL to a 32x32px thumbnail of the user's current avatar :>json str description: profile description :>json int follower_count: number of followers :>json int friend_count: number of friends (a.k.a. followees) :>json int id: unique identifier for profile :>json bool is_stub: indicates that this is a stub profile, e.g. related to another profile but has not been fully imported :>json bool is_interesting: indicates whether this profile has been marked as interesting. The value can be null. :>json str join_date: the date this profile joined its social network (ISO-8601) :>json list labels: list of labels for this profile :>json int label[n].id: the unique id for this label :>json str label[n].name: the label :>json str last_update: the last time that information about this profile was retrieved from the social media site (ISO-8601) :>json str location: geographic location provided by the user, as free text :>json str name: the full name provided by this user :>json int post_count: the number of posts made by this profile :>json bool private: true if this is a private account (i.e. not world- readable) :>json str site: machine-readable site name that this profile belongs to :>json str site_name: human-readable site name that this profile belongs to :>json str time_zone: the user's provided time zone as free text :>json str upstream_id: the user ID assigned by the social site :>json str url: URL endpoint for retriving more data about this profile :>json str username: the current username for this profile :>json list usernames: list of known usernames for this profile :>json str usernames[n].end_date: the last known date this username was used for this profile :>json str usernames[n].start_date: the first known date this username was used for this profile :>json str usernames[n].username: a username used for this profile :status 202: accepted for background processing :status 400: invalid request body :status 401: authentication required ''' redis = worker.get_redis() # Get profile. id_ = get_int_arg('id_', id_) current_avatar_id = self._current_avatar_subquery() profile, avatar = g.db.query(Profile, Avatar) \ .outerjoin(Avatar, Avatar.id == current_avatar_id) \ .filter(Profile.id == id_).first() if profile is None: raise NotFound("Profile '%s' does not exist." % id_) request_json = request.get_json() # Validate put data and set attributes # Only 'is_interesting' and 'labels' are modifiable if 'is_interesting' in request_json: if isinstance(request_json['is_interesting'], bool): profile.is_interesting = request_json['is_interesting'] elif request_json['is_interesting'] is None: profile.is_interesting = None else: raise BadRequest("Attribute 'is_interesting' is type boolean," " or can be set as null") # labels expects the string 'name' rather than id, to avoid the need to # create labels before adding them. if 'labels' in request_json: labels = [] if isinstance(request_json['labels'], list): for label_json in request_json['labels']: if 'name' in label_json: label = g.db.query(Label) \ .filter(Label.name==label_json['name']) \ .first() if label is None: try: label = Label( name=label_json['name'].lower().strip() ) g.db.add(label) g.db.flush() redis.publish( 'label', json.dumps(label.as_dict()) ) except IntegrityError: g.db.rollback() raise BadRequest('Label could not be saved') except AssertionError: g.db.rollback() raise BadRequest( '"{}" contains non-alphanumeric character' .format( label_json['name'] ) ) labels.append(label) else: raise BadRequest("Label 'name' is required") profile.labels = labels else: raise BadRequest("'labels' must be a list") response = profile.as_dict() response['url'] = url_for('ProfileView:get', id_=profile.id) # Save the profile try: g.db.commit() redis.publish('profile_update', json.dumps(profile.as_dict())) except DBAPIError as e: g.db.rollback() raise BadRequest('Profile could not be saved') # Create usernames list. usernames = list() for username in profile.usernames: if username.end_date is not None: end_date = username.end_date.isoformat() else: end_date = None if username.start_date is not None: start_date = username.start_date.isoformat() else: start_date = None usernames.append({ 'end_date': end_date, 'username': username.username, 'start_date': start_date, }) response['usernames'] = usernames # Create avatar attributes. if avatar is not None: response['avatar_url'] = url_for( 'FileView:get', id_=avatar.file.id ) response['avatar_thumb_url'] = url_for( 'FileView:get', id_=avatar.thumb_file.id ) else: response['avatar_url'] = url_for( 'static', filename='img/default_user.png' ) response['avatar_thumb_url'] = url_for( 'static', filename='img/default_user_thumb.png' ) # Send response. return jsonify(**response)
def put(self, id_): ''' Update proxy identified by `id`. **Example Request** .. sourcecode:: json PUT /api/proxies/id { "protocol": "http", "host": "192.168.0.2", "port": 80, "username": "******", "password": "******", "active": true, } **Example Response** .. sourcecode:: json { "id": 1, "protocol": "http", "host": "192.168.0.22", "port": 80, "username": "******", "password": "******", "active": true, }, :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json str protocol: protocol of proxy address :>json str host: host of proxy address :>json int port: port of proxy address :>json str username: username of proxy :>json str password: password of proxy :>json bool active: proxy active status :>header Content-Type: application/json :>json int id: unique identifier :>json str protocol: protocol of proxy address :>json str host: host of proxy address :>json int port: port of proxy address :>json str username: username of proxy :>json str password: password of proxy :>json bool active: proxy active status :status 200: ok :status 401: authentication required :status 403: must be an administrator ''' # Get proxy id_ = get_int_arg('id_', id_) proxy = g.db.query(Proxy).filter(Proxy.id == id_).first() if proxy is None: raise NotFound("Proxy '%s' does not exist." % id_) # Validate request json request_json = request.get_json() validate_request_json(request_json, PROXY_ATTRS) # Update proxy proxy.protocol = request_json['protocol'] proxy.host = request_json['host'] proxy.port = request_json['port'] proxy.active = request_json['active'] try: proxy.username = request_json['username'] except KeyError: pass try: proxy.password = request_json['password'] except KeyError: pass # Save the updated proxy try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='proxy', message={ 'proxy': proxy.as_dict(), 'status': 'updated', 'resource': None }) response = jsonify(proxy.as_dict()) response.status_code = 200 # Send response return response
def put(self, id_): """ Update the label identified by `id`. **Example Request** .. sourcecode:: json { {"name": "gender"}, } **Example Response** .. sourcecode:: json { "id": "2", "name": "gender", "url": "https://quickpin/api/label/1", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json str name: the value of the name attribute :>header Content-Type: application/json :>json int id: unique identifier for label :>json str name: the label name :>json str url: URL endpoint for retriving more data about this label :status 202: created :status 400: invalid request body :status 401: authentication required """ # Get label. id_ = get_int_arg('id_', id_) label = g.db.query(Label).filter(Label.id == id_).first() if label is None: raise NotFound("Label '%s' does not exist." % id_) request_json = request.get_json() # Validate data and set attributes if 'name' in request_json: if request_json['name'].strip() != '': label.name = request_json['name'].lower().strip() else: raise BadRequest('Attribute "name" cannot be an empty string') else: raise BadRequest('Attribue "name" is required') # Save the updated label try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) response = label.as_dict() response['url'] = url_for('LabelView:get', id_=label.id) # Send response. return jsonify(**response)
def put(self, id_): ''' Update the note identified by `id`. **Example Request** .. sourcecode:: json { { "category": "user annotation", "body": "This profile belongs to two interesting networks", "profile_id": "25 ", }, } **Example Response** .. sourcecode:: json { "id": "2", "category": "user annotation", "body": "This profile belongs to an interesting network", "profile_id": "25 ", "created_at": "2015-12-14T16:23:18.101558", "url": "https://quickpin/api/note/2", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json int id: unique identifier for the note :>json str category: the user-defined category of this note :>json str body: the note :>json str profile_id: the unique id of the profile this note belongs to :>json str created_at: the iso-formatted creation time of the note :>json str url: API endpoint URL for this note object :status 202: created :status 400: invalid request body :status 401: authentication required ''' # Get note. id_ = get_int_arg('id_', id_) note = g.db.query(ProfileNote).filter(ProfileNote.id == id_).first() if note is None: raise NotFound("Note '%s' does not exist." % id_) redis = worker.get_redis() request_json = request.get_json() # Validate data and set attributes if 'category' in request_json: if request_json['category'].strip() != '': note.category = request_json['category'].lower().strip() if 'body' in request_json: if request_json['body'].strip() != '': note.body = request_json['body'].strip() else: raise BadRequest('Attribute "name" cannot be an empty string') # Save the updated note try: g.db.commit() except DBAPIError: g.db.rollback() raise BadRequest('Could not update note.') # Generate SSE redis.publish('profile_notes', json.dumps(note.as_dict())) response = note.as_dict() response['url'] = url_for('ProfileNoteView:get', id_=note.id) # Send response. return jsonify(**response)
def post_jobs_for_site(self, site_id): """ Request background jobs for site identified by `id`. **Example Request** ..sourcode:: json { "jobs": [ { "name": "test", }, ... ] } **Example Response** .. sourcecode:: json { "tracker_ids": { "1": "tracker.12344565", } } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list jobs: a list of jobs to schedule :>json string jobs[n].name: name of job :>header Content-Type: application/json :>json array tracker_ids: array of worker tracking ids {site ID: tracker ID} :status 202: scheduled :status 400: invalid request body :status 401: authentication required """ request_attrs = { 'jobs': { 'type': list, 'required': True }, } job_attrs = { 'name': { 'type': str, 'required': True }, } available_jobs = ['test'] tracker_ids = dict() # Get site. site_id = get_int_arg('site_id', site_id) site = g.db.query(Site).filter(Site.id == site_id).first() # Validate if site is None: raise NotFound("Site '%s' does not exist." % site_id) request_json = request.get_json() validate_request_json(request_json, request_attrs) for job in request_json['jobs']: validate_json_attr('name', job_attrs, job) if job['name'] not in available_jobs: raise BadRequest('`{}` does not exist in available' ' jobs: {}'.format(job['name'], ','.join(available_jobs))) # Schedule jobs for job in request_json['jobs']: tracker_id = 'tracker.{}'.format(random_string(10)) tracker_ids[site.id] = tracker_id if job['name'] == 'test': description = 'Testing site "{}"'.format(site.name) worker.scrape.test_site.enqueue( site_id=site.id, tracker_id=tracker_id, jobdesc=description, user_id=g.user.id, ) response = jsonify(tracker_ids=tracker_ids) response.status_code = 202 return response
def put(self, id_): ''' Update the site identified by `id`. **Example Request** ..sourcecode:: json { "name": "bebo", "url": "http://bebo.com/usernames/search=%s", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "test_username_pos": "bob", "test_username_ne": "adfjf393rfjffkjd", "headers": {"referer": "http://www.google.com"}, "censor_images": false, "wait_time": 5, "use_proxy": false, } **Example Response** ..sourcecode:: json { "id": 2, "name": "bebo", "search_text": "Bebo User Page.</title>", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "url": "https://bebo.com/usernames/search=%s", "test_username_pos": "bob", "test_username_neg": "adfjf393rfjffkjd", "test_status": "f", "tested_at": "2016-01-01T00:00:00.000000+00:00", "headers": {"referer": "http://www.google.com"}, "censor_images": false, "wait_time": 5, "use_proxy": false, }, :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json string name: name of site :<json string url: username search url for the site :<json string test_username_pos: username that exists on site (used for testing) :<json string test_username_neg: username that does not exist on site (used for testing) :<json array headers: custom headers :<json bool censor_images: whether to censor images from this profile :<json int wait_time: time (in seconds) to wait for updates after page is loaded :<json bool use_proxy: whether to proxy requests for this profile URL :>header Content-Type: application/json :>json int id: unique identifier for site :>json str name: name of site :>json str url: username search url for the site :>json int status_code: the status code to check for determining a match (nullable) :>json string match_type: type of match (see get_match_types() for valid match types) (nullable) :>json string match_expr: expression to use for determining a page match (nullable) :>json str test_status: results of username test :>json str tested_at: timestamp of last test :>json str test_username_pos: username that exists on site (used for testing) :>json str test_username_neg: username that does not exist on site (used for testing) :>json array headers: custom headers :>json bool censor_images: whether to censor images from this profile :>json int wait_time: time (in seconds) to wait for updates after page is loaded :>json bool use_proxy: whether to proxy requests for this profile URL :status 202: updated :status 400: invalid request body :status 401: authentication required :status 404: site does not exist ''' # Get site. id_ = get_int_arg('id_', id_) site = g.db.query(Site).filter(Site.id == id_).first() if site is None: raise NotFound("Site '%s' does not exist." % id_) request_json = request.get_json() # Validate data and set attributes if 'name' in request_json: validate_json_attr('name', _site_attrs, request_json) site.name = request_json['name'].strip() if 'url' in request_json: validate_json_attr('url', _site_attrs, request_json) site.url = request_json['url'].lower().strip() if 'match_expr' in request_json: validate_json_attr('match_expr', _site_attrs, request_json) site.match_expr = request_json['match_expr'] if 'match_type' in request_json: validate_json_attr('match_type', _site_attrs, request_json) site.match_type = request_json['match_type'].strip() if 'status_code' in request_json: validate_json_attr('status_code', _site_attrs, request_json) status = request_json['status_code'] site.status_code = None if status is None else int(status) if (request_json['match_type'] is None or request_json['match_expr'] is None) and \ request_json['status_code'] is None: raise BadRequest('At least one of the ' 'following is required: ' 'status code or page match.') if 'test_username_pos' in request_json: validate_json_attr('test_username_pos', _site_attrs, request_json) site.test_username_pos = ( request_json['test_username_pos'].lower().strip()) if 'test_username_neg' in request_json: validate_json_attr('test_username_neg', _site_attrs, request_json) site.test_username_neg = ( request_json['test_username_neg'].lower().strip()) if 'headers' in request_json: validate_json_attr('headers', _site_attrs, request_json) site.headers = request_json['headers'] if 'censor_images' in request_json: validate_json_attr('censor_images', _site_attrs, request_json) site.censor_images = request_json['censor_images'] if 'use_proxy' in request_json: validate_json_attr('use_proxy', _site_attrs, request_json) site.use_proxy = request_json['use_proxy'] if 'wait_time' in request_json: validate_json_attr('wait_time', _site_attrs, request_json) site.wait_time = request_json['wait_time'] # Save the updated site try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='site', message={ 'site': site.as_dict(), 'status': 'updated', 'resource': None }) response = jsonify(site.as_dict()) response.status_code = 200 # Send response. return response
def get(self, id_): ''' .. http:get:: /api/profile/(int:id_) Get the profile identified by `id`. **Example Response** .. sourcecode:: json { "avatar_url": "https://quickpin/api/file/1", "avatar_thumb_url": "https://quickpin/api/file/2", "description": "A human being.", "follower_count": 71, "friend_count": 28, "id": 1, "is_stub": false, "is_interesting": false, "join_date": "2012-01-30T15:11:35", "labels": [ { "id": 1, "name": "male" }, ], "last_update": "2015-08-18T10:51:16", "location": "Washington, DC", "name": "John Doe", "post_count": 1666, "private": false, "score": "-2.0621606863", "site": "twitter", "site_name": "Twitter", "time_zone": "Central Time (US & Canada)", "upstream_id": "11009418", "url": "https://quickpin/api/profile/1", "username": "******", "usernames": [ { "end_date": "2012-06-30T15:00:00", "start_date": "2012-01-01T12:00:00", "username": "******" }, ... ] } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str avatar_url: URL to the user's current avatar :>json str avatar_thumb_url: URL to a 32x32px thumbnail of the user's current avatar :>json str description: profile description :>json int follower_count: number of followers :>json int friend_count: number of friends (a.k.a. followees) :>json int id: unique identifier for profile :>json bool is_stub: indicates that this is a stub profile, e.g. related to another profile but has not been fully imported :>json bool is_interesting: indicates whether this profile has been marked as interesting. The value can be null. :>json str join_date: the date this profile joined its social network (ISO-8601) :>json list labels: list of labels for this profile :>json int label[n].id: the unique id for this label :>json str label[n].name: the label :>json str last_update: the last time that information about this profile was retrieved from the social media site (ISO-8601) :>json str location: geographic location provided by the user, as free text :>json str name: the full name provided by this user :>json int note[n].body: the text body of of the note :>json str note[n].category: the category of the note. :>json str note[n].created_at: time at which the note was created. :>json int post_count: the number of posts made by this profile :>json bool private: true if this is a private account (i.e. not world- readable) :>json str score: user defined score for this profile :>json str site: machine-readable site name that this profile belongs to :>json str site_name: human-readable site name that this profile belongs to :>json str time_zone: the user's provided time zone as free text :>json str upstream_id: the user ID assigned by the social site :>json str url: URL endpoint for retriving more data about this profile :>json str username: the current username for this profile :>json list usernames: list of known usernames for this profile :>json str usernames[n].end_date: the last known date this username was used for this profile :>json str usernames[n].start_date: the first known date this username was used for this profile :>json str usernames[n].username: a username used for this profile :status 200: ok :status 400: invalid argument[s] :status 401: authentication required :status 404: user does not exist ''' # Get profile. id_ = get_int_arg('id_', id_) try: profile, avatar = g.db.query(Profile, Avatar) \ .outerjoin(Profile.current_avatar) \ .filter(Profile.id == id_).one() except NoResultFound: raise NotFound("Profile '%s' does not exist." % id_) response = profile.as_dict() response['url'] = url_for('ProfileView:get', id_=profile.id) # Create usernames list. usernames = list() for username in profile.usernames: if username.end_date is not None: end_date = username.end_date.isoformat() else: end_date = None if username.start_date is not None: start_date = username.start_date.isoformat() else: start_date = None usernames.append({ 'end_date': end_date, 'username': username.username, 'start_date': start_date, }) response['usernames'] = usernames # Create avatar attributes. if avatar is not None: response['avatar_url'] = url_for( 'FileView:get', id_=avatar.file.id ) response['avatar_thumb_url'] = url_for( 'FileView:get', id_=avatar.thumb_file.id ) else: response['avatar_url'] = url_for( 'static', filename='img/default_user.png' ) response['avatar_thumb_url'] = url_for( 'static', filename='img/default_user_thumb.png' ) # Send response. return jsonify(**response)
def put(self, id_): ''' Update the profile identified by `id` with submitted data. The following attributes are modifiable: * is_interesting * labels * score **Example Request** .. sourcecode:: json { "is_interesting": true, "labels": [ {"name": "male"}, {"name": "british"}, ... ], "score": 2323.0, ... } **Example Response** .. sourcecode:: json { "avatar_url": "https://quickpin/api/file/1", "avatar_thumb_url": "https://quickpin/api/file/2", "description": "A human being.", "follower_count": 71, "friend_count": 28, "id": 1, "is_stub": false, "is_interesting": true, "join_date": "2012-01-30T15:11:35", "labels": [ { "id": 1, "name": "male" }, { "id": 2, "name": "british" }, ], "last_update": "2015-08-18T10:51:16", "location": "Washington, DC", "name": "John Doe", "post_count": 1666, "private": false, "score": "-2.0621606863", "site": "twitter", "site_name": "Twitter", "time_zone": "Central Time (US & Canada)", "upstream_id": "11009418", "url": "https://quickpin/api/profile/1", "username": "******", "usernames": [ { "end_date": "2012-06-30T15:00:00", "start_date": "2012-01-01T12:00:00", "username": "******" }, ... ] } :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json bool is_interesting: whether profile is marked as interesting :<json list labels: list of profile labels :<json float score: profile score :>header Content-Type: application/json :>json str avatar_url: URL to the user's current avatar :>json str avatar_thumb_url: URL to a 32x32px thumbnail of the user's current avatar :>json str description: profile description :>json int follower_count: number of followers :>json int friend_count: number of friends (a.k.a. followees) :>json int id: unique identifier for profile :>json bool is_stub: indicates that this is a stub profile, e.g. related to another profile but has not been fully imported :>json bool is_interesting: indicates whether this profile has been marked as interesting. The value can be null. :>json str join_date: the date this profile joined its social network (ISO-8601) :>json list labels: list of labels for this profile :>json int label[n].id: the unique id for this label :>json str label[n].name: the label :>json str last_update: the last time that information about this profile was retrieved from the social media site (ISO-8601) :>json str location: geographic location provided by the user, as free text :>json str name: the full name provided by this user :>json int note[n].id: the unique id for this note :>json int note[n].category: the user-defined category of this note :>json int note[n].body: the user-defined text-body of this note :>json int post_count: the number of posts made by this profile :>json bool private: true if this is a private account (i.e. not world- readable) :>json str score: user-defined score for this profile. Can be null. :>json str site: machine-readable site name that this profile belongs to :>json str site_name: human-readable site name that this profile belongs to :>json str time_zone: the user's provided time zone as free text :>json str upstream_id: the user ID assigned by the social site :>json str url: URL endpoint for retriving more data about this profile :>json str username: the current username for this profile :>json list usernames: list of known usernames for this profile :>json str usernames[n].end_date: the last known date this username was used for this profile :>json str usernames[n].start_date: the first known date this username was used for this profile :>json str usernames[n].username: a username used for this profile :status 202: accepted for background processing :status 400: invalid request body :status 401: authentication required ''' redis = worker.get_redis() # Get profile. id_ = get_int_arg('id_', id_) profile, avatar = g.db.query(Profile, Avatar) \ .outerjoin(Profile.current_avatar) \ .filter(Profile.id == id_).first() if profile is None: raise NotFound("Profile '%s' does not exist." % id_) request_json = request.get_json() # Validate put data and set attributes # Only 'is_interesting', 'score', and 'labels' are modifiable if 'is_interesting' in request_json: if isinstance(request_json['is_interesting'], bool): profile.is_interesting = request_json['is_interesting'] elif request_json['is_interesting'] is None: profile.is_interesting = None else: raise BadRequest("'is_interesting' is a boolean (true or false.") if 'score' in request_json: if request_json['score'] is None: profile.score = None else: try: profile.score = float(request_json['score']) except: raise BadRequest("'score' must be a decimal number.") # labels expects the string 'name' rather than id, to avoid the need to # create labels before adding them. if 'labels' in request_json: labels = [] if isinstance(request_json['labels'], list): for label_json in request_json['labels']: if 'name' in label_json: label = g.db.query(Label) \ .filter(Label.name==label_json['name']) \ .first() if label is None: try: label = Label( name=label_json['name'].lower().strip() ) g.db.add(label) g.db.flush() redis.publish( 'label', json.dumps(label.as_dict()) ) except IntegrityError: g.db.rollback() raise BadRequest('Label could not be saved') except AssertionError: g.db.rollback() raise BadRequest( '"{}" contains non-alphanumeric character' .format( label_json['name'] ) ) labels.append(label) else: raise BadRequest("Label 'name' is required") profile.labels = labels else: raise BadRequest("'labels' must be a list") response = profile.as_dict() response['url'] = url_for('ProfileView:get', id_=profile.id) # Save the profile try: g.db.commit() redis.publish('profile', json.dumps(profile.as_dict())) except DBAPIError as e: g.db.rollback() raise BadRequest('Profile could not be saved') # Create usernames list. usernames = list() for username in profile.usernames: if username.end_date is not None: end_date = username.end_date.isoformat() else: end_date = None if username.start_date is not None: start_date = username.start_date.isoformat() else: start_date = None usernames.append({ 'end_date': end_date, 'username': username.username, 'start_date': start_date, }) response['usernames'] = usernames # Create avatar attributes. if avatar is not None: response['avatar_url'] = url_for( 'FileView:get', id_=avatar.file.id ) response['avatar_thumb_url'] = url_for( 'FileView:get', id_=avatar.thumb_file.id ) else: response['avatar_url'] = url_for( 'static', filename='img/default_user.png' ) response['avatar_thumb_url'] = url_for( 'static', filename='img/default_user_thumb.png' ) # Send response. return jsonify(**response)
def put(self, id_): ''' Update the category identified by `id`. **Example Request** ..sourcode:: json { { "name": "priority sites" "sites": [1,5] }, } **Example Response** ..sourcecode:: json { "id": 1, "name": "priority sites", "sites": [ { "category": "books", "id": 1, "name": "aNobil", "search_text": "- aNobii</title>", "status_code": 200, "url": "http://www.anobii.com/%s/books" }, { "category": "coding", "id": 5, "name": "bitbucket", "search_text": "\"username\":", "status_code": 200, "url": "https://bitbucket.org/api/2.0/users/%s" }, ... ] }, :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json str name: the value of the name attribute :>header Content-Type: application/json :>json int id: unique identifier for category :>json str name: the category name :>json list sites: list of sites associated with this category :>json str sites[n].category: the site category :>json str sites[n].id: the unique id for site :>json str sites[n].name: the site name :>json str sites[n].search_text: string search pattern :>json str sites[n].status_code: server response code for site :>json str sites[n].url: the site url :status 200: updated :status 400: invalid request body :status 401: authentication required ''' editable_fields = ['name', 'sites'] # Get category. id_ = get_int_arg('id_', id_) category = g.db.query(Category).filter(Category.id == id_).first() if category is None: raise NotFound("Category '%s' does not exist." % id_) request_json = request.get_json() # Validate data and set attributes if request_json is None: raise BadRequest("Specify at least one editable field: {}".format( editable_fields)) for field in request_json: if field not in editable_fields: raise BadRequest( "'{}' is not one of the editable fields: {}".format( field, editable_fields)) if 'name' in request_json: validate_json_attr('name', GROUP_ATTRS, request_json) category.name = request_json['name'].strip() if 'sites' in request_json: try: request_site_ids = [int(s) for s in request_json['sites']] except ValueError: raise BadRequest('Sites must be a list of integer site ids') if len(request_site_ids) == 0: raise BadRequest('Categorys must have at least one site') sites = g.db.query(Site) \ .filter(Site.id.in_(request_site_ids)) \ .all() site_ids = [site.id for site in sites] missing_sites = list(set(request_site_ids) - set(site_ids)) if len(missing_sites) > 0: raise BadRequest('Site ids "{}" do not exist'.format( ','.join(missing_sites))) else: category.sites = sites # Save the updated category g.db.add(category) try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='category', message={ 'id': category.id, 'name': category.name, 'status': 'updated', 'resource': url_for('CategoryView:get', id_=category.id) }) response = category.as_dict() response['url-for'] = url_for('CategoryView:get', id_=category.id) # Send response. return jsonify(**response)
def get(self, id_): ''' Get the profile identified by `id`. **Example Response** .. sourcecode:: json { "avatar_url": "https://quickpin/api/file/1", "avatar_thumb_url": "https://quickpin/api/file/2", "description": "A human being.", "follower_count": 71, "friend_count": 28, "id": 1, "is_stub": false, "is_interesting": false, "join_date": "2012-01-30T15:11:35", "labels": [ { "id": 1, "name": "male" }, ], "last_update": "2015-08-18T10:51:16", "location": "Washington, DC", "name": "John Doe", "post_count": 1666, "private": false, "site": "twitter", "site_name": "Twitter", "time_zone": "Central Time (US & Canada)", "upstream_id": "11009418", "url": "https://quickpin/api/profile/1", "username": "******", "usernames": [ { "end_date": "2012-06-30T15:00:00", "start_date": "2012-01-01T12:00:00", "username": "******" }, ... ] } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str avatar_url: URL to the user's current avatar :>json str avatar_thumb_url: URL to a 32x32px thumbnail of the user's current avatar :>json str description: profile description :>json int follower_count: number of followers :>json int friend_count: number of friends (a.k.a. followees) :>json int id: unique identifier for profile :>json bool is_stub: indicates that this is a stub profile, e.g. related to another profile but has not been fully imported :>json bool is_interesting: indicates whether this profile has been marked as interesting. The value can be null. :>json str join_date: the date this profile joined its social network (ISO-8601) :>json list labels: list of labels for this profile :>json int label[n].id: the unique id for this label :>json str label[n].name: the label :>json str last_update: the last time that information about this profile was retrieved from the social media site (ISO-8601) :>json str location: geographic location provided by the user, as free text :>json str name: the full name provided by this user :>json int post_count: the number of posts made by this profile :>json bool private: true if this is a private account (i.e. not world- readable) :>json str site: machine-readable site name that this profile belongs to :>json str site_name: human-readable site name that this profile belongs to :>json str time_zone: the user's provided time zone as free text :>json str upstream_id: the user ID assigned by the social site :>json str url: URL endpoint for retriving more data about this profile :>json str username: the current username for this profile :>json list usernames: list of known usernames for this profile :>json str usernames[n].end_date: the last known date this username was used for this profile :>json str usernames[n].start_date: the first known date this username was used for this profile :>json str usernames[n].username: a username used for this profile :status 200: ok :status 400: invalid argument[s] :status 401: authentication required :status 404: user does not exist ''' # Get profile. id_ = get_int_arg('id_', id_) current_avatar_id = self._current_avatar_subquery() profile, avatar = g.db.query(Profile, Avatar) \ .outerjoin(Avatar, Avatar.id == current_avatar_id) \ .filter(Profile.id == id_).first() if profile is None: raise NotFound("Profile '%s' does not exist." % id_) response = profile.as_dict() response['url'] = url_for('ProfileView:get', id_=profile.id) # Create usernames list. usernames = list() for username in profile.usernames: if username.end_date is not None: end_date = username.end_date.isoformat() else: end_date = None if username.start_date is not None: start_date = username.start_date.isoformat() else: start_date = None usernames.append({ 'end_date': end_date, 'username': username.username, 'start_date': start_date, }) response['usernames'] = usernames # Create avatar attributes. if avatar is not None: response['avatar_url'] = url_for( 'FileView:get', id_=avatar.file.id ) response['avatar_thumb_url'] = url_for( 'FileView:get', id_=avatar.thumb_file.id ) else: response['avatar_url'] = url_for( 'static', filename='img/default_user.png' ) response['avatar_thumb_url'] = url_for( 'static', filename='img/default_user_thumb.png' ) # Send response. return jsonify(**response)
def put(self, id_): ''' Update the label identified by `id`. **Example Request** ..sourcode:: json { {"name": "gender"}, } **Example Response** ..sourcecode:: json { "id": "2", "name": "gender", "url": "https://quickpin/api/label/1", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json str name: the value of the name attribute :>header Content-Type: application/json :>json int id: unique identifier for label :>json str name: the label name :>json str url: URL endpoint for retriving more data about this label :status 202: created :status 400: invalid request body :status 401: authentication required ''' # Get label. id_ = get_int_arg('id_', id_) label = g.db.query(Label).filter(Label.id == id_).first() if label is None: raise NotFound("Label '%s' does not exist." % id_) request_json = request.get_json() # Validate data and set attributes if 'name' in request_json: if request_json['name'].strip() != '': label.name = request_json['name'].lower().strip() else: raise BadRequest('Attribute "name" cannot be an empty string') else: raise BadRequest('Attribue "name" is required') # Save the updated label try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) response = label.as_dict() response['url'] = url_for('LabelView:get', id_=label.id) # Send response. return jsonify(**response)