def delete(self, id_): ''' Delete proxy identified by `id_`. ''' # Get proxy. id_ = get_int_arg('id_', id_) proxy = g.db.query(Proxy).filter(Proxy.id == id_).first() if proxy is None: raise NotFound("Proxy '%s' does not exist." % id_) # Delete proxy try: g.db.delete(proxy) g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest('Could not delete proxy.') # Send redis notifications notify_mask_client(channel='proxy', message={ 'proxy': proxy.as_dict(), 'status': 'deleted', 'resource': None }) message = 'Proxy id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete site identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) site = g.db.query(Site).filter(Site.id == id_).first() if site is None: raise NotFound("Site '%s' does not exist." % id_) # Delete site try: g.db.delete(site) g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest('"{}" must be removed from all groups ' 'before deleting.'.format(site.name)) # Send redis notifications notify_mask_client(channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'deleted', 'resource': None }) message = 'Site id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete the category identified by `id`. **Example Response** ..sourcecode:: json { "message": "Category `main` deleted", } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>header Content-Type: application/json :>json str message: the API response message :status 200: deleted :status 400: invalid request body :status 401: authentication required :status 404: category does not exist ''' # Get label. id_ = get_int_arg('id_', id_) category = g.db.query(Category).filter(Category.id == id_).first() if category is None: raise NotFound("Category '%s' does not exist." % id_) # Delete label g.db.delete(category) try: g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest( 'You must delete archived results that' 'use category "{}" before it can be deleted.'.format( category.name)) except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='category', message={ 'id': category.id, 'name': category.name, 'status': 'deleted', 'resource': None }) message = 'Category id "{}" deleted'.format(category.id) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete archive identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) archive = g.db.query(Archive).filter(Archive.id == id_).filter( Archive.user_id == g.user.id).first() if archive is None: raise NotFound("Archive '%s' does not exist." % id_) # Delete site try: g.db.delete(archive) g.db.commit() except IntegrityError: g.db.rollback() raise BadRequest('Could not delete archive.') # Send redis notifications notify_mask_client(channel='archive', message={ 'id': archive.id, 'name': archive.username, 'status': 'deleted', 'resource': None }) message = 'Archive id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def delete(self, id_): ''' Delete site identified by `id_`. ''' # Get site. id_ = get_int_arg('id_', id_) site = g.db.query(Site).filter(Site.id == id_).first() if site is None: raise NotFound("Site '%s' does not exist." % id_) try: # Remove site from categories categories = g.db.query(Category).filter( Category.sites.contains(site)).all() for category in categories: category.sites.remove(site) # Delete site g.db.delete(site) g.db.commit() except Exception as e: g.db.rollback() raise BadRequest(e) # Send redis notifications notify_mask_client(channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'deleted', 'resource': None }) message = 'Site id "{}" deleted'.format(id_) response = jsonify(message=message) response.status_code = 200 return response
def put(self, id_): ''' Update proxy identified by `id`. **Example Request** .. sourcecode:: json PUT /api/proxies/id { "protocol": "http", "host": "192.168.0.2", "port": 80, "username": "******", "password": "******", "active": true, } **Example Response** .. sourcecode:: json { "id": 1, "protocol": "http", "host": "192.168.0.22", "port": 80, "username": "******", "password": "******", "active": true, }, :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json str protocol: protocol of proxy address :>json str host: host of proxy address :>json int port: port of proxy address :>json str username: username of proxy :>json str password: password of proxy :>json bool active: proxy active status :>header Content-Type: application/json :>json int id: unique identifier :>json str protocol: protocol of proxy address :>json str host: host of proxy address :>json int port: port of proxy address :>json str username: username of proxy :>json str password: password of proxy :>json bool active: proxy active status :status 200: ok :status 401: authentication required :status 403: must be an administrator ''' # Get proxy id_ = get_int_arg('id_', id_) proxy = g.db.query(Proxy).filter(Proxy.id == id_).first() if proxy is None: raise NotFound("Proxy '%s' does not exist." % id_) # Validate request json request_json = request.get_json() validate_request_json(request_json, PROXY_ATTRS) # Update proxy proxy.protocol = request_json['protocol'] proxy.host = request_json['host'] proxy.port = request_json['port'] proxy.active = request_json['active'] try: proxy.username = request_json['username'] except KeyError: pass try: proxy.password = request_json['password'] except KeyError: pass # Save the updated proxy try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='proxy', message={ 'proxy': proxy.as_dict(), 'status': 'updated', 'resource': None }) response = jsonify(proxy.as_dict()) response.status_code = 200 # Send response return response
def post(self): ''' Create new proxies. **Example Request** .. sourcecode:: json { "proxies": [ { "protocol": "http", "host": "192.168.0.2", "port": 80, "username": "******", "password": "******", "active": true, }, ... ] } **Example Response** .. sourcecode:: json { "message": "1 proxy created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json list proxies: list of proxies :<json str proxies[n]["protocol"]: protocol of proxy address :<json str proxies[n]["host"]: host of proxy address :<json int proxies[n]["port"]: port of proxy address :<json str proxies[n]["username"]: username of proxy :<json str proxies[n]["password"]: password of proxy :<json bool proxies[n]["active"]: proxy active status :>header Content-Type: application/json :>json string message: API response message :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() proxies = [] # Ensure all data is valid before db operations for proxy_json in request_json['proxies']: validate_request_json(proxy_json, PROXY_ATTRS) # Save proxies for proxy_json in request_json['proxies']: proxy = Proxy(protocol=proxy_json['protocol'].lower().strip(), host=proxy_json['host'].lower().strip(), port=proxy_json['port'], active=proxy_json['active']) # Username is optional, and can be None try: proxy.username = proxy_json['username'].lower().strip() except KeyError: pass except AttributeError: proxy.username = None # Password is optional, and can be None try: proxy.password = proxy_json['password'].strip() except KeyError: pass except AttributeError: proxy.password = None g.db.add(proxy) try: g.db.flush() proxies.append(proxy) except IntegrityError: g.db.rollback() raise BadRequest('Proxy {}://{}:{} already exists.'.format( proxy.protocol, proxy.host, proxy.port)) g.db.commit() # Send redis notifications for proxy in proxies: notify_mask_client(channel='proxy', message={ 'proxy': proxy.as_dict(), 'status': 'created', 'resource': None }) message = '{} new proxies created'.format(len(request_json['proxies'])) response = jsonify(message=message) response.status_code = 202 return response
def put(self, id_): ''' Update the site identified by `id`. **Example Request** ..sourcecode:: json { "name": "bebo", "url": "http://bebo.com/usernames/search=%s", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "test_username_pos": "bob", "test_username_ne": "adfjf393rfjffkjd", "headers": {"referer": "http://www.google.com"}, "censor_images": false, "wait_time": 5, "use_proxy": false, } **Example Response** ..sourcecode:: json { "id": 2, "name": "bebo", "search_text": "Bebo User Page.</title>", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "url": "https://bebo.com/usernames/search=%s", "test_username_pos": "bob", "test_username_neg": "adfjf393rfjffkjd", "test_status": "f", "tested_at": "2016-01-01T00:00:00.000000+00:00", "headers": {"referer": "http://www.google.com"}, "censor_images": false, "wait_time": 5, "use_proxy": false, }, :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json string name: name of site :<json string url: username search url for the site :<json string test_username_pos: username that exists on site (used for testing) :<json string test_username_neg: username that does not exist on site (used for testing) :<json array headers: custom headers :<json bool censor_images: whether to censor images from this profile :<json int wait_time: time (in seconds) to wait for updates after page is loaded :<json bool use_proxy: whether to proxy requests for this profile URL :>header Content-Type: application/json :>json int id: unique identifier for site :>json str name: name of site :>json str url: username search url for the site :>json int status_code: the status code to check for determining a match (nullable) :>json string match_type: type of match (see get_match_types() for valid match types) (nullable) :>json string match_expr: expression to use for determining a page match (nullable) :>json str test_status: results of username test :>json str tested_at: timestamp of last test :>json str test_username_pos: username that exists on site (used for testing) :>json str test_username_neg: username that does not exist on site (used for testing) :>json array headers: custom headers :>json bool censor_images: whether to censor images from this profile :>json int wait_time: time (in seconds) to wait for updates after page is loaded :>json bool use_proxy: whether to proxy requests for this profile URL :status 202: updated :status 400: invalid request body :status 401: authentication required :status 404: site does not exist ''' # Get site. id_ = get_int_arg('id_', id_) site = g.db.query(Site).filter(Site.id == id_).first() if site is None: raise NotFound("Site '%s' does not exist." % id_) request_json = request.get_json() # Validate data and set attributes if 'name' in request_json: validate_json_attr('name', _site_attrs, request_json) site.name = request_json['name'].strip() if 'url' in request_json: validate_json_attr('url', _site_attrs, request_json) site.url = request_json['url'].lower().strip() if 'match_expr' in request_json: validate_json_attr('match_expr', _site_attrs, request_json) site.match_expr = request_json['match_expr'] if 'match_type' in request_json: validate_json_attr('match_type', _site_attrs, request_json) site.match_type = request_json['match_type'].strip() if 'status_code' in request_json: validate_json_attr('status_code', _site_attrs, request_json) status = request_json['status_code'] site.status_code = None if status is None else int(status) if (request_json['match_type'] is None or request_json['match_expr'] is None) and \ request_json['status_code'] is None: raise BadRequest('At least one of the ' 'following is required: ' 'status code or page match.') if 'test_username_pos' in request_json: validate_json_attr('test_username_pos', _site_attrs, request_json) site.test_username_pos = ( request_json['test_username_pos'].lower().strip()) if 'test_username_neg' in request_json: validate_json_attr('test_username_neg', _site_attrs, request_json) site.test_username_neg = ( request_json['test_username_neg'].lower().strip()) if 'headers' in request_json: validate_json_attr('headers', _site_attrs, request_json) site.headers = request_json['headers'] if 'censor_images' in request_json: validate_json_attr('censor_images', _site_attrs, request_json) site.censor_images = request_json['censor_images'] if 'use_proxy' in request_json: validate_json_attr('use_proxy', _site_attrs, request_json) site.use_proxy = request_json['use_proxy'] if 'wait_time' in request_json: validate_json_attr('wait_time', _site_attrs, request_json) site.wait_time = request_json['wait_time'] # Save the updated site try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='site', message={ 'site': site.as_dict(), 'status': 'updated', 'resource': None }) response = jsonify(site.as_dict()) response.status_code = 200 # Send response. return response
def post(self): ''' Create new sites to included in username searches. **Example Request** .. sourcecode:: json { "sites": [ { "name": "about.me", "url": "http://about.me/%s", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "test_username_pos": "john", "test_username_neg": "dPGMFrf72SaS", "headers": {"referer": "http://www.google.com"}, "censor_images": false, "wait_time": 5, "use_proxy": false, }, ... ] } **Example Response** .. sourcecode:: json { "message": "1 site created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json list sites: a list of sites to create :<json string sites[n].name: name of site :<json string sites[n].url: username search url for the site :<json int sites[n].status_code: the status code to check for determining a match (nullable) :<json string sites[n].match_type: type of match (see get_match_types() for valid match types) (nullable) :<json string sites[n].match_expr: expression to use for determining a page match (nullable) :<json string sites[n].test_username_pos: username that exists on site (used for testing) :<json string sites[n].test_username_neg: username that does not exist on site (used for testing) :<json array sites[n].headers: custom headers :<json bool sites[n].censor_images: whether to censor images from this profile :<json int sites[n].wait_time: time (in seconds) to wait for updates after page is loaded :<json bool sites[n].use_proxy: whether to proxy requests for this profile URL :>header Content-Type: application/json :>json string message: API response message :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() sites = [] # Ensure all data is valid before db operations for site_json in request_json['sites']: validate_request_json(site_json, _site_attrs) if (site_json['match_type'] is None or site_json['match_expr'] is None) and \ site_json['status_code'] is None: raise BadRequest('At least one of the ' 'following is required: ' 'status code or page match.') if '%s' not in site_json['url']: raise BadRequest('URL must contain replacement character %s') # Save sites for site_json in request_json['sites']: test_username_pos = site_json['test_username_pos'].lower().strip() site = Site(name=site_json['name'].strip(), url=site_json['url'].lower().strip(), test_username_pos=test_username_pos) site.status_code = site_json['status_code'] site.match_expr = site_json['match_expr'] site.match_type = site_json['match_type'] if 'test_username_neg' in site_json: site.test_username_neg = site_json['test_username_neg'] \ .lower().strip(), if 'headers' in site_json: site.headers = site_json['headers'] g.db.add(site) try: g.db.flush() sites.append(site) except IntegrityError: g.db.rollback() raise BadRequest('Site URL {} already exists.'.format( site.url)) g.db.commit() # Send redis notifications for site in sites: notify_mask_client(channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'created', 'resource': None }) message = '{} new sites created'.format(len(request_json['sites'])) response = jsonify(message=message) response.status_code = 202 return response
def post(self): ''' Create a category. **Example Request** ..sourcode:: json { "categories": [ { "name": "gender", "sites": [1, 2, 7] }, ... ] } **Example Response** ..sourcecode:: json { "message": "2 new categories created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list categories: a list of categories to create :>json str categories[n].name: name of category to create :>header Content-Type: application/json :>json str message: api response message :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() categories = list() # Validate input for category_json in request_json['categories']: validate_request_json(category_json, GROUP_ATTRS) try: request_site_ids = [int(s) for s in category_json['sites']] except TypeError: raise BadRequest('Sites must be integer site ids') if len(request_site_ids) == 0: raise BadRequest('At least one site is required.') sites = g.db.query(Site)\ .filter(Site.id.in_(request_site_ids))\ .all() site_ids = [site.id for site in sites] missing_sites = list(set(request_site_ids) - set(site_ids)) if len(missing_sites) > 0: raise BadRequest('Site ids {} do not exist'.format(','.join( str(s) for s in missing_sites))) # Create categories for category_json in request_json['categories']: try: category = Category(name=category_json['name'].strip(), sites=sites) g.db.add(category) g.db.flush() # Create dict for API JSON response category_dict = category.as_dict() # Add a link to the created category category_dict['url-for'] = url_for('CategoryView:get', id_=category.id) categories.append(category_dict) except IntegrityError: g.db.rollback() raise BadRequest('Category "{}" already exists'.format( category.name)) # Save categories g.db.commit() # Send redis notifications for category in categories: notify_mask_client(channel='category', message={ 'id': category['id'], 'name': category['name'], 'status': 'created', 'resource': category['url-for'] }) message = '{} new categories created' \ .format(len(request_json['categories'])) response = jsonify(message=message, categories=categories) response.status_code = 200 return response
def put(self, id_): ''' Update the category identified by `id`. **Example Request** ..sourcode:: json { { "name": "priority sites" "sites": [1,5] }, } **Example Response** ..sourcecode:: json { "id": 1, "name": "priority sites", "sites": [ { "category": "books", "id": 1, "name": "aNobil", "search_text": "- aNobii</title>", "status_code": 200, "url": "http://www.anobii.com/%s/books" }, { "category": "coding", "id": 5, "name": "bitbucket", "search_text": "\"username\":", "status_code": 200, "url": "https://bitbucket.org/api/2.0/users/%s" }, ... ] }, :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json str name: the value of the name attribute :>header Content-Type: application/json :>json int id: unique identifier for category :>json str name: the category name :>json list sites: list of sites associated with this category :>json str sites[n].category: the site category :>json str sites[n].id: the unique id for site :>json str sites[n].name: the site name :>json str sites[n].search_text: string search pattern :>json str sites[n].status_code: server response code for site :>json str sites[n].url: the site url :status 200: updated :status 400: invalid request body :status 401: authentication required ''' editable_fields = ['name', 'sites'] # Get category. id_ = get_int_arg('id_', id_) category = g.db.query(Category).filter(Category.id == id_).first() if category is None: raise NotFound("Category '%s' does not exist." % id_) request_json = request.get_json() # Validate data and set attributes if request_json is None: raise BadRequest("Specify at least one editable field: {}".format( editable_fields)) for field in request_json: if field not in editable_fields: raise BadRequest( "'{}' is not one of the editable fields: {}".format( field, editable_fields)) if 'name' in request_json: validate_json_attr('name', GROUP_ATTRS, request_json) category.name = request_json['name'].strip() if 'sites' in request_json: try: request_site_ids = [int(s) for s in request_json['sites']] except ValueError: raise BadRequest('Sites must be a list of integer site ids') if len(request_site_ids) == 0: raise BadRequest('Categorys must have at least one site') sites = g.db.query(Site) \ .filter(Site.id.in_(request_site_ids)) \ .all() site_ids = [site.id for site in sites] missing_sites = list(set(request_site_ids) - set(site_ids)) if len(missing_sites) > 0: raise BadRequest('Site ids "{}" do not exist'.format( ','.join(missing_sites))) else: category.sites = sites # Save the updated category g.db.add(category) try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='category', message={ 'id': category.id, 'name': category.name, 'status': 'updated', 'resource': url_for('CategoryView:get', id_=category.id) }) response = category.as_dict() response['url-for'] = url_for('CategoryView:get', id_=category.id) # Send response. return jsonify(**response)
def post(self): ''' Create new sites to included in username searches. **Example Request** .. sourcecode:: json { "sites": [ { "name": "about.me", "url": "http://about.me/%s", "category": "social", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "test_username_pos": "john", "test_username_neg": "dPGMFrf72SaS" }, ... ] } **Example Response** .. sourcecode:: json { "message": "1 site created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list sites: a list of sites to create :>json string sites[n].name: name of site :>json string sites[n].url: username search url for the site :>json string sites[n].category: category of the site :>json int sites[n].status_code: the status code to check for determining a match (nullable) :>json string sites[n].match_type: type of match (see get_match_types() for valid match types) (nullable) :>json string sites[n].match_expr: expression to use for determining a page match (nullable) :>json string sites[n].test_username_pos: username that exists on site (used for testing) :>json string sites[n].test_username_neg: username that does not exist on site (used for testing) :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() sites = [] # Ensure all data is valid before db operations for site_json in request_json['sites']: validate_request_json(site_json, SITE_ATTRS) if (site_json['match_type'] is None or \ site_json['match_expr'] is None) and \ site_json['status_code'] is None: raise BadRequest('At least one of the following is required: ' 'status code or page match.') # Save sites for site_json in request_json['sites']: test_username_pos = site_json['test_username_pos'].lower().strip() site = Site(name=site_json['name'].lower().strip(), url=site_json['url'].lower().strip(), category=site_json['category'].lower().strip(), test_username_pos=test_username_pos) site.status_code = site_json['status_code'] site.match_expr = site_json['match_expr'] site.match_type = site_json['match_type'] if 'test_username_neg' in site_json: site.test_username_neg = site_json['test_username_neg'] \ .lower().strip(), g.db.add(site) try: g.db.flush() sites.append(site) except IntegrityError: g.db.rollback() raise BadRequest( 'Site URL {} already exists.'.format(site.url) ) g.db.commit() # Send redis notifications for site in sites: notify_mask_client( channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'created', 'resource': None } ) message = '{} new sites created'.format(len(request_json['sites'])) response = jsonify(message=message) response.status_code = 202 return response