def post(self): ''' Request search of usernames. **Example Request** .. sourcecode:: json { "usernames": [ "johndoe", "janedoe", ... ], "category": 3, "test": False, } **Example Response** .. sourcecode:: json { "tracker_ids": { "johndoe": "tracker.12344565", } } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list usernames: a list of usernames to search for :>json int category: ID of site category to use (optional) :>json int site: ID of site to search (optional) :>json bool test: test results (optional, default: false) :>header Content-Type: application/json :>json list jobs: list of worker jobs :>json list jobs[n].id: unique id of this job :>json list jobs[n].usename: username target of this job :status 202: accepted for background processing :status 400: invalid request body :status 401: authentication required ''' test = False category = None category_id = None jobs = [] tracker_ids = dict() redis = g.redis request_json = request.get_json() site = None if 'usernames' not in request_json: raise BadRequest('`usernames` is required') validate_request_json(request_json, _username_attrs) if len(request_json['usernames']) == 0: raise BadRequest('At least one username is required') if 'category' in request_json and 'site' in request_json: raise BadRequest('Supply either `category` or `site`.') if 'category' in request_json: category_id = request_json['category'] category = g.db.query(Category) \ .filter(Category.id == category_id).first() if category is None: raise NotFound("Category '%s' does not exist." % category_id) else: category_id = category.id if 'site' in request_json: site_id = request_json['site'] site = g.db.query(Site).filter(Site.id == site_id).first() if site is None: raise NotFound("Site '%s' does not exist." % site_id) if 'test' in request_json: test = request_json['test'] if category: sites = category.sites elif site: sites = g.db.query(Site).filter(Site.id == site.id).all() else: sites = g.db.query(Site).all() # Only check valid sites. valid_sites = [] for site in sites: if site.valid: valid_sites.append(site) # sites = sites.filter(Site.valid == True).all() # noqa if len(valid_sites) == 0: raise NotFound('No valid sites to check') for username in request_json['usernames']: # Create an object in redis to track the number of sites completed # in this search. tracker_id = 'tracker.{}'.format(random_string(10)) tracker_ids[username] = tracker_id redis.set(tracker_id, 0) redis.expire(tracker_id, 600) total = len(valid_sites) # Queue a job for each site. for site in valid_sites: description = 'Checking {} for user "{}"'.format(site.name, username) job = worker.scrape.check_username.enqueue( username=username, site_id=site.id, category_id=category_id, total=total, tracker_id=tracker_id, test=test, jobdesc=description, timeout=_redis_worker['username_timeout'], user_id=g.user.id ) jobs.append({ 'id': job.id, 'username': username, 'category': category_id, }) response = jsonify(tracker_ids=tracker_ids) response.status_code = 202 return response
def post(self): ''' Create new proxies. **Example Request** .. sourcecode:: json { "proxies": [ { "protocol": "http", "host": "192.168.0.2", "port": 80, "username": "******", "password": "******", "active": true, }, ... ] } **Example Response** .. sourcecode:: json { "message": "1 proxy created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json list proxies: list of proxies :<json str proxies[n]["protocol"]: protocol of proxy address :<json str proxies[n]["host"]: host of proxy address :<json int proxies[n]["port"]: port of proxy address :<json str proxies[n]["username"]: username of proxy :<json str proxies[n]["password"]: password of proxy :<json bool proxies[n]["active"]: proxy active status :>header Content-Type: application/json :>json string message: API response message :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() proxies = [] # Ensure all data is valid before db operations for proxy_json in request_json['proxies']: validate_request_json(proxy_json, PROXY_ATTRS) # Save proxies for proxy_json in request_json['proxies']: proxy = Proxy(protocol=proxy_json['protocol'].lower().strip(), host=proxy_json['host'].lower().strip(), port=proxy_json['port'], active=proxy_json['active']) # Username is optional, and can be None try: proxy.username = proxy_json['username'].lower().strip() except KeyError: pass except AttributeError: proxy.username = None # Password is optional, and can be None try: proxy.password = proxy_json['password'].strip() except KeyError: pass except AttributeError: proxy.password = None g.db.add(proxy) try: g.db.flush() proxies.append(proxy) except IntegrityError: g.db.rollback() raise BadRequest('Proxy {}://{}:{} already exists.'.format( proxy.protocol, proxy.host, proxy.port)) g.db.commit() # Send redis notifications for proxy in proxies: notify_mask_client(channel='proxy', message={ 'proxy': proxy.as_dict(), 'status': 'created', 'resource': None }) message = '{} new proxies created'.format(len(request_json['proxies'])) response = jsonify(message=message) response.status_code = 202 return response
def put(self, id_): ''' Update proxy identified by `id`. **Example Request** .. sourcecode:: json PUT /api/proxies/id { "protocol": "http", "host": "192.168.0.2", "port": 80, "username": "******", "password": "******", "active": true, } **Example Response** .. sourcecode:: json { "id": 1, "protocol": "http", "host": "192.168.0.22", "port": 80, "username": "******", "password": "******", "active": true, }, :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json str protocol: protocol of proxy address :>json str host: host of proxy address :>json int port: port of proxy address :>json str username: username of proxy :>json str password: password of proxy :>json bool active: proxy active status :>header Content-Type: application/json :>json int id: unique identifier :>json str protocol: protocol of proxy address :>json str host: host of proxy address :>json int port: port of proxy address :>json str username: username of proxy :>json str password: password of proxy :>json bool active: proxy active status :status 200: ok :status 401: authentication required :status 403: must be an administrator ''' # Get proxy id_ = get_int_arg('id_', id_) proxy = g.db.query(Proxy).filter(Proxy.id == id_).first() if proxy is None: raise NotFound("Proxy '%s' does not exist." % id_) # Validate request json request_json = request.get_json() validate_request_json(request_json, PROXY_ATTRS) # Update proxy proxy.protocol = request_json['protocol'] proxy.host = request_json['host'] proxy.port = request_json['port'] proxy.active = request_json['active'] try: proxy.username = request_json['username'] except KeyError: pass try: proxy.password = request_json['password'] except KeyError: pass # Save the updated proxy try: g.db.commit() except DBAPIError as e: g.db.rollback() raise BadRequest('Database error: {}'.format(e)) # Send redis notifications notify_mask_client(channel='proxy', message={ 'proxy': proxy.as_dict(), 'status': 'updated', 'resource': None }) response = jsonify(proxy.as_dict()) response.status_code = 200 # Send response return response
def post(self): ''' Create new sites to included in username searches. **Example Request** .. sourcecode:: json { "sites": [ { "name": "about.me", "url": "http://about.me/%s", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "test_username_pos": "john", "test_username_neg": "dPGMFrf72SaS", "headers": {"referer": "http://www.google.com"}, "censor_images": false, "wait_time": 5, "use_proxy": false, }, ... ] } **Example Response** .. sourcecode:: json { "message": "1 site created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json list sites: a list of sites to create :<json string sites[n].name: name of site :<json string sites[n].url: username search url for the site :<json int sites[n].status_code: the status code to check for determining a match (nullable) :<json string sites[n].match_type: type of match (see get_match_types() for valid match types) (nullable) :<json string sites[n].match_expr: expression to use for determining a page match (nullable) :<json string sites[n].test_username_pos: username that exists on site (used for testing) :<json string sites[n].test_username_neg: username that does not exist on site (used for testing) :<json array sites[n].headers: custom headers :<json bool sites[n].censor_images: whether to censor images from this profile :<json int sites[n].wait_time: time (in seconds) to wait for updates after page is loaded :<json bool sites[n].use_proxy: whether to proxy requests for this profile URL :>header Content-Type: application/json :>json string message: API response message :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() sites = [] # Ensure all data is valid before db operations for site_json in request_json['sites']: validate_request_json(site_json, _site_attrs) if (site_json['match_type'] is None or site_json['match_expr'] is None) and \ site_json['status_code'] is None: raise BadRequest('At least one of the ' 'following is required: ' 'status code or page match.') if '%s' not in site_json['url']: raise BadRequest('URL must contain replacement character %s') # Save sites for site_json in request_json['sites']: test_username_pos = site_json['test_username_pos'].lower().strip() site = Site(name=site_json['name'].strip(), url=site_json['url'].lower().strip(), test_username_pos=test_username_pos) site.status_code = site_json['status_code'] site.match_expr = site_json['match_expr'] site.match_type = site_json['match_type'] if 'test_username_neg' in site_json: site.test_username_neg = site_json['test_username_neg'] \ .lower().strip(), if 'headers' in site_json: site.headers = site_json['headers'] g.db.add(site) try: g.db.flush() sites.append(site) except IntegrityError: g.db.rollback() raise BadRequest('Site URL {} already exists.'.format( site.url)) g.db.commit() # Send redis notifications for site in sites: notify_mask_client(channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'created', 'resource': None }) message = '{} new sites created'.format(len(request_json['sites'])) response = jsonify(message=message) response.status_code = 202 return response
def post_jobs_for_site(self, site_id): """ Request background jobs for site identified by `id`. **Example Request** ..sourcode:: json { "jobs": [ { "name": "test", }, ... ] } **Example Response** .. sourcecode:: json { "tracker_ids": { "1": "tracker.12344565", } } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list jobs: a list of jobs to schedule :>json string jobs[n].name: name of job :>header Content-Type: application/json :>json array tracker_ids: array of worker tracking ids {site ID: tracker ID} :status 202: scheduled :status 400: invalid request body :status 401: authentication required """ request_attrs = { 'jobs': { 'type': list, 'required': True }, } job_attrs = { 'name': { 'type': str, 'required': True }, } available_jobs = ['test'] tracker_ids = dict() # Get site. site_id = get_int_arg('site_id', site_id) site = g.db.query(Site).filter(Site.id == site_id).first() # Validate if site is None: raise NotFound("Site '%s' does not exist." % site_id) request_json = request.get_json() validate_request_json(request_json, request_attrs) for job in request_json['jobs']: validate_json_attr('name', job_attrs, job) if job['name'] not in available_jobs: raise BadRequest('`{}` does not exist in available' ' jobs: {}'.format(job['name'], ','.join(available_jobs))) # Schedule jobs for job in request_json['jobs']: tracker_id = 'tracker.{}'.format(random_string(10)) tracker_ids[site.id] = tracker_id if job['name'] == 'test': description = 'Testing site "{}"'.format(site.name) worker.scrape.test_site.enqueue( site_id=site.id, tracker_id=tracker_id, jobdesc=description, user_id=g.user.id, ) response = jsonify(tracker_ids=tracker_ids) response.status_code = 202 return response
def post(self): ''' Request search of usernames. **Example Request** .. sourcecode:: json { "usernames": [ "johndoe", "janedoe", ... ], "group": 3, "test": False, } **Example Response** .. sourcecode:: json { "tracker_ids": { "johndoe": "tracker.12344565", } } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list usernames: a list of usernames to search for :>json int group: ID of site group to use (optional) :>json int site: ID of site to search (optional) :>json bool test: test results (optional, default: false) :>header Content-Type: application/json :>json list jobs: list of worker jobs :>json list jobs[n].id: unique id of this job :>json list jobs[n].usename: username target of this job :status 202: accepted for background processing :status 400: invalid request body :status 401: authentication required ''' test = False group = None group_id = None jobs = [] tracker_ids = dict() redis = g.redis request_json = request.get_json() site = None if 'usernames' not in request_json: raise BadRequest('`usernames` is required') validate_request_json(request_json, USERNAME_ATTRS) if len(request_json['usernames']) == 0: raise BadRequest('At least one username is required') if 'group' in request_json and 'site' in request_json: raise BadRequest('Supply either `group` or `site`.') if 'group' in request_json: group_id = request_json['group'] group = g.db.query(Group).filter(Group.id == group_id).first() if group is None: raise NotFound("Group '%s' does not exist." % group_id) else: group_id = group.id if 'site' in request_json: site_id = request_json['site'] site = g.db.query(Site).filter(Site.id == site_id).first() if site is None: raise NotFound("Site '%s' does not exist." % site_id) if 'test' in request_json: test = request_json['test'] if group: sites = group.sites elif site: sites = g.db.query(Site).filter(Site.id == site.id) else: sites = g.db.query(Site) # Only check valid sites. sites = sites.filter(Site.valid == True).all() # noqa if len(sites) == 0: raise NotFound('No valid sites to check') for username in request_json['usernames']: # Create an object in redis to track the number of sites completed # in this search. tracker_id = 'tracker.{}'.format(random_string(10)) tracker_ids[username] = tracker_id redis.set(tracker_id, 0) redis.expire(tracker_id, 600) total = len(sites) # Queue a job for each site. for site in sites: job_id = app.queue.schedule_username( username=username, site=site, group_id=group_id, total=total, tracker_id=tracker_id, test=test ) jobs.append({ 'id': job_id, 'username': username, 'group': group_id, }) response = jsonify(tracker_ids=tracker_ids) response.status_code = 202 return response
def post_jobs_for_sites(self): """ Request background jobs for all sites. **Example Request** ..sourcode:: json { "jobs": [ { "name": "test", }, ... ] } **Example Response** .. sourcecode:: json { "tracker_ids": { "1": "tracker.12344565", } } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list jobs: a list of jobs to schedule :>json string jobs[n].name: name of job :>header Content-Type: application/json :>json array tracker_ids: array of worker tracking ids :status 202: scheduled :status 400: invalid request body :status 401: authentication required """ request_attrs = { 'jobs': { 'type': list, 'required': True }, } job_attrs = { 'name': { 'type': str, 'required': True }, } available_jobs = ['test'] tracker_ids = dict() request_json = request.get_json() validate_request_json(request_json, request_attrs) for job in request_json['jobs']: validate_json_attr('name', job_attrs, job) if job['name'] not in available_jobs: raise BadRequest('`{}` does not exist in available' ' jobs: {}'.format(job['name'], ','.join(available_jobs))) # Get sites. sites = g.db.query(Site).all() # Schedule jobs for job in request_json['jobs']: for site in sites: tracker_id = 'tracker.{}'.format(random_string(10)) tracker_ids[site.id] = tracker_id if job['name'] == 'test': app.queue.schedule_site_test( site=site, tracker_id=tracker_id, ) response = jsonify(tracker_ids=tracker_ids) response.status_code = 202 return response
def post(self): ''' Process user payment. **Example Request** .. sourcecode:: json { "user_id": 1, "stripe_token": "tok_1A9VDuL25MRJTn0APWrFQrN6", "credits": 400 "currency": "usd", "description": "200 credits for $20", } **Example Response** .. sourcecode:: json { "message": "200 credits added." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :<json int user_id: the user ID :<json str stripe_token: the stripe payment token :<json int credits: the purchase credits :>header Content-Type: application/json :>json string message: API response message :status 200: ok :status 400: invalid request body :status 401: authentication required :status 403: not authorized to make the requested changes ''' # Validate json input request_json = request.get_json() validate_request_json(request_json, _payment_attrs) user = g.db.query(User).filter( User.id == request_json['user_id']).first() if g.user.id != user.id: raise Forbidden('You may only purchase credits for ' 'your own account.') # Configure stripe client try: stripe.api_key = get_config(session=g.db, key='stripe_secret_key', required=True).value except Exception as e: raise ServiceUnavailable(e) key = 'credit_cost' credit_cost = g.db.query(Configuration) \ .filter(Configuration.key == key) \ .first() if credit_cost is None: raise NotFound( 'There is no configuration item named "{}".'.format(key)) # Stripe token is created client-side using Stripe.js token = request_json['stripe_token'] # Get payment paremeters credits = int(request_json['credits']) description = request_json['description'] currency = request_json['currency'] costs = self._get_costs(credit_cost.value) # Calculate credit amount. try: amount = costs[credits] # credits = list(costs.keys())[list( # costs.values()).index(int(amount))] except IndexError: raise BadRequest('Invalid credit amount.') try: # Charge the user's card: charge = stripe.Charge.create(amount=amount, currency=currency, description=description, source=token) except stripe.error.CardError as e: # Since it's a decline, stripe.error.CardError will be caught body = e.json_body err = body['error'] raise BadRequest('Card error: {}'.format(err['message'])) except stripe.error.RateLimitError as e: # Too many requests made to the API too quickly body = e.json_body err = body['error'] raise BadRequest('Rate limit error: {}'.format(err['message'])) except stripe.error.InvalidRequestError as e: # Invalid parameters were supplied to Stripe's API body = e.json_body err = body['error'] raise BadRequest('Invalid parameters: {}'.format(err['message'])) except stripe.error.AuthenticationError as e: # Authentication with Stripe's API failed # (maybe API keys changed recently) body = e.json_body err = body['error'] raise ServiceUnavailable('Stripe authentication error: {}'.format( err['message'])) except stripe.error.APIConnectionError as e: # Network communication with Stripe failed body = e.json_body err = body['error'] raise ServiceUnavailable( 'Stripe API communication failed: {}'.format(err['message'])) except stripe.error.StripeError as e: # Generic error body = e.json_body err = body['error'] raise ServiceUnavailable('Stripe error: {}'.format(err['message'])) except Exception as e: # Something else happened, completely unrelated to Stripe raise ServiceUnavailable('Error: {}'.format(e)) user.credits += credits g.db.commit() g.redis.publish('user', json.dumps(user.as_dict())) message = '{} credits added.'.format(amount) response = jsonify(message=message) response.status_code = 202 return response
def post(self): ''' Create a category. **Example Request** ..sourcode:: json { "categories": [ { "name": "gender", "sites": [1, 2, 7] }, ... ] } **Example Response** ..sourcecode:: json { "message": "2 new categories created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list categories: a list of categories to create :>json str categories[n].name: name of category to create :>header Content-Type: application/json :>json str message: api response message :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() categories = list() # Validate input for category_json in request_json['categories']: validate_request_json(category_json, GROUP_ATTRS) try: request_site_ids = [int(s) for s in category_json['sites']] except TypeError: raise BadRequest('Sites must be integer site ids') if len(request_site_ids) == 0: raise BadRequest('At least one site is required.') sites = g.db.query(Site)\ .filter(Site.id.in_(request_site_ids))\ .all() site_ids = [site.id for site in sites] missing_sites = list(set(request_site_ids) - set(site_ids)) if len(missing_sites) > 0: raise BadRequest('Site ids {} do not exist'.format(','.join( str(s) for s in missing_sites))) # Create categories for category_json in request_json['categories']: try: category = Category(name=category_json['name'].strip(), sites=sites) g.db.add(category) g.db.flush() # Create dict for API JSON response category_dict = category.as_dict() # Add a link to the created category category_dict['url-for'] = url_for('CategoryView:get', id_=category.id) categories.append(category_dict) except IntegrityError: g.db.rollback() raise BadRequest('Category "{}" already exists'.format( category.name)) # Save categories g.db.commit() # Send redis notifications for category in categories: notify_mask_client(channel='category', message={ 'id': category['id'], 'name': category['name'], 'status': 'created', 'resource': category['url-for'] }) message = '{} new categories created' \ .format(len(request_json['categories'])) response = jsonify(message=message, categories=categories) response.status_code = 200 return response
def post(self): ''' Create new sites to included in username searches. **Example Request** .. sourcecode:: json { "sites": [ { "name": "about.me", "url": "http://about.me/%s", "category": "social", "status_code": 200, "match_type": "text", "match_expr": "Foo Bar Baz", "test_username_pos": "john", "test_username_neg": "dPGMFrf72SaS" }, ... ] } **Example Response** .. sourcecode:: json { "message": "1 site created." } :<header Content-Type: application/json :<header X-Auth: the client's auth token :>json list sites: a list of sites to create :>json string sites[n].name: name of site :>json string sites[n].url: username search url for the site :>json string sites[n].category: category of the site :>json int sites[n].status_code: the status code to check for determining a match (nullable) :>json string sites[n].match_type: type of match (see get_match_types() for valid match types) (nullable) :>json string sites[n].match_expr: expression to use for determining a page match (nullable) :>json string sites[n].test_username_pos: username that exists on site (used for testing) :>json string sites[n].test_username_neg: username that does not exist on site (used for testing) :status 200: created :status 400: invalid request body :status 401: authentication required ''' request_json = request.get_json() sites = [] # Ensure all data is valid before db operations for site_json in request_json['sites']: validate_request_json(site_json, SITE_ATTRS) if (site_json['match_type'] is None or \ site_json['match_expr'] is None) and \ site_json['status_code'] is None: raise BadRequest('At least one of the following is required: ' 'status code or page match.') # Save sites for site_json in request_json['sites']: test_username_pos = site_json['test_username_pos'].lower().strip() site = Site(name=site_json['name'].lower().strip(), url=site_json['url'].lower().strip(), category=site_json['category'].lower().strip(), test_username_pos=test_username_pos) site.status_code = site_json['status_code'] site.match_expr = site_json['match_expr'] site.match_type = site_json['match_type'] if 'test_username_neg' in site_json: site.test_username_neg = site_json['test_username_neg'] \ .lower().strip(), g.db.add(site) try: g.db.flush() sites.append(site) except IntegrityError: g.db.rollback() raise BadRequest( 'Site URL {} already exists.'.format(site.url) ) g.db.commit() # Send redis notifications for site in sites: notify_mask_client( channel='site', message={ 'id': site.id, 'name': site.name, 'status': 'created', 'resource': None } ) message = '{} new sites created'.format(len(request_json['sites'])) response = jsonify(message=message) response.status_code = 202 return response