def post(self):
'''
Request search of usernames.
**Example Request**
.. sourcecode:: json
{
"usernames": [
"johndoe",
"janedoe",
...
],
"category": 3,
"test": False,
}
**Example Response**
.. sourcecode:: json
{
"tracker_ids": {
"johndoe": "tracker.12344565",
}
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list usernames: a list of usernames to search for
:>json int category: ID of site category to use (optional)
:>json int site: ID of site to search (optional)
:>json bool test: test results (optional, default: false)
:>header Content-Type: application/json
:>json list jobs: list of worker jobs
:>json list jobs[n].id: unique id of this job
:>json list jobs[n].usename: username target of this job
:status 202: accepted for background processing
:status 400: invalid request body
:status 401: authentication required
'''
test = False
category = None
category_id = None
jobs = []
tracker_ids = dict()
redis = g.redis
request_json = request.get_json()
site = None
if 'usernames' not in request_json:
raise BadRequest('`usernames` is required')
validate_request_json(request_json, _username_attrs)
if len(request_json['usernames']) == 0:
raise BadRequest('At least one username is required')
if 'category' in request_json and 'site' in request_json:
raise BadRequest('Supply either `category` or `site`.')
if 'category' in request_json:
category_id = request_json['category']
category = g.db.query(Category) \
.filter(Category.id == category_id).first()
if category is None:
raise NotFound("Category '%s' does not exist." % category_id)
else:
category_id = category.id
if 'site' in request_json:
site_id = request_json['site']
site = g.db.query(Site).filter(Site.id == site_id).first()
if site is None:
raise NotFound("Site '%s' does not exist." % site_id)
if 'test' in request_json:
test = request_json['test']
if category:
sites = category.sites
elif site:
sites = g.db.query(Site).filter(Site.id == site.id).all()
else:
sites = g.db.query(Site).all()
# Only check valid sites.
valid_sites = []
for site in sites:
if site.valid:
valid_sites.append(site)
# sites = sites.filter(Site.valid == True).all() # noqa
if len(valid_sites) == 0:
raise NotFound('No valid sites to check')
for username in request_json['usernames']:
# Create an object in redis to track the number of sites completed
# in this search.
tracker_id = 'tracker.{}'.format(random_string(10))
tracker_ids[username] = tracker_id
redis.set(tracker_id, 0)
redis.expire(tracker_id, 600)
total = len(valid_sites)
# Queue a job for each site.
for site in valid_sites:
description = 'Checking {} for user "{}"'.format(site.name,
username)
job = worker.scrape.check_username.enqueue(
username=username,
site_id=site.id,
category_id=category_id,
total=total,
tracker_id=tracker_id,
test=test,
jobdesc=description,
timeout=_redis_worker['username_timeout'],
user_id=g.user.id
)
jobs.append({
'id': job.id,
'username': username,
'category': category_id,
})
response = jsonify(tracker_ids=tracker_ids)
response.status_code = 202
return response
def post(self):
'''
Create new proxies.
**Example Request**
.. sourcecode:: json
{
"proxies": [
{
"protocol": "http",
"host": "192.168.0.2",
"port": 80,
"username": "******",
"password": "******",
"active": true,
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"message": "1 proxy created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:<json list proxies: list of proxies
:<json str proxies[n]["protocol"]: protocol of proxy address
:<json str proxies[n]["host"]: host of proxy address
:<json int proxies[n]["port"]: port of proxy address
:<json str proxies[n]["username"]: username of proxy
:<json str proxies[n]["password"]: password of proxy
:<json bool proxies[n]["active"]: proxy active status
:>header Content-Type: application/json
:>json string message: API response message
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
proxies = []
# Ensure all data is valid before db operations
for proxy_json in request_json['proxies']:
validate_request_json(proxy_json, PROXY_ATTRS)
# Save proxies
for proxy_json in request_json['proxies']:
proxy = Proxy(protocol=proxy_json['protocol'].lower().strip(),
host=proxy_json['host'].lower().strip(),
port=proxy_json['port'],
active=proxy_json['active'])
# Username is optional, and can be None
try:
proxy.username = proxy_json['username'].lower().strip()
except KeyError:
pass
except AttributeError:
proxy.username = None
# Password is optional, and can be None
try:
proxy.password = proxy_json['password'].strip()
except KeyError:
pass
except AttributeError:
proxy.password = None
g.db.add(proxy)
try:
g.db.flush()
proxies.append(proxy)
except IntegrityError:
g.db.rollback()
raise BadRequest('Proxy {}://{}:{} already exists.'.format(
proxy.protocol, proxy.host, proxy.port))
g.db.commit()
# Send redis notifications
for proxy in proxies:
notify_mask_client(channel='proxy',
message={
'proxy': proxy.as_dict(),
'status': 'created',
'resource': None
})
message = '{} new proxies created'.format(len(request_json['proxies']))
response = jsonify(message=message)
response.status_code = 202
return response
def put(self, id_):
'''
Update proxy identified by `id`.
**Example Request**
.. sourcecode:: json
PUT /api/proxies/id
{
"protocol": "http",
"host": "192.168.0.2",
"port": 80,
"username": "******",
"password": "******",
"active": true,
}
**Example Response**
.. sourcecode:: json
{
"id": 1,
"protocol": "http",
"host": "192.168.0.22",
"port": 80,
"username": "******",
"password": "******",
"active": true,
},
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json str protocol: protocol of proxy address
:>json str host: host of proxy address
:>json int port: port of proxy address
:>json str username: username of proxy
:>json str password: password of proxy
:>json bool active: proxy active status
:>header Content-Type: application/json
:>json int id: unique identifier
:>json str protocol: protocol of proxy address
:>json str host: host of proxy address
:>json int port: port of proxy address
:>json str username: username of proxy
:>json str password: password of proxy
:>json bool active: proxy active status
:status 200: ok
:status 401: authentication required
:status 403: must be an administrator
'''
# Get proxy
id_ = get_int_arg('id_', id_)
proxy = g.db.query(Proxy).filter(Proxy.id == id_).first()
if proxy is None:
raise NotFound("Proxy '%s' does not exist." % id_)
# Validate request json
request_json = request.get_json()
validate_request_json(request_json, PROXY_ATTRS)
# Update proxy
proxy.protocol = request_json['protocol']
proxy.host = request_json['host']
proxy.port = request_json['port']
proxy.active = request_json['active']
try:
proxy.username = request_json['username']
except KeyError:
pass
try:
proxy.password = request_json['password']
except KeyError:
pass
# Save the updated proxy
try:
g.db.commit()
except DBAPIError as e:
g.db.rollback()
raise BadRequest('Database error: {}'.format(e))
# Send redis notifications
notify_mask_client(channel='proxy',
message={
'proxy': proxy.as_dict(),
'status': 'updated',
'resource': None
})
response = jsonify(proxy.as_dict())
response.status_code = 200
# Send response
return response
def post(self):
'''
Create new sites to included in username searches.
**Example Request**
.. sourcecode:: json
{
"sites": [
{
"name": "about.me",
"url": "http://about.me/%s",
"status_code": 200,
"match_type": "text",
"match_expr": "Foo Bar Baz",
"test_username_pos": "john",
"test_username_neg": "dPGMFrf72SaS",
"headers": {"referer": "http://www.google.com"},
"censor_images": false,
"wait_time": 5,
"use_proxy": false,
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"message": "1 site created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:<json list sites: a list of sites to create
:<json string sites[n].name: name of site
:<json string sites[n].url: username search url for the site
:<json int sites[n].status_code: the status code to check for
determining a match (nullable)
:<json string sites[n].match_type: type of match (see get_match_types()
for valid match types) (nullable)
:<json string sites[n].match_expr: expression to use for determining
a page match (nullable)
:<json string sites[n].test_username_pos: username that exists on site
(used for testing)
:<json string sites[n].test_username_neg: username that does not exist
on site (used for testing)
:<json array sites[n].headers: custom headers
:<json bool sites[n].censor_images: whether to censor images
from this profile
:<json int sites[n].wait_time: time (in seconds) to wait for updates
after page is loaded
:<json bool sites[n].use_proxy: whether to proxy requests
for this profile URL
:>header Content-Type: application/json
:>json string message: API response message
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
sites = []
# Ensure all data is valid before db operations
for site_json in request_json['sites']:
validate_request_json(site_json, _site_attrs)
if (site_json['match_type'] is None or
site_json['match_expr'] is None) and \
site_json['status_code'] is None:
raise BadRequest('At least one of the '
'following is required: '
'status code or page match.')
if '%s' not in site_json['url']:
raise BadRequest('URL must contain replacement character %s')
# Save sites
for site_json in request_json['sites']:
test_username_pos = site_json['test_username_pos'].lower().strip()
site = Site(name=site_json['name'].strip(),
url=site_json['url'].lower().strip(),
test_username_pos=test_username_pos)
site.status_code = site_json['status_code']
site.match_expr = site_json['match_expr']
site.match_type = site_json['match_type']
if 'test_username_neg' in site_json:
site.test_username_neg = site_json['test_username_neg'] \
.lower().strip(),
if 'headers' in site_json:
site.headers = site_json['headers']
g.db.add(site)
try:
g.db.flush()
sites.append(site)
except IntegrityError:
g.db.rollback()
raise BadRequest('Site URL {} already exists.'.format(
site.url))
g.db.commit()
# Send redis notifications
for site in sites:
notify_mask_client(channel='site',
message={
'id': site.id,
'name': site.name,
'status': 'created',
'resource': None
})
message = '{} new sites created'.format(len(request_json['sites']))
response = jsonify(message=message)
response.status_code = 202
return response
def post_jobs_for_site(self, site_id):
"""
Request background jobs for site identified by `id`.
**Example Request**
..sourcode:: json
{
"jobs": [
{
"name": "test",
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"tracker_ids": {
"1": "tracker.12344565",
}
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list jobs: a list of jobs to schedule
:>json string jobs[n].name: name of job
:>header Content-Type: application/json
:>json array tracker_ids: array of worker tracking ids
{site ID: tracker ID}
:status 202: scheduled
:status 400: invalid request body
:status 401: authentication required
"""
request_attrs = {
'jobs': {
'type': list,
'required': True
},
}
job_attrs = {
'name': {
'type': str,
'required': True
},
}
available_jobs = ['test']
tracker_ids = dict()
# Get site.
site_id = get_int_arg('site_id', site_id)
site = g.db.query(Site).filter(Site.id == site_id).first()
# Validate
if site is None:
raise NotFound("Site '%s' does not exist." % site_id)
request_json = request.get_json()
validate_request_json(request_json, request_attrs)
for job in request_json['jobs']:
validate_json_attr('name', job_attrs, job)
if job['name'] not in available_jobs:
raise BadRequest('`{}` does not exist in available'
' jobs: {}'.format(job['name'],
','.join(available_jobs)))
# Schedule jobs
for job in request_json['jobs']:
tracker_id = 'tracker.{}'.format(random_string(10))
tracker_ids[site.id] = tracker_id
if job['name'] == 'test':
description = 'Testing site "{}"'.format(site.name)
worker.scrape.test_site.enqueue(
site_id=site.id,
tracker_id=tracker_id,
jobdesc=description,
user_id=g.user.id,
)
response = jsonify(tracker_ids=tracker_ids)
response.status_code = 202
return response
def post(self):
'''
Request search of usernames.
**Example Request**
.. sourcecode:: json
{
"usernames": [
"johndoe",
"janedoe",
...
],
"group": 3,
"test": False,
}
**Example Response**
.. sourcecode:: json
{
"tracker_ids": {
"johndoe": "tracker.12344565",
}
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list usernames: a list of usernames to search for
:>json int group: ID of site group to use (optional)
:>json int site: ID of site to search (optional)
:>json bool test: test results (optional, default: false)
:>header Content-Type: application/json
:>json list jobs: list of worker jobs
:>json list jobs[n].id: unique id of this job
:>json list jobs[n].usename: username target of this job
:status 202: accepted for background processing
:status 400: invalid request body
:status 401: authentication required
'''
test = False
group = None
group_id = None
jobs = []
tracker_ids = dict()
redis = g.redis
request_json = request.get_json()
site = None
if 'usernames' not in request_json:
raise BadRequest('`usernames` is required')
validate_request_json(request_json, USERNAME_ATTRS)
if len(request_json['usernames']) == 0:
raise BadRequest('At least one username is required')
if 'group' in request_json and 'site' in request_json:
raise BadRequest('Supply either `group` or `site`.')
if 'group' in request_json:
group_id = request_json['group']
group = g.db.query(Group).filter(Group.id == group_id).first()
if group is None:
raise NotFound("Group '%s' does not exist." % group_id)
else:
group_id = group.id
if 'site' in request_json:
site_id = request_json['site']
site = g.db.query(Site).filter(Site.id == site_id).first()
if site is None:
raise NotFound("Site '%s' does not exist." % site_id)
if 'test' in request_json:
test = request_json['test']
if group:
sites = group.sites
elif site:
sites = g.db.query(Site).filter(Site.id == site.id)
else:
sites = g.db.query(Site)
# Only check valid sites.
sites = sites.filter(Site.valid == True).all() # noqa
if len(sites) == 0:
raise NotFound('No valid sites to check')
for username in request_json['usernames']:
# Create an object in redis to track the number of sites completed
# in this search.
tracker_id = 'tracker.{}'.format(random_string(10))
tracker_ids[username] = tracker_id
redis.set(tracker_id, 0)
redis.expire(tracker_id, 600)
total = len(sites)
# Queue a job for each site.
for site in sites:
job_id = app.queue.schedule_username(
username=username,
site=site,
group_id=group_id,
total=total,
tracker_id=tracker_id,
test=test
)
jobs.append({
'id': job_id,
'username': username,
'group': group_id,
})
response = jsonify(tracker_ids=tracker_ids)
response.status_code = 202
return response
def post_jobs_for_sites(self):
"""
Request background jobs for all sites.
**Example Request**
..sourcode:: json
{
"jobs": [
{
"name": "test",
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"tracker_ids": {
"1": "tracker.12344565",
}
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list jobs: a list of jobs to schedule
:>json string jobs[n].name: name of job
:>header Content-Type: application/json
:>json array tracker_ids: array of worker tracking ids
:status 202: scheduled
:status 400: invalid request body
:status 401: authentication required
"""
request_attrs = {
'jobs': {
'type': list,
'required': True
},
}
job_attrs = {
'name': {
'type': str,
'required': True
},
}
available_jobs = ['test']
tracker_ids = dict()
request_json = request.get_json()
validate_request_json(request_json, request_attrs)
for job in request_json['jobs']:
validate_json_attr('name', job_attrs, job)
if job['name'] not in available_jobs:
raise BadRequest('`{}` does not exist in available'
' jobs: {}'.format(job['name'],
','.join(available_jobs)))
# Get sites.
sites = g.db.query(Site).all()
# Schedule jobs
for job in request_json['jobs']:
for site in sites:
tracker_id = 'tracker.{}'.format(random_string(10))
tracker_ids[site.id] = tracker_id
if job['name'] == 'test':
app.queue.schedule_site_test(
site=site,
tracker_id=tracker_id,
)
response = jsonify(tracker_ids=tracker_ids)
response.status_code = 202
return response
def post(self):
'''
Process user payment.
**Example Request**
.. sourcecode:: json
{
"user_id": 1,
"stripe_token": "tok_1A9VDuL25MRJTn0APWrFQrN6",
"credits": 400
"currency": "usd",
"description": "200 credits for $20",
}
**Example Response**
.. sourcecode:: json
{
"message": "200 credits added."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:<json int user_id: the user ID
:<json str stripe_token: the stripe payment token
:<json int credits: the purchase credits
:>header Content-Type: application/json
:>json string message: API response message
:status 200: ok
:status 400: invalid request body
:status 401: authentication required
:status 403: not authorized to make the requested changes
'''
# Validate json input
request_json = request.get_json()
validate_request_json(request_json, _payment_attrs)
user = g.db.query(User).filter(
User.id == request_json['user_id']).first()
if g.user.id != user.id:
raise Forbidden('You may only purchase credits for '
'your own account.')
# Configure stripe client
try:
stripe.api_key = get_config(session=g.db,
key='stripe_secret_key',
required=True).value
except Exception as e:
raise ServiceUnavailable(e)
key = 'credit_cost'
credit_cost = g.db.query(Configuration) \
.filter(Configuration.key == key) \
.first()
if credit_cost is None:
raise NotFound(
'There is no configuration item named "{}".'.format(key))
# Stripe token is created client-side using Stripe.js
token = request_json['stripe_token']
# Get payment paremeters
credits = int(request_json['credits'])
description = request_json['description']
currency = request_json['currency']
costs = self._get_costs(credit_cost.value)
# Calculate credit amount.
try:
amount = costs[credits]
# credits = list(costs.keys())[list(
# costs.values()).index(int(amount))]
except IndexError:
raise BadRequest('Invalid credit amount.')
try:
# Charge the user's card:
charge = stripe.Charge.create(amount=amount,
currency=currency,
description=description,
source=token)
except stripe.error.CardError as e:
# Since it's a decline, stripe.error.CardError will be caught
body = e.json_body
err = body['error']
raise BadRequest('Card error: {}'.format(err['message']))
except stripe.error.RateLimitError as e:
# Too many requests made to the API too quickly
body = e.json_body
err = body['error']
raise BadRequest('Rate limit error: {}'.format(err['message']))
except stripe.error.InvalidRequestError as e:
# Invalid parameters were supplied to Stripe's API
body = e.json_body
err = body['error']
raise BadRequest('Invalid parameters: {}'.format(err['message']))
except stripe.error.AuthenticationError as e:
# Authentication with Stripe's API failed
# (maybe API keys changed recently)
body = e.json_body
err = body['error']
raise ServiceUnavailable('Stripe authentication error: {}'.format(
err['message']))
except stripe.error.APIConnectionError as e:
# Network communication with Stripe failed
body = e.json_body
err = body['error']
raise ServiceUnavailable(
'Stripe API communication failed: {}'.format(err['message']))
except stripe.error.StripeError as e:
# Generic error
body = e.json_body
err = body['error']
raise ServiceUnavailable('Stripe error: {}'.format(err['message']))
except Exception as e:
# Something else happened, completely unrelated to Stripe
raise ServiceUnavailable('Error: {}'.format(e))
user.credits += credits
g.db.commit()
g.redis.publish('user', json.dumps(user.as_dict()))
message = '{} credits added.'.format(amount)
response = jsonify(message=message)
response.status_code = 202
return response
def post(self):
'''
Create a category.
**Example Request**
..sourcode:: json
{
"categories": [
{
"name": "gender",
"sites": [1, 2, 7]
},
...
]
}
**Example Response**
..sourcecode:: json
{
"message": "2 new categories created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list categories: a list of categories to create
:>json str categories[n].name: name of category to create
:>header Content-Type: application/json
:>json str message: api response message
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
categories = list()
# Validate input
for category_json in request_json['categories']:
validate_request_json(category_json, GROUP_ATTRS)
try:
request_site_ids = [int(s) for s in category_json['sites']]
except TypeError:
raise BadRequest('Sites must be integer site ids')
if len(request_site_ids) == 0:
raise BadRequest('At least one site is required.')
sites = g.db.query(Site)\
.filter(Site.id.in_(request_site_ids))\
.all()
site_ids = [site.id for site in sites]
missing_sites = list(set(request_site_ids) - set(site_ids))
if len(missing_sites) > 0:
raise BadRequest('Site ids {} do not exist'.format(','.join(
str(s) for s in missing_sites)))
# Create categories
for category_json in request_json['categories']:
try:
category = Category(name=category_json['name'].strip(),
sites=sites)
g.db.add(category)
g.db.flush()
# Create dict for API JSON response
category_dict = category.as_dict()
# Add a link to the created category
category_dict['url-for'] = url_for('CategoryView:get',
id_=category.id)
categories.append(category_dict)
except IntegrityError:
g.db.rollback()
raise BadRequest('Category "{}" already exists'.format(
category.name))
# Save categories
g.db.commit()
# Send redis notifications
for category in categories:
notify_mask_client(channel='category',
message={
'id': category['id'],
'name': category['name'],
'status': 'created',
'resource': category['url-for']
})
message = '{} new categories created' \
.format(len(request_json['categories']))
response = jsonify(message=message, categories=categories)
response.status_code = 200
return response
def post(self):
'''
Create new sites to included in username searches.
**Example Request**
.. sourcecode:: json
{
"sites": [
{
"name": "about.me",
"url": "http://about.me/%s",
"category": "social",
"status_code": 200,
"match_type": "text",
"match_expr": "Foo Bar Baz",
"test_username_pos": "john",
"test_username_neg": "dPGMFrf72SaS"
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"message": "1 site created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list sites: a list of sites to create
:>json string sites[n].name: name of site
:>json string sites[n].url: username search url for the site
:>json string sites[n].category: category of the site
:>json int sites[n].status_code: the status code to check for
determining a match (nullable)
:>json string sites[n].match_type: type of match (see get_match_types()
for valid match types) (nullable)
:>json string sites[n].match_expr: expression to use for determining
a page match (nullable)
:>json string sites[n].test_username_pos: username that exists on site
(used for testing)
:>json string sites[n].test_username_neg: username that does not exist
on site (used for testing)
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
sites = []
# Ensure all data is valid before db operations
for site_json in request_json['sites']:
validate_request_json(site_json, SITE_ATTRS)
if (site_json['match_type'] is None or \
site_json['match_expr'] is None) and \
site_json['status_code'] is None:
raise BadRequest('At least one of the following is required: '
'status code or page match.')
# Save sites
for site_json in request_json['sites']:
test_username_pos = site_json['test_username_pos'].lower().strip()
site = Site(name=site_json['name'].lower().strip(),
url=site_json['url'].lower().strip(),
category=site_json['category'].lower().strip(),
test_username_pos=test_username_pos)
site.status_code = site_json['status_code']
site.match_expr = site_json['match_expr']
site.match_type = site_json['match_type']
if 'test_username_neg' in site_json:
site.test_username_neg = site_json['test_username_neg'] \
.lower().strip(),
g.db.add(site)
try:
g.db.flush()
sites.append(site)
except IntegrityError:
g.db.rollback()
raise BadRequest(
'Site URL {} already exists.'.format(site.url)
)
g.db.commit()
# Send redis notifications
for site in sites:
notify_mask_client(
channel='site',
message={
'id': site.id,
'name': site.name,
'status': 'created',
'resource': None
}
)
message = '{} new sites created'.format(len(request_json['sites']))
response = jsonify(message=message)
response.status_code = 202
return response