def delete(self, id_):
'''
Delete proxy identified by `id_`.
'''
# Get proxy.
id_ = get_int_arg('id_', id_)
proxy = g.db.query(Proxy).filter(Proxy.id == id_).first()
if proxy is None:
raise NotFound("Proxy '%s' does not exist." % id_)
# Delete proxy
try:
g.db.delete(proxy)
g.db.commit()
except IntegrityError:
g.db.rollback()
raise BadRequest('Could not delete proxy.')
# Send redis notifications
notify_mask_client(channel='proxy',
message={
'proxy': proxy.as_dict(),
'status': 'deleted',
'resource': None
})
message = 'Proxy id "{}" deleted'.format(id_)
response = jsonify(message=message)
response.status_code = 200
return response
def delete(self, id_):
'''
Delete site identified by `id_`.
'''
# Get site.
id_ = get_int_arg('id_', id_)
site = g.db.query(Site).filter(Site.id == id_).first()
if site is None:
raise NotFound("Site '%s' does not exist." % id_)
# Delete site
try:
g.db.delete(site)
g.db.commit()
except IntegrityError:
g.db.rollback()
raise BadRequest('"{}" must be removed from all groups '
'before deleting.'.format(site.name))
# Send redis notifications
notify_mask_client(channel='site',
message={
'id': site.id,
'name': site.name,
'status': 'deleted',
'resource': None
})
message = 'Site id "{}" deleted'.format(id_)
response = jsonify(message=message)
response.status_code = 200
return response
def delete(self, id_):
'''
Delete the category identified by `id`.
**Example Response**
..sourcecode:: json
{
"message": "Category `main` deleted",
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>header Content-Type: application/json
:>json str message: the API response message
:status 200: deleted
:status 400: invalid request body
:status 401: authentication required
:status 404: category does not exist
'''
# Get label.
id_ = get_int_arg('id_', id_)
category = g.db.query(Category).filter(Category.id == id_).first()
if category is None:
raise NotFound("Category '%s' does not exist." % id_)
# Delete label
g.db.delete(category)
try:
g.db.commit()
except IntegrityError:
g.db.rollback()
raise BadRequest(
'You must delete archived results that'
'use category "{}" before it can be deleted.'.format(
category.name))
except DBAPIError as e:
g.db.rollback()
raise BadRequest('Database error: {}'.format(e))
# Send redis notifications
notify_mask_client(channel='category',
message={
'id': category.id,
'name': category.name,
'status': 'deleted',
'resource': None
})
message = 'Category id "{}" deleted'.format(category.id)
response = jsonify(message=message)
response.status_code = 200
return response
def delete(self, id_):
'''
Delete archive identified by `id_`.
'''
# Get site.
id_ = get_int_arg('id_', id_)
archive = g.db.query(Archive).filter(Archive.id == id_).filter(
Archive.user_id == g.user.id).first()
if archive is None:
raise NotFound("Archive '%s' does not exist." % id_)
# Delete site
try:
g.db.delete(archive)
g.db.commit()
except IntegrityError:
g.db.rollback()
raise BadRequest('Could not delete archive.')
# Send redis notifications
notify_mask_client(channel='archive',
message={
'id': archive.id,
'name': archive.username,
'status': 'deleted',
'resource': None
})
message = 'Archive id "{}" deleted'.format(id_)
response = jsonify(message=message)
response.status_code = 200
return response
def delete(self, id_):
'''
Delete site identified by `id_`.
'''
# Get site.
id_ = get_int_arg('id_', id_)
site = g.db.query(Site).filter(Site.id == id_).first()
if site is None:
raise NotFound("Site '%s' does not exist." % id_)
try:
# Remove site from categories
categories = g.db.query(Category).filter(
Category.sites.contains(site)).all()
for category in categories:
category.sites.remove(site)
# Delete site
g.db.delete(site)
g.db.commit()
except Exception as e:
g.db.rollback()
raise BadRequest(e)
# Send redis notifications
notify_mask_client(channel='site',
message={
'id': site.id,
'name': site.name,
'status': 'deleted',
'resource': None
})
message = 'Site id "{}" deleted'.format(id_)
response = jsonify(message=message)
response.status_code = 200
return response
def put(self, id_):
'''
Update proxy identified by `id`.
**Example Request**
.. sourcecode:: json
PUT /api/proxies/id
{
"protocol": "http",
"host": "192.168.0.2",
"port": 80,
"username": "******",
"password": "******",
"active": true,
}
**Example Response**
.. sourcecode:: json
{
"id": 1,
"protocol": "http",
"host": "192.168.0.22",
"port": 80,
"username": "******",
"password": "******",
"active": true,
},
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json str protocol: protocol of proxy address
:>json str host: host of proxy address
:>json int port: port of proxy address
:>json str username: username of proxy
:>json str password: password of proxy
:>json bool active: proxy active status
:>header Content-Type: application/json
:>json int id: unique identifier
:>json str protocol: protocol of proxy address
:>json str host: host of proxy address
:>json int port: port of proxy address
:>json str username: username of proxy
:>json str password: password of proxy
:>json bool active: proxy active status
:status 200: ok
:status 401: authentication required
:status 403: must be an administrator
'''
# Get proxy
id_ = get_int_arg('id_', id_)
proxy = g.db.query(Proxy).filter(Proxy.id == id_).first()
if proxy is None:
raise NotFound("Proxy '%s' does not exist." % id_)
# Validate request json
request_json = request.get_json()
validate_request_json(request_json, PROXY_ATTRS)
# Update proxy
proxy.protocol = request_json['protocol']
proxy.host = request_json['host']
proxy.port = request_json['port']
proxy.active = request_json['active']
try:
proxy.username = request_json['username']
except KeyError:
pass
try:
proxy.password = request_json['password']
except KeyError:
pass
# Save the updated proxy
try:
g.db.commit()
except DBAPIError as e:
g.db.rollback()
raise BadRequest('Database error: {}'.format(e))
# Send redis notifications
notify_mask_client(channel='proxy',
message={
'proxy': proxy.as_dict(),
'status': 'updated',
'resource': None
})
response = jsonify(proxy.as_dict())
response.status_code = 200
# Send response
return response
def post(self):
'''
Create new proxies.
**Example Request**
.. sourcecode:: json
{
"proxies": [
{
"protocol": "http",
"host": "192.168.0.2",
"port": 80,
"username": "******",
"password": "******",
"active": true,
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"message": "1 proxy created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:<json list proxies: list of proxies
:<json str proxies[n]["protocol"]: protocol of proxy address
:<json str proxies[n]["host"]: host of proxy address
:<json int proxies[n]["port"]: port of proxy address
:<json str proxies[n]["username"]: username of proxy
:<json str proxies[n]["password"]: password of proxy
:<json bool proxies[n]["active"]: proxy active status
:>header Content-Type: application/json
:>json string message: API response message
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
proxies = []
# Ensure all data is valid before db operations
for proxy_json in request_json['proxies']:
validate_request_json(proxy_json, PROXY_ATTRS)
# Save proxies
for proxy_json in request_json['proxies']:
proxy = Proxy(protocol=proxy_json['protocol'].lower().strip(),
host=proxy_json['host'].lower().strip(),
port=proxy_json['port'],
active=proxy_json['active'])
# Username is optional, and can be None
try:
proxy.username = proxy_json['username'].lower().strip()
except KeyError:
pass
except AttributeError:
proxy.username = None
# Password is optional, and can be None
try:
proxy.password = proxy_json['password'].strip()
except KeyError:
pass
except AttributeError:
proxy.password = None
g.db.add(proxy)
try:
g.db.flush()
proxies.append(proxy)
except IntegrityError:
g.db.rollback()
raise BadRequest('Proxy {}://{}:{} already exists.'.format(
proxy.protocol, proxy.host, proxy.port))
g.db.commit()
# Send redis notifications
for proxy in proxies:
notify_mask_client(channel='proxy',
message={
'proxy': proxy.as_dict(),
'status': 'created',
'resource': None
})
message = '{} new proxies created'.format(len(request_json['proxies']))
response = jsonify(message=message)
response.status_code = 202
return response
def put(self, id_):
'''
Update the site identified by `id`.
**Example Request**
..sourcecode:: json
{
"name": "bebo",
"url": "http://bebo.com/usernames/search=%s",
"status_code": 200,
"match_type": "text",
"match_expr": "Foo Bar Baz",
"test_username_pos": "bob",
"test_username_ne": "adfjf393rfjffkjd",
"headers": {"referer": "http://www.google.com"},
"censor_images": false,
"wait_time": 5,
"use_proxy": false,
}
**Example Response**
..sourcecode:: json
{
"id": 2,
"name": "bebo",
"search_text": "Bebo User Page.</title>",
"status_code": 200,
"match_type": "text",
"match_expr": "Foo Bar Baz",
"url": "https://bebo.com/usernames/search=%s",
"test_username_pos": "bob",
"test_username_neg": "adfjf393rfjffkjd",
"test_status": "f",
"tested_at": "2016-01-01T00:00:00.000000+00:00",
"headers": {"referer": "http://www.google.com"},
"censor_images": false,
"wait_time": 5,
"use_proxy": false,
},
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:<json string name: name of site
:<json string url: username search url for the site
:<json string test_username_pos: username that exists on site
(used for testing)
:<json string test_username_neg: username that does not
exist on site (used for testing)
:<json array headers: custom headers
:<json bool censor_images: whether to censor images from this profile
:<json int wait_time: time (in seconds) to wait for updates
after page is loaded
:<json bool use_proxy: whether to proxy requests for this profile URL
:>header Content-Type: application/json
:>json int id: unique identifier for site
:>json str name: name of site
:>json str url: username search url for the site
:>json int status_code: the status code to check for
determining a match (nullable)
:>json string match_type: type of match (see get_match_types()
for valid match types) (nullable)
:>json string match_expr: expression to use for determining
a page match (nullable)
:>json str test_status: results of username test
:>json str tested_at: timestamp of last test
:>json str test_username_pos: username that exists on site
(used for testing)
:>json str test_username_neg: username that does not
exist on site (used for testing)
:>json array headers: custom headers
:>json bool censor_images: whether to censor images from this profile
:>json int wait_time: time (in seconds) to wait for updates after
page is loaded
:>json bool use_proxy: whether to proxy requests for this profile URL
:status 202: updated
:status 400: invalid request body
:status 401: authentication required
:status 404: site does not exist
'''
# Get site.
id_ = get_int_arg('id_', id_)
site = g.db.query(Site).filter(Site.id == id_).first()
if site is None:
raise NotFound("Site '%s' does not exist." % id_)
request_json = request.get_json()
# Validate data and set attributes
if 'name' in request_json:
validate_json_attr('name', _site_attrs, request_json)
site.name = request_json['name'].strip()
if 'url' in request_json:
validate_json_attr('url', _site_attrs, request_json)
site.url = request_json['url'].lower().strip()
if 'match_expr' in request_json:
validate_json_attr('match_expr', _site_attrs, request_json)
site.match_expr = request_json['match_expr']
if 'match_type' in request_json:
validate_json_attr('match_type', _site_attrs, request_json)
site.match_type = request_json['match_type'].strip()
if 'status_code' in request_json:
validate_json_attr('status_code', _site_attrs, request_json)
status = request_json['status_code']
site.status_code = None if status is None else int(status)
if (request_json['match_type'] is None or
request_json['match_expr'] is None) and \
request_json['status_code'] is None:
raise BadRequest('At least one of the '
'following is required: '
'status code or page match.')
if 'test_username_pos' in request_json:
validate_json_attr('test_username_pos', _site_attrs, request_json)
site.test_username_pos = (
request_json['test_username_pos'].lower().strip())
if 'test_username_neg' in request_json:
validate_json_attr('test_username_neg', _site_attrs, request_json)
site.test_username_neg = (
request_json['test_username_neg'].lower().strip())
if 'headers' in request_json:
validate_json_attr('headers', _site_attrs, request_json)
site.headers = request_json['headers']
if 'censor_images' in request_json:
validate_json_attr('censor_images', _site_attrs, request_json)
site.censor_images = request_json['censor_images']
if 'use_proxy' in request_json:
validate_json_attr('use_proxy', _site_attrs, request_json)
site.use_proxy = request_json['use_proxy']
if 'wait_time' in request_json:
validate_json_attr('wait_time', _site_attrs, request_json)
site.wait_time = request_json['wait_time']
# Save the updated site
try:
g.db.commit()
except DBAPIError as e:
g.db.rollback()
raise BadRequest('Database error: {}'.format(e))
# Send redis notifications
notify_mask_client(channel='site',
message={
'site': site.as_dict(),
'status': 'updated',
'resource': None
})
response = jsonify(site.as_dict())
response.status_code = 200
# Send response.
return response
def post(self):
'''
Create new sites to included in username searches.
**Example Request**
.. sourcecode:: json
{
"sites": [
{
"name": "about.me",
"url": "http://about.me/%s",
"status_code": 200,
"match_type": "text",
"match_expr": "Foo Bar Baz",
"test_username_pos": "john",
"test_username_neg": "dPGMFrf72SaS",
"headers": {"referer": "http://www.google.com"},
"censor_images": false,
"wait_time": 5,
"use_proxy": false,
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"message": "1 site created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:<json list sites: a list of sites to create
:<json string sites[n].name: name of site
:<json string sites[n].url: username search url for the site
:<json int sites[n].status_code: the status code to check for
determining a match (nullable)
:<json string sites[n].match_type: type of match (see get_match_types()
for valid match types) (nullable)
:<json string sites[n].match_expr: expression to use for determining
a page match (nullable)
:<json string sites[n].test_username_pos: username that exists on site
(used for testing)
:<json string sites[n].test_username_neg: username that does not exist
on site (used for testing)
:<json array sites[n].headers: custom headers
:<json bool sites[n].censor_images: whether to censor images
from this profile
:<json int sites[n].wait_time: time (in seconds) to wait for updates
after page is loaded
:<json bool sites[n].use_proxy: whether to proxy requests
for this profile URL
:>header Content-Type: application/json
:>json string message: API response message
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
sites = []
# Ensure all data is valid before db operations
for site_json in request_json['sites']:
validate_request_json(site_json, _site_attrs)
if (site_json['match_type'] is None or
site_json['match_expr'] is None) and \
site_json['status_code'] is None:
raise BadRequest('At least one of the '
'following is required: '
'status code or page match.')
if '%s' not in site_json['url']:
raise BadRequest('URL must contain replacement character %s')
# Save sites
for site_json in request_json['sites']:
test_username_pos = site_json['test_username_pos'].lower().strip()
site = Site(name=site_json['name'].strip(),
url=site_json['url'].lower().strip(),
test_username_pos=test_username_pos)
site.status_code = site_json['status_code']
site.match_expr = site_json['match_expr']
site.match_type = site_json['match_type']
if 'test_username_neg' in site_json:
site.test_username_neg = site_json['test_username_neg'] \
.lower().strip(),
if 'headers' in site_json:
site.headers = site_json['headers']
g.db.add(site)
try:
g.db.flush()
sites.append(site)
except IntegrityError:
g.db.rollback()
raise BadRequest('Site URL {} already exists.'.format(
site.url))
g.db.commit()
# Send redis notifications
for site in sites:
notify_mask_client(channel='site',
message={
'id': site.id,
'name': site.name,
'status': 'created',
'resource': None
})
message = '{} new sites created'.format(len(request_json['sites']))
response = jsonify(message=message)
response.status_code = 202
return response
def post(self):
'''
Create a category.
**Example Request**
..sourcode:: json
{
"categories": [
{
"name": "gender",
"sites": [1, 2, 7]
},
...
]
}
**Example Response**
..sourcecode:: json
{
"message": "2 new categories created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list categories: a list of categories to create
:>json str categories[n].name: name of category to create
:>header Content-Type: application/json
:>json str message: api response message
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
categories = list()
# Validate input
for category_json in request_json['categories']:
validate_request_json(category_json, GROUP_ATTRS)
try:
request_site_ids = [int(s) for s in category_json['sites']]
except TypeError:
raise BadRequest('Sites must be integer site ids')
if len(request_site_ids) == 0:
raise BadRequest('At least one site is required.')
sites = g.db.query(Site)\
.filter(Site.id.in_(request_site_ids))\
.all()
site_ids = [site.id for site in sites]
missing_sites = list(set(request_site_ids) - set(site_ids))
if len(missing_sites) > 0:
raise BadRequest('Site ids {} do not exist'.format(','.join(
str(s) for s in missing_sites)))
# Create categories
for category_json in request_json['categories']:
try:
category = Category(name=category_json['name'].strip(),
sites=sites)
g.db.add(category)
g.db.flush()
# Create dict for API JSON response
category_dict = category.as_dict()
# Add a link to the created category
category_dict['url-for'] = url_for('CategoryView:get',
id_=category.id)
categories.append(category_dict)
except IntegrityError:
g.db.rollback()
raise BadRequest('Category "{}" already exists'.format(
category.name))
# Save categories
g.db.commit()
# Send redis notifications
for category in categories:
notify_mask_client(channel='category',
message={
'id': category['id'],
'name': category['name'],
'status': 'created',
'resource': category['url-for']
})
message = '{} new categories created' \
.format(len(request_json['categories']))
response = jsonify(message=message, categories=categories)
response.status_code = 200
return response
def put(self, id_):
'''
Update the category identified by `id`.
**Example Request**
..sourcode:: json
{
{
"name": "priority sites"
"sites": [1,5]
},
}
**Example Response**
..sourcecode:: json
{
"id": 1,
"name": "priority sites",
"sites": [
{
"category": "books",
"id": 1,
"name": "aNobil",
"search_text": "- aNobii</title>",
"status_code": 200,
"url": "http://www.anobii.com/%s/books"
},
{
"category": "coding",
"id": 5,
"name": "bitbucket",
"search_text": "\"username\":",
"status_code": 200,
"url": "https://bitbucket.org/api/2.0/users/%s"
},
...
]
},
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json str name: the value of the name attribute
:>header Content-Type: application/json
:>json int id: unique identifier for category
:>json str name: the category name
:>json list sites: list of sites associated with this category
:>json str sites[n].category: the site category
:>json str sites[n].id: the unique id for site
:>json str sites[n].name: the site name
:>json str sites[n].search_text: string search pattern
:>json str sites[n].status_code: server response code for site
:>json str sites[n].url: the site url
:status 200: updated
:status 400: invalid request body
:status 401: authentication required
'''
editable_fields = ['name', 'sites']
# Get category.
id_ = get_int_arg('id_', id_)
category = g.db.query(Category).filter(Category.id == id_).first()
if category is None:
raise NotFound("Category '%s' does not exist." % id_)
request_json = request.get_json()
# Validate data and set attributes
if request_json is None:
raise BadRequest("Specify at least one editable field: {}".format(
editable_fields))
for field in request_json:
if field not in editable_fields:
raise BadRequest(
"'{}' is not one of the editable fields: {}".format(
field, editable_fields))
if 'name' in request_json:
validate_json_attr('name', GROUP_ATTRS, request_json)
category.name = request_json['name'].strip()
if 'sites' in request_json:
try:
request_site_ids = [int(s) for s in request_json['sites']]
except ValueError:
raise BadRequest('Sites must be a list of integer site ids')
if len(request_site_ids) == 0:
raise BadRequest('Categorys must have at least one site')
sites = g.db.query(Site) \
.filter(Site.id.in_(request_site_ids)) \
.all()
site_ids = [site.id for site in sites]
missing_sites = list(set(request_site_ids) - set(site_ids))
if len(missing_sites) > 0:
raise BadRequest('Site ids "{}" do not exist'.format(
','.join(missing_sites)))
else:
category.sites = sites
# Save the updated category
g.db.add(category)
try:
g.db.commit()
except DBAPIError as e:
g.db.rollback()
raise BadRequest('Database error: {}'.format(e))
# Send redis notifications
notify_mask_client(channel='category',
message={
'id':
category.id,
'name':
category.name,
'status':
'updated',
'resource':
url_for('CategoryView:get', id_=category.id)
})
response = category.as_dict()
response['url-for'] = url_for('CategoryView:get', id_=category.id)
# Send response.
return jsonify(**response)
def post(self):
'''
Create new sites to included in username searches.
**Example Request**
.. sourcecode:: json
{
"sites": [
{
"name": "about.me",
"url": "http://about.me/%s",
"category": "social",
"status_code": 200,
"match_type": "text",
"match_expr": "Foo Bar Baz",
"test_username_pos": "john",
"test_username_neg": "dPGMFrf72SaS"
},
...
]
}
**Example Response**
.. sourcecode:: json
{
"message": "1 site created."
}
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:>json list sites: a list of sites to create
:>json string sites[n].name: name of site
:>json string sites[n].url: username search url for the site
:>json string sites[n].category: category of the site
:>json int sites[n].status_code: the status code to check for
determining a match (nullable)
:>json string sites[n].match_type: type of match (see get_match_types()
for valid match types) (nullable)
:>json string sites[n].match_expr: expression to use for determining
a page match (nullable)
:>json string sites[n].test_username_pos: username that exists on site
(used for testing)
:>json string sites[n].test_username_neg: username that does not exist
on site (used for testing)
:status 200: created
:status 400: invalid request body
:status 401: authentication required
'''
request_json = request.get_json()
sites = []
# Ensure all data is valid before db operations
for site_json in request_json['sites']:
validate_request_json(site_json, SITE_ATTRS)
if (site_json['match_type'] is None or \
site_json['match_expr'] is None) and \
site_json['status_code'] is None:
raise BadRequest('At least one of the following is required: '
'status code or page match.')
# Save sites
for site_json in request_json['sites']:
test_username_pos = site_json['test_username_pos'].lower().strip()
site = Site(name=site_json['name'].lower().strip(),
url=site_json['url'].lower().strip(),
category=site_json['category'].lower().strip(),
test_username_pos=test_username_pos)
site.status_code = site_json['status_code']
site.match_expr = site_json['match_expr']
site.match_type = site_json['match_type']
if 'test_username_neg' in site_json:
site.test_username_neg = site_json['test_username_neg'] \
.lower().strip(),
g.db.add(site)
try:
g.db.flush()
sites.append(site)
except IntegrityError:
g.db.rollback()
raise BadRequest(
'Site URL {} already exists.'.format(site.url)
)
g.db.commit()
# Send redis notifications
for site in sites:
notify_mask_client(
channel='site',
message={
'id': site.id,
'name': site.name,
'status': 'created',
'resource': None
}
)
message = '{} new sites created'.format(len(request_json['sites']))
response = jsonify(message=message)
response.status_code = 202
return response
