def add_logentry():
"""Creating new log entry and saving it to DB.
Returns False, if validation had problems with validating inputs,
returns False, if client tries to send not a json,
otherwise return True and ObjectId.
"""
if request.headers['Content-Type'] == 'application/json':
errors = []
request_data = request.json
# Lets check selected level
level = ""
if "level" not in request_data:
errors.append({'level': 'Field required.'})
else:
level = request_data['level']
# Is level in level list in config file?
if level not in config['level']:
errors.append({'level': 'Unknown level type.'})
# Checking owner present (required)
owner = ""
if "owner" not in request_data:
errors.append({'owner': 'Field required.'})
else:
owner = request_data['owner']
# Checking data present (required)
data = ""
if "data" not in request_data:
errors.append({'data': 'Field required.'})
else:
data = request_data['data']
tags = []
# Tags isn't required. If it present lets try to convert it to python-list.
# If successfully - add it to entry. If not - return full error and don't create entry in DB.
if "tags" in request_data:
tags = request.json['tags']
if not isinstance(tags, list):
errors.append({'tags': 'Tags must be an array.'})
if not errors:
entry = LogEntry(level, owner, data, tags)
id_or_error = entry.save()
if not isinstance(id_or_error, ObjectId):
return jsonify({'OK': False, 'error': id_or_error})
# ___str___ is a string representation of JS ObjectID from MongoDB.
return jsonify({'OK': True, 'id': id_or_error.__str__()})
else:
return jsonify({"OK": False, 'errors': errors})
else:
#TODO Here should be NORMAL exception.
return jsonify({"errors": ["415 Unsupported Media Type. \"application/json\" required.\n",]})
def files_types(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
result = simple_group('file', 'content_guess', mongodb)
return jsonify(result, response)
def hpfeeds(mongodb):
try:
auth.require(role='access_all')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
mongo_keys = {'_id', 'id', 'channel'}
#intersection
common_keys = (set(query_keys) & mongo_keys)
try:
for item in common_keys:
if item.endswith('_id'):
query_dict[item] = ObjectId(request.query[item])
elif item == 'id':
query_dict['_' + item] = ObjectId(request.query[item])
else:
query_dict[item] = request.query[item]
except InvalidId:
abort(400, 'Not a valid ObjectId.')
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
result = list(mongodb['hpfeed'].find(query_dict).sort('timestamp',
-1).limit(limit))
return jsonify({'hpfeeds': result}, response)
def get_files(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
if 'hash' in query_keys:
hash_length = len(request.query['hash'])
if hash_length is 128:
query_dict['hashes.sha512'] = request.query['hash']
elif hash_length is 40:
query_dict['hashes.sha1'] = request.query['hash']
elif hash_length is 32:
query_dict['hashes.md5'] = request.query['hash']
else:
abort(400, '{0} could not be recognized as a supported hash. Currently supported hashes are: SHA1, SHA512 and MD5. ')
else:
abort(400, 'Only supported query parameter is "hash"')
p_limit = {'_id': False}
if 'no_data' in query_keys:
p_limit['data'] = False
result = list(mongodb['file'].find(query_dict, fields=p_limit).limit(limit))
return jsonify({'files': result}, response)
def files_types(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
result = simple_group('file', 'content_guess', mongodb)
return jsonify(result, response)
def urls(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
if 'url_regex' in query_keys:
query_dict['url'] = {'$regex': request.query.url_regex}
if 'hash' in query_keys:
hash_length = len(request.query['hash'])
if hash_length is 128:
query_dict['extractions.hashes.sha512'] = request.query['hash']
elif hash_length is 40:
query_dict['extractions.hashes.sha1'] = request.query['hash']
elif hash_length is 32:
query_dict['extractions.hashes.md5'] = request.query['hash']
else:
abort(400, '{0} could not be recognized as a supported hash. Currently supported hashes are: SHA1, SHA512 and MD5. ')
result = list(mongodb['url'].find(query_dict, fields={'_id': False}).limit(limit))
return jsonify({'urls': result}, response)
def __hello_world(request, query_params=None, **kwargs):
events = MotionEvent.objects.all()
retVal = {
'events': [e.to_dict() for e in events]
};
return HttpResponse(jsonify(retVal, query_params), mimetype="application/json")
def hpfeeds(mongodb):
try:
auth.require(role='access_all')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
mongo_keys = {'_id', 'id', 'channel'}
#intersection
common_keys = (set(query_keys) & mongo_keys)
try:
for item in common_keys:
if item.endswith('_id'):
query_dict[item] = ObjectId(request.query[item])
elif item == 'id':
query_dict['_' + item] = ObjectId(request.query[item])
else:
query_dict[item] = request.query[item]
except InvalidId:
abort(400, 'Not a valid ObjectId.')
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
result = list(mongodb['hpfeed'].find(query_dict).sort('timestamp', -1).limit(limit))
return jsonify({'hpfeeds': result}, response)
def put_detail(self, collection, pk):
"""Updates whole document."""
collection = self.get_collection(collection)
collection.update({"_id": ObjectId(pk)},
request.json)
response.status = 202
return jsonify(request.json)
def hpfeeds(mongodb):
try:
auth.require(role="access_all")
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
mongo_keys = {"_id", "id", "channel"}
# intersection
common_keys = set(query_keys) & mongo_keys
try:
for item in common_keys:
if item.endswith("_id"):
query_dict[item] = ObjectId(request.query[item])
elif item == "id":
query_dict["_" + item] = ObjectId(request.query[item])
else:
query_dict[item] = request.query[item]
except InvalidId:
abort(400, "Not a valid ObjectId.")
if "limit" in query_keys:
limit = int(request.query.limit)
else:
limit = 50
result = list(mongodb["hpfeed"].find(query_dict).sort("timestamp", -1).limit(limit))
return jsonify({"hpfeeds": result}, response)
def urls(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
if 'url_regex' in query_keys:
query_dict['url'] = {'$regex': request.query.url_regex}
if 'hash' in query_keys:
hash_length = len(request.query['hash'])
if hash_length is 128:
query_dict['extractions.hashes.sha512'] = request.query['hash']
elif hash_length is 40:
query_dict['extractions.hashes.sha1'] = request.query['hash']
elif hash_length is 32:
query_dict['extractions.hashes.md5'] = request.query['hash']
else:
abort(
400,
'{0} could not be recognized as a supported hash. Currently supported hashes are: SHA1, SHA512 and MD5. '
)
result = list(mongodb['url'].find(query_dict, fields={
'_id': False
}).limit(limit))
return jsonify({'urls': result}, response)
def get_levels_list():
"""Levels list
Returns level list from config file as a json-list.
"""
levels_list = config['level']
return jsonify({'level': levels_list})
def create_charge():
"""Returns POST Data."""
extracted = h.get_dict('url', 'args', 'form', 'data', 'origin', 'headers',
'files', 'json')
charge_entry = Charge.from_dict(extracted['json'])
db.session.add(charge_entry)
db.session.commit()
return h.jsonify(extracted)
def files_types(mongodb):
try:
auth.require(role="access_normalized")
except AAAException as e:
return HTTPError(401, e.message)
return HTTPError(410, "This part of the API has been temporarily disabled to due to performance issues.")
result = simple_group("file", "content_guess", mongodb)
return jsonify(result, response)
def create_charge():
"""Returns POST Data."""
extracted = h.get_dict(
'url', 'args', 'form', 'data', 'origin', 'headers', 'files', 'json')
charge_entry = Charge.from_dict(extracted['json'])
db.session.add(charge_entry)
db.session.commit()
return h.jsonify(extracted)
def get_levels_list():
"""Levels list
Returns level list from config file as a json-list.
"""
levels_list = config['level']
return jsonify({'level': levels_list})
def ajax(*args, **kwargs):
#app. logger.debug('ajax:',someth)
#if request.method == 'POST':
#app.logger.debug(request.form.get('lat'))
#app.logger.debug(request.form.get('lng'))
if request.method == 'GET':
query=db.session.query(Projects).order_by(db.desc('date_created'))
#jsony(query)
#qqq = query.all()
projs=[]
for qq in query.all():
#for q in qq:
projs.append(qq.json())
if kwargs:
query=Projects.query.get(kwargs['proj_id'])
projs=query.json()
return jsonify(result = projs)
return jsonify(status='success')
def put_detail(self, collection, pk):
"""Updates whole document."""
collection = self.get_collection(collection)
if '_id' in request.json:
# we are ignoring id fields of bundle,
# because _id field is immutable
del request.json['_id']
collection.update({"_id": ObjectId(pk)}, request.json)
response.status = 202
return jsonify(request.json)
def put_detail(self, collection, pk):
"""Updates whole document."""
collection = self.get_collection(collection)
if '_id' in request.json:
# we are ignoring id fields of bundle,
# because _id field is immutable
del request.json['_id']
collection.update({"_id": ObjectId(pk)}, request.json)
response.status = 202
return jsonify(request.json)
def files_types(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
return HTTPError(
410,
'This part of the API has been temporarily disabled to due to performance issues.'
)
result = simple_group('file', 'content_guess', mongodb)
return jsonify(result, response)
def index():
"""Status page
Generating status (main) page and returns it.
"""
response = {'name': "simplelogs",
'type': "API",
'status': "available",
'version': VERSION}
return jsonify(response)
def session_protocols(mongodb):
"""
Returns a grouped list of all protocols intercepted.
Example:
{"protocols": [{"count": 680, "protocol": "http"},
{"count": 125, "protocol": "ssh},
{"count": 74, "protocol": "imap}]}
"""
auth.require(role='access_normalized')
result = simple_group('session', 'protocol', mongodb)
return jsonify(result, response)
def session_protocols(mongodb):
"""
Returns a grouped list of all protocols intercepted.
Example:
{"protocols": [{"count": 680, "protocol": "http"},
{"count": 125, "protocol": "ssh},
{"count": 74, "protocol": "imap}]}
"""
auth.require(role='access_normalized')
result = simple_group('session', 'protocol', mongodb)
return jsonify(result, response)
def session_protocols(mongodb):
"""
Returns a grouped list of all protocols intercepted.
Example:
{"protocols": [{"count": 680, "protocol": "http"},
{"count": 125, "protocol": "ssh},
{"count": 74, "protocol": "imap}]}
"""
return HTTPError(410, 'This part of the API has been temporarily disabled to due to performance issues.')
auth.require(role='access_normalized')
result = simple_group('session', 'protocol', mongodb)
return jsonify(result, response)
def index():
"""Status page
Generating status (main) page and returns it.
"""
response = {
'name': "simplelogs",
'type': "API",
'status': "available",
'version': VERSION
}
return jsonify(response)
def display_session(request, offset, query_params=None):
# get the id number of the session the user wants to view
try:
offset = int(offset)
except ValueError:
raise Http404()
# retreived the session and all its motion events
s = Session.objects.all().filter(id=offset)
events = MotionEvent.objects.all().filter(sessionId=s)
retVal = {
'Motion Events': [e.to_dict() for e in events]
}
return HttpResponse(jsonify(retVal, query_params), mimetype="application/json")
def hpfeeds(mongodb):
try:
auth.require(role='access_all')
except AAAException as e:
return HTTPError(401, e.message)
tmp_result = mongodb['daily_stats'].find_one({'_id': 'total'})
del tmp_result['_id']
result = []
for key, value in tmp_result.items():
result.append({'channel': key, 'count': value})
return jsonify({'stats': result}, response)
def hpfeeds(mongodb):
try:
auth.require(role='access_all')
except AAAException as e:
return HTTPError(401, e.message)
tmp_result = mongodb['daily_stats'].find_one({'_id': 'total'})
del tmp_result['_id']
result = []
for key, value in tmp_result.items():
result.append({'channel': key, 'count': value})
return jsonify({'stats': result}, response)
def hpfeeds(mongodb):
try:
auth.require(role="access_all")
except AAAException as e:
return HTTPError(401, e.message)
tmp_result = mongodb["daily_stats"].find_one({"_id": "total"})
del tmp_result["_id"]
result = []
for key, value in tmp_result.items():
result.append({"channel": key, "count": value})
return jsonify({"stats": result}, response)
def get_list(self, collection):
"""Returns paginated objects."""
collection = self.get_collection(collection)
limit = int_or_default(request.query.limit, 20)
offset = int_or_default(request.query.offset, 0)
query = self.get_query()
cursor = collection.find(query)
meta = {
"limit": limit,
"offset": offset,
"total_count": cursor.count(),
}
objects = cursor.skip(offset).limit(limit)
objects = map(self.get_bundler(collection), objects)
return jsonify({"meta": meta, "objects": objects})
def display_heatmap(request, offset, query_params=None):
# the session id number chosen
try:
offset = int(offset)
except ValueError:
raise Http404()
s = Session.objects.get(id=offset)
events = MotionEvent.objects.all().filter(sessionId=s)
size = len(events)
# formats the motion event data into a json object to use
# to generate the heatmap display
retVal = {
'data': [d.to_heatmap() for d in events]
}
motion = jsonify(retVal, query_params)
return render_to_response('heatmap.html',{'retVal':retVal, 'session':s, 'motion':motion, 'events':events, 'size':size})
def sessions_get_by_query(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
mongo_keys = {
'id', '_id', 'protocol', 'source_ip', 'source_port', 'destination_ip',
'destination_port', 'honeypot'
}
#intersection
common_keys = (set(query_keys) & mongo_keys)
for item in common_keys:
if item.endswith('_id'):
query_dict[item] = ObjectId(request.query[item])
elif item is 'id':
query_dict['_' + item] = ObjectId(request.query[item])
elif item.endswith('_port'):
query_dict[item] = int(request.query[item])
else:
query_dict[item] = request.query[item]
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
#remove ip of honeypot if user is not authorized to see it
u = auth.current_user.role
lvl = auth._store.roles[u]
needed_lvl = auth._store.roles['access_normalized']
p_limit = {'_id': False}
if lvl < needed_lvl:
p_limit = {'destination_ip': False}
result = list(mongodb['session'].find(spec=query_dict,
fields=p_limit).limit(limit))
return jsonify({'sessions': result}, response)
def login():
email = request.form.get('email', None)
password = request.form.get('password', None)
if not email or not password:
return json.dumps({'error': INCORRECT_EMAIL_PASSWORD})
# grab user from database based on credentials
user = db.users.find_one({'email': email})
if not user:
return json.dumps({'error': INCORRECT_EMAIL_PASSWORD})
hashed_password = user['hashed_password']
if bcrypt.hashpw(password, hashed_password) == hashed_password:
# abstract into pre-serialize user
del user['hashed_password']
user['logged_in'] = True
session['user'] = str(user['_id'])
# return user object dump
return jsonify(user)
else:
return json.dumps({'error': INCORRECT_EMAIL_PASSWORD})
def get_list(self, collection):
"""Returns paginated objects."""
collection = self.get_collection(collection)
limit = int_or_default(request.query.limit, 20)
offset = int_or_default(request.query.offset, 0)
query = self.get_query()
cursor = collection.find(query)
meta = {
"limit": limit,
"offset": offset,
"total_count": cursor.count(),
}
objects = cursor.skip(offset).limit(limit)
objects = map(self.get_bundler(collection), objects)
return jsonify({"meta": meta,
"objects": objects})
def get_dorks(mongodb):
try:
auth.require(role='public')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
#set default parameters
sort_key = 'count'
sort_order = -1
limit = 200
if 'sort_by' in query_keys:
sort_key = request.query.sort_by
if 'sort_order' in query_keys:
try:
sort_order = int(request.query.sort_order)
except ValueError:
raise HTTPError(400, 'sort_order must be an integer.')
if 'regex' in query_keys:
query_dict['content'] = {'$regex': request.query.regex}
#inurl, intitle, etc.
if 'type' in query_keys:
query_dict['type'] = request.query.type
if 'limit' in query_keys:
limit = int(request.query.limit)
result = list(mongodb['dork'].find(query_dict).sort(
sort_key, sort_order).limit(limit))
#delete mongo _id - better way?
for entry in result:
entry['firsttime'] = entry['_id'].generation_time
del entry['_id']
return jsonify({'dorks': result}, response)
def get_dorks(mongodb):
try:
auth.require(role='public')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
#set default parameters
sort_key = 'count'
sort_order = -1
limit = 200
if 'sort_by' in query_keys:
sort_key = request.query.sort_by
if 'sort_order' in query_keys:
try:
sort_order = int (request.query.sort_order)
except ValueError:
raise HTTPError(400, 'sort_order must be an integer.')
if 'regex' in query_keys:
query_dict['content'] = {'$regex': request.query.regex}
#inurl, intitle, etc.
if 'type' in query_keys:
query_dict['type'] = request.query.type
if 'limit' in query_keys:
limit = int(request.query.limit)
result = list(mongodb['dork'].find(query_dict).sort(sort_key, sort_order).limit(limit))
#delete mongo _id - better way?
for entry in result:
entry['firsttime'] = entry['_id'].generation_time
del entry['_id']
return jsonify({'dorks': result}, response)
def hpfeeds(mongodb):
try:
auth.require(role='access_all')
except AAAException as e:
return HTTPError(401, e.message)
if 'date' in request.query and 'channel' in request.query:
query = {'date': request.query.date, 'channel': request.query.channel}
elif 'date' in request.query:
query = {'date': request.query.date}
elif 'channel' in request.query:
query = {'channel': request.query.channel}
else:
abort(404, 'muhaha')
results = list(mongodb['daily_stats'].find(query))
for result in results:
del result['_id']
return jsonify({'stats': results}, response)
def hpfeeds(mongodb):
try:
auth.require(role="access_all")
except AAAException as e:
return HTTPError(401, e.message)
if "date" in request.query and "channel" in request.query:
query = {"date": request.query.date, "channel": request.query.channel}
elif "date" in request.query:
query = {"date": request.query.date}
elif "channel" in request.query:
query = {"channel": request.query.channel}
else:
abort(404, "muhaha")
results = list(mongodb["daily_stats"].find(query))
for result in results:
del result["_id"]
return jsonify({"stats": results}, response)
def hpfeeds(mongodb):
try:
auth.require(role='access_all')
except AAAException as e:
return HTTPError(401, e.message)
if 'date' in request.query and 'channel' in request.query:
query = {'date': request.query.date, 'channel': request.query.channel}
elif 'date' in request.query:
query = {'date': request.query.date}
elif 'channel' in request.query:
query = {'channel': request.query.channel}
else:
abort(404, 'Bad Request')
results = list(mongodb['daily_stats'].find(query))
for result in results:
del result['_id']
return jsonify({'stats': results}, response)
def sessions_get_by_query(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
mongo_keys = {'id', '_id', 'protocol', 'source_ip', 'source_port', 'destination_ip', 'destination_port', 'honeypot'}
#intersection
common_keys = (set(query_keys) & mongo_keys)
for item in common_keys:
if item.endswith('_id'):
query_dict[item] = ObjectId(request.query[item])
elif item is 'id':
query_dict['_' + item] = ObjectId(request.query[item])
elif item.endswith('_port'):
query_dict[item] = int(request.query[item])
else:
query_dict[item] = request.query[item]
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
#remove ip of honeypot if user is not authorized to see it
u = auth.current_user.role
lvl = auth._store.roles[u]
needed_lvl = auth._store.roles['access_normalized']
if lvl < needed_lvl:
p_limit = {'destination_ip': False}
else:
p_limit = None
result = list(mongodb['session'].find(spec=query_dict, fields=p_limit).limit(limit))
return jsonify({'sessions': result}, response)
def get_files(mongodb):
try:
auth.require(role='access_normalized')
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
if 'limit' in query_keys:
limit = int(request.query.limit)
else:
limit = 50
if 'hash' in query_keys:
hash_length = len(request.query['hash'])
if hash_length is 128:
query_dict['hashes.sha512'] = request.query['hash']
elif hash_length is 40:
query_dict['hashes.sha1'] = request.query['hash']
elif hash_length is 32:
query_dict['hashes.md5'] = request.query['hash']
else:
abort(
400,
'{0} could not be recognized as a supported hash. Currently supported hashes are: SHA1, SHA512 and MD5. '
)
else:
abort(400, 'Only supported query parameter is "hash"')
p_limit = {'_id': False}
if 'no_data' in query_keys:
p_limit['data'] = False
result = list(mongodb['file'].find(query_dict,
fields=p_limit).limit(limit))
return jsonify({'files': result}, response)
def get_files(mongodb):
try:
auth.require(role="access_normalized")
except AAAException as e:
return HTTPError(401, e.message)
query_keys = request.query.keys()
query_dict = {}
if "limit" in query_keys:
limit = int(request.query.limit)
else:
limit = 50
if "hash" in query_keys:
hash_length = len(request.query["hash"])
if hash_length is 128:
query_dict["hashes.sha512"] = request.query["hash"]
elif hash_length is 40:
query_dict["hashes.sha1"] = request.query["hash"]
elif hash_length is 32:
query_dict["hashes.md5"] = request.query["hash"]
else:
abort(
400,
"{0} could not be recognized as a supported hash. Currently supported hashes are: SHA1, SHA512 and MD5. ",
)
else:
abort(400, 'Only supported query parameter is "hash"')
p_limit = {"_id": False}
if "no_data" in query_keys:
p_limit["data"] = False
result = list(mongodb["file"].find(query_dict, fields=p_limit).limit(limit))
return jsonify({"files": result}, response)
def post(self):
user = request.form.to_dict()
# get password and hash it
password = request.form.get('password', None)
confirm = request.form.get('confirm', None)
if password:
if password == confirm:
hashed_password = bcrypt.hashpw(password, bcrypt.gensalt())
user['hashed_password'] = hashed_password
del user['password']
del user['confirm']
# insert returns an ObjectId
user_id = str(db.users.insert(user))
# abstract into pre-serialize user
del user['hashed_password']
user['logged_in'] = True
session['user'] = user_id
return jsonify(user)
else:
# password != confirm
return json.dumps({'error': PASSWORDS_DO_NOT_MATCH})
else:
# no password was entered
return json.dumps({'error': NO_PASSWORD_PROVIDED})
def add_logentry():
"""Creating new log entry and saving it to DB.
Returns False, if validation had problems with validating inputs,
returns False, if client tries to send not a json,
otherwise return True and ObjectId.
"""
if request.headers['Content-Type'] == 'application/json':
errors = []
request_data = request.json
# Lets check selected level
level = ""
if "level" not in request_data:
errors.append({'level': 'Field required.'})
else:
level = request_data['level']
# Is level in level list in config file?
if level not in config['level']:
errors.append({'level': 'Unknown level type.'})
# Checking owner present (required)
owner = ""
if "owner" not in request_data:
errors.append({'owner': 'Field required.'})
else:
owner = request_data['owner']
# Checking data present (required)
data = ""
if "data" not in request_data:
errors.append({'data': 'Field required.'})
else:
data = request_data['data']
tags = []
# Tags isn't required. If it present lets try to convert it to python-list.
# If successfully - add it to entry. If not - return full error and don't create entry in DB.
if "tags" in request_data:
tags = request.json['tags']
if not isinstance(tags, list):
errors.append({'tags': 'Tags must be an array.'})
if not errors:
entry = LogEntry(level, owner, data, tags)
id_or_error = entry.save()
if not isinstance(id_or_error, ObjectId):
return jsonify({'OK': False, 'error': id_or_error})
# ___str___ is a string representation of JS ObjectID from MongoDB.
return jsonify({'OK': True, 'id': id_or_error.__str__()})
else:
return jsonify({"OK": False, 'errors': errors})
else:
#TODO Here should be NORMAL exception.
return jsonify({
"errors": [
"415 Unsupported Media Type. \"application/json\" required.\n",
]
})
def post_list(self, collection):
"""Creates new document"""
collection = self.get_collection(collection)
inserted = collection.insert(request.json)
response.status = 201
return jsonify({"_id": inserted})
def get_detail(self, collection, pk):
"""Returns a single document."""
cursor = self.get_collection(collection)
data = cursor.find_one({"_id": ObjectId(pk)}) or abort(404)
return jsonify(self.get_bundler(cursor)(data))
'Unsupported Media Type. \"application/json\" required.\n', 415)
entry = LogEntry()
try:
result = entry.get_entries(find=find,
sort=sort,
skip=skip,
limit=limit)
except ValueError, e:
raise InvalidAPIUsage(e.message, status_code=404)
except AttributeError, e:
raise InvalidAPIUsage(e.message, status_code=400)
except TypeError, e:
raise InvalidAPIUsage(e.message, status_code=400)
except AutoReconnect, e:
raise InvalidAPIUsage(e.message, status_code=500)
return jsonify(dict(OK=True, result=list(result)))
def get_owners():
"""Get owners list
Returns owners list
"""
entry = LogEntry()
try:
result = entry.get_owners()
except ValueError, e:
raise InvalidAPIUsage(e.message, status_code=404)
except AutoReconnect, e:
raise InvalidAPIUsage(e.message, status_code=500)
def error(self, message, error):
"""Returns the error response."""
return jsonify({"error": error.status_code, "message": message})
def get_detail(self, collection, pk):
"""Returns a single document."""
cursor = self.get_collection(collection)
data = cursor.find_one({"_id": ObjectId(pk)}) or abort(404)
return jsonify(self.get_bundler(cursor)(data))
def error(self, error, message=None):
"""Returns the error response."""
return jsonify({
"error": error.status_code,
"message": error.body or message
})
def scraper(url, html):
tree = fromstring(html)
stat_links = tree.xpath(
'//div[contains(@class,"flex-list")]//ul/li/a/@href')
domain = '{}://{}'.format(urlparse(url).scheme, urlparse(url).netloc)
if stat_links:
stat_links = [urljoin(domain, link) for link in stat_links]
while stat_links:
stat_url = stat_links.pop()
if 'statistics' in stat_url:
print('# Downloading files: ', stat_url)
dir_path = url[url.find('map'):].split('map')[-1]
dir_path = dir_path.split('?')[0]
dir_path = 'data{}/{}'.format(dir_path,
stat_url.rsplit('/', 2)[1])
dir_path = os.path.join(CURRENT_DIR, dir_path)
os.makedirs(dir_path, exist_ok=True)
# create a new instance of browser/driver
driver = get_driver(download_dir=dir_path)
# set the domain
driver.get('https://www.statista.com')
for ck in cookies:
if 'expiry' in ck:
del ck['expiry']
driver.add_cookie(ck)
throttle.wait(stat_url)
driver.get(stat_url)
time.sleep(2.5)
download_btns = driver.find_elements_by_css_selector(
'#download button.button')
print('...saving files')
for btn in download_btns:
try:
btn.click()
time.sleep(random.randint(2, 4))
except selenium.common.exceptions.ElementNotInteractableException:
print('Unable to download file: Upgrade to corporate')
except selenium.common.exceptions.ElementClickInterceptedException:
continue
except selenium.common.exceptions.StaleElementReferenceException:
stat_links.append(stat_url)
break
# Extract metadata
tree = fromstring(driver.page_source)
# source
src_keys = tree.xpath('//div[@id="source"]//dt//text()')
src_vals = tree.xpath('//div[@id="source"]//dd//text()')
src_vals = [v.strip() for v in src_vals]
src_metadata = jsonify(src_keys, src_vals)
# info
info_keys = tree.xpath('//div[@id="info"]//dt//text()')
info_keys = [key.strip() for key in info_keys]
info_vals = tree.xpath('//div[@id="info"]//dd//text()')
info_vals = [val.strip() for val in info_vals]
info_metadata = jsonify(info_keys, info_vals)
metadata = {**src_metadata, **info_metadata}
meta_dir = os.path.join(dir_path, 'metadata.json')
print('...saving metadata')
with open(meta_dir, 'w') as meta_file:
json.dump(metadata, meta_file)
time.sleep(2)
driver.quit()
def handle_invalid_usage(error):
response = jsonify(error.to_dict())
response.status_code = error.status_code
return response
def not_found(error):
return make_response(jsonify({'error': 'Not found'}), 404)
def post_list(self, collection):
"""Creates new document"""
collection = self.get_collection(collection)
inserted = collection.insert(request.json)
response.status = 201
return jsonify({"_id": inserted})
def put_detail(self, collection, pk):
"""Updates whole document."""
collection = self.get_collection(collection)
collection.update({"_id": ObjectId(pk)}, request.json)
response.status = 202
return jsonify(request.json)
