def rethinkdb_insert(self, req_msg):
"""
Inserts an event into RethinkDB.
"""
# Add timestamps in a format RethinkDB understands.
del req_msg["live"]
req_msg["timestamp"] = r.iso8601(req_msg["start"])
req_msg["timestamp_end"] = r.iso8601(req_msg["end"])
r.db("catcierge").table("events").insert(req_msg).run(self.rdb)
def rethinkdb_insert(self, req_msg):
"""
Inserts an event into RethinkDB.
"""
# Add timestamps in a format RethinkDB understands.
del req_msg["live"]
req_msg["timestamp"] = r.iso8601(req_msg["start"])
req_msg["timestamp_end"] = r.iso8601(req_msg["end"])
r.db("catcierge").table("events").insert(req_msg).run(self.rdb)
def to_rql_repr(self, value):
if isinstance(value, str):
return r.iso8601(value, default_timezone=self.DEFAULT_TIMEZONE).in_timezone(self.timezone)
elif isinstance(value, int) or isinstance(value, float):
return datetime.fromtimestamp(value).isoformat()
elif isinstance(value, dict):
return r.time(
value.get('year', None),
value.get('month', None),
value.get('day', None),
value.get('hour', None),
value.get('minute', None),
value.get('second', None),
value.get('timezone', self.timezone),
).in_timezone(self.timezone)
else:
return r.iso8601(value.isoformat(), default_timezone=self.DEFAULT_TIMEZONE).as_timezone(self.timezone)
def encode_rql_dates(data):
'''
Convert all datetime.date and datetime.datetime objects to RqlTzinfo objects
'''
for k,v in data.items():
if type(v) is dict:
encode_rql_dates(v)
elif type(v) is datetime.date or type(v) is datetime.datetime:
if type(v) is datetime.date:
v = datetime.datetime.combine(v, datetime.time())
data[k] = r.iso8601('{}Z'.format(v.isoformat()))
def contact(uid=None):
'''The contact interface to the DB'''
data = None
if request.json is not None:
data = schema.load(request.json).data
if 'dob' in data:
data['dob'] = r.iso8601(request.json['dob'])
print('Data:', data)
if request.method == 'GET':
return jsonify(get(uid=uid))
elif request.method == 'PUT' and data != None and 'dob' in data:
return jsonify(update(data))
elif request.method == 'POST' and data != None:
return jsonify(add(data))
elif request.method == 'DELETE' and uid != None:
return jsonify(delete(uid))
else:
return "I can't do that..."
def main(args):
with r.connect(host=DB_HOST, port=DB_PORT, db=DB_DB).repl() as conn:
today = datetime.now(r.make_timezone(TIMEZONE))
delta = today - timedelta(days=DAYS_AGO)
# map: seasons with episodes that aired within a timedelta, merged with series ID and name
# reduce: list concatenation
series = r.table('series').map(
lambda series:
series['seasons'].filter(
lambda season: season['episodes'].filter(
lambda ep: r.iso8601(ep['airdate'], default_timezone=TIMEZONE).during(delta, today)
).count().gt(0)
).pluck('season_number').merge(series.pluck('series_id', 'series_name'))
).reduce(lambda a, b: a + b).run()
if len(series) < 1:
print("No current series in database")
for s in series:
pick_cherries(s['series_id'], s['season_number'], outdb='{}:{}/{}'.format(DB_HOST, DB_PORT, DB_DB))
def create_analysis():
""" Processes the alert + creates an analysis when necessary """
analysis_data = flask.request.get_json()
if any(key not in analysis_data for key in ['src_ip', 'dest_ip', 'dest_port', 'detection', 'timestamp']):
flask.abort(500, 'Bad format of input data')
aggregate = False
alert_tuple = (analysis_data['src_ip'], analysis_data['dest_ip'], analysis_data['dest_port'])
for progress in r.table('analysis').filter(r.row['job_status']['state'] == 'PENDING').run(flask.g.rdb_conn):
if (progress['alert']['details']['src_ip'], progress['alert']['details']['dest_ip'], progress['alert']['details']['dest_port']) == alert_tuple:
aggregate = progress['id']
break
# Create `analysis` table if necessary
if 'analysis' not in r.table_list().run(flask.g.rdb_conn):
r.table_create('analysis').run(flask.g.rdb_conn)
if not aggregate:
# Alert creates a new analysis
analysis = {}
job = engine.analyze.apply_async((analysis_data, ), countdown=config['job_freeze'])
analysis['id'] = job.task_id
analysis['job_freeze'] = config['job_freeze']
analysis['date'] = r.now()
analysis['alert'] = {
'details': analysis_data,
'count': 1
}
analysis['job_status'] = {
'state': 'PENDING',
'eta': r.iso8601(str(arrow.utcnow().replace(seconds=config['job_freeze'])))
}
r.table('analysis').insert(analysis).run(flask.g.rdb_conn)
return flask.jsonify(job_id=job.task_id)
else:
# Alert is aggregated to the existing analysis
r.table('analysis').get(aggregate).update({'alert': {'count': r.row['alert']['count'].add(1)}}).run(flask.g.rdb_conn)
return flask.jsonify(job_id=aggregate)
async def ingresar(dictio):
dictio['version'] = isNone(dictio['version'], 'int')
dictio['epoch'] = isNone(dictio['epoch'], 'int')
dictio['no'] = isNone(dictio['no'], 'int')
dictio['retardo'] = isNone(dictio['retardo'], 'float')
# dictio['email_origen'] = isNone(dictio['email_origen'],'float')
dictio['latitud'] = isNone(dictio['latitud'], 'float')
dictio['longitud'] = isNone(dictio['longitud'], 'float')
dictio['dep'] = isNone(dictio['dep'], 'float')
dictio['m1_magnitud'] = isNone(dictio['m1_magnitud'], 'float')
dictio['up'] = isNone(dictio['up'], 'int')
dictio['fecha_origen'] = r.iso8601(dictio['fecha_origen'])
dictio['ano_sfile'] = isNone(dictio['ano_sfile'], 'int')
dictio['mes_sfile'] = isNone(dictio['mes_sfile'], 'int')
dictio['dia_sfile'] = isNone(dictio['dia_sfile'], 'int')
dictio['m5'] = isTrue(dictio['m5'])
dictio['m20'] = isTrue(dictio['m20'])
dictio['sensible'] = isTrue(dictio['sensible'])
# dictio['ok_envio_email'] = isTrue(dictio['ok_envio_email'])
dictio['operator'] = dictio['operator'].strip()
con = Continuos()
# print(dictio)
con.ejecutar({
'message': {
'table': 'analisis',
'option': 'insert',
'dictio': dictio
}
})
del (con)
def free_busy(self, user_id, from_date, to_date):
dtz = 'Z' # Default Timezone
selection = list(
self.event_table.filter({
"user_id": user_id
}).filter((r.row["start"] != r.expr(""))
& (r.row["end"] != r.expr(""))).filter(
r.iso8601(r.row['start'], default_timezone=dtz).
during(r.iso8601(from_date, default_timezone=dtz),
r.iso8601(to_date, default_timezone=dtz),
left_bound="closed",
right_bound="closed")
| r.iso8601(r.row['end'], default_timezone=dtz).
during(r.iso8601(from_date, default_timezone=dtz),
r.iso8601(to_date, default_timezone=dtz),
left_bound="closed",
right_bound="closed")).run(self.connection))
return selection
def timestamp_to_rethinkdb(timestamp):
i = timestamp.rfind('-')
return rethinkdb.iso8601(timestamp[:i].replace('-', '') + 'T' +
timestamp[i + 1:].replace(':', '') + 'Z')
def timestamp_to_rethinkdb(timestamp):
i = timestamp.rfind('-')
return rethinkdb.iso8601(timestamp[:i].replace('-','') + 'T' + timestamp[i+1:].replace(':','') + 'Z')
def analyze(self, analysis_data, pcap_path=None):
""" Analyses alert """
error = None
start_time = time.time()
# Create RethinkDB connection
rdb = r.connect(host=config['rethinkdb']['host'],
port=config['rethinkdb']['port'],
db=config['rethinkdb']['db'])
analysis = {}
# NOTE: r.now() cannot be used, because the result of the function is evaluated
# when query is run. This is OK for the first update function. However, the second
# run will replace this time, which is not intended.
analysis['job_start'] = r.iso8601(str(arrow.utcnow()))
analysis['job_status'] = r.literal({
'state': 'PROGRESS'
})
# Update the database entry for status update
r.table('analysis').get(self.request.id).update(analysis).run(rdb)
# pcap_path is not None when provided using Web form
if not pcap_path:
try:
analysis['pcap_file'] = request_pcap(analysis_data)
pcap_path = os.path.join(config['pcap_directory'], analysis['pcap_file'])
except (IOError, requests.ConnectionError) as e:
error = str(e)
pcap_path = None
# Check PCAP file permissions
if pcap_path and not os.access(pcap_path, os.F_OK & os.R_OK):
error = 'PCAP file can not be accessed'
# Check if retrieved PCAP is non-empty
if pcap_path and os.stat(pcap_path).st_size == 0:
error = 'PCAP file does not contain any data'
try:
modules = get_modules()
except (ImportError, ) as e:
error = 'Modules can not be imported. Reason: "{0}"'.format(str(e))
if error:
analysis['job_status'] = r.literal({
'state': 'ERROR',
'error': error
})
r.table('analysis').get(self.request.id).update(analysis).run(rdb)
rdb.close()
return error # end the Celery task
# Everything is ready = starting the analysis
analysis['results'] = {}
for mod in modules:
module_name = mod.__NAME__
try:
analysis['results'][module_name] = mod(analysis_data, pcap_path, config).bootstrap()
except (Exception, ) as e:
analysis['results'][module_name] = {'error': str(e)}
analysis['exec_time'] = round(time.time() - start_time, 2)
analysis['job_status'] = r.literal({
'state': 'SUCCESS'
})
r.table('analysis').get(self.request.id).update(analysis).run(rdb)
rdb.close()
def on_message(self, message):
"""
Websocket on message.
"""
range = json.loads(message)
logger.info("WS %s: %s" % (self.id, json.dumps(range, indent=4)))
# Return different queries depending on the length of the time range.
timediff = arrow.get(range["end"]) - arrow.get(range["start"])
if timediff.days >= 1:
logger.info("Time span more than 1 day (%s - %s) %s" %
(range["start"], range["end"], timediff))
global_start = arrow.get(range["start"])
global_end = arrow.get(range["end"])
events = []
# Based on the timeline time range shown, split it up into days
# and get the number of events for each day.
for day in arrow.Arrow.span_range("day", global_start, global_end):
start = day[0].isoformat()
end = day[1].isoformat()
# For each day do a query and count.
# TODO: Make all this into one query?
# http://www.rethinkdb.com/docs/cookbook/python/#implementing-pagination
num_events = r.db("catcierge").table("events").filter(
r.row["timestamp"].during(r.iso8601(start),
r.iso8601(end))).count().run(
self.rdb)
if num_events == 0:
continue
event = {
"type": "day",
"start": start,
"end": end,
"count": num_events
}
events.append(event)
for event in events:
self.send_catcierge_event(event)
# TODO: Add month aggregate...
else:
# Query RethinkDB for events in the given time range.
events = r.db("catcierge").table("events").filter(
r.row["timestamp"].during(r.iso8601(range["start"]),
r.iso8601(range["end"]))).run(
self.rdb)
for doc in events:
# Delete these stupid date things we have the same info in start/end
# so that we can turn the document into JSON.
del doc["timestamp"]
del doc["timestamp_end"]
jdoc = json.dumps(doc, indent=4)
self.send_catcierge_event(jdoc)
# pylint: disable=invalid-name
# pylint: disable=missing-docstring
# pylint: disable=trailing-whitespace
import rethinkdb as r
import logging
from datetime import datetime
from db_conf import get_conf
logging.basicConfig(filename='delete_old_tweets.log', level=logging.INFO)
logging.info(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
rdb_config = get_conf()
try:
with r.connect(**rdb_config) as conn:
q = r.table('tweets').filter(
lambda tweet: r.iso8601(tweet['created_at']).lt(r.now().date(
) - 30 * 24 * 60 * 60)).delete().run(conn)
except Exception as e:
logging.warning(
"Error executing query. No old Tweets deleted. Error message: %s", e)
else:
if q['errors'] != 0:
logging.warning("There was an error deleting %i Tweets", q['errors'])
else:
logging.info("Deleted %i Tweets", q['deleted'])
def on_message(self, message):
"""
Websocket on message.
"""
range = json.loads(message)
logger.info("WS %s: %s" % (self.id, json.dumps(range, indent=4)))
# Return different queries depending on the length of the time range.
timediff = arrow.get(range["end"]) - arrow.get(range["start"])
if timediff.days >= 1:
logger.info("Time span more than 1 day (%s - %s) %s" % (range["start"], range["end"], timediff))
global_start = arrow.get(range["start"])
global_end = arrow.get(range["end"])
events = []
# Based on the timeline time range shown, split it up into days
# and get the number of events for each day.
for day in arrow.Arrow.span_range("day", global_start, global_end):
start = day[0].isoformat()
end = day[1].isoformat()
# For each day do a query and count.
# TODO: Make all this into one query?
# http://www.rethinkdb.com/docs/cookbook/python/#implementing-pagination
num_events = (
r.db("catcierge")
.table("events")
.filter(r.row["timestamp"].during(r.iso8601(start), r.iso8601(end)))
.count()
.run(self.rdb)
)
if num_events == 0:
continue
event = {"type": "day", "start": start, "end": end, "count": num_events}
events.append(event)
for event in events:
self.send_catcierge_event(event)
# TODO: Add month aggregate...
else:
# Query RethinkDB for events in the given time range.
events = (
r.db("catcierge")
.table("events")
.filter(r.row["timestamp"].during(r.iso8601(range["start"]), r.iso8601(range["end"])))
.run(self.rdb)
)
for doc in events:
# Delete these stupid date things we have the same info in start/end
# so that we can turn the document into JSON.
del doc["timestamp"]
del doc["timestamp_end"]
jdoc = json.dumps(doc, indent=4)
self.send_catcierge_event(jdoc)
