def rethinkdb_insert(self, req_msg): """ Inserts an event into RethinkDB. """ # Add timestamps in a format RethinkDB understands. del req_msg["live"] req_msg["timestamp"] = r.iso8601(req_msg["start"]) req_msg["timestamp_end"] = r.iso8601(req_msg["end"]) r.db("catcierge").table("events").insert(req_msg).run(self.rdb)
def to_rql_repr(self, value): if isinstance(value, str): return r.iso8601(value, default_timezone=self.DEFAULT_TIMEZONE).in_timezone(self.timezone) elif isinstance(value, int) or isinstance(value, float): return datetime.fromtimestamp(value).isoformat() elif isinstance(value, dict): return r.time( value.get('year', None), value.get('month', None), value.get('day', None), value.get('hour', None), value.get('minute', None), value.get('second', None), value.get('timezone', self.timezone), ).in_timezone(self.timezone) else: return r.iso8601(value.isoformat(), default_timezone=self.DEFAULT_TIMEZONE).as_timezone(self.timezone)
def encode_rql_dates(data): ''' Convert all datetime.date and datetime.datetime objects to RqlTzinfo objects ''' for k,v in data.items(): if type(v) is dict: encode_rql_dates(v) elif type(v) is datetime.date or type(v) is datetime.datetime: if type(v) is datetime.date: v = datetime.datetime.combine(v, datetime.time()) data[k] = r.iso8601('{}Z'.format(v.isoformat()))
def contact(uid=None): '''The contact interface to the DB''' data = None if request.json is not None: data = schema.load(request.json).data if 'dob' in data: data['dob'] = r.iso8601(request.json['dob']) print('Data:', data) if request.method == 'GET': return jsonify(get(uid=uid)) elif request.method == 'PUT' and data != None and 'dob' in data: return jsonify(update(data)) elif request.method == 'POST' and data != None: return jsonify(add(data)) elif request.method == 'DELETE' and uid != None: return jsonify(delete(uid)) else: return "I can't do that..."
def main(args): with r.connect(host=DB_HOST, port=DB_PORT, db=DB_DB).repl() as conn: today = datetime.now(r.make_timezone(TIMEZONE)) delta = today - timedelta(days=DAYS_AGO) # map: seasons with episodes that aired within a timedelta, merged with series ID and name # reduce: list concatenation series = r.table('series').map( lambda series: series['seasons'].filter( lambda season: season['episodes'].filter( lambda ep: r.iso8601(ep['airdate'], default_timezone=TIMEZONE).during(delta, today) ).count().gt(0) ).pluck('season_number').merge(series.pluck('series_id', 'series_name')) ).reduce(lambda a, b: a + b).run() if len(series) < 1: print("No current series in database") for s in series: pick_cherries(s['series_id'], s['season_number'], outdb='{}:{}/{}'.format(DB_HOST, DB_PORT, DB_DB))
def create_analysis(): """ Processes the alert + creates an analysis when necessary """ analysis_data = flask.request.get_json() if any(key not in analysis_data for key in ['src_ip', 'dest_ip', 'dest_port', 'detection', 'timestamp']): flask.abort(500, 'Bad format of input data') aggregate = False alert_tuple = (analysis_data['src_ip'], analysis_data['dest_ip'], analysis_data['dest_port']) for progress in r.table('analysis').filter(r.row['job_status']['state'] == 'PENDING').run(flask.g.rdb_conn): if (progress['alert']['details']['src_ip'], progress['alert']['details']['dest_ip'], progress['alert']['details']['dest_port']) == alert_tuple: aggregate = progress['id'] break # Create `analysis` table if necessary if 'analysis' not in r.table_list().run(flask.g.rdb_conn): r.table_create('analysis').run(flask.g.rdb_conn) if not aggregate: # Alert creates a new analysis analysis = {} job = engine.analyze.apply_async((analysis_data, ), countdown=config['job_freeze']) analysis['id'] = job.task_id analysis['job_freeze'] = config['job_freeze'] analysis['date'] = r.now() analysis['alert'] = { 'details': analysis_data, 'count': 1 } analysis['job_status'] = { 'state': 'PENDING', 'eta': r.iso8601(str(arrow.utcnow().replace(seconds=config['job_freeze']))) } r.table('analysis').insert(analysis).run(flask.g.rdb_conn) return flask.jsonify(job_id=job.task_id) else: # Alert is aggregated to the existing analysis r.table('analysis').get(aggregate).update({'alert': {'count': r.row['alert']['count'].add(1)}}).run(flask.g.rdb_conn) return flask.jsonify(job_id=aggregate)
async def ingresar(dictio): dictio['version'] = isNone(dictio['version'], 'int') dictio['epoch'] = isNone(dictio['epoch'], 'int') dictio['no'] = isNone(dictio['no'], 'int') dictio['retardo'] = isNone(dictio['retardo'], 'float') # dictio['email_origen'] = isNone(dictio['email_origen'],'float') dictio['latitud'] = isNone(dictio['latitud'], 'float') dictio['longitud'] = isNone(dictio['longitud'], 'float') dictio['dep'] = isNone(dictio['dep'], 'float') dictio['m1_magnitud'] = isNone(dictio['m1_magnitud'], 'float') dictio['up'] = isNone(dictio['up'], 'int') dictio['fecha_origen'] = r.iso8601(dictio['fecha_origen']) dictio['ano_sfile'] = isNone(dictio['ano_sfile'], 'int') dictio['mes_sfile'] = isNone(dictio['mes_sfile'], 'int') dictio['dia_sfile'] = isNone(dictio['dia_sfile'], 'int') dictio['m5'] = isTrue(dictio['m5']) dictio['m20'] = isTrue(dictio['m20']) dictio['sensible'] = isTrue(dictio['sensible']) # dictio['ok_envio_email'] = isTrue(dictio['ok_envio_email']) dictio['operator'] = dictio['operator'].strip() con = Continuos() # print(dictio) con.ejecutar({ 'message': { 'table': 'analisis', 'option': 'insert', 'dictio': dictio } }) del (con)
def free_busy(self, user_id, from_date, to_date): dtz = 'Z' # Default Timezone selection = list( self.event_table.filter({ "user_id": user_id }).filter((r.row["start"] != r.expr("")) & (r.row["end"] != r.expr(""))).filter( r.iso8601(r.row['start'], default_timezone=dtz). during(r.iso8601(from_date, default_timezone=dtz), r.iso8601(to_date, default_timezone=dtz), left_bound="closed", right_bound="closed") | r.iso8601(r.row['end'], default_timezone=dtz). during(r.iso8601(from_date, default_timezone=dtz), r.iso8601(to_date, default_timezone=dtz), left_bound="closed", right_bound="closed")).run(self.connection)) return selection
def timestamp_to_rethinkdb(timestamp): i = timestamp.rfind('-') return rethinkdb.iso8601(timestamp[:i].replace('-', '') + 'T' + timestamp[i + 1:].replace(':', '') + 'Z')
def timestamp_to_rethinkdb(timestamp): i = timestamp.rfind('-') return rethinkdb.iso8601(timestamp[:i].replace('-','') + 'T' + timestamp[i+1:].replace(':','') + 'Z')
def analyze(self, analysis_data, pcap_path=None): """ Analyses alert """ error = None start_time = time.time() # Create RethinkDB connection rdb = r.connect(host=config['rethinkdb']['host'], port=config['rethinkdb']['port'], db=config['rethinkdb']['db']) analysis = {} # NOTE: r.now() cannot be used, because the result of the function is evaluated # when query is run. This is OK for the first update function. However, the second # run will replace this time, which is not intended. analysis['job_start'] = r.iso8601(str(arrow.utcnow())) analysis['job_status'] = r.literal({ 'state': 'PROGRESS' }) # Update the database entry for status update r.table('analysis').get(self.request.id).update(analysis).run(rdb) # pcap_path is not None when provided using Web form if not pcap_path: try: analysis['pcap_file'] = request_pcap(analysis_data) pcap_path = os.path.join(config['pcap_directory'], analysis['pcap_file']) except (IOError, requests.ConnectionError) as e: error = str(e) pcap_path = None # Check PCAP file permissions if pcap_path and not os.access(pcap_path, os.F_OK & os.R_OK): error = 'PCAP file can not be accessed' # Check if retrieved PCAP is non-empty if pcap_path and os.stat(pcap_path).st_size == 0: error = 'PCAP file does not contain any data' try: modules = get_modules() except (ImportError, ) as e: error = 'Modules can not be imported. Reason: "{0}"'.format(str(e)) if error: analysis['job_status'] = r.literal({ 'state': 'ERROR', 'error': error }) r.table('analysis').get(self.request.id).update(analysis).run(rdb) rdb.close() return error # end the Celery task # Everything is ready = starting the analysis analysis['results'] = {} for mod in modules: module_name = mod.__NAME__ try: analysis['results'][module_name] = mod(analysis_data, pcap_path, config).bootstrap() except (Exception, ) as e: analysis['results'][module_name] = {'error': str(e)} analysis['exec_time'] = round(time.time() - start_time, 2) analysis['job_status'] = r.literal({ 'state': 'SUCCESS' }) r.table('analysis').get(self.request.id).update(analysis).run(rdb) rdb.close()
def on_message(self, message): """ Websocket on message. """ range = json.loads(message) logger.info("WS %s: %s" % (self.id, json.dumps(range, indent=4))) # Return different queries depending on the length of the time range. timediff = arrow.get(range["end"]) - arrow.get(range["start"]) if timediff.days >= 1: logger.info("Time span more than 1 day (%s - %s) %s" % (range["start"], range["end"], timediff)) global_start = arrow.get(range["start"]) global_end = arrow.get(range["end"]) events = [] # Based on the timeline time range shown, split it up into days # and get the number of events for each day. for day in arrow.Arrow.span_range("day", global_start, global_end): start = day[0].isoformat() end = day[1].isoformat() # For each day do a query and count. # TODO: Make all this into one query? # http://www.rethinkdb.com/docs/cookbook/python/#implementing-pagination num_events = r.db("catcierge").table("events").filter( r.row["timestamp"].during(r.iso8601(start), r.iso8601(end))).count().run( self.rdb) if num_events == 0: continue event = { "type": "day", "start": start, "end": end, "count": num_events } events.append(event) for event in events: self.send_catcierge_event(event) # TODO: Add month aggregate... else: # Query RethinkDB for events in the given time range. events = r.db("catcierge").table("events").filter( r.row["timestamp"].during(r.iso8601(range["start"]), r.iso8601(range["end"]))).run( self.rdb) for doc in events: # Delete these stupid date things we have the same info in start/end # so that we can turn the document into JSON. del doc["timestamp"] del doc["timestamp_end"] jdoc = json.dumps(doc, indent=4) self.send_catcierge_event(jdoc)
# pylint: disable=invalid-name # pylint: disable=missing-docstring # pylint: disable=trailing-whitespace import rethinkdb as r import logging from datetime import datetime from db_conf import get_conf logging.basicConfig(filename='delete_old_tweets.log', level=logging.INFO) logging.info(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) rdb_config = get_conf() try: with r.connect(**rdb_config) as conn: q = r.table('tweets').filter( lambda tweet: r.iso8601(tweet['created_at']).lt(r.now().date( ) - 30 * 24 * 60 * 60)).delete().run(conn) except Exception as e: logging.warning( "Error executing query. No old Tweets deleted. Error message: %s", e) else: if q['errors'] != 0: logging.warning("There was an error deleting %i Tweets", q['errors']) else: logging.info("Deleted %i Tweets", q['deleted'])
def on_message(self, message): """ Websocket on message. """ range = json.loads(message) logger.info("WS %s: %s" % (self.id, json.dumps(range, indent=4))) # Return different queries depending on the length of the time range. timediff = arrow.get(range["end"]) - arrow.get(range["start"]) if timediff.days >= 1: logger.info("Time span more than 1 day (%s - %s) %s" % (range["start"], range["end"], timediff)) global_start = arrow.get(range["start"]) global_end = arrow.get(range["end"]) events = [] # Based on the timeline time range shown, split it up into days # and get the number of events for each day. for day in arrow.Arrow.span_range("day", global_start, global_end): start = day[0].isoformat() end = day[1].isoformat() # For each day do a query and count. # TODO: Make all this into one query? # http://www.rethinkdb.com/docs/cookbook/python/#implementing-pagination num_events = ( r.db("catcierge") .table("events") .filter(r.row["timestamp"].during(r.iso8601(start), r.iso8601(end))) .count() .run(self.rdb) ) if num_events == 0: continue event = {"type": "day", "start": start, "end": end, "count": num_events} events.append(event) for event in events: self.send_catcierge_event(event) # TODO: Add month aggregate... else: # Query RethinkDB for events in the given time range. events = ( r.db("catcierge") .table("events") .filter(r.row["timestamp"].during(r.iso8601(range["start"]), r.iso8601(range["end"]))) .run(self.rdb) ) for doc in events: # Delete these stupid date things we have the same info in start/end # so that we can turn the document into JSON. del doc["timestamp"] del doc["timestamp_end"] jdoc = json.dumps(doc, indent=4) self.send_catcierge_event(jdoc)