def test_nested_literal_throws_merge(self, conn):
with pytest.raises(RqlRuntimeError):
r.db('things').table('points').filter({
'id': 'one'
}).map(lambda doc: doc.merge(
{'points': r.literal({'pt1': r.literal({'z': 'z-1'})})})).run(
conn)
def publish(self, topic_key, payload):
'''Publish a message to this exchange on the given topic'''
self.assert_table()
# first try to just update an existing document
result = self.table.filter({
'topic': topic_key
if not isinstance(topic_key, dict)
else r.literal(topic_key),
}).update({
'payload': payload,
'updated_on': r.now(),
}).run(self.conn)
# If the topic doesn't exist yet, insert a new document. Note:
# it's possible someone else could have inserted it in the
# meantime and this would create a duplicate. That's a risk we
# take here. The consequence is that duplicated messages may
# be sent to the consumer.
if not result['replaced']:
result = self.table.insert({
'topic': topic_key,
'payload': payload,
'updated_on': r.now(),
}).run(self.conn)
def heartbeat_call(self, typeobj):
display_name = request.json['hostname']
logger.info('Processing heartbeat from {!r}'.format(request.headers.getlist("X-Forwarded-For")[0] if request.headers.getlist("X-Forwarded-For") else request.remote_addr))
agent_info = request.json['info']
slug = slugify(display_name)
server, lab = self._get_server(typeobj, slug)
server.display_name = display_name
server.errors = request.json.get('errors', [])
# Update/keep agent_info in the server object. We always keep
# a copy of the last keepalive even after creating sub-objects
# from it.
is_new = 'id' not in server
if is_new:
server.agent_info = agent_info
else:
server.agent_info = r.literal(agent_info)
server.last_seen = now()
server.status = 'online'
server.save()
self._update_sub_objects(server, typeobj, agent_info)
if is_new:
create_event(
obj_id = server.id,
user_id = None,
interested_ids = [server.id, lab.id],
title = 'Created **{}** {}'.format(server.display_name, typeobj.display_name['singular']),
)
return 'ok'
def _sync_sub_objects(self, server, subtype, get_provider_info_func, last_update):
'''Query for all subobjects with type `subtype` of the `server`.
The `last_update` is a `dict` of `slug -> fields`. If `slug`
doesn't exist, it's created with the desired `fields`. If it
exists, the current object is updated. If a subobject exists but
not found in `last_update` it's removed from the `server`.
'''
remaining = {subobj.id: subobj for subobj in server.get_children_with_subtype(subtype)}
existing_slugs = {subobj.slug: subobj for subobj in remaining.itervalues()}
for slug, expected_fields in last_update.iteritems():
if slug in existing_slugs:
subobj = existing_slugs[slug]
subobj.update(**expected_fields)
remaining.pop(subobj.id)
else:
subobj = create_object(parent=server, type=subtype, slug=slug)
subobj.update(**expected_fields)
provider_info = get_provider_info_func(subobj)
if 'id' in subobj:
subobj.provider = r.literal(provider_info)
else:
subobj.provider = provider_info
subobj.save()
for subobj in remaining.itervalues():
subobj.delete()
def publish(self, topic_key, payload):
'''Publish a message to this exchange on the given topic'''
self.assert_table()
# first try to just update an existing document
result = self.table.filter({
'topic':
topic_key
if not isinstance(topic_key, dict) else r.literal(topic_key),
}).update({
'payload': payload,
'updated_on': r.now(),
}).run(self.conn)
# If the topic doesn't exist yet, insert a new document. Note:
# it's possible someone else could have inserted it in the
# meantime and this would create a duplicate. That's a risk we
# take here. The consequence is that duplicated messages may
# be sent to the consumer.
if not result['replaced']:
result = self.table.insert({
'topic': topic_key,
'payload': payload,
'updated_on': r.now(),
}).run(self.conn)
def test_map_merge_literal(self, conn):
expected = {'id': 'one', 'points': {'pt1': {'z': 'z-1'}}}
result = r.db('things').table('points').filter({
'id': 'one'
}).map(
lambda doc: doc.merge({'points': {
'pt1': r.literal({'z': 'z-1'})
}})).run(conn)
assertEqual(expected, list(result)[0])
async def playlist_remove(self, ctx, playlist_name):
"""Remove a song from the playlist"""
playlists = (await r.table("playlists").get(str(ctx.author.id)
).run(self.bot.r_conn
))["playlists"]
playlist = playlists.get(playlist_name)
if not playlist:
return await ctx.send(
"You don't have a playlist with that name >~<")
msg = "**Type the number of a song to remove:**```\n"
for i, track in enumerate(playlist, start=1):
msg += "%s. %s\n" % (i, track["info"]["title"].replace(
"@", "@\u200B"))
msg += "```"
await ctx.send(msg)
def check(m):
return m.channel == ctx.channel and m.author == ctx.author
try:
x = await self.bot.wait_for("message", check=check, timeout=10.0)
except:
try:
await msg.delete()
except:
pass
return await ctx.send("Timed out.")
try:
x = int(x.content)
except:
return await ctx.send("Not a valid number, returning")
if x not in list(range(1, len(playlist) + 1)):
print(len(playlist))
return await ctx.send("Not a valid option, returning")
song = playlist[x - 1]
new_playlist = []
for track in playlist:
if not track == song:
new_playlist.append(track)
await ctx.send("Removed from playlist")
if new_playlist:
await r.table("playlists").get(str(ctx.author.id)).update({
"playlists": {
playlist_name: new_playlist
}
}).run(self.bot.r_conn)
else:
await r.table("playlists").get(str(ctx.author.id)).update({
"playlists": {
playlist_name: r.literal()
}
}).run(self.bot.r_conn)
def do_fix(db, collection=None):
if collection is None:
bad_meta, bad_tables = find_spurious_meta_and_tables(r.table('__METADATA__').run(db), r.table_list().run(db))
if len(bad_meta) == 0 and len(bad_tables) == 0:
return 0, 0
r.table('__METADATA__').get_all(*bad_meta).delete().run(db)
for table in bad_tables:
r.table_drop(table).run(db)
return len(bad_meta), len(bad_tables)
#else
check_collection_name(collection)
meta = r.table('__METADATA__').get(collection).run(db)
if meta is None:
raise BadCollection('collection {} does not exist.'.format(collection))
doing_init = meta.get('doing_init')
appending_filenames = meta.get('appending_filenames')
if not collection in r.table_list().run(db):
raise BadCollection("this is a spurious collection.")
if doing_init:
do_delete(db, collection)
return 'doing_init'
if appending_filenames:
bad_samples = [k for k in meta['samples'] if meta['samples'][k] in appending_filenames]
result = r.table(collection) \
.filter(r.row['IDs'].keys().set_intersection(appending_filenames) != [])\
.replace(lambda x: r.branch(x['IDs'].keys().set_difference(appending_filenames) == [],
None, # delete record
x.merge({
'IDs': r.literal(x['IDs'].without(appending_filenames)),
'QUALs': r.literal(x['QUALs'].without(appending_filenames)),
'FILTERs': r.literal(x['FILTERs'].without(appending_filenames)),
'INFOs': r.literal(x['INFOs'].without(appending_filenames)),
'samples': r.literal(x['samples'].without(bad_samples)),
}))).run(db)
r.table('__METADATA__').get(collection)\
.replace(lambda x: x.merge({
'vcfs': r.literal(x['vcfs'].without(appending_filenames)),
'samples': r.literal(x['samples'].without(bad_samples))
}).without('appending_filenames')).run(db)
return appending_filenames, bad_samples, result['deleted'], result['replaced']
return None
def save(self):
'''
Persist changes to rethinkdb. Updates only the fields that have
changed. Performs insert rather than update if the document has no
primary key or if the primary key is absent from the database.
If there have been any changes to nested fields, updates the first
level attribute. For example, if foo['bar']['baz']['quux'] has changed,
all of foo['bar'] is replaced, but foo['something_else'] is not
touched.
'''
should_insert = False
try:
self[self.pk_field] # raises KeyError if missing
if self._updates:
# r.literal() to replace, not merge with, nested fields
updates = {
field: r.literal(self._updates[field])
for field in self._updates
}
query = self.rr.table(self.table).get(
self.pk_value).update(updates)
result = query.run()
if result['skipped']: # primary key not found
should_insert = True
elif result['errors'] or result['deleted']:
raise Exception(
'unexpected result %s from rethinkdb query %s' %
(result, query))
if not should_insert and self._deletes:
query = self.rr.table(self.table).replace(
r.row.without(self._deletes))
result = query.run()
if result['errors']: # primary key not found
should_insert = True
elif result['replaced'] != 1:
raise Exception(
'unexpected result %s from rethinkdb query %s' %
(result, query))
except KeyError:
should_insert = True
if should_insert:
query = self.rr.table(self.table).insert(self)
result = query.run()
if result['inserted'] != 1:
raise Exception(
'unexpected result %s from rethinkdb query %s' %
(result, query))
if 'generated_keys' in result:
dict.__setitem__(self, self.pk_field,
result['generated_keys'][0])
self._clear_updates()
def update_table_field(table, id_doc, field, value, merge_dict=True):
r_conn = new_rethink_connection()
rtable = r.table(table)
if merge_dict is True:
result = rtable.get(id_doc).update(
{field: value}).run(r_conn)
else:
result = rtable.get(id_doc).update(
{field: r.literal(value)}).run(r_conn)
close_rethink_connection(r_conn)
return result
def update_table_field(table, id_doc, field, value, merge_dict=True):
r_conn = new_rethink_connection()
rtable = r.table(table)
if merge_dict is True:
result = rtable.get(id_doc).update({field: value}).run(r_conn)
else:
result = rtable.get(id_doc).update({
field: r.literal(value)
}).run(r_conn)
close_rethink_connection(r_conn)
return result
def test_update_literal(self, conn):
expected = {'id': 'one', 'points': {'pt1': {'z': 'z-1'}}}
r.db('things').table('points').filter({
'id': 'one'
}).update({
'points': r.literal({'pt1': {
'z': 'z-1'
}})
}).run(conn)
result = r.db('things').table('points').get('one').run(conn)
assertEqual(expected, result)
async def playlist_delete(self, ctx, playlist_name):
"""Delete a playlist"""
playlists = (await r.table("playlists").get(str(ctx.author.id)
).run(self.bot.r_conn
))["playlists"]
if not playlists.get(playlist_name):
return await ctx.send(
"You don't have a playlist with that name >~<")
await r.table("playlists").get(str(ctx.author.id)).update({
"playlists": {
playlist_name: r.literal()
}
}).run(self.bot.r_conn)
await ctx.send("Deleted playlist.")
def _set_record(self,
chat: typing.Union[int, str, None] = None,
user: typing.Union[int, str, None] = None,
state=None,
data=None,
update_data: typing.Optional[bool] = None):
"""
Make record in RethinkDB
:param chat: Chat id
:param user: User id
:param state: New state
:param data: New data
:param update_data: If true, updates data in DB, else rewrites data
:return:
"""
if not data:
data = {}
chat, user = map(str, self.check_address(chat, user))
if r.table('states').get(chat).run(self._connection):
if update_data:
r.table('states').get(chat).update({
user: {
'data': data
}
}).run(self._connection)
else:
r.table('states').get(chat).update({
user: {
'state': state,
'data': r.literal(data)
}
}).run(self._connection)
else:
r.table('states').insert({
'id': chat,
user: {
'state': state,
'data': data
}
}).run(self._connection)
def delete_attr(self, obj):
'''Remove a user-defined attribute from an object.'''
require_user()
lab = obj.get_parent_object()
typeobj = obj.get_type_object()
try:
attr_slug = request.json['slug']
except LookupError as error:
flask_abort(httplib.BAD_REQUEST, 'Missing "{}" parameter'.format(error))
attr_type = self._get_typeobj_attr(typeobj, attr_slug)
if 'attrs' in obj and attr_slug in obj.attrs:
obj.attrs = r.literal({slug: value for slug, value in obj.attrs.iteritems() if slug != attr_slug})
obj.save()
create_event(
obj_id = obj.id,
user_id = current_identity.id,
interested_ids = [obj.id, lab.id],
title = 'Removed the **{}** attribute from **{}**'.format(attr_type['display_name'], obj.display_name),
)
return None, httplib.NO_CONTENT
async def set_bucket(self,
*,
chat: typing.Union[str, int, None] = None,
user: typing.Union[str, int, None] = None,
bucket: typing.Dict = None):
chat, user = map(str, self.check_address(chat=chat, user=user))
async with self.connection() as conn:
if await r.table(self._table).get(chat).run(conn):
await r.table(self._table).get(chat).update({
user: {
'bucket': r.literal(bucket)
}
}).run(conn)
else:
await r.table(self._table).insert({
'id': chat,
user: {
'bucket': bucket
}
}).run(conn)
async def set_data(self,
*,
chat: typing.Union[str, int, None] = None,
user: typing.Union[str, int, None] = None,
data: typing.Dict = None):
chat, user = map(str, self.check_address(chat=chat, user=user))
conn = await self.connection()
if await r.table(self._table).get(chat).run(conn):
await r.table(self._table).get(chat).update({
user: {
'data': r.literal(data)
}
}).run(conn)
else:
await r.table(self._table).insert({
'id': chat,
user: {
'data': data
}
}).run(conn)
def analyze(self, analysis_data, pcap_path=None):
""" Analyses alert """
error = None
start_time = time.time()
# Create RethinkDB connection
rdb = r.connect(host=config['rethinkdb']['host'],
port=config['rethinkdb']['port'],
db=config['rethinkdb']['db'])
analysis = {}
# NOTE: r.now() cannot be used, because the result of the function is evaluated
# when query is run. This is OK for the first update function. However, the second
# run will replace this time, which is not intended.
analysis['job_start'] = r.iso8601(str(arrow.utcnow()))
analysis['job_status'] = r.literal({
'state': 'PROGRESS'
})
# Update the database entry for status update
r.table('analysis').get(self.request.id).update(analysis).run(rdb)
# pcap_path is not None when provided using Web form
if not pcap_path:
try:
analysis['pcap_file'] = request_pcap(analysis_data)
pcap_path = os.path.join(config['pcap_directory'], analysis['pcap_file'])
except (IOError, requests.ConnectionError) as e:
error = str(e)
pcap_path = None
# Check PCAP file permissions
if pcap_path and not os.access(pcap_path, os.F_OK & os.R_OK):
error = 'PCAP file can not be accessed'
# Check if retrieved PCAP is non-empty
if pcap_path and os.stat(pcap_path).st_size == 0:
error = 'PCAP file does not contain any data'
try:
modules = get_modules()
except (ImportError, ) as e:
error = 'Modules can not be imported. Reason: "{0}"'.format(str(e))
if error:
analysis['job_status'] = r.literal({
'state': 'ERROR',
'error': error
})
r.table('analysis').get(self.request.id).update(analysis).run(rdb)
rdb.close()
return error # end the Celery task
# Everything is ready = starting the analysis
analysis['results'] = {}
for mod in modules:
module_name = mod.__NAME__
try:
analysis['results'][module_name] = mod(analysis_data, pcap_path, config).bootstrap()
except (Exception, ) as e:
analysis['results'][module_name] = {'error': str(e)}
analysis['exec_time'] = round(time.time() - start_time, 2)
analysis['job_status'] = r.literal({
'state': 'SUCCESS'
})
r.table('analysis').get(self.request.id).update(analysis).run(rdb)
rdb.close()
