def save(self, db_conn):
"""
Overwrite save method to add to Elasticsearch.
"""
# TODO-2 should we validate the save worked before going to ES?
from database.topic import get_topic, deliver_topic
from database.user import get_user, deliver_user
data = json_prep(self.deliver())
topic = get_topic({'id': self['topic_id']}, db_conn)
if topic:
data['topic'] = json_prep(deliver_topic(topic))
user = get_user({'id': self['user_id']}, db_conn)
if user:
data['user'] = json_prep(deliver_user(user))
es.index(
index='entity',
doc_type='post',
body=data,
id=self['id'],
)
return super().save(db_conn)
def save(self):
"""
Overwrite save method to add to Elasticsearch.
"""
# TODO-2 should we validate the save worked before going to ES?
from models.topic import Topic
from models.user import User
data = json_prep(self.deliver())
topic = Topic.get(id=self['topic_id'])
if topic:
data['topic'] = json_prep(topic.deliver())
user = User.get(id=self['user_id'])
if user:
data['user'] = json_prep(user.deliver())
es.index(
index='entity',
doc_type='post',
body=data,
id=self['id'],
)
return super().save()
def test_json_prep():
"""
Expect to convert dates in objects to ISO strings.
"""
assert isinstance(util.json_prep({'a': datetime.now()})['a'], str)
assert isinstance(
util.json_prep({
'a': {
'a': datetime.now()
},
})['a']['a'], str)
def test_json_prep():
"""
Expect to convert dates in objects to ISO strings.
"""
from datetime import datetime
assert isinstance(util.json_prep({
'a': datetime.now()
})['a'], str)
def add_topic_to_es(topic):
"""
Add the topic to ElasticSearch.
"""
data = json_prep(deliver_topic(topic))
es.index(
index='entity',
doc_type='topic',
body=data,
id=data['id'],
)
def save(self):
"""
Overwrite save method to add to Elasticsearch.
"""
# TODO-2 should we validate the save worked before going to ES?
data = json_prep(self.deliver())
data["avatar"] = self.get_avatar()
es.index(index="entity", doc_type="user", body=data, id=self["id"])
return super().save()
def add_topic_to_es(topic):
"""
Add the topic to ElasticSearch.
"""
data = json_prep(deliver_topic(topic))
return es.index(
index='entity',
doc_type='topic',
body=data,
id=convert_uuid_to_slug(data['id']),
)
def add_user_to_es(user):
"""
Add the user to Elasticsearch.
"""
data = json_prep(deliver_user(user))
data['avatar'] = get_avatar(user['email'])
return es.index(
index='entity',
doc_type='user',
body=data,
id=convert_uuid_to_slug(data['id']),
)
def add_post_to_es(db_conn, post):
"""
Upsert the post data into elasticsearch.
"""
from database.topic import get_topic, deliver_topic
from database.user import get_user, deliver_user
data = json_prep(deliver_post(post))
topic = get_topic(db_conn, {'id': post['topic_id']})
if topic:
data['topic'] = json_prep(deliver_topic(topic))
user = get_user(db_conn, {'id': post['user_id']})
if user:
data['user'] = json_prep(deliver_user(user))
return es.index(
index='entity',
doc_type='post',
body=data,
id=convert_uuid_to_slug(post['id']),
)
def save(self, db_conn):
"""
Overwrite save method to add to Elasticsearch.
"""
# TODO-2 should we validate the save worked before going to ES?
es.index(
index='entity',
doc_type='topic',
body=json_prep(self.deliver()),
id=self['id'],
)
return super().save(db_conn)
def save_entity_to_es(kind, data):
"""
Overwrite save method to add to Elasticsearch.
"""
# NB use deliver_thing(entity) BEFORE calling this function
body = json_prep(data)
if data['status'] == 'accepted':
return es.index(
index='entity',
doc_type=kind,
body=body,
id=convert_uuid_to_slug(data['entity_id']),
)
def add_user_to_es(user):
"""
Add the user to Elasticsearch.
"""
data = json_prep(deliver_user(user))
data['avatar'] = get_avatar(user['email'])
return es.index(
index='entity',
doc_type='user',
body=data,
id=data['id'],
)
def save(self):
"""
Overwrite save method to add to Elasticsearch.
"""
# TODO-2 should we validate the save worked before going to ES?
es.index(
index='entity',
doc_type='topic',
body=json_prep(self.deliver()),
id=self['id'],
)
return super().save()
def save(self, db_conn):
"""
Overwrite save method to add to Elasticsearch.
"""
# TODO-2 should we validate the save worked before going to ES?
doc_type = self.__class__.__name__.lower()
if 'card' in doc_type:
doc_type = 'card'
if self['status'] == 'accepted':
es.index(
index='entity',
doc_type=doc_type,
body=json_prep(self.deliver()),
id=self['entity_id'],
)
return super().save(db_conn)
def es_populate():
setup_db()
db_conn = make_db_connection()
# Empty the database
es.indices.delete(index='entity', ignore=[400, 404])
# Add users
users = r.table('users').run(db_conn)
for user in users:
data = pick(json_prep(user), ('id', 'name'))
data['avatar'] = get_avatar(user['email'])
es.index(
index='entity',
doc_type='user',
body=data,
id=user['id'],
)
# Add units
units = (r.table('units')
.filter(r.row['status'].eq('accepted'))
.group('entity_id')
.max('created')
.default(None)
.ungroup()
.map(r.row['reduction'])
.run(db_conn))
for unit in units:
es.index(
index='entity',
doc_type='unit',
body=json_prep(unit),
id=unit['entity_id'],
)
# Add cards
cards = (r.table('cards')
.filter(r.row['status'].eq('accepted'))
.group('entity_id')
.max('created')
.default(None)
.ungroup()
.map(r.row['reduction'])
.run(db_conn))
for card in cards:
es.index(
index='entity',
doc_type='card',
body=json_prep(card),
id=card['entity_id'],
)
# Add subjects
subjects = (r.table('subjects')
.filter(r.row['status'].eq('accepted'))
.group('entity_id')
.max('created')
.default(None)
.ungroup()
.map(r.row['reduction'])
.run(db_conn))
for subject in subjects:
es.index(
index='entity',
doc_type='subject',
body=json_prep(subject),
id=subject['entity_id'],
)
# Add topics
topics = r.table('topics').run(db_conn)
for topic in topics:
es.index(
index='entity',
doc_type='topic',
body=json_prep(topic),
id=topic['id'],
)
# Add posts
posts = r.table('posts').run(db_conn)
for post in posts:
data = json_prep(post)
topic = (r.table('topics')
.get(data['topic_id'])
.run(db_conn))
user = (r.table('users')
.get(data['user_id'])
.run(db_conn))
data['topic'] = json_prep(topic)
data['user'] = pick(json_prep(user), ('id', 'name'))
es.index(
index='entity',
doc_type='post',
body=data,
id=post['id'],
)
close_db_connection(db_conn)
def es_populate():
db_conn = make_db_connection()
# Empty the database
es.indices.delete(index='entity', ignore=[400, 404])
# Add users
cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
with cur:
cur.execute("SELECT * FROM users;")
data = cur.fetchall()
users = [row for row in data]
db_conn.commit()
for user in users:
data = pick(json_prep(user), ('id', 'name'))
data['avatar'] = get_avatar(user['email'])
es.index(
index='entity',
doc_type='user',
body=data,
id=convert_uuid_to_slug(user['id']),
)
# Add units
cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
with cur:
cur.execute("""
SELECT DISTINCT ON (entity_id) *
FROM units
WHERE status = 'accepted'
ORDER BY entity_id, created DESC;
""")
data = cur.fetchall()
units = [row for row in data]
db_conn.commit()
for unit in units:
es.index(
index='entity',
doc_type='unit',
body=json_prep(unit),
id=convert_uuid_to_slug(unit['entity_id']),
)
# Add cards
cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
with cur:
cur.execute("""
SELECT DISTINCT ON (entity_id) *
FROM cards
WHERE status = 'accepted'
ORDER BY entity_id, created DESC;
""")
data = cur.fetchall()
cards = [row for row in data]
db_conn.commit()
for card in cards:
es.index(
index='entity',
doc_type='card',
body=json_prep(card),
id=convert_uuid_to_slug(card['entity_id']),
)
# Add subjects
cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
with cur:
cur.execute("""
SELECT DISTINCT ON (entity_id) *
FROM subjects
WHERE status = 'accepted'
ORDER BY entity_id, created DESC;
""")
data = cur.fetchall()
subjects = [row for row in data]
db_conn.commit()
for subject in subjects:
es.index(
index='entity',
doc_type='subject',
body=json_prep(subject),
id=convert_uuid_to_slug(subject['entity_id']),
)
"""
TODO-1 fix these
# Add topics
topics = r.table('topics').run(db_conn)
for topic in topics:
es.index(
index='entity',
doc_type='topic',
body=json_prep(topic),
id=topic['id'],
)
# Add posts
posts = r.table('posts').run(db_conn)
for post in posts:
data = json_prep(post)
topic = (r.table('topics')
.get(data['topic_id'])
.run(db_conn))
user = (r.table('users')
.get(data['user_id'])
.run(db_conn))
data['topic'] = json_prep(topic)
data['user'] = pick(json_prep(user), ('id', 'name'))
es.index(
index='entity',
doc_type='post',
body=data,
id=post['id'],
)
"""
close_db_connection(db_conn)
'id': 'eileen',
'created': r.time(2014, 1, 1, 'Z'),
'modified': r.time(2014, 1, 1, 'Z'),
'name': 'eileen',
'email': '*****@*****.**',
'password': bcrypt.encrypt('example1'),
'settings': {
'email_frequency': 'daily',
'view_sets': 'public',
'view_follows': 'public',
}
}]).run(database.db_conn))
users = database.db.table('users').run(database.db_conn)
for user in users:
data = pick(json_prep(user), ('id', 'name'))
data['avatar'] = get_avatar(user['email'])
es.index(
index='entity',
doc_type='user',
body=data,
id=user['id'],
)
(database.db.table('units').insert([{
'id': 'plus-1',
'created': r.time(2014, 1, 1, 'Z'),
'modified': r.time(2014, 1, 1, 'Z'),
'entity_id': 'plus',
'previous_id': None,
'language': 'en',
'created': r.time(2014, 1, 1, 'Z'),
'modified': r.time(2014, 1, 1, 'Z'),
'name': 'eileen',
'email': '*****@*****.**',
'password': bcrypt.encrypt('example1'),
'settings': {
'email_frequency': 'daily',
'view_sets': 'public',
'view_follows': 'public',
}
}])
.run(database.db_conn))
users = database.db.table('users').run(database.db_conn)
for user in users:
data = pick(json_prep(user), ('id', 'name'))
data['avatar'] = get_avatar(user['email'])
es.index(
index='entity',
doc_type='user',
body=data,
id=user['id'],
)
(database.db.table('units')
.insert([{
'id': 'plus-1',
'created': r.time(2014, 1, 1, 'Z'),
'modified': r.time(2014, 1, 1, 'Z'),
'entity_id': 'plus',
'previous_id': None,
