def create_views(self, db):
# https://markhaa.se/posts/couchdb-views-in-python/
# create the two views we will need (by week and by suburb)
# note that if they already exist in the DB it won't do anything
view_by_week = ViewDefinition('sentimentDocs',
'byWeek',
MAP_BY_WEEK,
reduce_fun=REDUCE_STATS)
view_by_week.get_doc(self.DB[db])
view_by_week.sync(self.DB[db])
view_by_suburb = ViewDefinition('sentimentDocs',
'bySuburb',
MAP_BY_SUBURB,
reduce_fun=REDUCE_STATS)
view_by_suburb.get_doc(self.DB[db])
view_by_suburb.sync(self.DB[db])
view_by_week = ViewDefinition('sentimentDocs',
'byWeek2',
MAP_BY_WEEK2,
reduce_fun=REDUCE_STATS)
view_by_week.get_doc(self.DB[db])
view_by_week.sync(self.DB[db])
view_by_suburb = ViewDefinition('sentimentDocs',
'bySuburb2',
MAP_BY_SUBURB2,
reduce_fun=REDUCE_STATS)
view_by_suburb.get_doc(self.DB[db])
view_by_suburb.sync(self.DB[db])
def _create_views(self):
# twitter/count_type
count_type_map = 'function(doc) { emit([doc.type, doc.id], 1); }'
count_type_reduce = 'function(keys, values) { return sum(values); }'
view = ViewDefinition('twitter',
'count_type',
count_type_map,
reduce_fun=count_type_reduce)
view.sync(self.db)
# The unique key for each tweet is doc.id. The key (a 64-bit long) is represented as a string because
# the largest JavaScript long is 2^53.
# A problem arises because keys are sorted as strings and a doc.id may have fewer digits but a larger
# leading digit. So, it is sorted in the wrong order.
# The solutions is to zero-pad the doc.id to fill 19 digits. (The max 64-bit long - 2^63 - has 19 digits.)
# That is why we emit the doc.id key as ("0000000000000000000"+doc.id).slice(-19).
# twitter/get_tweets
get_tweets = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit(("0000000000000000000"+doc.id).slice(-19), doc); }'
view = ViewDefinition('twitter', 'get_tweets', get_tweets)
view.sync(self.db)
# twitter/get_tweets_by_date (sort by date and tweet id)
get_tweets_by_date = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit((new Date(doc.created_at).getTime())+"-"+("0000000000000000000"+doc.id).slice(-19), doc); }'
view = ViewDefinition('twitter', 'get_tweets_by_date',
get_tweets_by_date)
view.sync(self.db)
# twitter/get_users
get_users = 'function(doc) { if (doc.type == "TWITTER_USER") emit(doc.id, doc); }'
view = ViewDefinition('twitter', 'get_users', get_users)
view.sync(self.db)
def setUp(self):
couch.create(self.test_db_name)
self.testDesign = ViewDefinition(
'test0', 'test', '''\function(doc) { emit(null, doc); }''')
self.testDesign2 = ViewDefinition(
'test1', 'test', '''function(doc) {
emit('hello world', 'I\nam\na\nmulti-line\nand\ttabbed\tstring');
}''')
def createViews(db):
generalViewCode = '''
function(doc) {
if(doc.type == "token") {
if(%s) {
emit(doc._id, doc._id);
}
}
}
'''
# todo View
todoCondition = 'doc.lock == 0 && doc.done == 0'
todo_view = ViewDefinition('Monitor', 'todo',
generalViewCode % (todoCondition))
todo_view.sync(db)
# locked View
lockedCondition = 'doc.lock > 0 && doc.done == 0'
locked_view = ViewDefinition('Monitor', 'locked',
generalViewCode % (lockedCondition))
locked_view.sync(db)
# done View
doneCondition = 'doc.lock > 0 && doc.done > 0 && doc.output == 0'
done_view = ViewDefinition('Monitor', 'done',
generalViewCode % (doneCondition))
done_view.sync(db)
#
errorCondition = 'doc.lock > 0 && doc.done > 0 && doc.output > 0'
error_view = ViewDefinition('Monitor', 'error',
generalViewCode % (errorCondition))
error_view.sync(db)
# overview_total View -- lists all views and the number of tokens in each view
overviewMapCode = '''
function(doc) {
if(doc.type == "token") {
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.done == 0) {
emit('locked', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0) {
emit('done', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output > 0) {
emit('error', 1);
}
}
}
'''
overviewReduceCode = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition('Monitor', 'overview_total',
overviewMapCode, overviewReduceCode)
overview_total_view.sync(db)
def main():
server = couchdb.Server('http://localhost:5984')
dbname = 'test'
try:
server.delete(dbname)
print "Deleted %s" % dbname
except:
pass
ttime = 1334187642
db = server.create(dbname)
data = [('1', 'one', 'foo', ttime), ('2', 'two', 'bar', ttime),
('3', 'three', 'foo', ttime), ('4', 'three', 'bar', ttime),
('5', 'three', 'bar', ttime + 86400)]
for d in data:
key, tag, txt, tt = d
print "Input", key, tag, txt, tt
row = {'tag': tag, 'text': txt.lower(), 'time': tt}
db[key] = row
# query
_date = "2012-04-11"
dt = datetime.strptime(_date, "%Y-%m-%d").utctimetuple()
#dt = datetime.strptime(_date,"%Y-%m-%d")
#stime=int(time.mktime(dt.utctimetuple()))
# etime=stime+86400-1
view = ViewDefinition('index',
'daily_tags',
time_tag_mapper,
language='python')
view.sync(db)
# tags = [row.value for row in db.view('index/daily_tags',startkey=stime,endkey=etime)]
tags = [
row.value for row in db.view('index/daily_tags',
key=[dt.tm_year, dt.tm_mon, dt.tm_mday])
]
tags = list(set(tags))
print "Tags today", set(tags)
print ""
view = ViewDefinition('index',
'daily_tagcount',
tag_mapper,
reduce_fun=tag_sumreducer,
language='python')
view.sync(db)
#d = dt.timetuple()
d = dt
for tag in sorted(tags):
# tag_counts = [ (row.key, row.value) for row in db.view('index/tagcount', group=True) ]
_key = [d.tm_year, d.tm_mon, d.tm_mday, tag]
tag_count = [(row.value)
for row in db.view('index/daily_tagcount', key=_key)][0]
print "Found %d %s on %s-%s-%s " % (tag_count, tag, _key[0], _key[1],
_key[2])
def all_view_ViewDefinition(resource):
return ViewDefinition(
resource, 'all', '''function(doc) {
if(doc.doc_type == '%(resource)s') {
emit(doc.%(doc_id)sID, null);
}
}''' % dict(resource=resource[:-1].title(), doc_id=resource[:-1]))
def get_twitter_id_from_couch(database):
id_list = []
view_result = database.view('_design/twitter/_view/byTwitterID')
try:
view_result.total_rows
except:
view = ViewDefinition(
'twitter', 'byTwitterID', '''function(doc) {
if (doc.type === "sentiment"){
emit(doc._id, doc.id);
}
}''')
view.get_doc(database)
view.sync(database)
view_result = database.view('_design/twitter/_view/byTwitterID')
for item in view_result:
twitter_id = item.value
if twitter_id not in id_list:
id_list.append(twitter_id)
return id_list
def get_tweet_ids_by_city(database, city):
id_list = []
view_result = database.view('_design/twitter/_view/tweetIdByCity?key="' +
city + '"')
try:
view_result.total_rows
except:
view = ViewDefinition(
'twitter', 'tweetIdByCity', '''function(doc) {
if (doc.type === "tweet"){
emit(doc.city, doc.id);
}
}''')
view.get_doc(database)
view.sync(database)
view_result = database.view(
'_design/twitter/_view/tweetIdByCity?key=\"' + city + '\"')
if not view_result.total_rows:
return []
for item in view_result:
tweet_id = item.value
if tweet_id not in id_list:
id_list.append(tweet_id)
return id_list
def _prepare_couchdb(self):
server = Server(self.couch_url,
session=Session(retry_delays=range(10)))
try:
if self.db_name not in server:
self.db = server.create(self.db_name)
else:
self.db = server[self.db_name]
except Exception as e:
LOGGER.error('Database error: {}'.format(repr(e)))
raise
by_date_modified_view = ViewDefinition(
self.resource, 'by_dateModified', '''function(doc) {
if (doc.doc_type == '%(resource)s') {
var fields=['%(doc_type)sID'], data={};
for (var i in fields) {
if (doc[fields[i]]) {
data[fields[i]] = doc[fields[i]]
}
}
emit(doc.dateModified, data);
}}''' % dict(resource=self.resource[:-1].title(),
doc_type=self.resource[:-1])
)
by_date_modified_view.sync(self.db)
validate_doc = self.db.get(VALIDATE_BULK_DOCS_ID,
{'_id': VALIDATE_BULK_DOCS_ID})
if validate_doc.get('validate_doc_update') != VALIDATE_BULK_DOCS_UPDATE:
validate_doc['validate_doc_update'] = VALIDATE_BULK_DOCS_UPDATE
self.db.save(validate_doc)
LOGGER.info('Validate document update view saved.')
else:
LOGGER.info('Validate document update view already exist.')
def make_view(self, name, javascript_function):
try:
mapper = """function(doc){emit(doc.id_str, doc.id_str);}"""
vw = ViewDefinition('index', name, mapper)
vw.sync(self.dao)
except Exception as e:
print("Error making maxid view %s" % e)
def add_overview_view(self):
overviewMapCode = '''
function(doc) {
if(doc.type == "%s") {
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.done == 0) {
emit('locked', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0) {
emit('done', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output > 0) {
emit('error', 1);
}
}
}
'''
overviewReduceCode = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition(self.t_type, 'overview_total',
overviewMapCode % (self.t_type),
overviewReduceCode)
self.views['overview_total'] = overview_total_view
overview_total_view.sync(self.db)
def create_view(db, view_tuples):
for vt in view_tuples:
try:
view = ViewDefinition(vt[0], vt[1], vt[2], vt[3])
view.sync(db)
except:
pass
def setUpModule():
"""Create test databases in local server"""
if not has_couchdb:
return
server = couchdb.Server()
## Create databases
for x in DATABASES:
if not server.__contains__(x):
LOG.info("Creating database {}".format(x))
server.create(x)
## Create views for flowcells and samples
for dbname in DATABASES:
dblab = dbname.replace("-test", "")
db = server[dbname]
for k, v in VIEWS[dblab].items():
for title, view in v.items():
viewdef = ViewDefinition(k, title, view)
viewdef.sync(db)
## Create and upload project summary
with open(os.path.join(filedir, "data", "config",
"project_summary.yaml")) as fh:
prj_sum = yaml.load(fh)
db = server["samples-test"]
p_con = ProjectSummaryConnection(dbname="projects-test",
username="******",
password="******")
for p in prj_sum:
prj = ProjectSummaryDocument(**p)
p_con.save(prj, key="project_name")
def test_view(self):
# This test does quite a bit. First, create 4 test records.
# Then, create a view that will emit those records and insert that into
# the db. Finally, call our cushion.view object and compare results.
self._save_some_data({'foo': 1, 'bar': 'a'})
self._save_some_data({'foo': 2, 'bar': 'a'})
self._save_some_data({'foo': 3, 'bar': 'b'})
self._save_some_data({'foo': 4, 'bar': 'b'})
fake_map = """ function (doc) { emit(doc['bar'], doc); } """
# we're going to use python-couchdb's dynamic view loader stuff here
from couchdb.design import ViewDefinition
from couchdb.client import Server
global baseurl
cdb = Server(baseurl)
couchdb = cdb[self.dbname]
view_defn = ViewDefinition('test',
'view',
map_fun=fake_map,
language='javascript')
view_defn.sync(couchdb)
self.cushion.view(self.dbname, 'test/view', self.stop, key='b')
records = self.wait()
self.assertTrue(len(records) == 2)
def couch_get_view(self, db, get_token):
view = ViewDefinition(
'%sview' % db.name, '%s' % db.name,
'''function(doc) {if (doc._id == "%s") emit(doc);}''' % get_token)
view.sync(db)
for res in db.view('_design/%sview/_view/%s' % (db.name, db.name)):
return {res.id: res.key}
def create_views(self):
print("++ Creating views")
view = ViewDefinition(
'get_crashes', 'all', '''function(doc) {
emit(doc.crashhash,doc.crashcount);
}''')
view.get_doc(self.db)
view.sync(self.db)
def createView(self, viewName):
view = ViewDefinition(
'tweets', viewName, '''function(doc) {
emit(doc.text, doc.coordinates);
}''')
view.get_doc(self.db)
view.sync(self.db)
print("create view user, view should be created")
def __get__(self, instance, cls=None):
if self.wrapper is DEFAULT:
wrapper = cls._wrap_row
else:
wrapper = self.wrapper
return ViewDefinition(self.design, self.name, self.map_fun,
self.reduce_fun, language=self.language,
wrapper=wrapper, **self.defaults)
def _create_view(self):
view_map = 'function(doc) { if(doc.bag_of_words) { for (word in doc.bag_of_words) { emit([doc.bag_of_words[word], doc.polarity, doc.suburb],1)}}}'
view_reduce = '_sum'
view = ViewDefinition('application',
'mainview',
view_map,
reduce_fun=view_reduce)
view.sync(self.DBRef)
def createViewUsers(self):
view = ViewDefinition(
'tweets', "users", '''function(doc) {
emit(doc.user["id"], doc.done);
}''')
view.get_doc(self.db)
view.sync(self.db)
print("user view should be created")
def _times_view():
return ViewDefinition(
'couchdyno_times', 'times', '''
function(doc) {
if(doc._id.indexOf("couchdyno_") === 0 && doc.ts) {
emit(doc.ts, null);
}
}
''')
def __init__(self):
self.server = couchdb.Server(self.url)
if self.dbname in self.server:
self.db = self.server[self.dbname]
else:
self.db = self.server.create(self.dbname)
self.viewEventDoc = ViewDefinition('index', 'eventDocView',
events_data_selector)
self.viewEventDoc.sync(self.db)
self.viewEventName = ViewDefinition('index', 'eventNameView',
events_names_selector)
self.viewEventName.sync(self.db)
self.viewNeighborhoodShape = ViewDefinition(
'index', 'neighborhoodsShapesView', neighborhoods_shapes_selector)
self.viewNeighborhoodShape.sync(self.db)
self.viewTrafficEvents = ViewDefinition('index', 'trafficEventsView',
traffic_events_selector)
self.viewTrafficEvents.sync(self.db)
def _bootstrap(self):
if self.dbid not in self.server:
self.server.create(self.dbid)
if self.keydbid not in self.server:
self.server.create(self.keydbid)
self.db = self.server[self.dbid]
self.keydb = self.server[self.keydbid]
self.log_view = ViewDefinition(self.LOG_DDOCID, self.LOG_VIEW_NAME,
self.LOG_VIEW_CODE)
self.log_view.sync(self.db)
def _conflicts_view():
return ViewDefinition('couchdyno_conflicts',
'conflicts',
'''
function(doc) {
if(doc._id.indexOf("couchdyno_") === 0 && doc._conflicts) {
emit(doc._id, doc._conflicts.length);
}
}
''',
reduce_fun='_sum')
def create_db(name):
dbm = get_db_manager('http://localhost:5984/', name)
views = []
for v in view_js.keys():
funcs = view_js[v]
map = (funcs['map'] if 'map' in funcs else None)
reduce = (funcs['reduce'] if 'reduce' in funcs else None)
views.append(ViewDefinition(v, v, map, reduce))
ViewDefinition.sync_many(dbm.database, views)
return dbm
def by_dateModified_view_ViewDefinition(resource):
fields = _get_fields(resource)
return ViewDefinition(resource, 'by_dateModified', '''function(doc) {
if(doc.doc_type == '%(resource)s' && doc.status != 'draft') {
var fields=%(fields)s, data={};
for (var i in fields) {
if (doc[fields[i]]) {
data[fields[i]] = doc[fields[i]]
}
}
emit(doc.dateModified, data);
}
}''' % dict(resource=resource[:-1].title(), fields=repr(fields)))
def createViews_per_OBSID(db, OBSID):
OBSIDViewCode = '''
function(doc) {
if(doc.type == "token") {
if (doc.OBSID == "%s"){
emit(doc._id, doc._id);
}
}
}
'''
# obsID View
obsid_view = ViewDefinition('Observations', OBSID, OBSIDViewCode % (OBSID))
obsid_view.sync(db)
def _init_terminal_contracts_view(self):
terminal_contracts_view = ViewDefinition(
'contract_views',
'has_terminal_status',
'''function(doc) {
if (doc.status == 'pending.terminated' || doc.status == 'pending.unsuccessful') {
emit(doc._id, doc.status);
}
}
'''
)
terminal_contracts_view.sync(self._db)
return terminal_contracts_view
def __get__(self, instance, cls=None):
if self.wrapper is DEFAULT:
def wrapper(row):
if row.doc is not None:
return cls.wrap(row.doc)
data = row.value
data['_id'] = row.id
return cls.wrap(data)
else:
wrapper = self.wrapper
return ViewDefinition(self.design, self.name, self.map_fun,
self.reduce_fun, language=self.language,
wrapper=wrapper, **self.defaults)
def test_by_local_seq_view_ViewDefinition(resource='tenders'):
changes_fields = _get_changes_fields(resource)
return ViewDefinition(resource, 'test_by_local_seq', '''function(doc) {
if(doc.doc_type == '%(resource)s' && doc.status != 'draft' && doc.mode == 'test') {
var fields=%(fields)s, data={};
for (var i in fields) {
if (doc[fields[i]]) {
data[fields[i]] = doc[fields[i]]
}
}
emit(doc._local_seq, data);
}
}''' % dict(resource=resource[:-1].title(), fields=repr(changes_fields)))