def add_view(self, view_name="test_view",
cond='doc.lock > 0 && doc.done > 0 && doc.output < 0 ',
emit_value='doc._id', emit_value2='doc._id'):
"""Adds a view to the db, needs a view name and a condition.
Emits all tokens with the type of TokenHandler.t_type, that also match the condition
:param view_name: The name of the new view to be created
:type view_name: str
:param cond: A string containing the condition which all tokens of the view must match.
It can include boolean operators '>', '<'and '&&'.
The token's fields are refered to as 'doc.field'
:type cond: str
:param emit_value: The (first) emit value that is returne by the view.
If you look on couchdb/request the tokens from the view, you'll get two values.
This will be the first (typically the token's ID)
:type emit_value: str
:param emit_value2: The second emit value. You can thus return the token's ID
and its status for example
:type emit_value2: str
"""
general_view_code = '''
function(doc) {
if(doc.type == "%s") {
if(%s) {
emit(%s, %s );
}
}
}
'''
view = ViewDefinition(self.t_type, view_name, general_view_code % (
self.t_type, cond, emit_value, emit_value2))
self.views[view_name] = view
view.sync(self.database)
def get_id_map():
"""A map function that stores a view of the id->sn conversion for
all entries of a specific type.
"""
tp = session['info']['type']
gr = session['group']
db = get_db()
# Ask for the view.
res = db.view('_design/content/_view/' + '_'.join([gr, tp, 'id_map']))
# See if the view is really there.
try:
len(res)
return res
except:
mapfun = """
function(doc) {
if (doc.type == '""" + session['info']['type'] + """')
emit(doc.id, doc.sn)
}"""
view = ViewDefinition('content', '_'.join([gr, tp, 'id_map']), mapfun)
view.sync(db)
return db.view('_design/content/_view/' + '_'.join([gr, tp, 'id_map']))
def get_twitter_id_from_couch(database):
id_list = []
view_result = database.view('_design/twitter/_view/byTwitterID')
try:
view_result.total_rows
except:
view = ViewDefinition(
'twitter', 'byTwitterID', '''function(doc) {
if (doc.type === "sentiment"){
emit(doc._id, doc.id);
}
}''')
view.get_doc(database)
view.sync(database)
view_result = database.view('_design/twitter/_view/byTwitterID')
for item in view_result:
twitter_id = item.value
if twitter_id not in id_list:
id_list.append(twitter_id)
return id_list
def setUpModule():
"""Create test databases in local server"""
if not has_couchdb:
return
server = couchdb.Server()
## Create databases
for x in DATABASES:
if not server.__contains__(x):
LOG.info("Creating database {}".format(x))
server.create(x)
## Create views for flowcells and samples
for dbname in DATABASES:
dblab = dbname.replace("-test", "")
db = server[dbname]
for k, v in VIEWS[dblab].items():
for title, view in v.items():
viewdef = ViewDefinition(k, title, view)
viewdef.sync(db)
## Create and upload project summary
with open(os.path.join(filedir, "data", "config", "project_summary.yaml")) as fh:
prj_sum = yaml.load(fh)
db = server["samples-test"]
p_con = ProjectSummaryConnection(dbname="projects-test", username="******", password="******")
for p in prj_sum:
prj = ProjectSummaryDocument(**p)
p_con.save(prj, key="project_name")
def get_attribute(attr):
"""A map function that stores an attribute for all entries of a
specific type.
"""
tp = session['info']['type']
gr = session['group']
db = get_db()
# Ask for the view.
res = db.view('_design/attributes/_view/' + '_'.join([gr, tp, attr]))
# See if the view is really there.
try:
len(res)
return res
except:
mapfun = """
function(doc) {
if (doc.type == '""" + session['info']['type'] + """')
emit(doc['""" + attr + """'], doc.id)
}"""
view = ViewDefinition('attributes', '_'.join([gr, tp, attr]), mapfun)
view.sync(db)
return db.view('_design/attributes/_view/' + '_'.join([gr, tp, attr]))
def connect_db(db_name):
"""Connect to the database if open, and start database if not running."""
try:
client = couchdb.Server()
except:
subprocess.call(['couchdb', '-b'])
time.sleep(2)
client = couchdb.Server()
try:
db = client[db_name]
except:
client.create(db_name)
db = client[db_name]
toc = {}
toc['n_runs'] = 0
toc['_id'] = 'toc'
db.save(toc)
#create permanent view to all if one doesn't exist
if '_design/all' not in db:
view_def = ViewDefinition('all', 'all','''
function(doc) {
if( doc.run_number )
emit(parseInt(doc.run_number), doc);
}''')
view_def.sync(db)
return db
def make_view(self, name, javascript_function):
try:
mapper = """function(doc){emit(doc.id_str, doc.id_str);}"""
vw = ViewDefinition('index', name, mapper)
vw.sync(self.dao)
except Exception as e:
print "Error making maxid view %s" % e
def add_overview_view(self):
overviewMapCode = '''
function(doc) {
if(doc.type == "%s") {
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.done == 0) {
emit('locked', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0) {
emit('done', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output > 0) {
emit('error', 1);
}
}
}
'''
overviewReduceCode = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition(self.t_type, 'overview_total',
overviewMapCode % (self.t_type),
overviewReduceCode)
self.views['overview_total'] = overview_total_view
overview_total_view.sync(self.db)
def _prepare_couchdb(self):
server = Server(self.couch_url,
session=Session(retry_delays=range(10)))
try:
if self.db_name not in server:
self.db = server.create(self.db_name)
else:
self.db = server[self.db_name]
except Exception as e:
LOGGER.error('Database error: {}'.format(repr(e)))
raise
by_date_modified_view = ViewDefinition(
self.resource, 'by_dateModified', '''function(doc) {
if (doc.doc_type == '%(resource)s') {
var fields=['%(doc_type)sID'], data={};
for (var i in fields) {
if (doc[fields[i]]) {
data[fields[i]] = doc[fields[i]]
}
}
emit(doc.dateModified, data);
}}''' % dict(resource=self.resource[:-1].title(),
doc_type=self.resource[:-1])
)
by_date_modified_view.sync(self.db)
validate_doc = self.db.get(VALIDATE_BULK_DOCS_ID,
{'_id': VALIDATE_BULK_DOCS_ID})
if validate_doc.get('validate_doc_update') != VALIDATE_BULK_DOCS_UPDATE:
validate_doc['validate_doc_update'] = VALIDATE_BULK_DOCS_UPDATE
self.db.save(validate_doc)
LOGGER.info('Validate document update view saved.')
else:
LOGGER.info('Validate document update view already exist.')
def get_tweet_ids_by_city(database, city):
id_list = []
view_result = database.view('_design/twitter/_view/tweetIdByCity?key="' +
city + '"')
try:
view_result.total_rows
except:
view = ViewDefinition(
'twitter', 'tweetIdByCity', '''function(doc) {
if (doc.type === "tweet"){
emit(doc.city, doc.id);
}
}''')
view.get_doc(database)
view.sync(database)
view_result = database.view(
'_design/twitter/_view/tweetIdByCity?key=\"' + city + '\"')
if not view_result.total_rows:
return []
for item in view_result:
tweet_id = item.value
if tweet_id not in id_list:
id_list.append(tweet_id)
return id_list
def test_view(self):
# This test does quite a bit. First, create 4 test records.
# Then, create a view that will emit those records and insert that into
# the db. Finally, call our cushion.view object and compare results.
self._save_some_data({'foo': 1, 'bar': 'a'})
self._save_some_data({'foo': 2, 'bar': 'a'})
self._save_some_data({'foo': 3, 'bar': 'b'})
self._save_some_data({'foo': 4, 'bar': 'b'})
fake_map = """ function (doc) { emit(doc['bar'], doc); } """
# we're going to use python-couchdb's dynamic view loader stuff here
from couchdb.design import ViewDefinition
from couchdb.client import Server
global baseurl
cdb = Server(baseurl)
couchdb = cdb[self.dbname]
view_defn = ViewDefinition(
'test', 'view',
map_fun = fake_map,
language = 'javascript' )
view_defn.sync(couchdb)
self.cushion.view(self.dbname, 'test/view', self.stop, key='b')
records = self.wait()
self.assertTrue(len(records) == 2)
def test_view(self):
# This test does quite a bit. First, create 4 test records.
# Then, create a view that will emit those records and insert that into
# the db. Finally, call our cushion.view object and compare results.
self._save_some_data({'foo': 1, 'bar': 'a'})
self._save_some_data({'foo': 2, 'bar': 'a'})
self._save_some_data({'foo': 3, 'bar': 'b'})
self._save_some_data({'foo': 4, 'bar': 'b'})
fake_map = """ function (doc) { emit(doc['bar'], doc); } """
# we're going to use python-couchdb's dynamic view loader stuff here
from couchdb.design import ViewDefinition
from couchdb.client import Server
global baseurl
cdb = Server(baseurl)
couchdb = cdb[self.dbname]
view_defn = ViewDefinition('test',
'view',
map_fun=fake_map,
language='javascript')
view_defn.sync(couchdb)
self.cushion.view(self.dbname, 'test/view', self.stop, key='b')
records = self.wait()
self.assertTrue(len(records) == 2)
def make_view(self, name, javascript_function):
try:
mapper = """function(doc){emit(doc.id_str, doc.id_str);}"""
vw = ViewDefinition('index', name, mapper)
vw.sync(self.dao)
except Exception as e:
print("Error making maxid view %s" % e)
def create_view(db, view_tuples):
for vt in view_tuples:
try:
view = ViewDefinition(vt[0], vt[1], vt[2], vt[3])
view.sync(db)
except:
pass
def setUpModule():
"""Create test databases in local server"""
if not has_couchdb:
return
server = couchdb.Server()
## Create databases
for x in DATABASES:
if not server.__contains__(x):
LOG.info("Creating database {}".format(x))
server.create(x)
## Create views for flowcells and samples
for dbname in DATABASES:
dblab = dbname.replace("-test", "")
db = server[dbname]
for k, v in VIEWS[dblab].items():
for title, view in v.items():
viewdef = ViewDefinition(k, title, view)
viewdef.sync(db)
## Create and upload project summary
with open(os.path.join(filedir, "data", "config",
"project_summary.yaml")) as fh:
prj_sum = yaml.load(fh)
db = server["samples-test"]
p_con = ProjectSummaryConnection(dbname="projects-test",
username="******",
password="******")
for p in prj_sum:
prj = ProjectSummaryDocument(**p)
p_con.save(prj, key="project_name")
def createView(self, viewName):
view = ViewDefinition(
'tweets', viewName, '''function(doc) {
emit(doc.text, doc.coordinates);
}''')
view.get_doc(self.db)
view.sync(self.db)
print("create view user, view should be created")
def _create_view(self):
view_map = 'function(doc) { if(doc.bag_of_words) { for (word in doc.bag_of_words) { emit([doc.bag_of_words[word], doc.polarity, doc.suburb],1)}}}'
view_reduce = '_sum'
view = ViewDefinition('application',
'mainview',
view_map,
reduce_fun=view_reduce)
view.sync(self.DBRef)
def sync_views(self):
for url, view in self.config.viewdata["views"].items():
segments = url.split("/")
designdoc = segments[0]
name = "/".join(segments[1:])
view = ViewDefinition(designdoc, name, view[0], view[1])
view.get_doc(self.db)
view.sync(self.db)
def createViewUsers(self):
view = ViewDefinition(
'tweets', "users", '''function(doc) {
emit(doc.user["id"], doc.done);
}''')
view.get_doc(self.db)
view.sync(self.db)
print("user view should be created")
def couch_get_view(self, db, get_token):
view = ViewDefinition(
'%sview' % db.name, '%s' % db.name,
'''function(doc) {if (doc._id == "%s") emit(doc);}''' % get_token)
view.sync(db)
for res in db.view('_design/%sview/_view/%s' % (db.name, db.name)):
return {res.id: res.key}
def create_views(self):
print("++ Creating views")
view = ViewDefinition(
'get_crashes', 'all', '''function(doc) {
emit(doc.crashhash,doc.crashcount);
}''')
view.get_doc(self.db)
view.sync(self.db)
def createViews(db):
generalViewCode = '''
function(doc) {
if(doc.type == "token") {
if(%s) {
emit(doc._id, doc._id);
}
}
}
'''
# todo View
todoCondition = 'doc.lock == 0 && doc.done == 0'
todo_view = ViewDefinition('Monitor', 'todo',
generalViewCode % (todoCondition))
todo_view.sync(db)
# locked View
lockedCondition = 'doc.lock > 0 && doc.done == 0'
locked_view = ViewDefinition('Monitor', 'locked',
generalViewCode % (lockedCondition))
locked_view.sync(db)
# done View
doneCondition = 'doc.lock > 0 && doc.done > 0 && doc.output == 0'
done_view = ViewDefinition('Monitor', 'done',
generalViewCode % (doneCondition))
done_view.sync(db)
#
errorCondition = 'doc.lock > 0 && doc.done > 0 && doc.output > 0'
error_view = ViewDefinition('Monitor', 'error',
generalViewCode % (errorCondition))
error_view.sync(db)
# overview_total View -- lists all views and the number of tokens in each view
overviewMapCode = '''
function(doc) {
if(doc.type == "token") {
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.done == 0) {
emit('locked', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0) {
emit('done', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output > 0) {
emit('error', 1);
}
}
}
'''
overviewReduceCode = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition('Monitor', 'overview_total',
overviewMapCode, overviewReduceCode)
overview_total_view.sync(db)
def main():
server = couchdb.Server('http://localhost:5984')
dbname = 'test'
try:
server.delete(dbname)
print "Deleted %s" % dbname
except:
pass
ttime = 1334187642
db = server.create(dbname)
data = [('1', 'one', 'foo', ttime), ('2', 'two', 'bar', ttime),
('3', 'three', 'foo', ttime), ('4', 'three', 'bar', ttime),
('5', 'three', 'bar', ttime + 86400)]
for d in data:
key, tag, txt, tt = d
print "Input", key, tag, txt, tt
row = {'tag': tag, 'text': txt.lower(), 'time': tt}
db[key] = row
# query
_date = "2012-04-11"
dt = datetime.strptime(_date, "%Y-%m-%d").utctimetuple()
#dt = datetime.strptime(_date,"%Y-%m-%d")
#stime=int(time.mktime(dt.utctimetuple()))
# etime=stime+86400-1
view = ViewDefinition('index',
'daily_tags',
time_tag_mapper,
language='python')
view.sync(db)
# tags = [row.value for row in db.view('index/daily_tags',startkey=stime,endkey=etime)]
tags = [
row.value for row in db.view('index/daily_tags',
key=[dt.tm_year, dt.tm_mon, dt.tm_mday])
]
tags = list(set(tags))
print "Tags today", set(tags)
print ""
view = ViewDefinition('index',
'daily_tagcount',
tag_mapper,
reduce_fun=tag_sumreducer,
language='python')
view.sync(db)
#d = dt.timetuple()
d = dt
for tag in sorted(tags):
# tag_counts = [ (row.key, row.value) for row in db.view('index/tagcount', group=True) ]
_key = [d.tm_year, d.tm_mon, d.tm_mday, tag]
tag_count = [(row.value)
for row in db.view('index/daily_tagcount', key=_key)][0]
print "Found %d %s on %s-%s-%s " % (tag_count, tag, _key[0], _key[1],
_key[2])
def add_view(self, view, map_fun, reduce_fun=None, design_doc="Monitor",
*args, **kwargs):
""" Add a view to the database
All extra parameters are passed to couchdb.design.ViewDefinition
:param view: name of the view
:param map_fun: string of the javascript map function
:param reduce_fun: string of the javascript reduce function (optional)
"""
definition = ViewDefinition(
design_doc, view, map_fun, reduce_fun, *args, **kwargs)
definition.sync(self.db)
def getDocs():
DB = 'trafico_chile'
server = couchdb.Server('http://localhost:5984')
db = server[DB]
view = ViewDefinition('trafico_chile','by_date_time',sinceDateMapper,language='python')
view.sync(db)
docs = []
for row in db.view('trafico_chile/by_date_time',startkey=list(time.gmtime(time.time()-3600.*50.))):
_last_update = list(time.gmtime(time.time()))
docs.append(db.get(row.id))
return docs
def test_views_creation(self):
book_definition = yaml.load( open(DATADIR%'test_couchish_book.yaml').read() )
dvd_definition = yaml.load( open(DATADIR%'test_couchish_dvd.yaml').read() )
post_definition = yaml.load( open(DATADIR%'test_couchish_post.yaml').read() )
author_definition = yaml.load( open(DATADIR%'test_couchish_author.yaml').read() )
views_definition = yaml.load( open(DATADIR%'test_couchish_views.yaml').read() )
models_definition = {'book': book_definition, 'author': author_definition,'post': post_definition, 'dvd': dvd_definition}
viewdata = get_views(models_definition, views_definition)
for url, view in viewdata['views'].items():
designdoc = url.split('/')[0]
view = ViewDefinition(designdoc, url, view[0])
view.get_doc(self.db)
view.sync(self.db)
def _init_terminal_contracts_view(self):
terminal_contracts_view = ViewDefinition(
'contract_views',
'has_terminal_status',
'''function(doc) {
if (doc.status == 'pending.terminated' || doc.status == 'pending.unsuccessful') {
emit(doc._id, doc.status);
}
}
'''
)
terminal_contracts_view.sync(self._db)
return terminal_contracts_view
def createViews_per_OBSID(db, OBSID):
OBSIDViewCode = '''
function(doc) {
if(doc.type == "token") {
if (doc.OBSID == "%s"){
emit(doc._id, doc._id);
}
}
}
'''
# obsID View
obsid_view = ViewDefinition('Observations', OBSID, OBSIDViewCode % (OBSID))
obsid_view.sync(db)
def p20141208(self, request, *args):
from MyRingCouchDB import MyRingCouchDB
from MyRingUser import MyRingUser
from env_config import COUCHDB_USER, COUCHDB_PASS
from couchdb.design import ViewDefinition
MCD = MyRingCouchDB()
self.couch = MCD.instantiate_couchdb_as_admin()
self.couch.resource.credentials = (COUCHDB_USER, COUCHDB_PASS)
print('patch_20141208')
print(current_user.username)
#db = self.couch[self.user_database]
#user = MyRingUser.load(db,current_user)
user = self.MAM.select_user(current_user.username)
if user:
for ring in user['rings']:
db_ringname = current_user.username + '_' + str(
ring['ringname'])
print(db_ringname)
db2 = self.couch[db_ringname]
if not RingsSchema.load(db2, 'schema'):
schema = RingsSchema.load(db2, 'blueprint')
print('Schema:', schema)
schema._id = 'schema'
schema.store(db2)
view = ViewDefinition(
'ring', 'schema', '''
function(doc) {
if(doc._id=='schema'){
emit(doc._id, doc)
}
}
''')
view.get_doc(db2)
view.sync(db2)
d = {
'rq': current_user,
'template': 'avispa_rest/tools/flashresponsejson.html'
}
return d
def _create_views(self):
# twitter/count_type
count_type_map = 'function(doc) { emit([doc.type, doc.id], 1); }'
count_type_reduce = 'function(keys, values) { return sum(values); }'
view = ViewDefinition('twitter',
'count_type',
count_type_map,
reduce_fun=count_type_reduce)
view.sync(self.db)
# The unique key for each tweet is doc.id. The key (a 64-bit long) is represented as a string because
# the largest JavaScript long is 2^53.
# A problem arises because keys are sorted as strings and a doc.id may have fewer digits but a larger
# leading digit. So, it is sorted in the wrong order.
# The solutions is to zero-pad the doc.id to fill 19 digits. (The max 64-bit long - 2^63 - has 19 digits.)
# That is why we emit the doc.id key as ("0000000000000000000"+doc.id).slice(-19).
# twitter/get_tweets
get_tweets = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit(("0000000000000000000"+doc.id).slice(-19), doc); }'
view = ViewDefinition('twitter', 'get_tweets', get_tweets)
view.sync(self.db)
# twitter/get_tweets_by_date (sort by date and tweet id)
get_tweets_by_date = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit((new Date(doc.created_at).getTime())+"-"+("0000000000000000000"+doc.id).slice(-19), doc); }'
view = ViewDefinition('twitter', 'get_tweets_by_date',
get_tweets_by_date)
view.sync(self.db)
# twitter/get_users
get_users = 'function(doc) { if (doc.type == "TWITTER_USER") emit(doc.id, doc); }'
view = ViewDefinition('twitter', 'get_users', get_users)
view.sync(self.db)
def _create_views(self):
# twitter/count_type
count_type_map = 'function(doc) { emit([doc.type, doc.id], 1); }'
count_type_reduce = 'function(keys, values) { return sum(values); }'
view = ViewDefinition('twitter', 'count_type', count_type_map, reduce_fun=count_type_reduce)
view.sync(self.db)
# The unique key for each tweet is doc.id. The key (a 64-bit long) is represented as a string because
# the largest JavaScript long is 2^53.
# A problem arises because keys are sorted as strings and a doc.id may have fewer digits but a larger
# leading digit. So, it is sorted in the wrong order.
# The solutions is to zero-pad the doc.id to fill 19 digits. (The max 64-bit long - 2^63 - has 19 digits.)
# That is why we emit the doc.id key as ("0000000000000000000"+doc.id).slice(-19).
# twitter/get_tweets
get_tweets = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit(("0000000000000000000"+doc.id).slice(-19), doc); }'
view = ViewDefinition('twitter', 'get_tweets', get_tweets)
view.sync(self.db)
# twitter/get_tweets_by_date (sort by date and tweet id)
get_tweets_by_date = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit((new Date(doc.created_at).getTime())+"-"+("0000000000000000000"+doc.id).slice(-19), doc); }'
view = ViewDefinition('twitter', 'get_tweets_by_date', get_tweets_by_date)
view.sync(self.db)
# twitter/get_users
get_users = 'function(doc) { if (doc.type == "TWITTER_USER") emit(doc.id, doc); }'
view = ViewDefinition('twitter', 'get_users', get_users)
view.sync(self.db)
def create_views(self, db):
# https://markhaa.se/posts/couchdb-views-in-python/
# create the two views we will need (by week and by suburb)
# note that if they already exist in the DB it won't do anything
view_by_week = ViewDefinition('sentimentDocs',
'byWeek',
MAP_BY_WEEK,
reduce_fun=REDUCE_STATS)
view_by_week.get_doc(self.DB[db])
view_by_week.sync(self.DB[db])
view_by_suburb = ViewDefinition('sentimentDocs',
'bySuburb',
MAP_BY_SUBURB,
reduce_fun=REDUCE_STATS)
view_by_suburb.get_doc(self.DB[db])
view_by_suburb.sync(self.DB[db])
view_by_week = ViewDefinition('sentimentDocs',
'byWeek2',
MAP_BY_WEEK2,
reduce_fun=REDUCE_STATS)
view_by_week.get_doc(self.DB[db])
view_by_week.sync(self.DB[db])
view_by_suburb = ViewDefinition('sentimentDocs',
'bySuburb2',
MAP_BY_SUBURB2,
reduce_fun=REDUCE_STATS)
view_by_suburb.get_doc(self.DB[db])
view_by_suburb.sync(self.DB[db])
def db(request):
SERVER = couchdb.Server(SERVER_URL)
def delete():
if DB_NAME in SERVER:
del SERVER[DB_NAME]
delete()
db = SERVER.create(DB_NAME)
view = ViewDefinition(
'index',
'by_date',
map_fun="""function (doc) { emit(doc.id, doc.date); }""")
view.sync(db)
request.addfinalizer(delete)
def db(request):
SERVER = couchdb.Server(couchdb_url)
def delete():
if DB_NAME in SERVER:
del SERVER[DB_NAME]
delete()
db = SERVER.create(DB_NAME)
view = ViewDefinition(
'test', 'all', """
function (doc) { emit(doc._id, doc._id); }
""")
view.sync(db)
request.addfinalizer(delete)
def map_reduce(couchServer,database,designDocument,viewName,map_fun,reduce_fun):
couch=couchServer
print(couch)
db=couch[database]
print(db)
designDocument=designDocument
viewName=viewName
map_fun=map_fun
reduce_fun=reduce_fun
view=ViewDefinition(design=designDocument,name=viewName,
map_fun=map_fun,
reduce_fun=reduce_fun)
view.sync(db)
print('complete!')
def add_mapreduce_view(self, view_name="test_mapred_view",
cond='doc.PIPELINE_STEP == "pref_cal1" '):
"""
While the overview_view is applied to all the tokens in the design document,
this 'mapreduce' view is useful if instead of regular view, you want to filter
the tokens and display the user with a 'mini-oververview' view.
This way you can check the status of a subset of the tokens.
"""
overview_map_code = '''
function(doc) {
if(doc.type == "%s" )
if(%s){
{
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.status == 'downloading' ) {
emit('downloading', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0 ) {
emit('done', 1);
}
if(doc.lock > 0 && doc.output != 0 && doc.output != "" ) {
emit('error', 1);
}
if(doc.lock > 0 && doc.status == 'launched' ) {
emit('waiting', 1);
}
if(doc.lock > 0 && doc.done==0 && doc.status!='downloading' ) {
emit('running', 1);
}
}
}
}
'''
overview_reduce_code = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition(self.t_type, view_name,
overview_map_code % (
self.t_type, cond),
overview_reduce_code)
self.views['overview_total'] = overview_total_view
overview_total_view.sync(self.database)
def test_load_couchdb(self):
"""
Test that resource instances and tiles load into a mobile survey's couchdb
"""
post_data = {
"datadownloadconfig": {"count":1000,"download":True,"resources":['d84a098c-368b-11e8-bafc-c4b301baab9f'],"custom":""},
"startdate":"2018-01-29",
"tilecache":"",
"enddate":"2018-03-30",
"createdby_id":1,
"bounds":{"type":"FeatureCollection","features":[]},
"cards":['fe035187-368b-11e8-bf56-c4b301baab9f', '4215f135-369c-11e8-9544-c4b301baab9f'],
"lasteditedby_id":1,
"groups":[],
"active":False,
"users":[],
"id": str(self.mobile_survey.id),
"name":"test survey",
"description":"a description"
}
response_json = self.post_mobile_survey(post_data)
couch = couchdb.Server(settings.COUCHDB_URL)
resources = 0
tiles = 0
if 'project_' + str(self.mobile_survey.id) in couch:
db = couch['project_' + str(self.mobile_survey.id)]
view = ViewDefinition('tests', 'all', '''function(doc) {emit(doc._id, null);}''')
view.get_doc(db)
view.sync(db)
for item in db.view('_design/tests/_view/all', include_docs=True):
if item.doc['type'] == 'tile':
tiles += 1
if item.doc['type'] == 'resource':
resources += 1
# tile_count = len(db.find({'selector': {'type': 'tile'}}))
# resource_count = len(db.find({'selector': {'type': 'resource'}}))
else:
print '{0} is not in couch'.format('project_' + str(self.mobile_survey.id))
self.assertEqual(tiles, 2)
self.assertEqual(resources,1)
def main():
server = couchdb.Server('http://localhost:5984')
dbname = 'test'
try:
server.delete(dbname)
print "Deleted %s"%dbname
except:
pass
ttime = 1334187642
db = server.create(dbname)
data = [('1','one', 'foo', ttime),
('2','two','bar', ttime),
('3','three','foo',ttime),
('4','three','bar',ttime),
('5','three','bar', ttime+86400)]
for d in data:
key, tag, txt, tt = d
print "Input",key, tag, txt, tt
row = {'tag':tag, 'text':txt.lower(), 'time': tt}
db[key] = row
# query
_date = "2012-04-11"
dt = datetime.strptime(_date,"%Y-%m-%d").utctimetuple()
#dt = datetime.strptime(_date,"%Y-%m-%d")
#stime=int(time.mktime(dt.utctimetuple()))
# etime=stime+86400-1
view = ViewDefinition('index','daily_tags',time_tag_mapper, language='python')
view.sync(db)
# tags = [row.value for row in db.view('index/daily_tags',startkey=stime,endkey=etime)]
tags = [row.value for row in db.view('index/daily_tags',key=[dt.tm_year, dt.tm_mon, dt.tm_mday])]
tags = list(set(tags))
print "Tags today",set(tags)
print ""
view = ViewDefinition('index','daily_tagcount', tag_mapper, reduce_fun=tag_sumreducer,
language='python')
view.sync(db)
#d = dt.timetuple()
d = dt
for tag in sorted(tags):
# tag_counts = [ (row.key, row.value) for row in db.view('index/tagcount', group=True) ]
_key = [d.tm_year, d.tm_mon, d.tm_mday, tag]
tag_count = [ (row.value) for row in db.view('index/daily_tagcount', key=_key) ][0]
print "Found %d %s on %s-%s-%s "%(tag_count,tag,_key[0],_key[1],_key[2])
def getDocs(self):
DB = 'trafico_chile'
server = couchdb.Server('http://localhost:5984')
db = server[DB]
view = ViewDefinition('trafico_chile',
'by_date_time',
sinceDateMapper,
language='python')
view.sync(db)
docs = []
for row in db.view('trafico_chile/by_date_time',
startkey=list(
time.gmtime(time.time() - 3600. * 2000))):
self._last_update = time.strftime('%d_%b_%Y_%H_%M_%S',
list(time.gmtime(time.time())))
docs.append(db.get(row.id))
return docs
def add_view(self,
v_name="test_view",
cond='doc.lock > 0 && doc.done > 0 && doc.output < 0 '):
"""Adds a view to the db, needs a view name and a condition. Emits all tokens with
the type of the current Token_Handler
"""
generalViewCode = '''
function(doc) {
if(doc.type == "%s") {
if(%s) {
emit(doc._id, doc._id);
}
}
}
'''
view = ViewDefinition(self.t_type, v_name,
generalViewCode % (self.t_type, cond))
self.views[v_name] = view
view.sync(self.db)
def test_custom_view(self):
cview = ViewDefinition('cview', 'cview',
"""function(doc) { emit(doc.name, 1); }""")
cview.sync(self.db)
init_length = len(self.db.view('_design/cview/_view/cview'))
test_values = [
(init_length, False),
(init_length + 1, True),
(init_length + 2, False),
]
target = CouchCountTarget(self.couch_client, DATABASE, init_length + 1,
'_design/cview/_view/cview')
for (d, result), doc in zip(test_values, TEST_DOCS):
self.assertEqual(result, target.exists())
self.db.save(doc)
def test_delete_all_in_view():
from melkman.db.util import delete_all_in_view
from couchdb.schema import Document
from couchdb.design import ViewDefinition
db = make_db()
view_bad = ViewDefinition('test_daiv', 'bad_items',
'''
function(doc) {
if (doc.badflag == true) {
emit(true, null);
}
}
''')
view_bad.sync(db)
for i in range(10):
doc = Document('doc_%d' % i)
doc['foo'] = i
if i % 2 == 1:
doc['badflag'] = True
doc.store(db)
for i in range(10):
assert 'doc_%d' % i in db
delete_all_in_view(db, view_bad)
for i in range(10):
doc_id = 'doc_%d' % i
if i % 2 == 0:
assert doc_id in db, 'expected %s in db' % doc_id
else:
assert doc_id not in db, 'expected %s not in db' % doc_id
def add_overview_view(self):
""" Helper function that creates the Map-reduce view which makes it easy to count
the number of jobs in the 'locked','todo','downloading','error' and 'running' states
"""
overview_map_code = '''
function(doc) {
if(doc.type == "%s") {
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.status == 'downloading' ) {
emit('downloading', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0 ) {
emit('done', 1);
}
if(doc.lock > 0 && doc.output != 0 ) {
emit('error', 1);
}
if(doc.lock > 0 && doc.status == 'launched' ) {
emit('waiting', 1);
}
if(doc.lock > 0 && doc.done==0 && doc.status!='downloading' ) {
emit('running', 1);
}
}
}
'''
overview_reduce_code = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition(self.t_type, 'overview_total',
overview_map_code % (self.t_type),
overview_reduce_code)
self.views['overview_total'] = overview_total_view
overview_total_view.sync(self.database)
def fetch_docs_of_type(doc_type, db, since=None):
LOG.debug(
f'Fetching all docs of type {doc_type} from database {db.name} for first time'
)
if since is None:
map_func = docs_of_type_view_template.substitute(doc_type=doc_type)
map_func = "".join(map_func.split())
view = ViewDefinition('hotspur', doc_type, map_func)
view.sync(db)
results = view(db, update_seq=True)
return {
"docs": [row.value for row in results.rows],
"last_seq": results.update_seq
}
else:
update = db.changes(since=since,
include_docs=True,
filter="_view",
view=f'hotspur/{doc_type}')
return {
"docs": [row["doc"] for row in update["results"]],
"last_seq": update["last_seq"]
}
def multiply(self, m1, m2):
'''Perform matrix multiplication between m1 and m2 using MapReduce'''
product = []
for row in range(1, self.dim+1):
for col in range(1, self.dim+1):
#maps proper row and column of m1 and m2
map_fun = ('''function(doc) {
for (i=0;i < doc.elements.length; i++) {
if((doc.elements[i][0]== %s && doc._id == '%s')|(doc.elements[i][1]== %s && doc._id=='%s')) {
emit(null, doc.elements[i]);
}
}
}''' % (str(row), m1.id, str(col), m2.id) )
#calculates dot product of the row and column
red = ('''function(key, data) {
var total = 0;
var dim = %s
for(i=0; i < dim; i++) {
total += data[i][2] * data[i+dim][2];
}
return total;
}''' % self.dim)
#creates a permanent view in Couch
view = ViewDefinition('oper','multiply',map_fun,reduce_fun=red)
view.sync(self.matrix_db)
design = "_design/oper/_view/multiply"
for result in self.matrix_db.view(design):
product.append([row, col, result.value])
#prints the result
print m1.id, 'X', m2.id
print_matrix(product)
def _create_views(self):
# twitter/count_type
count_type_map = 'function(doc) { emit([doc.type, doc.id], 1); }'
count_type_reduce = 'function(keys, values) { return sum(values); }'
view = ViewDefinition('twitter', 'count_type', count_type_map, reduce_fun=count_type_reduce)
view.sync(self.db)
# twitter/get_tweets
get_tweets = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit(doc.id, doc); }'
view = ViewDefinition('twitter', 'get_tweet', get_tweets)
view.sync(self.db)
# twitter/get_users
get_users = 'function(doc) { if (doc.type == "TWITTER_USER") emit(doc.id, doc); }'
view = ViewDefinition('twitter', 'get_users', get_users)
view.sync(self.db)
def view_checkins_date():
view = ViewDefinition('index', 'map_checkins_date', map_checkins_date, language='python')
view.sync(couch[prefix + 'checkins'])
@classmethod
def load(cls, id, db=projects):
return super(Project, cls).load(db, id)
@classmethod
def load_or_create(cls, id, db=projects):
project = cls.load(id, db)
if not project:
project = cls(id)
return project
project_names = ViewDefinition("projects", "all", '''\
function(doc) {
if (doc.project) {
emit(doc.project, 1);
}
}''',
'''\
function(keys, values, rereduce) {
return sum(values);
}''')
project_names.sync(timesheets)
Timesheet._all_timesheets.sync(timesheets)
Timesheet._by_date.sync(timesheets)
Timesheet._by_invoice.sync(timesheets)
Timesheet._by_project.sync(timesheets)
Timesheet._by_project_unbilled.sync(timesheets)
Timesheet._by_date_unbilled.sync(timesheets)
function(doc) {
emit(1, parseInt(doc._id));
}""",
"""\
function(keys, values, rereduce) {
var max = 0;
for (var i in values) {
if (values[i] > max) {
max = values[i];
}
}
return max;
}""",
)
project_names.sync(timesheets)
type_names.sync(timesheets)
Timesheet._all_timesheets.sync(timesheets)
Timesheet._by_date.sync(timesheets)
Timesheet._by_invoice.sync(timesheets)
Timesheet._by_project.sync(timesheets)
Timesheet._by_project_unbilled.sync(timesheets)
Timesheet._by_date_unbilled.sync(timesheets)
ProjectType._by_project_type.sync(project_types)
last_invoice_num.sync(invoices)
Invoice._all_invoices.sync(invoices)
Invoice._by_project.sync(invoices)
project_names = Project.project_list()
for name in project_names:
doc = db.get(adoc)
doc['sent_type'] = some_dic[adoc]
db.save(doc)
#not currently working
rows = db.view('_all_docs', keys=['key1', 'key2', 'missing'], include_docs=True)
docs = [row.doc for row in rows]
#create a view
#pass javascript
code = '''function(doc) { if(doc.frm == "*****@*****.**") emit(doc.frm, null); }'''
results = db.query(code)
for res in results:
print res.key
rpt_view = ViewDefinition('reports', 'fromemail', '''function(doc) { if(doc.frm == "*****@*****.**") emit(doc.frm, null); }''')
rpt_view.sync(db)
for res in db.view("_design/reports/_view/fromemail"):
print res.id, res.key
#class of document for updating
class Jobtobedone(Document):
frm = TextField()
to = ListField(TextField())
sub = TextField()
added = DateTimeField(default=datetime.now())
def view_toplevel_categories():
view = ViewDefinition('index', 'map_toplevel_categories', map_toplevel_categories, reduce_values, language='python')
view.sync(couch['categories'])
def view_checkins_count():
view = ViewDefinition('index', 'map_checkins_count', map_checkins_count, reduce_sum, language='python')
view.sync(couch[prefix + 'checkins'])
def create_view(self, design, name, map_fun, reduce_fun=None, language="javascript", wrapper=None, **defaults):
"""Just uses ViewDefinition but makes sure it uses it"""
view = ViewDefinition(design, name, map_fun, reduce_fun=None, language="javascript", wrapper=None, **defaults)
view.sync(self.db)
db = server[DB]
# Query out the documents at a given group level of interest
# Group by year, month, day
docs = db.view('index/doc_count_by_date_time', group_level=3)
# Now, load the documents keyed by [year, month, day] into a new database
db_scratch = server.create(DB + '-num-per-day')
db_scratch.update(docs)
def transposeMapper(doc):
yield (doc['value'], doc['key'])
view = ViewDefinition('index',
'num_per_day',
transposeMapper,
language='python')
view.sync(db_scratch)
fields = ['Date', 'Count']
pt = PrettyTable(fields=fields)
[pt.set_field_align(f, 'l') for f in fields]
for row in db_scratch.view('index/num_per_day'):
if row.key > 10: # display stats where more than 10 messages were sent
pt.add_row(['-'.join([str(i) for i in row.value]), row.key])
def _sync_test_views(self):
dashboard_view = ViewDefinition("record_docs", "by_ingestion_sequence_include_status",
"""function(doc) { if (doc.type == 'record') { emit(doc.ingestionSequence, doc.status) } }""")
dashboard_view.sync(self.dashboard_db)
def view_venues():
view = ViewDefinition('index', 'map_venues', map_venues, reduce_sum, language='python')
view.sync(couch[prefix + 'checkins'])
# -*- coding: utf-8 -*-
import sys
import couchdb
from couchdb.design import ViewDefinition
import json
from pa_setting import dbname
server = couchdb.Server('http://localhost:5984')
db = server[dbname]
def dateTimeToDocMapper(doc):
from dateutil.parser import parse
from datetime import datetime as dt
if doc.get('created_at'):
# [year, month, day, hour, min, sec]
_date = list(dt.timetuple(parse(doc['created_at']))[:-3])
yield (_date, doc)
view = ViewDefinition('index', 'by_date_time', dateTimeToDocMapper, language='python')
view.sync(db)
def create_views_in_db_hunters( db ):
view = ViewDefinition(
'words',
'random',
'''function( doc ) {
emit( Math.random(), doc.words );
}''')
view.sync( db )
view2 = ViewDefinition(
'words',
'all_occurrences',
'''function( doc ) {
var words = doc.words;
for(word in words){
emit( [word, words[word]], {'username':doc.username, 'image_url':doc.image_url['120px'] } );
}
}''')
view2.sync( db )
view3 = ViewDefinition(
'list',
'by_username',
'''function( doc ) {
emit( doc.username, doc );
}''')
view3.sync( db )
view4 = ViewDefinition(
'list',
'by_votes_count',
'''function( doc ) {
emit( doc.votes_count, [doc.username, doc.image_url["120px"] ] );
}''')
view4.sync( db )
view5 = ViewDefinition(
'list',
'by_posts_count',
'''function( doc ) {
emit( doc.posts_count, [doc.username, doc.image_url["120px"] ] );
}''')
view5.sync( db )
view6 = ViewDefinition(
'list',
'by_average_votes',
'''function( doc ) {
var average = doc.votes_count/doc.posts_count;
emit( Math.round(average), [doc.username, doc.image_url["120px"] ] );
}''')
view6.sync( db )
# tweet value.
def id_mapper(doc):
yield (None, doc['id'])
# Find the maximum tweet id
def max_finding_reducer(keys, values, rereduce):
return max(values)
view = ViewDefinition('index',
'max_tweet_id',
id_mapper,
max_finding_reducer,
language='python')
view.sync(db)
try:
KW['since_id'] = int([_id for _id in db.view('index/max_tweet_id')
][0].value)
except IndexError, e:
KW['since_id'] = 1
# Harvest tweets for the given timeline.
# For friend and home timelines, the unofficial limitation is about 800 statuses
# although other documentation may state otherwise. The public timeline only returns
# 20 statuses and gets updated every 60 seconds, so consider using the streaming API
# for public statuses. See http://bit.ly/fgJrAx
# Note that the count and since_id params have no effect for the public timeline
page_num = 1
while page_num <= MAX_PAGES:
@classmethod
def load(cls, id, db=invoices):
return super(Invoice, cls).load(db, id)
@classmethod
def next_invoice_number(cls):
result = last_invoice_num(invoices)
if result.rows:
return result.rows[0]['value'] + 1
return 1
last_invoice_num = ViewDefinition("last_num", "all", '''\
function(doc) {
emit(1, parseInt(doc._id));
}''',
'''\
function(keys, values, rereduce) {
var max = 0;
for (var i in values) {
if (values[i] > max) {
max = values[i];
}
}
return max;
}''')
last_invoice_num.sync(invoices)
Invoice._all_invoices.sync(invoices)
Invoice._by_project.sync(invoices)
