def setUpModule():
"""Create test databases in local server"""
if not has_couchdb:
return
server = couchdb.Server()
## Create databases
for x in DATABASES:
if not server.__contains__(x):
LOG.info("Creating database {}".format(x))
server.create(x)
## Create views for flowcells and samples
for dbname in DATABASES:
dblab = dbname.replace("-test", "")
db = server[dbname]
for k, v in VIEWS[dblab].items():
for title, view in v.items():
viewdef = ViewDefinition(k, title, view)
viewdef.sync(db)
## Create and upload project summary
with open(os.path.join(filedir, "data", "config", "project_summary.yaml")) as fh:
prj_sum = yaml.load(fh)
db = server["samples-test"]
p_con = ProjectSummaryConnection(dbname="projects-test", username="******", password="******")
for p in prj_sum:
prj = ProjectSummaryDocument(**p)
p_con.save(prj, key="project_name")
def test_view(self):
# This test does quite a bit. First, create 4 test records.
# Then, create a view that will emit those records and insert that into
# the db. Finally, call our cushion.view object and compare results.
self._save_some_data({'foo': 1, 'bar': 'a'})
self._save_some_data({'foo': 2, 'bar': 'a'})
self._save_some_data({'foo': 3, 'bar': 'b'})
self._save_some_data({'foo': 4, 'bar': 'b'})
fake_map = """ function (doc) { emit(doc['bar'], doc); } """
# we're going to use python-couchdb's dynamic view loader stuff here
from couchdb.design import ViewDefinition
from couchdb.client import Server
global baseurl
cdb = Server(baseurl)
couchdb = cdb[self.dbname]
view_defn = ViewDefinition(
'test', 'view',
map_fun = fake_map,
language = 'javascript' )
view_defn.sync(couchdb)
self.cushion.view(self.dbname, 'test/view', self.stop, key='b')
records = self.wait()
self.assertTrue(len(records) == 2)
def make_view(self, name, javascript_function):
try:
mapper = """function(doc){emit(doc.id_str, doc.id_str);}"""
vw = ViewDefinition('index', name, mapper)
vw.sync(self.dao)
except Exception as e:
print("Error making maxid view %s" % e)
def test_view(self):
# This test does quite a bit. First, create 4 test records.
# Then, create a view that will emit those records and insert that into
# the db. Finally, call our cushion.view object and compare results.
self._save_some_data({'foo': 1, 'bar': 'a'})
self._save_some_data({'foo': 2, 'bar': 'a'})
self._save_some_data({'foo': 3, 'bar': 'b'})
self._save_some_data({'foo': 4, 'bar': 'b'})
fake_map = """ function (doc) { emit(doc['bar'], doc); } """
# we're going to use python-couchdb's dynamic view loader stuff here
from couchdb.design import ViewDefinition
from couchdb.client import Server
global baseurl
cdb = Server(baseurl)
couchdb = cdb[self.dbname]
view_defn = ViewDefinition('test',
'view',
map_fun=fake_map,
language='javascript')
view_defn.sync(couchdb)
self.cushion.view(self.dbname, 'test/view', self.stop, key='b')
records = self.wait()
self.assertTrue(len(records) == 2)
def add_view(self, view_name="test_view",
cond='doc.lock > 0 && doc.done > 0 && doc.output < 0 ',
emit_value='doc._id', emit_value2='doc._id'):
"""Adds a view to the db, needs a view name and a condition.
Emits all tokens with the type of TokenHandler.t_type, that also match the condition
:param view_name: The name of the new view to be created
:type view_name: str
:param cond: A string containing the condition which all tokens of the view must match.
It can include boolean operators '>', '<'and '&&'.
The token's fields are refered to as 'doc.field'
:type cond: str
:param emit_value: The (first) emit value that is returne by the view.
If you look on couchdb/request the tokens from the view, you'll get two values.
This will be the first (typically the token's ID)
:type emit_value: str
:param emit_value2: The second emit value. You can thus return the token's ID
and its status for example
:type emit_value2: str
"""
general_view_code = '''
function(doc) {
if(doc.type == "%s") {
if(%s) {
emit(%s, %s );
}
}
}
'''
view = ViewDefinition(self.t_type, view_name, general_view_code % (
self.t_type, cond, emit_value, emit_value2))
self.views[view_name] = view
view.sync(self.database)
def make_view(self, name, javascript_function):
try:
mapper = """function(doc){emit(doc.id_str, doc.id_str);}"""
vw = ViewDefinition('index', name, mapper)
vw.sync(self.dao)
except Exception as e:
print "Error making maxid view %s" % e
def connect_db(db_name):
"""Connect to the database if open, and start database if not running."""
try:
client = couchdb.Server()
except:
subprocess.call(['couchdb', '-b'])
time.sleep(2)
client = couchdb.Server()
try:
db = client[db_name]
except:
client.create(db_name)
db = client[db_name]
toc = {}
toc['n_runs'] = 0
toc['_id'] = 'toc'
db.save(toc)
#create permanent view to all if one doesn't exist
if '_design/all' not in db:
view_def = ViewDefinition('all', 'all','''
function(doc) {
if( doc.run_number )
emit(parseInt(doc.run_number), doc);
}''')
view_def.sync(db)
return db
def add_overview_view(self):
overviewMapCode = '''
function(doc) {
if(doc.type == "%s") {
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.done == 0) {
emit('locked', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0) {
emit('done', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output > 0) {
emit('error', 1);
}
}
}
'''
overviewReduceCode = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition(self.t_type, 'overview_total',
overviewMapCode % (self.t_type),
overviewReduceCode)
self.views['overview_total'] = overview_total_view
overview_total_view.sync(self.db)
def sync(self):
# Synchronizing all couchdb views of the document class
views = []
# Spare a lit bit of space for Javascript views
import re
def _minify(language, code):
if language == "javascript":
return re.compile(r" ?(\W) ?").sub(lambda match: match.group(1),
code)
else:
return code
for attr in self.doc_class.__dict__:
value = getattr(self.doc_class, attr)
if isinstance(value, ViewDefinition):
if value.map_fun:
value.map_fun = _minify(value.language, value.map_fun)
if value.reduce_fun:
value.reduce_fun = _minify(value.language, value.reduce_fun)
views.append(value)
elif isinstance(value, FilterFunction):
value[value.language] = _minify(value.language, value[value.language])
ViewDefinition.sync_many(self.database, views)
sync_docs(self.database, [self.doc_class])
def sync(self, app):
"""
This syncs the database for the given app. It will first make sure the
database exists, then synchronize all the views and run all the
callbacks with the connected database.
It will run any callbacks registered with `on_sync`, and when the
views are being synchronized, if a method called `update_design_doc`
exists on the manager, it will be called before every design document
is updated.
:param app: The application to synchronize with.
"""
server_url = app.config['COUCHDB_SERVER']
db_name = app.config['COUCHDB_DATABASE']
server = couchdb.Server(server_url)
if 'COUCHDB_USERNAME' in app.config and 'COUCHDB_PASSWORD' in app.config:
server.resource.credentials = (app.config['COUCHDB_USERNAME'],
app.config['COUCHDB_PASSWORD'])
if db_name not in server:
db = server.create(db_name)
else:
db = server[db_name]
OldViewDefinition.sync_many(db,
tuple(self.all_viewdefs()),
callback=getattr(self, 'update_design_doc',
None))
for callback in self.sync_callbacks:
callback(db)
def __init__(self, db_url, name=None):
url = "{}/{}".format(db_url, name)
get_or_create(db_url, name)
super(TendersStorage, self).__init__(url=url)
ViewDefinition.sync_many(self, [tenders_all,
tenders_date_modified,
tenders_date_modified_for_package])
def setUpModule():
"""Create test databases in local server"""
if not has_couchdb:
return
server = couchdb.Server()
## Create databases
for x in DATABASES:
if not server.__contains__(x):
LOG.info("Creating database {}".format(x))
server.create(x)
## Create views for flowcells and samples
for dbname in DATABASES:
dblab = dbname.replace("-test", "")
db = server[dbname]
for k, v in VIEWS[dblab].items():
for title, view in v.items():
viewdef = ViewDefinition(k, title, view)
viewdef.sync(db)
## Create and upload project summary
with open(os.path.join(filedir, "data", "config",
"project_summary.yaml")) as fh:
prj_sum = yaml.load(fh)
db = server["samples-test"]
p_con = ProjectSummaryConnection(dbname="projects-test",
username="******",
password="******")
for p in prj_sum:
prj = ProjectSummaryDocument(**p)
p_con.save(prj, key="project_name")
def _prepare_couchdb(self):
server = Server(self.couch_url,
session=Session(retry_delays=range(10)))
try:
if self.db_name not in server:
self.db = server.create(self.db_name)
else:
self.db = server[self.db_name]
except Exception as e:
LOGGER.error('Database error: {}'.format(repr(e)))
raise
by_date_modified_view = ViewDefinition(
self.resource, 'by_dateModified', '''function(doc) {
if (doc.doc_type == '%(resource)s') {
var fields=['%(doc_type)sID'], data={};
for (var i in fields) {
if (doc[fields[i]]) {
data[fields[i]] = doc[fields[i]]
}
}
emit(doc.dateModified, data);
}}''' % dict(resource=self.resource[:-1].title(),
doc_type=self.resource[:-1])
)
by_date_modified_view.sync(self.db)
validate_doc = self.db.get(VALIDATE_BULK_DOCS_ID,
{'_id': VALIDATE_BULK_DOCS_ID})
if validate_doc.get('validate_doc_update') != VALIDATE_BULK_DOCS_UPDATE:
validate_doc['validate_doc_update'] = VALIDATE_BULK_DOCS_UPDATE
self.db.save(validate_doc)
LOGGER.info('Validate document update view saved.')
else:
LOGGER.info('Validate document update view already exist.')
def sync_db(db):
ViewDefinition.sync_many(db, [
all_doc_tags,
Article.all_months, Article.all_tags, Article.by_month,
Article.by_tag, Article.by_time, Article.by_slug,
Comment.by_time, Comment.comment_count, Comment.by_anytime,
Documentation.by_path, Documentation.ids_for_version,
Documentation.doc_key,
Human.by_displayname, Human.by_email, Human.by_email_token,
Human.by_openid, Human.by_password_token,
Paste.by_author, Paste.by_tag, Paste.all_tags, Paste.by_time,
Paste.by_old_id, Paste.by_tag_time, Paste.by_session_id,
Rating.all_raters,
Snippet.by_date, Snippet.by_author, Snippet.by_slug, Snippet.by_title,
Snippet.by_author_id, Snippet.by_tag, Snippet.all_tags,
Snippet.author_totals,
Traceback.by_uuid, Traceback.by_time, Traceback.by_session_id,
])
def get_db(server, db_name):
try:
db = server[db_name]
except couchdb.ResourceNotFound:
db = server.create(db_name)
ViewDefinition.sync_many(db, db_views, remove_missing=True)
return db
def sync(self, app):
"""
This syncs the database for the given app. It will first make sure the
database exists, then synchronize all the views and run all the
callbacks with the connected database.
It will run any callbacks registered with `on_sync`, and when the
views are being synchronized, if a method called `update_design_doc`
exists on the manager, it will be called before every design document
is updated.
:param app: The application to synchronize with.
"""
server_url = app.config['COUCHDB_SERVER']
db_name = app.config['COUCHDB_DATABASE']
server = couchdb.Server(server_url)
if 'COUCHDB_USERNAME' in app.config and 'COUCHDB_PASSWORD' in app.config:
server.resource.credentials = (app.config['COUCHDB_USERNAME'], app.config['COUCHDB_PASSWORD'])
if db_name not in server:
db = server.create(db_name)
else:
db = server[db_name]
OldViewDefinition.sync_many(
db, tuple(self.all_viewdefs()),
callback=getattr(self, 'update_design_doc', None)
)
for callback in self.sync_callbacks:
callback(db)
def get_id_map():
"""A map function that stores a view of the id->sn conversion for
all entries of a specific type.
"""
tp = session['info']['type']
gr = session['group']
db = get_db()
# Ask for the view.
res = db.view('_design/content/_view/' + '_'.join([gr, tp, 'id_map']))
# See if the view is really there.
try:
len(res)
return res
except:
mapfun = """
function(doc) {
if (doc.type == '""" + session['info']['type'] + """')
emit(doc.id, doc.sn)
}"""
view = ViewDefinition('content', '_'.join([gr, tp, 'id_map']), mapfun)
view.sync(db)
return db.view('_design/content/_view/' + '_'.join([gr, tp, 'id_map']))
def get_attribute(attr):
"""A map function that stores an attribute for all entries of a
specific type.
"""
tp = session['info']['type']
gr = session['group']
db = get_db()
# Ask for the view.
res = db.view('_design/attributes/_view/' + '_'.join([gr, tp, attr]))
# See if the view is really there.
try:
len(res)
return res
except:
mapfun = """
function(doc) {
if (doc.type == '""" + session['info']['type'] + """')
emit(doc['""" + attr + """'], doc.id)
}"""
view = ViewDefinition('attributes', '_'.join([gr, tp, attr]), mapfun)
view.sync(db)
return db.view('_design/attributes/_view/' + '_'.join([gr, tp, attr]))
def create_couchdb_views(app, created_models, verbosity, **kwargs):
ViewDefinition.sync_many(db, [item_type_date, by_date])
ContentType.objects.get_or_create(
name='CouchDB Item',
app_label='couch_lifestream',
model='couchdbitem'
)
def create_view(db, view_tuples):
for vt in view_tuples:
try:
view = ViewDefinition(vt[0], vt[1], vt[2], vt[3])
view.sync(db)
except:
pass
def sync_design_databases(server: Server, db_names: dict):
# specific databases views
for db_key, views in VIEWS.items():
views.append(conflicts_view) # should be in every db
db_name = db_names[db_key]
ViewDefinition.sync_many(server[db_name],
views,
callback=add_index_options)
def sync_views(self):
for url, view in self.config.viewdata["views"].items():
segments = url.split("/")
designdoc = segments[0]
name = "/".join(segments[1:])
view = ViewDefinition(designdoc, name, view[0], view[1])
view.get_doc(self.db)
view.sync(self.db)
def __init__(self, host_url, db_name):
server = couchdb.Server(host_url)
if server.uuids():
self.uid = server.uuids()[0]
self.db = get_or_create_db(server, db_name)
ViewDefinition.sync_many(self.db, [releases_ocid, releases_id])
LOGGER.info("Starting storage: {}".format(self.db.info()))
def couch_get_view(self, db, get_token):
view = ViewDefinition(
'%sview' % db.name, '%s' % db.name,
'''function(doc) {if (doc._id == "%s") emit(doc);}''' % get_token)
view.sync(db)
for res in db.view('_design/%sview/_view/%s' % (db.name, db.name)):
return {res.id: res.key}
def setUp(self):
couch.create(self.test_db_name)
self.testDesign = ViewDefinition(
'test0', 'test', '''\function(doc) { emit(null, doc); }''')
self.testDesign2 = ViewDefinition(
'test1', 'test', '''function(doc) {
emit('hello world', 'I\nam\na\nmulti-line\nand\ttabbed\tstring');
}''')
def _create_view(self):
view_map = 'function(doc) { if(doc.bag_of_words) { for (word in doc.bag_of_words) { emit([doc.bag_of_words[word], doc.polarity, doc.suburb],1)}}}'
view_reduce = '_sum'
view = ViewDefinition('application',
'mainview',
view_map,
reduce_fun=view_reduce)
view.sync(self.DBRef)
def setUp(self):
self.server = Server('http://localhost:5984')
self.db_name = 'test_%s' % uuid4()
self.server.create(self.db_name)
db = self.server[self.db_name]
ViewDefinition.sync_many(db, couchdb_views)
self.db = CouchDBDatabase(db, lambda: six.text_type(uuid4()))
super(TestCouchDBBackend, self).setUp()
def upload_views(db_name, map_funcs):
"""
Wrapper to syncronize a view
@author BrendonCrawford
"""
ViewDefinition.sync_many(SERVER[db_name], map_funcs, True)
print "Uploaded views: <%s>" % db_name
return True
def upload_views(db_name, map_funcs):
"""
Wrapper to syncronize a view
@author BrendonCrawford
"""
ViewDefinition.sync_many(SERVER[db_name], map_funcs, True)
print "Uploaded views: <%s>" % db_name
return True
def _sync_views(self):
ViewDefinition.sync_many(self.adb, VIEWS)
_id = '_design/report'
original = self.adb.get(_id)
original['views']['lib'] = {
'jsonpatch': jsonpatch,
'tenders': tenders_lib,
'bids': bids_lib
}
self.adb.save(original)
def _sync_views(self):
ViewDefinition.sync_many(self.adb, VIEWS)
_id = '_design/report'
original = self.adb.get(_id)
original['views']['lib'] = {
'jsonpatch': jsonpatch,
'tenders': tenders_lib,
'bids': bids_lib
}
self.adb.save(original)
def _bootstrap(self):
if self.dbid not in self.server:
self.server.create(self.dbid)
if self.keydbid not in self.server:
self.server.create(self.keydbid)
self.db = self.server[self.dbid]
self.keydb = self.server[self.keydbid]
self.log_view = ViewDefinition(self.LOG_DDOCID, self.LOG_VIEW_NAME,
self.LOG_VIEW_CODE)
self.log_view.sync(self.db)
def add_view(self, view, map_fun, reduce_fun=None, design_doc="Monitor",
*args, **kwargs):
""" Add a view to the database
All extra parameters are passed to couchdb.design.ViewDefinition
:param view: name of the view
:param map_fun: string of the javascript map function
:param reduce_fun: string of the javascript reduce function (optional)
"""
definition = ViewDefinition(
design_doc, view, map_fun, reduce_fun, *args, **kwargs)
definition.sync(self.db)
def create_db(name):
dbm = get_db_manager('http://localhost:5984/', name)
views = []
for v in view_js.keys():
funcs = view_js[v]
map = (funcs['map'] if 'map' in funcs else None)
reduce = (funcs['reduce'] if 'reduce' in funcs else None)
views.append(ViewDefinition(v, v, map, reduce))
ViewDefinition.sync_many(dbm.database, views)
return dbm
def getDocs():
DB = 'trafico_chile'
server = couchdb.Server('http://localhost:5984')
db = server[DB]
view = ViewDefinition('trafico_chile','by_date_time',sinceDateMapper,language='python')
view.sync(db)
docs = []
for row in db.view('trafico_chile/by_date_time',startkey=list(time.gmtime(time.time()-3600.*50.))):
_last_update = list(time.gmtime(time.time()))
docs.append(db.get(row.id))
return docs
def _init_terminal_contracts_view(self):
terminal_contracts_view = ViewDefinition(
'contract_views',
'has_terminal_status',
'''function(doc) {
if (doc.status == 'pending.terminated' || doc.status == 'pending.unsuccessful') {
emit(doc._id, doc.status);
}
}
'''
)
terminal_contracts_view.sync(self._db)
return terminal_contracts_view
def createViews_per_OBSID(db, OBSID):
OBSIDViewCode = '''
function(doc) {
if(doc.type == "token") {
if (doc.OBSID == "%s"){
emit(doc._id, doc._id);
}
}
}
'''
# obsID View
obsid_view = ViewDefinition('Observations', OBSID, OBSIDViewCode % (OBSID))
obsid_view.sync(db)
def action_syncviews(): """ Synchronize views to CouchDB. """ from webui import couchdb_views from couchdb.design import ViewDefinition import webui.db try: ViewDefinition.sync_many(webui.db.database, couchdb_views.view_definitions) except AttributeError: print 'Error: CouchDB must not be running' sys.exit(1)
def test_views_creation(self):
book_definition = yaml.load( open(DATADIR%'test_couchish_book.yaml').read() )
dvd_definition = yaml.load( open(DATADIR%'test_couchish_dvd.yaml').read() )
post_definition = yaml.load( open(DATADIR%'test_couchish_post.yaml').read() )
author_definition = yaml.load( open(DATADIR%'test_couchish_author.yaml').read() )
views_definition = yaml.load( open(DATADIR%'test_couchish_views.yaml').read() )
models_definition = {'book': book_definition, 'author': author_definition,'post': post_definition, 'dvd': dvd_definition}
viewdata = get_views(models_definition, views_definition)
for url, view in viewdata['views'].items():
designdoc = url.split('/')[0]
view = ViewDefinition(designdoc, url, view[0])
view.get_doc(self.db)
view.sync(self.db)
def db(request):
SERVER = couchdb.Server(couchdb_url)
def delete():
if DB_NAME in SERVER:
del SERVER[DB_NAME]
delete()
db = SERVER.create(DB_NAME)
view = ViewDefinition(
'test', 'all', """
function (doc) { emit(doc._id, doc._id); }
""")
view.sync(db)
request.addfinalizer(delete)
def db(request):
SERVER = couchdb.Server(SERVER_URL)
def delete():
if DB_NAME in SERVER:
del SERVER[DB_NAME]
delete()
db = SERVER.create(DB_NAME)
view = ViewDefinition(
'index',
'by_date',
map_fun="""function (doc) { emit(doc.id, doc.date); }""")
view.sync(db)
request.addfinalizer(delete)
def map_reduce(couchServer,database,designDocument,viewName,map_fun,reduce_fun):
couch=couchServer
print(couch)
db=couch[database]
print(db)
designDocument=designDocument
viewName=viewName
map_fun=map_fun
reduce_fun=reduce_fun
view=ViewDefinition(design=designDocument,name=viewName,
map_fun=map_fun,
reduce_fun=reduce_fun)
view.sync(db)
print('complete!')
def all_view_ViewDefinition(resource):
return ViewDefinition(
resource, 'all', '''function(doc) {
if(doc.doc_type == '%(resource)s') {
emit(doc.%(doc_id)sID, null);
}
}''' % dict(resource=resource[:-1].title(), doc_id=resource[:-1]))
def add_mapreduce_view(self, view_name="test_mapred_view",
cond='doc.PIPELINE_STEP == "pref_cal1" '):
"""
While the overview_view is applied to all the tokens in the design document,
this 'mapreduce' view is useful if instead of regular view, you want to filter
the tokens and display the user with a 'mini-oververview' view.
This way you can check the status of a subset of the tokens.
"""
overview_map_code = '''
function(doc) {
if(doc.type == "%s" )
if(%s){
{
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.status == 'downloading' ) {
emit('downloading', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0 ) {
emit('done', 1);
}
if(doc.lock > 0 && doc.output != 0 && doc.output != "" ) {
emit('error', 1);
}
if(doc.lock > 0 && doc.status == 'launched' ) {
emit('waiting', 1);
}
if(doc.lock > 0 && doc.done==0 && doc.status!='downloading' ) {
emit('running', 1);
}
}
}
}
'''
overview_reduce_code = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition(self.t_type, view_name,
overview_map_code % (
self.t_type, cond),
overview_reduce_code)
self.views['overview_total'] = overview_total_view
overview_total_view.sync(self.database)
def __get__(self, instance, cls=None):
if self.wrapper is DEFAULT:
wrapper = cls._wrap_row
else:
wrapper = self.wrapper
return ViewDefinition(self.design, self.name, self.map_fun,
self.reduce_fun, language=self.language,
wrapper=wrapper, **self.defaults)
def main():
server = couchdb.Server('http://localhost:5984')
dbname = 'test'
try:
server.delete(dbname)
print "Deleted %s" % dbname
except:
pass
ttime = 1334187642
db = server.create(dbname)
data = [('1', 'one', 'foo', ttime), ('2', 'two', 'bar', ttime),
('3', 'three', 'foo', ttime), ('4', 'three', 'bar', ttime),
('5', 'three', 'bar', ttime + 86400)]
for d in data:
key, tag, txt, tt = d
print "Input", key, tag, txt, tt
row = {'tag': tag, 'text': txt.lower(), 'time': tt}
db[key] = row
# query
_date = "2012-04-11"
dt = datetime.strptime(_date, "%Y-%m-%d").utctimetuple()
#dt = datetime.strptime(_date,"%Y-%m-%d")
#stime=int(time.mktime(dt.utctimetuple()))
# etime=stime+86400-1
view = ViewDefinition('index',
'daily_tags',
time_tag_mapper,
language='python')
view.sync(db)
# tags = [row.value for row in db.view('index/daily_tags',startkey=stime,endkey=etime)]
tags = [
row.value for row in db.view('index/daily_tags',
key=[dt.tm_year, dt.tm_mon, dt.tm_mday])
]
tags = list(set(tags))
print "Tags today", set(tags)
print ""
view = ViewDefinition('index',
'daily_tagcount',
tag_mapper,
reduce_fun=tag_sumreducer,
language='python')
view.sync(db)
#d = dt.timetuple()
d = dt
for tag in sorted(tags):
# tag_counts = [ (row.key, row.value) for row in db.view('index/tagcount', group=True) ]
_key = [d.tm_year, d.tm_mon, d.tm_mday, tag]
tag_count = [(row.value)
for row in db.view('index/daily_tagcount', key=_key)][0]
print "Found %d %s on %s-%s-%s " % (tag_count, tag, _key[0], _key[1],
_key[2])
def _times_view():
return ViewDefinition(
'couchdyno_times', 'times', '''
function(doc) {
if(doc._id.indexOf("couchdyno_") === 0 && doc.ts) {
emit(doc.ts, null);
}
}
''')
def __call__(self, db=None, **options):
"""
This executes the view with the given database. If a database is not
given, the thread-local database (``g.couch``) is used.
:param db: The database to use, if necessary.
:param options: Options to pass to the view.
"""
return OldViewDefinition.__call__(self, db or g.couch, **options)
def getDocs(self):
DB = 'trafico_chile'
server = couchdb.Server('http://localhost:5984')
db = server[DB]
view = ViewDefinition('trafico_chile',
'by_date_time',
sinceDateMapper,
language='python')
view.sync(db)
docs = []
for row in db.view('trafico_chile/by_date_time',
startkey=list(
time.gmtime(time.time() - 3600. * 2000))):
self._last_update = time.strftime('%d_%b_%Y_%H_%M_%S',
list(time.gmtime(time.time())))
docs.append(db.get(row.id))
return docs
def __call__(self, db=None, **options):
"""
This executes the view with the given database. If a database is not
given, the thread-local database (``g.couch``) is used.
:param db: The database to use, if necessary.
:param options: Options to pass to the view.
"""
return OldViewDefinition.__call__(self, db or g.couch, **options)
def test_custom_view(self):
cview = ViewDefinition('cview', 'cview',
"""function(doc) { emit(doc.name, 1); }""")
cview.sync(self.db)
init_length = len(self.db.view('_design/cview/_view/cview'))
test_values = [
(init_length, False),
(init_length + 1, True),
(init_length + 2, False),
]
target = CouchCountTarget(self.couch_client, DATABASE, init_length + 1,
'_design/cview/_view/cview')
for (d, result), doc in zip(test_values, TEST_DOCS):
self.assertEqual(result, target.exists())
self.db.save(doc)
def _bootstrap(self):
if self.dbid not in self.server:
self.server.create(self.dbid)
if self.keydbid not in self.server:
self.server.create(self.keydbid)
self.db = self.server[self.dbid]
self.keydb = self.server[self.keydbid]
self.log_view = ViewDefinition(self.LOG_DDOCID,
self.LOG_VIEW_NAME, self.LOG_VIEW_CODE)
self.log_view.sync(self.db)
def add_view(self,
v_name="test_view",
cond='doc.lock > 0 && doc.done > 0 && doc.output < 0 '):
"""Adds a view to the db, needs a view name and a condition. Emits all tokens with
the type of the current Token_Handler
"""
generalViewCode = '''
function(doc) {
if(doc.type == "%s") {
if(%s) {
emit(doc._id, doc._id);
}
}
}
'''
view = ViewDefinition(self.t_type, v_name,
generalViewCode % (self.t_type, cond))
self.views[v_name] = view
view.sync(self.db)
def _create_views(self):
# twitter/count_type
count_type_map = 'function(doc) { emit([doc.type, doc.id], 1); }'
count_type_reduce = 'function(keys, values) { return sum(values); }'
view = ViewDefinition('twitter', 'count_type', count_type_map, reduce_fun=count_type_reduce)
view.sync(self.db)
# twitter/get_tweets
get_tweets = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit(doc.id, doc); }'
view = ViewDefinition('twitter', 'get_tweet', get_tweets)
view.sync(self.db)
# twitter/get_users
get_users = 'function(doc) { if (doc.type == "TWITTER_USER") emit(doc.id, doc); }'
view = ViewDefinition('twitter', 'get_users', get_users)
view.sync(self.db)
def sync(self, app=None):
"""
This syncs the database for the given app. It will first make sure the
database exists, then synchronize all the views and run all the
callbacks with the connected database.
It will run any callbacks registered with `on_sync`, and when the
views are being synchronized, if a method called `update_design_doc`
exists on the manager, it will be called before every design document
is updated.
:param app: The application to synchronize with.
"""
db = self.db
CouchDBViewDefinition.sync_many(
db, tuple(self.all_viewdefs()),
callback=getattr(self, 'update_design_doc', None)
)
for callback in self.sync_callbacks:
callback(db)
def sync_log_views(self, log_db):
"""
Import ViewDefinitions for the log database and sync them.
"""
print 'Importing view maker functions'
# Import all view generator functions from the couchdb.views module
from votersdaily_web.api.couchdb import log_views
view_makers = [
v for k, v in log_views.__dict__.items() if k.find('make_views') == 0]
view_definitions = []
for maker in view_makers:
print 'Executing %s()' % maker.__name__
view_definitions.extend(maker(log_db))
print 'Syncing a total of %i views to CouchDB' % len(view_definitions)
ViewDefinition.sync_many(log_db, view_definitions, remove_missing=True)
print 'Finished'
def OnLogin(self, event):
self.user = User()
with dialog( dict(dialog = LoginDialog, user = self.user)) as val:
"""
do validation here
"""
if self.user.username == FAKE_USER and self.user.password == FAKE_PASSWORD:
try:
s = Server(self.URL)
blog = s.create(BLOG)
dlg = wx.MessageDialog(self, "Database {0} does not exist. Do you want to create it?".format(BLOG), "Database not found", style = wx.YES_NO)
if dlg.ShowModal() == wx.ID_YES:
from couchdb.design import ViewDefinition
ViewDefinition.sync_many( blog, [Design.all, Design.by_date, Design.by_author, Design.tags, Design.attachments])
p = Post()
p.author = self.user.username
p.subject = "Welcome Post"
p.content = "First Post. See that a <b>screenshot</b> of your computer is included as attachment."
p.date = datetime.now()
p.tags = ["GENERAL", "WELCOME"]
p.store(blog)
sfile = REGEXP.sub("","screenshot{0}".format(datetime.now()))
sfile = "{0}.png".format(sfile)
screenshot = Screenshot(filename = sfile)
doc = blog[p.id]
f = open(sfile,"rb")
blog.put_attachment(doc,f, sfile)
f.close()
else:
del s[BLOG]
dlg.Destroy()
self.Close()
dlg.Destroy()
except:
pass
else:
self.user.username = None
if not self.user.username:
self.Close()
def setup_app(command, conf, vars):
"""Place any commands to setup troppotardi here"""
config = load_environment(conf.global_conf, conf.local_conf)
print "Syncing the couchdb database..."
db = Database(config['couchdb_uri'])
ViewDefinition.sync_many(db, [
Image.pending_by_time, Image.by_day, Image.deleted_by_time,
User.by_time, User.by_username,
Email.by_time,
])
if not list(User.by_username(db)):
print "Creating first user - username and password \"admin\""
admin = User()
admin.username = "******"
admin.password = "******"
admin.email = "*****@*****.**"
admin.role = "Admin"
admin.store(db, revised_by=False)
print "Done."
def test_delete_all_in_view():
from melkman.db.util import delete_all_in_view
from couchdb.schema import Document
from couchdb.design import ViewDefinition
db = make_db()
view_bad = ViewDefinition('test_daiv', 'bad_items',
'''
function(doc) {
if (doc.badflag == true) {
emit(true, null);
}
}
''')
view_bad.sync(db)
for i in range(10):
doc = Document('doc_%d' % i)
doc['foo'] = i
if i % 2 == 1:
doc['badflag'] = True
doc.store(db)
for i in range(10):
assert 'doc_%d' % i in db
delete_all_in_view(db, view_bad)
for i in range(10):
doc_id = 'doc_%d' % i
if i % 2 == 0:
assert doc_id in db, 'expected %s in db' % doc_id
else:
assert doc_id not in db, 'expected %s not in db' % doc_id
def add_overview_view(self):
""" Helper function that creates the Map-reduce view which makes it easy to count
the number of jobs in the 'locked','todo','downloading','error' and 'running' states
"""
overview_map_code = '''
function(doc) {
if(doc.type == "%s") {
if (doc.lock == 0 && doc.done == 0){
emit('todo', 1);
}
if(doc.lock > 0 && doc.status == 'downloading' ) {
emit('downloading', 1);
}
if(doc.lock > 0 && doc.done > 0 && doc.output == 0 ) {
emit('done', 1);
}
if(doc.lock > 0 && doc.output != 0 ) {
emit('error', 1);
}
if(doc.lock > 0 && doc.status == 'launched' ) {
emit('waiting', 1);
}
if(doc.lock > 0 && doc.done==0 && doc.status!='downloading' ) {
emit('running', 1);
}
}
}
'''
overview_reduce_code = '''
function (key, values, rereduce) {
return sum(values);
}
'''
overview_total_view = ViewDefinition(self.t_type, 'overview_total',
overview_map_code % (self.t_type),
overview_reduce_code)
self.views['overview_total'] = overview_total_view
overview_total_view.sync(self.database)
