def add_view(self, view_name="test_view", cond='doc.lock > 0 && doc.done > 0 && doc.output < 0 ', emit_value='doc._id', emit_value2='doc._id'): """Adds a view to the db, needs a view name and a condition. Emits all tokens with the type of TokenHandler.t_type, that also match the condition :param view_name: The name of the new view to be created :type view_name: str :param cond: A string containing the condition which all tokens of the view must match. It can include boolean operators '>', '<'and '&&'. The token's fields are refered to as 'doc.field' :type cond: str :param emit_value: The (first) emit value that is returne by the view. If you look on couchdb/request the tokens from the view, you'll get two values. This will be the first (typically the token's ID) :type emit_value: str :param emit_value2: The second emit value. You can thus return the token's ID and its status for example :type emit_value2: str """ general_view_code = ''' function(doc) { if(doc.type == "%s") { if(%s) { emit(%s, %s ); } } } ''' view = ViewDefinition(self.t_type, view_name, general_view_code % ( self.t_type, cond, emit_value, emit_value2)) self.views[view_name] = view view.sync(self.database)
def get_id_map(): """A map function that stores a view of the id->sn conversion for all entries of a specific type. """ tp = session['info']['type'] gr = session['group'] db = get_db() # Ask for the view. res = db.view('_design/content/_view/' + '_'.join([gr, tp, 'id_map'])) # See if the view is really there. try: len(res) return res except: mapfun = """ function(doc) { if (doc.type == '""" + session['info']['type'] + """') emit(doc.id, doc.sn) }""" view = ViewDefinition('content', '_'.join([gr, tp, 'id_map']), mapfun) view.sync(db) return db.view('_design/content/_view/' + '_'.join([gr, tp, 'id_map']))
def get_twitter_id_from_couch(database): id_list = [] view_result = database.view('_design/twitter/_view/byTwitterID') try: view_result.total_rows except: view = ViewDefinition( 'twitter', 'byTwitterID', '''function(doc) { if (doc.type === "sentiment"){ emit(doc._id, doc.id); } }''') view.get_doc(database) view.sync(database) view_result = database.view('_design/twitter/_view/byTwitterID') for item in view_result: twitter_id = item.value if twitter_id not in id_list: id_list.append(twitter_id) return id_list
def setUpModule(): """Create test databases in local server""" if not has_couchdb: return server = couchdb.Server() ## Create databases for x in DATABASES: if not server.__contains__(x): LOG.info("Creating database {}".format(x)) server.create(x) ## Create views for flowcells and samples for dbname in DATABASES: dblab = dbname.replace("-test", "") db = server[dbname] for k, v in VIEWS[dblab].items(): for title, view in v.items(): viewdef = ViewDefinition(k, title, view) viewdef.sync(db) ## Create and upload project summary with open(os.path.join(filedir, "data", "config", "project_summary.yaml")) as fh: prj_sum = yaml.load(fh) db = server["samples-test"] p_con = ProjectSummaryConnection(dbname="projects-test", username="******", password="******") for p in prj_sum: prj = ProjectSummaryDocument(**p) p_con.save(prj, key="project_name")
def get_attribute(attr): """A map function that stores an attribute for all entries of a specific type. """ tp = session['info']['type'] gr = session['group'] db = get_db() # Ask for the view. res = db.view('_design/attributes/_view/' + '_'.join([gr, tp, attr])) # See if the view is really there. try: len(res) return res except: mapfun = """ function(doc) { if (doc.type == '""" + session['info']['type'] + """') emit(doc['""" + attr + """'], doc.id) }""" view = ViewDefinition('attributes', '_'.join([gr, tp, attr]), mapfun) view.sync(db) return db.view('_design/attributes/_view/' + '_'.join([gr, tp, attr]))
def connect_db(db_name): """Connect to the database if open, and start database if not running.""" try: client = couchdb.Server() except: subprocess.call(['couchdb', '-b']) time.sleep(2) client = couchdb.Server() try: db = client[db_name] except: client.create(db_name) db = client[db_name] toc = {} toc['n_runs'] = 0 toc['_id'] = 'toc' db.save(toc) #create permanent view to all if one doesn't exist if '_design/all' not in db: view_def = ViewDefinition('all', 'all',''' function(doc) { if( doc.run_number ) emit(parseInt(doc.run_number), doc); }''') view_def.sync(db) return db
def make_view(self, name, javascript_function): try: mapper = """function(doc){emit(doc.id_str, doc.id_str);}""" vw = ViewDefinition('index', name, mapper) vw.sync(self.dao) except Exception as e: print "Error making maxid view %s" % e
def add_overview_view(self): overviewMapCode = ''' function(doc) { if(doc.type == "%s") { if (doc.lock == 0 && doc.done == 0){ emit('todo', 1); } if(doc.lock > 0 && doc.done == 0) { emit('locked', 1); } if(doc.lock > 0 && doc.done > 0 && doc.output == 0) { emit('done', 1); } if(doc.lock > 0 && doc.done > 0 && doc.output > 0) { emit('error', 1); } } } ''' overviewReduceCode = ''' function (key, values, rereduce) { return sum(values); } ''' overview_total_view = ViewDefinition(self.t_type, 'overview_total', overviewMapCode % (self.t_type), overviewReduceCode) self.views['overview_total'] = overview_total_view overview_total_view.sync(self.db)
def _prepare_couchdb(self): server = Server(self.couch_url, session=Session(retry_delays=range(10))) try: if self.db_name not in server: self.db = server.create(self.db_name) else: self.db = server[self.db_name] except Exception as e: LOGGER.error('Database error: {}'.format(repr(e))) raise by_date_modified_view = ViewDefinition( self.resource, 'by_dateModified', '''function(doc) { if (doc.doc_type == '%(resource)s') { var fields=['%(doc_type)sID'], data={}; for (var i in fields) { if (doc[fields[i]]) { data[fields[i]] = doc[fields[i]] } } emit(doc.dateModified, data); }}''' % dict(resource=self.resource[:-1].title(), doc_type=self.resource[:-1]) ) by_date_modified_view.sync(self.db) validate_doc = self.db.get(VALIDATE_BULK_DOCS_ID, {'_id': VALIDATE_BULK_DOCS_ID}) if validate_doc.get('validate_doc_update') != VALIDATE_BULK_DOCS_UPDATE: validate_doc['validate_doc_update'] = VALIDATE_BULK_DOCS_UPDATE self.db.save(validate_doc) LOGGER.info('Validate document update view saved.') else: LOGGER.info('Validate document update view already exist.')
def get_tweet_ids_by_city(database, city): id_list = [] view_result = database.view('_design/twitter/_view/tweetIdByCity?key="' + city + '"') try: view_result.total_rows except: view = ViewDefinition( 'twitter', 'tweetIdByCity', '''function(doc) { if (doc.type === "tweet"){ emit(doc.city, doc.id); } }''') view.get_doc(database) view.sync(database) view_result = database.view( '_design/twitter/_view/tweetIdByCity?key=\"' + city + '\"') if not view_result.total_rows: return [] for item in view_result: tweet_id = item.value if tweet_id not in id_list: id_list.append(tweet_id) return id_list
def test_view(self): # This test does quite a bit. First, create 4 test records. # Then, create a view that will emit those records and insert that into # the db. Finally, call our cushion.view object and compare results. self._save_some_data({'foo': 1, 'bar': 'a'}) self._save_some_data({'foo': 2, 'bar': 'a'}) self._save_some_data({'foo': 3, 'bar': 'b'}) self._save_some_data({'foo': 4, 'bar': 'b'}) fake_map = """ function (doc) { emit(doc['bar'], doc); } """ # we're going to use python-couchdb's dynamic view loader stuff here from couchdb.design import ViewDefinition from couchdb.client import Server global baseurl cdb = Server(baseurl) couchdb = cdb[self.dbname] view_defn = ViewDefinition( 'test', 'view', map_fun = fake_map, language = 'javascript' ) view_defn.sync(couchdb) self.cushion.view(self.dbname, 'test/view', self.stop, key='b') records = self.wait() self.assertTrue(len(records) == 2)
def test_view(self): # This test does quite a bit. First, create 4 test records. # Then, create a view that will emit those records and insert that into # the db. Finally, call our cushion.view object and compare results. self._save_some_data({'foo': 1, 'bar': 'a'}) self._save_some_data({'foo': 2, 'bar': 'a'}) self._save_some_data({'foo': 3, 'bar': 'b'}) self._save_some_data({'foo': 4, 'bar': 'b'}) fake_map = """ function (doc) { emit(doc['bar'], doc); } """ # we're going to use python-couchdb's dynamic view loader stuff here from couchdb.design import ViewDefinition from couchdb.client import Server global baseurl cdb = Server(baseurl) couchdb = cdb[self.dbname] view_defn = ViewDefinition('test', 'view', map_fun=fake_map, language='javascript') view_defn.sync(couchdb) self.cushion.view(self.dbname, 'test/view', self.stop, key='b') records = self.wait() self.assertTrue(len(records) == 2)
def make_view(self, name, javascript_function): try: mapper = """function(doc){emit(doc.id_str, doc.id_str);}""" vw = ViewDefinition('index', name, mapper) vw.sync(self.dao) except Exception as e: print("Error making maxid view %s" % e)
def create_view(db, view_tuples): for vt in view_tuples: try: view = ViewDefinition(vt[0], vt[1], vt[2], vt[3]) view.sync(db) except: pass
def createView(self, viewName): view = ViewDefinition( 'tweets', viewName, '''function(doc) { emit(doc.text, doc.coordinates); }''') view.get_doc(self.db) view.sync(self.db) print("create view user, view should be created")
def _create_view(self): view_map = 'function(doc) { if(doc.bag_of_words) { for (word in doc.bag_of_words) { emit([doc.bag_of_words[word], doc.polarity, doc.suburb],1)}}}' view_reduce = '_sum' view = ViewDefinition('application', 'mainview', view_map, reduce_fun=view_reduce) view.sync(self.DBRef)
def sync_views(self): for url, view in self.config.viewdata["views"].items(): segments = url.split("/") designdoc = segments[0] name = "/".join(segments[1:]) view = ViewDefinition(designdoc, name, view[0], view[1]) view.get_doc(self.db) view.sync(self.db)
def createViewUsers(self): view = ViewDefinition( 'tweets', "users", '''function(doc) { emit(doc.user["id"], doc.done); }''') view.get_doc(self.db) view.sync(self.db) print("user view should be created")
def couch_get_view(self, db, get_token): view = ViewDefinition( '%sview' % db.name, '%s' % db.name, '''function(doc) {if (doc._id == "%s") emit(doc);}''' % get_token) view.sync(db) for res in db.view('_design/%sview/_view/%s' % (db.name, db.name)): return {res.id: res.key}
def create_views(self): print("++ Creating views") view = ViewDefinition( 'get_crashes', 'all', '''function(doc) { emit(doc.crashhash,doc.crashcount); }''') view.get_doc(self.db) view.sync(self.db)
def createViews(db): generalViewCode = ''' function(doc) { if(doc.type == "token") { if(%s) { emit(doc._id, doc._id); } } } ''' # todo View todoCondition = 'doc.lock == 0 && doc.done == 0' todo_view = ViewDefinition('Monitor', 'todo', generalViewCode % (todoCondition)) todo_view.sync(db) # locked View lockedCondition = 'doc.lock > 0 && doc.done == 0' locked_view = ViewDefinition('Monitor', 'locked', generalViewCode % (lockedCondition)) locked_view.sync(db) # done View doneCondition = 'doc.lock > 0 && doc.done > 0 && doc.output == 0' done_view = ViewDefinition('Monitor', 'done', generalViewCode % (doneCondition)) done_view.sync(db) # errorCondition = 'doc.lock > 0 && doc.done > 0 && doc.output > 0' error_view = ViewDefinition('Monitor', 'error', generalViewCode % (errorCondition)) error_view.sync(db) # overview_total View -- lists all views and the number of tokens in each view overviewMapCode = ''' function(doc) { if(doc.type == "token") { if (doc.lock == 0 && doc.done == 0){ emit('todo', 1); } if(doc.lock > 0 && doc.done == 0) { emit('locked', 1); } if(doc.lock > 0 && doc.done > 0 && doc.output == 0) { emit('done', 1); } if(doc.lock > 0 && doc.done > 0 && doc.output > 0) { emit('error', 1); } } } ''' overviewReduceCode = ''' function (key, values, rereduce) { return sum(values); } ''' overview_total_view = ViewDefinition('Monitor', 'overview_total', overviewMapCode, overviewReduceCode) overview_total_view.sync(db)
def main(): server = couchdb.Server('http://localhost:5984') dbname = 'test' try: server.delete(dbname) print "Deleted %s" % dbname except: pass ttime = 1334187642 db = server.create(dbname) data = [('1', 'one', 'foo', ttime), ('2', 'two', 'bar', ttime), ('3', 'three', 'foo', ttime), ('4', 'three', 'bar', ttime), ('5', 'three', 'bar', ttime + 86400)] for d in data: key, tag, txt, tt = d print "Input", key, tag, txt, tt row = {'tag': tag, 'text': txt.lower(), 'time': tt} db[key] = row # query _date = "2012-04-11" dt = datetime.strptime(_date, "%Y-%m-%d").utctimetuple() #dt = datetime.strptime(_date,"%Y-%m-%d") #stime=int(time.mktime(dt.utctimetuple())) # etime=stime+86400-1 view = ViewDefinition('index', 'daily_tags', time_tag_mapper, language='python') view.sync(db) # tags = [row.value for row in db.view('index/daily_tags',startkey=stime,endkey=etime)] tags = [ row.value for row in db.view('index/daily_tags', key=[dt.tm_year, dt.tm_mon, dt.tm_mday]) ] tags = list(set(tags)) print "Tags today", set(tags) print "" view = ViewDefinition('index', 'daily_tagcount', tag_mapper, reduce_fun=tag_sumreducer, language='python') view.sync(db) #d = dt.timetuple() d = dt for tag in sorted(tags): # tag_counts = [ (row.key, row.value) for row in db.view('index/tagcount', group=True) ] _key = [d.tm_year, d.tm_mon, d.tm_mday, tag] tag_count = [(row.value) for row in db.view('index/daily_tagcount', key=_key)][0] print "Found %d %s on %s-%s-%s " % (tag_count, tag, _key[0], _key[1], _key[2])
def add_view(self, view, map_fun, reduce_fun=None, design_doc="Monitor", *args, **kwargs): """ Add a view to the database All extra parameters are passed to couchdb.design.ViewDefinition :param view: name of the view :param map_fun: string of the javascript map function :param reduce_fun: string of the javascript reduce function (optional) """ definition = ViewDefinition( design_doc, view, map_fun, reduce_fun, *args, **kwargs) definition.sync(self.db)
def getDocs(): DB = 'trafico_chile' server = couchdb.Server('http://localhost:5984') db = server[DB] view = ViewDefinition('trafico_chile','by_date_time',sinceDateMapper,language='python') view.sync(db) docs = [] for row in db.view('trafico_chile/by_date_time',startkey=list(time.gmtime(time.time()-3600.*50.))): _last_update = list(time.gmtime(time.time())) docs.append(db.get(row.id)) return docs
def test_views_creation(self): book_definition = yaml.load( open(DATADIR%'test_couchish_book.yaml').read() ) dvd_definition = yaml.load( open(DATADIR%'test_couchish_dvd.yaml').read() ) post_definition = yaml.load( open(DATADIR%'test_couchish_post.yaml').read() ) author_definition = yaml.load( open(DATADIR%'test_couchish_author.yaml').read() ) views_definition = yaml.load( open(DATADIR%'test_couchish_views.yaml').read() ) models_definition = {'book': book_definition, 'author': author_definition,'post': post_definition, 'dvd': dvd_definition} viewdata = get_views(models_definition, views_definition) for url, view in viewdata['views'].items(): designdoc = url.split('/')[0] view = ViewDefinition(designdoc, url, view[0]) view.get_doc(self.db) view.sync(self.db)
def _init_terminal_contracts_view(self): terminal_contracts_view = ViewDefinition( 'contract_views', 'has_terminal_status', '''function(doc) { if (doc.status == 'pending.terminated' || doc.status == 'pending.unsuccessful') { emit(doc._id, doc.status); } } ''' ) terminal_contracts_view.sync(self._db) return terminal_contracts_view
def createViews_per_OBSID(db, OBSID): OBSIDViewCode = ''' function(doc) { if(doc.type == "token") { if (doc.OBSID == "%s"){ emit(doc._id, doc._id); } } } ''' # obsID View obsid_view = ViewDefinition('Observations', OBSID, OBSIDViewCode % (OBSID)) obsid_view.sync(db)
def p20141208(self, request, *args): from MyRingCouchDB import MyRingCouchDB from MyRingUser import MyRingUser from env_config import COUCHDB_USER, COUCHDB_PASS from couchdb.design import ViewDefinition MCD = MyRingCouchDB() self.couch = MCD.instantiate_couchdb_as_admin() self.couch.resource.credentials = (COUCHDB_USER, COUCHDB_PASS) print('patch_20141208') print(current_user.username) #db = self.couch[self.user_database] #user = MyRingUser.load(db,current_user) user = self.MAM.select_user(current_user.username) if user: for ring in user['rings']: db_ringname = current_user.username + '_' + str( ring['ringname']) print(db_ringname) db2 = self.couch[db_ringname] if not RingsSchema.load(db2, 'schema'): schema = RingsSchema.load(db2, 'blueprint') print('Schema:', schema) schema._id = 'schema' schema.store(db2) view = ViewDefinition( 'ring', 'schema', ''' function(doc) { if(doc._id=='schema'){ emit(doc._id, doc) } } ''') view.get_doc(db2) view.sync(db2) d = { 'rq': current_user, 'template': 'avispa_rest/tools/flashresponsejson.html' } return d
def _create_views(self): # twitter/count_type count_type_map = 'function(doc) { emit([doc.type, doc.id], 1); }' count_type_reduce = 'function(keys, values) { return sum(values); }' view = ViewDefinition('twitter', 'count_type', count_type_map, reduce_fun=count_type_reduce) view.sync(self.db) # The unique key for each tweet is doc.id. The key (a 64-bit long) is represented as a string because # the largest JavaScript long is 2^53. # A problem arises because keys are sorted as strings and a doc.id may have fewer digits but a larger # leading digit. So, it is sorted in the wrong order. # The solutions is to zero-pad the doc.id to fill 19 digits. (The max 64-bit long - 2^63 - has 19 digits.) # That is why we emit the doc.id key as ("0000000000000000000"+doc.id).slice(-19). # twitter/get_tweets get_tweets = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit(("0000000000000000000"+doc.id).slice(-19), doc); }' view = ViewDefinition('twitter', 'get_tweets', get_tweets) view.sync(self.db) # twitter/get_tweets_by_date (sort by date and tweet id) get_tweets_by_date = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit((new Date(doc.created_at).getTime())+"-"+("0000000000000000000"+doc.id).slice(-19), doc); }' view = ViewDefinition('twitter', 'get_tweets_by_date', get_tweets_by_date) view.sync(self.db) # twitter/get_users get_users = 'function(doc) { if (doc.type == "TWITTER_USER") emit(doc.id, doc); }' view = ViewDefinition('twitter', 'get_users', get_users) view.sync(self.db)
def create_views(self, db): # https://markhaa.se/posts/couchdb-views-in-python/ # create the two views we will need (by week and by suburb) # note that if they already exist in the DB it won't do anything view_by_week = ViewDefinition('sentimentDocs', 'byWeek', MAP_BY_WEEK, reduce_fun=REDUCE_STATS) view_by_week.get_doc(self.DB[db]) view_by_week.sync(self.DB[db]) view_by_suburb = ViewDefinition('sentimentDocs', 'bySuburb', MAP_BY_SUBURB, reduce_fun=REDUCE_STATS) view_by_suburb.get_doc(self.DB[db]) view_by_suburb.sync(self.DB[db]) view_by_week = ViewDefinition('sentimentDocs', 'byWeek2', MAP_BY_WEEK2, reduce_fun=REDUCE_STATS) view_by_week.get_doc(self.DB[db]) view_by_week.sync(self.DB[db]) view_by_suburb = ViewDefinition('sentimentDocs', 'bySuburb2', MAP_BY_SUBURB2, reduce_fun=REDUCE_STATS) view_by_suburb.get_doc(self.DB[db]) view_by_suburb.sync(self.DB[db])
def db(request): SERVER = couchdb.Server(SERVER_URL) def delete(): if DB_NAME in SERVER: del SERVER[DB_NAME] delete() db = SERVER.create(DB_NAME) view = ViewDefinition( 'index', 'by_date', map_fun="""function (doc) { emit(doc.id, doc.date); }""") view.sync(db) request.addfinalizer(delete)
def db(request): SERVER = couchdb.Server(couchdb_url) def delete(): if DB_NAME in SERVER: del SERVER[DB_NAME] delete() db = SERVER.create(DB_NAME) view = ViewDefinition( 'test', 'all', """ function (doc) { emit(doc._id, doc._id); } """) view.sync(db) request.addfinalizer(delete)
def map_reduce(couchServer,database,designDocument,viewName,map_fun,reduce_fun): couch=couchServer print(couch) db=couch[database] print(db) designDocument=designDocument viewName=viewName map_fun=map_fun reduce_fun=reduce_fun view=ViewDefinition(design=designDocument,name=viewName, map_fun=map_fun, reduce_fun=reduce_fun) view.sync(db) print('complete!')
def add_mapreduce_view(self, view_name="test_mapred_view", cond='doc.PIPELINE_STEP == "pref_cal1" '): """ While the overview_view is applied to all the tokens in the design document, this 'mapreduce' view is useful if instead of regular view, you want to filter the tokens and display the user with a 'mini-oververview' view. This way you can check the status of a subset of the tokens. """ overview_map_code = ''' function(doc) { if(doc.type == "%s" ) if(%s){ { if (doc.lock == 0 && doc.done == 0){ emit('todo', 1); } if(doc.lock > 0 && doc.status == 'downloading' ) { emit('downloading', 1); } if(doc.lock > 0 && doc.done > 0 && doc.output == 0 ) { emit('done', 1); } if(doc.lock > 0 && doc.output != 0 && doc.output != "" ) { emit('error', 1); } if(doc.lock > 0 && doc.status == 'launched' ) { emit('waiting', 1); } if(doc.lock > 0 && doc.done==0 && doc.status!='downloading' ) { emit('running', 1); } } } } ''' overview_reduce_code = ''' function (key, values, rereduce) { return sum(values); } ''' overview_total_view = ViewDefinition(self.t_type, view_name, overview_map_code % ( self.t_type, cond), overview_reduce_code) self.views['overview_total'] = overview_total_view overview_total_view.sync(self.database)
def test_load_couchdb(self): """ Test that resource instances and tiles load into a mobile survey's couchdb """ post_data = { "datadownloadconfig": {"count":1000,"download":True,"resources":['d84a098c-368b-11e8-bafc-c4b301baab9f'],"custom":""}, "startdate":"2018-01-29", "tilecache":"", "enddate":"2018-03-30", "createdby_id":1, "bounds":{"type":"FeatureCollection","features":[]}, "cards":['fe035187-368b-11e8-bf56-c4b301baab9f', '4215f135-369c-11e8-9544-c4b301baab9f'], "lasteditedby_id":1, "groups":[], "active":False, "users":[], "id": str(self.mobile_survey.id), "name":"test survey", "description":"a description" } response_json = self.post_mobile_survey(post_data) couch = couchdb.Server(settings.COUCHDB_URL) resources = 0 tiles = 0 if 'project_' + str(self.mobile_survey.id) in couch: db = couch['project_' + str(self.mobile_survey.id)] view = ViewDefinition('tests', 'all', '''function(doc) {emit(doc._id, null);}''') view.get_doc(db) view.sync(db) for item in db.view('_design/tests/_view/all', include_docs=True): if item.doc['type'] == 'tile': tiles += 1 if item.doc['type'] == 'resource': resources += 1 # tile_count = len(db.find({'selector': {'type': 'tile'}})) # resource_count = len(db.find({'selector': {'type': 'resource'}})) else: print '{0} is not in couch'.format('project_' + str(self.mobile_survey.id)) self.assertEqual(tiles, 2) self.assertEqual(resources,1)
def main(): server = couchdb.Server('http://localhost:5984') dbname = 'test' try: server.delete(dbname) print "Deleted %s"%dbname except: pass ttime = 1334187642 db = server.create(dbname) data = [('1','one', 'foo', ttime), ('2','two','bar', ttime), ('3','three','foo',ttime), ('4','three','bar',ttime), ('5','three','bar', ttime+86400)] for d in data: key, tag, txt, tt = d print "Input",key, tag, txt, tt row = {'tag':tag, 'text':txt.lower(), 'time': tt} db[key] = row # query _date = "2012-04-11" dt = datetime.strptime(_date,"%Y-%m-%d").utctimetuple() #dt = datetime.strptime(_date,"%Y-%m-%d") #stime=int(time.mktime(dt.utctimetuple())) # etime=stime+86400-1 view = ViewDefinition('index','daily_tags',time_tag_mapper, language='python') view.sync(db) # tags = [row.value for row in db.view('index/daily_tags',startkey=stime,endkey=etime)] tags = [row.value for row in db.view('index/daily_tags',key=[dt.tm_year, dt.tm_mon, dt.tm_mday])] tags = list(set(tags)) print "Tags today",set(tags) print "" view = ViewDefinition('index','daily_tagcount', tag_mapper, reduce_fun=tag_sumreducer, language='python') view.sync(db) #d = dt.timetuple() d = dt for tag in sorted(tags): # tag_counts = [ (row.key, row.value) for row in db.view('index/tagcount', group=True) ] _key = [d.tm_year, d.tm_mon, d.tm_mday, tag] tag_count = [ (row.value) for row in db.view('index/daily_tagcount', key=_key) ][0] print "Found %d %s on %s-%s-%s "%(tag_count,tag,_key[0],_key[1],_key[2])
def getDocs(self): DB = 'trafico_chile' server = couchdb.Server('http://localhost:5984') db = server[DB] view = ViewDefinition('trafico_chile', 'by_date_time', sinceDateMapper, language='python') view.sync(db) docs = [] for row in db.view('trafico_chile/by_date_time', startkey=list( time.gmtime(time.time() - 3600. * 2000))): self._last_update = time.strftime('%d_%b_%Y_%H_%M_%S', list(time.gmtime(time.time()))) docs.append(db.get(row.id)) return docs
def add_view(self, v_name="test_view", cond='doc.lock > 0 && doc.done > 0 && doc.output < 0 '): """Adds a view to the db, needs a view name and a condition. Emits all tokens with the type of the current Token_Handler """ generalViewCode = ''' function(doc) { if(doc.type == "%s") { if(%s) { emit(doc._id, doc._id); } } } ''' view = ViewDefinition(self.t_type, v_name, generalViewCode % (self.t_type, cond)) self.views[v_name] = view view.sync(self.db)
def test_custom_view(self): cview = ViewDefinition('cview', 'cview', """function(doc) { emit(doc.name, 1); }""") cview.sync(self.db) init_length = len(self.db.view('_design/cview/_view/cview')) test_values = [ (init_length, False), (init_length + 1, True), (init_length + 2, False), ] target = CouchCountTarget(self.couch_client, DATABASE, init_length + 1, '_design/cview/_view/cview') for (d, result), doc in zip(test_values, TEST_DOCS): self.assertEqual(result, target.exists()) self.db.save(doc)
def test_delete_all_in_view(): from melkman.db.util import delete_all_in_view from couchdb.schema import Document from couchdb.design import ViewDefinition db = make_db() view_bad = ViewDefinition('test_daiv', 'bad_items', ''' function(doc) { if (doc.badflag == true) { emit(true, null); } } ''') view_bad.sync(db) for i in range(10): doc = Document('doc_%d' % i) doc['foo'] = i if i % 2 == 1: doc['badflag'] = True doc.store(db) for i in range(10): assert 'doc_%d' % i in db delete_all_in_view(db, view_bad) for i in range(10): doc_id = 'doc_%d' % i if i % 2 == 0: assert doc_id in db, 'expected %s in db' % doc_id else: assert doc_id not in db, 'expected %s not in db' % doc_id
def add_overview_view(self): """ Helper function that creates the Map-reduce view which makes it easy to count the number of jobs in the 'locked','todo','downloading','error' and 'running' states """ overview_map_code = ''' function(doc) { if(doc.type == "%s") { if (doc.lock == 0 && doc.done == 0){ emit('todo', 1); } if(doc.lock > 0 && doc.status == 'downloading' ) { emit('downloading', 1); } if(doc.lock > 0 && doc.done > 0 && doc.output == 0 ) { emit('done', 1); } if(doc.lock > 0 && doc.output != 0 ) { emit('error', 1); } if(doc.lock > 0 && doc.status == 'launched' ) { emit('waiting', 1); } if(doc.lock > 0 && doc.done==0 && doc.status!='downloading' ) { emit('running', 1); } } } ''' overview_reduce_code = ''' function (key, values, rereduce) { return sum(values); } ''' overview_total_view = ViewDefinition(self.t_type, 'overview_total', overview_map_code % (self.t_type), overview_reduce_code) self.views['overview_total'] = overview_total_view overview_total_view.sync(self.database)
def fetch_docs_of_type(doc_type, db, since=None): LOG.debug( f'Fetching all docs of type {doc_type} from database {db.name} for first time' ) if since is None: map_func = docs_of_type_view_template.substitute(doc_type=doc_type) map_func = "".join(map_func.split()) view = ViewDefinition('hotspur', doc_type, map_func) view.sync(db) results = view(db, update_seq=True) return { "docs": [row.value for row in results.rows], "last_seq": results.update_seq } else: update = db.changes(since=since, include_docs=True, filter="_view", view=f'hotspur/{doc_type}') return { "docs": [row["doc"] for row in update["results"]], "last_seq": update["last_seq"] }
def multiply(self, m1, m2): '''Perform matrix multiplication between m1 and m2 using MapReduce''' product = [] for row in range(1, self.dim+1): for col in range(1, self.dim+1): #maps proper row and column of m1 and m2 map_fun = ('''function(doc) { for (i=0;i < doc.elements.length; i++) { if((doc.elements[i][0]== %s && doc._id == '%s')|(doc.elements[i][1]== %s && doc._id=='%s')) { emit(null, doc.elements[i]); } } }''' % (str(row), m1.id, str(col), m2.id) ) #calculates dot product of the row and column red = ('''function(key, data) { var total = 0; var dim = %s for(i=0; i < dim; i++) { total += data[i][2] * data[i+dim][2]; } return total; }''' % self.dim) #creates a permanent view in Couch view = ViewDefinition('oper','multiply',map_fun,reduce_fun=red) view.sync(self.matrix_db) design = "_design/oper/_view/multiply" for result in self.matrix_db.view(design): product.append([row, col, result.value]) #prints the result print m1.id, 'X', m2.id print_matrix(product)
def _create_views(self): # twitter/count_type count_type_map = 'function(doc) { emit([doc.type, doc.id], 1); }' count_type_reduce = 'function(keys, values) { return sum(values); }' view = ViewDefinition('twitter', 'count_type', count_type_map, reduce_fun=count_type_reduce) view.sync(self.db) # twitter/get_tweets get_tweets = 'function(doc) { if (doc.type == "TWITTER_STATUS") emit(doc.id, doc); }' view = ViewDefinition('twitter', 'get_tweet', get_tweets) view.sync(self.db) # twitter/get_users get_users = 'function(doc) { if (doc.type == "TWITTER_USER") emit(doc.id, doc); }' view = ViewDefinition('twitter', 'get_users', get_users) view.sync(self.db)
def view_checkins_date(): view = ViewDefinition('index', 'map_checkins_date', map_checkins_date, language='python') view.sync(couch[prefix + 'checkins'])
@classmethod def load(cls, id, db=projects): return super(Project, cls).load(db, id) @classmethod def load_or_create(cls, id, db=projects): project = cls.load(id, db) if not project: project = cls(id) return project project_names = ViewDefinition("projects", "all", '''\ function(doc) { if (doc.project) { emit(doc.project, 1); } }''', '''\ function(keys, values, rereduce) { return sum(values); }''') project_names.sync(timesheets) Timesheet._all_timesheets.sync(timesheets) Timesheet._by_date.sync(timesheets) Timesheet._by_invoice.sync(timesheets) Timesheet._by_project.sync(timesheets) Timesheet._by_project_unbilled.sync(timesheets) Timesheet._by_date_unbilled.sync(timesheets)
function(doc) { emit(1, parseInt(doc._id)); }""", """\ function(keys, values, rereduce) { var max = 0; for (var i in values) { if (values[i] > max) { max = values[i]; } } return max; }""", ) project_names.sync(timesheets) type_names.sync(timesheets) Timesheet._all_timesheets.sync(timesheets) Timesheet._by_date.sync(timesheets) Timesheet._by_invoice.sync(timesheets) Timesheet._by_project.sync(timesheets) Timesheet._by_project_unbilled.sync(timesheets) Timesheet._by_date_unbilled.sync(timesheets) ProjectType._by_project_type.sync(project_types) last_invoice_num.sync(invoices) Invoice._all_invoices.sync(invoices) Invoice._by_project.sync(invoices) project_names = Project.project_list() for name in project_names:
doc = db.get(adoc) doc['sent_type'] = some_dic[adoc] db.save(doc) #not currently working rows = db.view('_all_docs', keys=['key1', 'key2', 'missing'], include_docs=True) docs = [row.doc for row in rows] #create a view #pass javascript code = '''function(doc) { if(doc.frm == "*****@*****.**") emit(doc.frm, null); }''' results = db.query(code) for res in results: print res.key rpt_view = ViewDefinition('reports', 'fromemail', '''function(doc) { if(doc.frm == "*****@*****.**") emit(doc.frm, null); }''') rpt_view.sync(db) for res in db.view("_design/reports/_view/fromemail"): print res.id, res.key #class of document for updating class Jobtobedone(Document): frm = TextField() to = ListField(TextField()) sub = TextField() added = DateTimeField(default=datetime.now())
def view_toplevel_categories(): view = ViewDefinition('index', 'map_toplevel_categories', map_toplevel_categories, reduce_values, language='python') view.sync(couch['categories'])
def view_checkins_count(): view = ViewDefinition('index', 'map_checkins_count', map_checkins_count, reduce_sum, language='python') view.sync(couch[prefix + 'checkins'])
def create_view(self, design, name, map_fun, reduce_fun=None, language="javascript", wrapper=None, **defaults): """Just uses ViewDefinition but makes sure it uses it""" view = ViewDefinition(design, name, map_fun, reduce_fun=None, language="javascript", wrapper=None, **defaults) view.sync(self.db)
db = server[DB] # Query out the documents at a given group level of interest # Group by year, month, day docs = db.view('index/doc_count_by_date_time', group_level=3) # Now, load the documents keyed by [year, month, day] into a new database db_scratch = server.create(DB + '-num-per-day') db_scratch.update(docs) def transposeMapper(doc): yield (doc['value'], doc['key']) view = ViewDefinition('index', 'num_per_day', transposeMapper, language='python') view.sync(db_scratch) fields = ['Date', 'Count'] pt = PrettyTable(fields=fields) [pt.set_field_align(f, 'l') for f in fields] for row in db_scratch.view('index/num_per_day'): if row.key > 10: # display stats where more than 10 messages were sent pt.add_row(['-'.join([str(i) for i in row.value]), row.key])
def _sync_test_views(self): dashboard_view = ViewDefinition("record_docs", "by_ingestion_sequence_include_status", """function(doc) { if (doc.type == 'record') { emit(doc.ingestionSequence, doc.status) } }""") dashboard_view.sync(self.dashboard_db)
def view_venues(): view = ViewDefinition('index', 'map_venues', map_venues, reduce_sum, language='python') view.sync(couch[prefix + 'checkins'])
# -*- coding: utf-8 -*- import sys import couchdb from couchdb.design import ViewDefinition import json from pa_setting import dbname server = couchdb.Server('http://localhost:5984') db = server[dbname] def dateTimeToDocMapper(doc): from dateutil.parser import parse from datetime import datetime as dt if doc.get('created_at'): # [year, month, day, hour, min, sec] _date = list(dt.timetuple(parse(doc['created_at']))[:-3]) yield (_date, doc) view = ViewDefinition('index', 'by_date_time', dateTimeToDocMapper, language='python') view.sync(db)
def create_views_in_db_hunters( db ): view = ViewDefinition( 'words', 'random', '''function( doc ) { emit( Math.random(), doc.words ); }''') view.sync( db ) view2 = ViewDefinition( 'words', 'all_occurrences', '''function( doc ) { var words = doc.words; for(word in words){ emit( [word, words[word]], {'username':doc.username, 'image_url':doc.image_url['120px'] } ); } }''') view2.sync( db ) view3 = ViewDefinition( 'list', 'by_username', '''function( doc ) { emit( doc.username, doc ); }''') view3.sync( db ) view4 = ViewDefinition( 'list', 'by_votes_count', '''function( doc ) { emit( doc.votes_count, [doc.username, doc.image_url["120px"] ] ); }''') view4.sync( db ) view5 = ViewDefinition( 'list', 'by_posts_count', '''function( doc ) { emit( doc.posts_count, [doc.username, doc.image_url["120px"] ] ); }''') view5.sync( db ) view6 = ViewDefinition( 'list', 'by_average_votes', '''function( doc ) { var average = doc.votes_count/doc.posts_count; emit( Math.round(average), [doc.username, doc.image_url["120px"] ] ); }''') view6.sync( db )
# tweet value. def id_mapper(doc): yield (None, doc['id']) # Find the maximum tweet id def max_finding_reducer(keys, values, rereduce): return max(values) view = ViewDefinition('index', 'max_tweet_id', id_mapper, max_finding_reducer, language='python') view.sync(db) try: KW['since_id'] = int([_id for _id in db.view('index/max_tweet_id') ][0].value) except IndexError, e: KW['since_id'] = 1 # Harvest tweets for the given timeline. # For friend and home timelines, the unofficial limitation is about 800 statuses # although other documentation may state otherwise. The public timeline only returns # 20 statuses and gets updated every 60 seconds, so consider using the streaming API # for public statuses. See http://bit.ly/fgJrAx # Note that the count and since_id params have no effect for the public timeline page_num = 1 while page_num <= MAX_PAGES:
@classmethod def load(cls, id, db=invoices): return super(Invoice, cls).load(db, id) @classmethod def next_invoice_number(cls): result = last_invoice_num(invoices) if result.rows: return result.rows[0]['value'] + 1 return 1 last_invoice_num = ViewDefinition("last_num", "all", '''\ function(doc) { emit(1, parseInt(doc._id)); }''', '''\ function(keys, values, rereduce) { var max = 0; for (var i in values) { if (values[i] > max) { max = values[i]; } } return max; }''') last_invoice_num.sync(invoices) Invoice._all_invoices.sync(invoices) Invoice._by_project.sync(invoices)