def connect_db(self): self.db = couchdb.Database( self.config.db_url, session=couchdb.Session(retry_delays=range(10))) self.adb = couchdb.Database( self.config.adb_url, session=couchdb.Session(retry_delays=range(10)))
def connect_to_couch(config_file): "Connects to the couch databases" f = open(config_file) config = yaml.load(f) f.close() admin_db = config["admin"]["counts_db"] editions_db = config["lists"]["editions_db"] works_db = config["lists"]["works_db"] seeds_db = config["lists"]["seeds_db"] logging.debug(" Admin Database is %s", admin_db) logging.debug(" Editions Database is %s", editions_db) logging.debug(" Works Database is %s", works_db) logging.debug(" Seeds Database is %s", seeds_db) return couchdb.Database(admin_db), couchdb.Database( editions_db), couchdb.Database(works_db), couchdb.Database(seeds_db)
def connect_to_couch(config_file): "Connects to the couch databases" f = open(config_file) config = yaml.load(f) f.close() admin_db = config["admin"]["counts_db"] return couchdb.Database(admin_db)
def watch(ddoc_dir, db_uri): ddocname = get_ddocname(ddoc_dir) db = couchdb.Database(db_uri) obs = Observer() q = Queue() e2q = Ev2Q(q) obs.schedule(e2q, ddoc_dir, recursive=False) try: obs.start() while True: # We need a timeout, or else we can't catch keyboard interrupts, &c. try: evtype, ev = q.get(timeout=0.5) except Empty: continue name = ev.src_path fullpath = name if evtype in ["created", "modified"] and valid_filename( os.path.basename(name)) and not os.path.isdir(fullpath): print evtype, fullpath ddoc = db[ddocname] db.put_attachment(ddoc, open(fullpath)) except KeyboardInterrupt: pass finally: obs.stop() obs.join()
def connect_to_tombstone(): global tombstone_db try: tombstone_db_uri = config.get("celery", {})["tombstone_db"] tombstone_db = couchdb.Database(tombstone_db_uri) except Exception, e: logger.warning("Couldn't connect to tombstone database", exc_info=True)
def get_stats(ndays=30): """Returns the stats for the past `ndays`""" admin_db = couchdb.Database(config.admin.counts_db) end = datetime.datetime.now().strftime("counts-%Y-%m-%d") start = (datetime.datetime.now() - datetime.timedelta(days=ndays)).strftime("counts-%Y-%m-%d") docs = [ x.doc for x in admin_db.view("_all_docs", startkey_docid=start, endkey_docid=end, include_docs=True) ] retval = dict(human_edits=Stats(docs, "human_edits", "human_edits"), bot_edits=Stats(docs, "bot_edits", "bot_edits"), lists=Stats(docs, "lists", "total_lists"), visitors=Stats(docs, "visitors", "visitors"), loans=Stats(docs, "loans", "loans"), members=Stats(docs, "members", "total_members"), works=Stats(docs, "works", "total_works"), editions=Stats(docs, "editions", "total_editions"), ebooks=Stats(docs, "ebooks", "total_ebooks"), covers=Stats(docs, "covers", "total_covers"), authors=Stats(docs, "authors", "total_authors"), subjects=Stats(docs, "subjects", "total_subjects")) return retval
def get_db_connection(self): host = self.config.get_option('db', 'host') port = self.config.get_option('db', 'port') user_name = self.config.get_option('user', 'username') user_password = self.config.get_option('user', 'password') db_name = self.config.get_option('db', 'name') self.db = couchdb.Database( create_db_url(host, port, user_name, user_password, db_name), session=couchdb.Session(retry_delays=range(10))) a_name = self.config.get_option('admin', 'username') a_password = self.config.get_option('admin', 'password') self.adb = couchdb.Database( create_db_url(host, port, a_name, a_password, db_name))
def insertDocumentCouchdb(envelope, config): try: conf = config['couchdb'] db = couchdb.Database(conf['dbUrl']) del envelope['_rev'] del envelope['_id'] db.save(envelope) except (Exception), exc: log.error("Error writing to mongo") processHarvestResult.retry(exc)
def runQuery(): # Setup from query3 code couchserver = couchdb.Server("http://couchdb:5984/") dbname = "dblp" if dbname in couchserver: db = couchserver[dbname] else: db = couchdb.Database("http://localhost:5984/%s/" % dbname) name = findProceedingsWithMaxAuthor(db) print 'Proceedings in 2004 with the most authors: %s' % name
def sync(ddoc_dir, db_uri): db = couchdb.Database(db_uri) ddocname = get_ddocname(ddoc_dir) if not ddocname in db: db[ddocname] = {"_id": ddocname} # Initialize for fname in os.listdir(ddoc_dir): if valid_filename(fname): ddoc = db[ddocname] db.put_attachment(ddoc, open(os.path.join(ddoc_dir, fname)))
def __init__( self, host, port, database, createifnone=True ): #-------------------------------------------------------------------------- """ Initalize a CouchDBManager object. :param host: host name or IP address as string :param port: port number on which couchDB is listening :param database: name of the database """ url = "http://%s:%s@%s:%s" % (self.username, self.password, host, str(port)) try: self.database = couchdb.Database("%s/%s" % (url, database)) self.database.info() except: if createifnone: couchserver = couchdb.Server(url) couchserver.create(database) self.database = couchdb.Database("%s/%s" % (url, database)) self.database.info() self.shutdown = False
def __init__(self, url="http://localhost:5984", db="test", username="", password=""): """Create a CouchClient object. :param url: the location where the CouchDB instance is located, including the port at which it's listening. Default: http://localhost:5984 :param db: the database to use. Default: test. """ self.server = Server(url=url) if username == "": self.db = self.server[db] else: self.db = couchdb.Database(url + "/" + db) self.db.resource.credentials = (username, password)
def main(configfile): url = get_couchdb_url(configfile) db = couchdb.Database(url) db = CachedDB(db) for row in read_log(): if row.get("action") == 'store.put' and row['data']['data'].get( 'type') == '/type/loan': loan_start(db, row) elif row.get("action") == 'store.delete' and is_uuid( row['data']['key']): loan_end(db, row) db.commit()
def connect_to_couchdb(url=credentials.URL, username=credentials.USERNAME, password=credentials.PWD, dbname=credentials.DBNAME): """ Connect to the database with the credentials set in "credentials.py" file. It returns a couchdb object (not truly connect) since couchdb is RESTFULL and no separate connection is made for authorization and authentication. """ db = couchdb.Database(url + "/" + dbname) db.resource.credentials = (username, password) print(("Connection established with: " + dbname)) return db
def _get_count_docs(ndays): """Returns the count docs from admin couchdb database. This function is memoized to avoid accessing couchdb for every request. """ admin_db = couchdb.Database(config.admin.counts_db) end = datetime.datetime.now().strftime("counts-%Y-%m-%d") start = (datetime.datetime.now() - datetime.timedelta(days = ndays)).strftime("counts-%Y-%m-%d") stats.begin("couchdb") docs = [x.doc for x in admin_db.view("_all_docs", startkey_docid = start, endkey_docid = end, include_docs = True)] stats.end() return docs
def get(self, dataservice="", view='', list=''): """GET /extract/id: Show a specific intem""" try: db_url = '/'.join([ appConfig['couchdb.url'], appConfig['couchdb.db.resourcedata'] ]) db = couchdb.Database(db_url) dsDocument = db['_design/' + dataservice] if "dataservices" not in dsDocument: abort(406, "Invalid Data Service") log.error("no dataservices element in document") urlBase = "_design/{0}/_list/{1}/{2}".format( dataservice, list, view) startKey, endKey, includeDocs = self._orderParmaByView( request.params, view) return self._processRequest(startKey, endKey, urlBase, includeDocs) except couchdb.ResourceNotFound as ex: abort(406, "Invalid Data Service") log.error(ex)
def get_couch_database(): global _couchdb if config.get("couchdb_database"): _couchdb = couchdb.Database(config.couchdb_database) return _couchdb
from redis import StrictRedis import redis import couchdb from pprint import pprint import math import mincemeat import json db = couchdb.Database("http://localhost:5984/lr-data") INDEX_DB = 0 r = StrictRedis(db=INDEX_DB) def count_map(k, v): yield v def count_reduce(k, vs): with open("counts/" + k, "w+") as f: f.write(str(sum(vs))) return sum(vs) def process_keys(): for n in xrange(ord('a'), ord('z')): for l in xrange(ord('a'), ord('z')): query = chr(n) + chr(l) + "*" for k in r.keys(query): yield k def process_key(k): r = StrictRedis(db=INDEX_DB) try:
# * Parikshit Diwan (Student ID: 1110497) # * Colin McLean (Student ID: 1139518) # * Matthias Bachfischer (Student ID: 1133751) # # Location: Melbourne # import couchdb from geopy import Point from geopy.geocoders import Nominatim import time geolocator = Nominatim(timeout=10) username = "******" password = "******" cdb = couchdb.Database("http://172.26.133.36:5984/twitter_raw_data") cdb.resource.credentials = (username, password) resultdb = couchdb.Database("http://172.26.133.36:5984/twitter_result_db") resultdb.resource.credentials = (username, password) mango = {'selector': {'Flag': 'N'}} def getZipCode(longfield, latfield, geolocation_enabled): location = geolocator.reverse(Point(latfield, longfield)) try: if geolocation_enabled.lower() == 'true': postcode = location.raw['address']['postcode'] return postcode else: postcode = location.raw['address']['postcode'] postcode = "1" + postcode return postcode
def get_seeds_engine(): """Returns updater.SeedsDB instance.""" db_url = config.get("lists", {}).get("seeds_db") db = couchdb.Database(db_url) return updater.SeedsDB(db, web.storage(db=get_works_db()))
#! /usr/bin/env python import sys import couchdb db = couchdb.Database(sys.argv[1]) del db[sys.argv[2]]
def connect_to_taskdb(): db_uri = config.get("celery", {})["tombstone_db"] return couchdb.Database(db_uri)
sys.exit(2) except getopt.GetoptError as err: print(err) sys.exit(2) for opt, arg in opts: if opt in ("-s", "--server"): SERVER = arg elif opt in ("-d", "--db"): DATA_BASE = arg elif opt in ("-p", "--page"): DOCS_PER_PAGE = arg elif opt in ("-k", "--start-page"): START_PAGE = arg db = couchdb.Database(SERVER + DATA_BASE) page_number = int(START_PAGE) docs_number = 1 skip = (page_number - 1) * int(DOCS_PER_PAGE) region_handler = SG_planning() total_time = 0.0 f = open('add.txt', 'a') while skip < docs_number: time_0 = time.time() rows = db.view('_all_docs', limit=DOCS_PER_PAGE, skip=skip, include_docs=True) docs_number = int(rows.total_rows)
###Python 3.4.3 ##cs410 project ##query for DBLP Connectivity: Is the DBPL graph connected? (That is, is there a path between any two objects?) ##December 5, 2018 import couchdb import numpy couchserver = couchdb.Server("http://couchdb:5984/") dbname = "cs410project" if dbname in couchserver: db = couchserver[dbname] else: db = couchdb.Database("http://localhost:5984/cs410project/") stringView = "design/viewID" startNodeName = "Moshe Y. Vardi" #if connected, the node chosen to start is arbitrary ##start_ID = "f3922219d063fe633641cd4b95e2ba54" #Moshe Y. Vardi mangoQ = {"selector": { "$and": [ { "type": { "$eq": "person" } }, { "name": { "$eq": startNodeName } }
##query for DBLP: At what level is Moshe Vardi from Michael J. Franklin? ##December 5, 2018 import couchdb import numpy goalValue = -3 cutoffValue = -2 failureValue = -1 couchserver = couchdb.Server("http://couchdb:5984/") dbname = "cs410project" #change to the name of the database you want to use if dbname in couchserver: db = couchserver[dbname] else: db = couchdb.Database("http://localhost:5984/" + dbname + "/") stringView = "design/viewID" #change to view url that emits the attribute 'id' as a key startNodeName = "Moshe Y. Vardi" #729526 = id, change to person name you want to start with goalNodeName = "Michael J. Franklin" #747452 = id, change to person name you want to use as a goal ##start_ID = "f3922219d063fe633641cd4b95e2ba54" #Moshe Y. Vardi ##goal_ID = "7eed831a4d055da93f33e9eff8d55916" #Michael J. Franklin mangoQ = { "selector": { "$and": [{ "type": { "$eq": "person" } }, { "name": { "$eq": startNodeName
if doc['type'] == "paper" or doc['type'] == "book" or doc[ 'type'] == "msthesis": if doc['id'] in papers: result += doc['authors'] return result def process(elimate, level): result = [x for x in level if x != elimate] result = list(set(result)) return result couchserver = couchdb.Server("http://couchdb:5984/") dbname = "dblp" db = couchdb.Database("http://localhost:5984/dblp/") #looking for papers and got level1 papers = [] elimate = "" for id in db: doc = db.get(id) if doc['type'] == "person" and doc['name'] == "Michael Stonebraker": papers = doc['author_of'] elimate = doc['id'] level1 = find(db, papers) level1 = process(elimate, level1) #looking for papers within level1 and got level2 papers = [] for id in db: doc = db.get(id) if doc['type'] == "person" and doc['id'] in level1:
def get_admin_couchdb(): db_url = config.get("admin", {}).get("counts_db") return db_url and couchdb.Database(db_url)
def __init__(self, url): self.db = couchdb.Database(url)
#!/usr/bin/python import couchdb, urllib2,time target_dist_url = 'http://localhost/distribute' couchdb_url = 'http://localhost:5984/resource_data' change_threshold = 100 sleep_time = 60 if __name__ == '__main__': db = couchdb.Database(couchdb_url) last_seq = '' while True: time.sleep(sleep_time) if last_seq <> '': changes = db.changes(since=last_seq) else: changes = db.changes() last_seq = changes['last_seq'] if len(changes['results']) >= change_threshold: urllib2.urlopen(target_dist_url,'test')
def run(self, url): db = couchdb.Database(url) for docs in web.group(self.read(), 1000): db.update(docs)