def connect_db(self):
self.db = couchdb.Database(
self.config.db_url,
session=couchdb.Session(retry_delays=range(10)))
self.adb = couchdb.Database(
self.config.adb_url,
session=couchdb.Session(retry_delays=range(10)))
def connect_to_couch(config_file):
"Connects to the couch databases"
f = open(config_file)
config = yaml.load(f)
f.close()
admin_db = config["admin"]["counts_db"]
editions_db = config["lists"]["editions_db"]
works_db = config["lists"]["works_db"]
seeds_db = config["lists"]["seeds_db"]
logging.debug(" Admin Database is %s", admin_db)
logging.debug(" Editions Database is %s", editions_db)
logging.debug(" Works Database is %s", works_db)
logging.debug(" Seeds Database is %s", seeds_db)
return couchdb.Database(admin_db), couchdb.Database(
editions_db), couchdb.Database(works_db), couchdb.Database(seeds_db)
def connect_to_couch(config_file):
"Connects to the couch databases"
f = open(config_file)
config = yaml.load(f)
f.close()
admin_db = config["admin"]["counts_db"]
return couchdb.Database(admin_db)
def watch(ddoc_dir, db_uri):
ddocname = get_ddocname(ddoc_dir)
db = couchdb.Database(db_uri)
obs = Observer()
q = Queue()
e2q = Ev2Q(q)
obs.schedule(e2q, ddoc_dir, recursive=False)
try:
obs.start()
while True:
# We need a timeout, or else we can't catch keyboard interrupts, &c.
try:
evtype, ev = q.get(timeout=0.5)
except Empty:
continue
name = ev.src_path
fullpath = name
if evtype in ["created", "modified"] and valid_filename(
os.path.basename(name)) and not os.path.isdir(fullpath):
print evtype, fullpath
ddoc = db[ddocname]
db.put_attachment(ddoc, open(fullpath))
except KeyboardInterrupt:
pass
finally:
obs.stop()
obs.join()
def connect_to_tombstone():
global tombstone_db
try:
tombstone_db_uri = config.get("celery", {})["tombstone_db"]
tombstone_db = couchdb.Database(tombstone_db_uri)
except Exception, e:
logger.warning("Couldn't connect to tombstone database", exc_info=True)
def get_stats(ndays=30):
"""Returns the stats for the past `ndays`"""
admin_db = couchdb.Database(config.admin.counts_db)
end = datetime.datetime.now().strftime("counts-%Y-%m-%d")
start = (datetime.datetime.now() -
datetime.timedelta(days=ndays)).strftime("counts-%Y-%m-%d")
docs = [
x.doc for x in admin_db.view("_all_docs",
startkey_docid=start,
endkey_docid=end,
include_docs=True)
]
retval = dict(human_edits=Stats(docs, "human_edits", "human_edits"),
bot_edits=Stats(docs, "bot_edits", "bot_edits"),
lists=Stats(docs, "lists", "total_lists"),
visitors=Stats(docs, "visitors", "visitors"),
loans=Stats(docs, "loans", "loans"),
members=Stats(docs, "members", "total_members"),
works=Stats(docs, "works", "total_works"),
editions=Stats(docs, "editions", "total_editions"),
ebooks=Stats(docs, "ebooks", "total_ebooks"),
covers=Stats(docs, "covers", "total_covers"),
authors=Stats(docs, "authors", "total_authors"),
subjects=Stats(docs, "subjects", "total_subjects"))
return retval
def get_db_connection(self):
host = self.config.get_option('db', 'host')
port = self.config.get_option('db', 'port')
user_name = self.config.get_option('user', 'username')
user_password = self.config.get_option('user', 'password')
db_name = self.config.get_option('db', 'name')
self.db = couchdb.Database(
create_db_url(host, port, user_name, user_password, db_name),
session=couchdb.Session(retry_delays=range(10)))
a_name = self.config.get_option('admin', 'username')
a_password = self.config.get_option('admin', 'password')
self.adb = couchdb.Database(
create_db_url(host, port, a_name, a_password, db_name))
def insertDocumentCouchdb(envelope, config):
try:
conf = config['couchdb']
db = couchdb.Database(conf['dbUrl'])
del envelope['_rev']
del envelope['_id']
db.save(envelope)
except (Exception), exc:
log.error("Error writing to mongo")
processHarvestResult.retry(exc)
def runQuery():
# Setup from query3 code
couchserver = couchdb.Server("http://couchdb:5984/")
dbname = "dblp"
if dbname in couchserver:
db = couchserver[dbname]
else:
db = couchdb.Database("http://localhost:5984/%s/" % dbname)
name = findProceedingsWithMaxAuthor(db)
print 'Proceedings in 2004 with the most authors: %s' % name
def sync(ddoc_dir, db_uri):
db = couchdb.Database(db_uri)
ddocname = get_ddocname(ddoc_dir)
if not ddocname in db:
db[ddocname] = {"_id": ddocname}
# Initialize
for fname in os.listdir(ddoc_dir):
if valid_filename(fname):
ddoc = db[ddocname]
db.put_attachment(ddoc, open(os.path.join(ddoc_dir, fname)))
def __init__( self, host, port, database, createifnone=True ):
#--------------------------------------------------------------------------
"""
Initalize a CouchDBManager object.
:param host: host name or IP address as string
:param port: port number on which couchDB is listening
:param database: name of the database
"""
url = "http://%s:%s@%s:%s" % (self.username, self.password, host, str(port))
try:
self.database = couchdb.Database("%s/%s" % (url, database))
self.database.info()
except:
if createifnone:
couchserver = couchdb.Server(url)
couchserver.create(database)
self.database = couchdb.Database("%s/%s" % (url, database))
self.database.info()
self.shutdown = False
def __init__(self, url="http://localhost:5984", db="test",
username="", password=""):
"""Create a CouchClient object.
:param url: the location where the CouchDB instance is located,
including the port at which it's listening. Default: http://localhost:5984
:param db: the database to use. Default: test.
"""
self.server = Server(url=url)
if username == "":
self.db = self.server[db]
else:
self.db = couchdb.Database(url + "/" + db)
self.db.resource.credentials = (username, password)
def main(configfile):
url = get_couchdb_url(configfile)
db = couchdb.Database(url)
db = CachedDB(db)
for row in read_log():
if row.get("action") == 'store.put' and row['data']['data'].get(
'type') == '/type/loan':
loan_start(db, row)
elif row.get("action") == 'store.delete' and is_uuid(
row['data']['key']):
loan_end(db, row)
db.commit()
def connect_to_couchdb(url=credentials.URL,
username=credentials.USERNAME,
password=credentials.PWD,
dbname=credentials.DBNAME):
"""
Connect to the database with the credentials set in "credentials.py" file.
It returns a couchdb object (not truly connect) since couchdb is RESTFULL and no
separate connection is made for authorization and authentication.
"""
db = couchdb.Database(url + "/" + dbname)
db.resource.credentials = (username, password)
print(("Connection established with: " + dbname))
return db
def _get_count_docs(ndays):
"""Returns the count docs from admin couchdb database.
This function is memoized to avoid accessing couchdb for every request.
"""
admin_db = couchdb.Database(config.admin.counts_db)
end = datetime.datetime.now().strftime("counts-%Y-%m-%d")
start = (datetime.datetime.now() - datetime.timedelta(days = ndays)).strftime("counts-%Y-%m-%d")
stats.begin("couchdb")
docs = [x.doc for x in admin_db.view("_all_docs",
startkey_docid = start,
endkey_docid = end,
include_docs = True)]
stats.end()
return docs
def get(self, dataservice="", view='', list=''):
"""GET /extract/id: Show a specific intem"""
try:
db_url = '/'.join([
appConfig['couchdb.url'], appConfig['couchdb.db.resourcedata']
])
db = couchdb.Database(db_url)
dsDocument = db['_design/' + dataservice]
if "dataservices" not in dsDocument:
abort(406, "Invalid Data Service")
log.error("no dataservices element in document")
urlBase = "_design/{0}/_list/{1}/{2}".format(
dataservice, list, view)
startKey, endKey, includeDocs = self._orderParmaByView(
request.params, view)
return self._processRequest(startKey, endKey, urlBase, includeDocs)
except couchdb.ResourceNotFound as ex:
abort(406, "Invalid Data Service")
log.error(ex)
def get_couch_database():
global _couchdb
if config.get("couchdb_database"):
_couchdb = couchdb.Database(config.couchdb_database)
return _couchdb
from redis import StrictRedis
import redis
import couchdb
from pprint import pprint
import math
import mincemeat
import json
db = couchdb.Database("http://localhost:5984/lr-data")
INDEX_DB = 0
r = StrictRedis(db=INDEX_DB)
def count_map(k, v):
yield v
def count_reduce(k, vs):
with open("counts/" + k, "w+") as f:
f.write(str(sum(vs)))
return sum(vs)
def process_keys():
for n in xrange(ord('a'), ord('z')):
for l in xrange(ord('a'), ord('z')):
query = chr(n) + chr(l) + "*"
for k in r.keys(query):
yield k
def process_key(k):
r = StrictRedis(db=INDEX_DB)
try:
# * Parikshit Diwan (Student ID: 1110497)
# * Colin McLean (Student ID: 1139518)
# * Matthias Bachfischer (Student ID: 1133751)
#
# Location: Melbourne
#
import couchdb
from geopy import Point
from geopy.geocoders import Nominatim
import time
geolocator = Nominatim(timeout=10)
username = "******"
password = "******"
cdb = couchdb.Database("http://172.26.133.36:5984/twitter_raw_data")
cdb.resource.credentials = (username, password)
resultdb = couchdb.Database("http://172.26.133.36:5984/twitter_result_db")
resultdb.resource.credentials = (username, password)
mango = {'selector': {'Flag': 'N'}}
def getZipCode(longfield, latfield, geolocation_enabled):
location = geolocator.reverse(Point(latfield, longfield))
try:
if geolocation_enabled.lower() == 'true':
postcode = location.raw['address']['postcode']
return postcode
else:
postcode = location.raw['address']['postcode']
postcode = "1" + postcode
return postcode
def get_seeds_engine():
"""Returns updater.SeedsDB instance."""
db_url = config.get("lists", {}).get("seeds_db")
db = couchdb.Database(db_url)
return updater.SeedsDB(db, web.storage(db=get_works_db()))
#! /usr/bin/env python import sys import couchdb db = couchdb.Database(sys.argv[1]) del db[sys.argv[2]]
def connect_to_taskdb():
db_uri = config.get("celery", {})["tombstone_db"]
return couchdb.Database(db_uri)
sys.exit(2)
except getopt.GetoptError as err:
print(err)
sys.exit(2)
for opt, arg in opts:
if opt in ("-s", "--server"):
SERVER = arg
elif opt in ("-d", "--db"):
DATA_BASE = arg
elif opt in ("-p", "--page"):
DOCS_PER_PAGE = arg
elif opt in ("-k", "--start-page"):
START_PAGE = arg
db = couchdb.Database(SERVER + DATA_BASE)
page_number = int(START_PAGE)
docs_number = 1
skip = (page_number - 1) * int(DOCS_PER_PAGE)
region_handler = SG_planning()
total_time = 0.0
f = open('add.txt', 'a')
while skip < docs_number:
time_0 = time.time()
rows = db.view('_all_docs',
limit=DOCS_PER_PAGE,
skip=skip,
include_docs=True)
docs_number = int(rows.total_rows)
###Python 3.4.3
##cs410 project
##query for DBLP Connectivity: Is the DBPL graph connected? (That is, is there a path between any two objects?)
##December 5, 2018
import couchdb
import numpy
couchserver = couchdb.Server("http://couchdb:5984/")
dbname = "cs410project"
if dbname in couchserver:
db = couchserver[dbname]
else:
db = couchdb.Database("http://localhost:5984/cs410project/")
stringView = "design/viewID"
startNodeName = "Moshe Y. Vardi" #if connected, the node chosen to start is arbitrary
##start_ID = "f3922219d063fe633641cd4b95e2ba54" #Moshe Y. Vardi
mangoQ = {"selector": {
"$and": [
{
"type": {
"$eq": "person"
}
},
{
"name": {
"$eq": startNodeName
}
}
##query for DBLP: At what level is Moshe Vardi from Michael J. Franklin?
##December 5, 2018
import couchdb
import numpy
goalValue = -3
cutoffValue = -2
failureValue = -1
couchserver = couchdb.Server("http://couchdb:5984/")
dbname = "cs410project" #change to the name of the database you want to use
if dbname in couchserver:
db = couchserver[dbname]
else:
db = couchdb.Database("http://localhost:5984/" + dbname + "/")
stringView = "design/viewID" #change to view url that emits the attribute 'id' as a key
startNodeName = "Moshe Y. Vardi" #729526 = id, change to person name you want to start with
goalNodeName = "Michael J. Franklin" #747452 = id, change to person name you want to use as a goal
##start_ID = "f3922219d063fe633641cd4b95e2ba54" #Moshe Y. Vardi
##goal_ID = "7eed831a4d055da93f33e9eff8d55916" #Michael J. Franklin
mangoQ = {
"selector": {
"$and": [{
"type": {
"$eq": "person"
}
}, {
"name": {
"$eq": startNodeName
if doc['type'] == "paper" or doc['type'] == "book" or doc[
'type'] == "msthesis":
if doc['id'] in papers:
result += doc['authors']
return result
def process(elimate, level):
result = [x for x in level if x != elimate]
result = list(set(result))
return result
couchserver = couchdb.Server("http://couchdb:5984/")
dbname = "dblp"
db = couchdb.Database("http://localhost:5984/dblp/")
#looking for papers and got level1
papers = []
elimate = ""
for id in db:
doc = db.get(id)
if doc['type'] == "person" and doc['name'] == "Michael Stonebraker":
papers = doc['author_of']
elimate = doc['id']
level1 = find(db, papers)
level1 = process(elimate, level1)
#looking for papers within level1 and got level2
papers = []
for id in db:
doc = db.get(id)
if doc['type'] == "person" and doc['id'] in level1:
def get_admin_couchdb():
db_url = config.get("admin", {}).get("counts_db")
return db_url and couchdb.Database(db_url)
def __init__(self, url):
self.db = couchdb.Database(url)
#!/usr/bin/python
import couchdb, urllib2,time
target_dist_url = 'http://localhost/distribute'
couchdb_url = 'http://localhost:5984/resource_data'
change_threshold = 100
sleep_time = 60
if __name__ == '__main__':
db = couchdb.Database(couchdb_url)
last_seq = ''
while True:
time.sleep(sleep_time)
if last_seq <> '':
changes = db.changes(since=last_seq)
else:
changes = db.changes()
last_seq = changes['last_seq']
if len(changes['results']) >= change_threshold:
urllib2.urlopen(target_dist_url,'test')
def run(self, url):
db = couchdb.Database(url)
for docs in web.group(self.read(), 1000):
db.update(docs)
