def connector(collection):
# Make a server connection
svr = Server()
if collection not in svr:
return svr.create(collection)
else:
return svr[collection]
def connector(collection):
# Make a server connection
svr = Server()
if collection not in svr:
return svr.create(collection)
else:
return svr[collection]
def __init__(self):
"""Initialises a new connection to couch and
creates a new mis databse if nonexistent"""
LOG.info('initialising database')
database_name = CONFIG.get('couchdb', 'database')
host = CONFIG.get('couchdb', 'hostname')
port = CONFIG.get('couchdb', 'port')
database_uri = 'http://' + host + ':' + port + '/'
self.server = Server(database_uri)
try:
# This statement appears to do nothing, but is used to
# make sure the database is reachable.
self.server.version
except AttributeError as error:
if not test_tcp_connection(host, int(port)):
LOG.critical("couchdb cannot be reached at " + database_uri)
exit(1)
else:
LOG.error('unknown AttributeError thrown')
raise error
try:
LOG.debug('opening database')
self.database = self.server[database_name]
except (ResourceNotFound):
LOG.info('creating database')
# The database didn't exist. Lets create it.
self.database = self.server.create(database_name)
self.create_views()
def get_db(self, db_uri, db_name):
server = Server(db_uri)
try:
db = server[db_name]
except ResourceNotFound:
db = server.create(db_name)
return db
def couch_connect():
server = Server()
try:
db = server.create('feedme')
except:
db = server['feedme']
return db
def __init__(self, mountpoint, uri=None, *args, **kw):
fuse.Fuse.__init__(self, *args, **kw)
self.fuse_args.mountpoint = mountpoint
if uri is not None:
self.server = Server(uri)
else:
self.server = Server()
def _open_couch_db(dbname):
server = Server(url=self._couch_url)
try:
server[dbname]
except ResourceNotFound:
server.create(dbname)
return server[dbname]
class CouchDBBackend(object):
@property
def db(self):
return self.server[self.db_name]
def __init__(self, config):
settings = config.registry.settings
self.config = config
self.server = Server(settings['backend.db_host'])
self.db_name = os.environ.get('DB_NAME', settings['backend.db_name'])
self.create_db_if_not_exist()
self.sync_views()
self.config.add_subscriber(self.add_db_to_request, NewRequest)
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.debug('Creating and using db "%s"' % self.db_name)
except PreconditionFailed:
logger.debug('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.db, docs)
def add_db_to_request(self, event):
event.request.db = Database(self.db)
def saveDocument(self):
server = Server()
server.resource.credentials = ('secret', 'aedca29aed23e103f27f8e29dd6ec473')
# create database
db = server.create("new.job")
# associate NewJob to the db
NewJob.set_db(db)
self.save()
def Create_Couchdb_Instance(dtbs_name):
server = Server("http://127.0.0.1:5984")
try:
db = server.create(dtbs_name)
except Exception:
db = server[dtbs_name]
return db
def initDatabase():
server = Server('http://localhost:5984')
del server['lazycrawler']
try:
db = server.create('lazycrawler')
except Exception:
db = server['lazycrawler']
return db
class CouchDBBackend(object):
@classmethod
def load_from_config(cls, config):
settings = config.registry.settings
return CouchDBBackend(
host=settings['backend.db_host'],
db_name=os.environ.get('DB_NAME', settings['backend.db_name']),
)
def __init__(self, host, db_name, id_generator):
self.server = Server(host)
self.db_name = db_name
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (host, e))
self._db = self.server[self.db_name]
self.sync_views()
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.info('Creating and using db "%s"' % self.db_name)
except PreconditionFailed:
logger.info('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.server[self.db_name], docs)
def __get_raw_user_token(self, user_id):
try:
return views.usertokens(self._db, key=user_id).rows[0].value
except IndexError:
raise UserIdNotFound(user_id)
def get_user_token(self, user_id):
"""Returns the information associated with a user token"""
usertoken = dict(**self.__get_raw_user_token(user_id))
return usertoken['token']
def add_token(self, user_id, token):
# Check that the token doesn't already exist.
try:
self.__get_raw_user_token(user_id)
raise UserIdAlreadyExist(user_id)
except UserIdNotFound:
pass
doc = dict(token=token, user_id=user_id, type='usertoken')
self._db.save(doc)
def __init__(self, config_path=None):
self.server = Server()
if config_path:
self.config = yaml.load(open(config_path, 'r').read())
if not all(db_name in self.server
for db_name in self.config['schema'].values()):
self.setup()
self.connect()
def __init__(self, db_uri):
local.application = self
server = Server(db_uri)
try:
db = server.create("urls")
except Exception:
db = server["urls"]
self.dispatch = SharedDataMiddleware(self.dispatch, {"/static": STATIC_PATH})
URL.db = db
def init_boards():
"""
:rtype : object
"""
server = Server()
try:
db = server.create('boards')
except Exception:
db = server['boards']
return db
def init_boards():
"""
:rtype : object
"""
server = Server()
try:
db = server.create("boards")
except Exception:
db = server["boards"]
return db
def __init__(self):
from couchdb.client import Server
couch = Server('http://localhost:5984')
print (couch)
try:
self.db = couch.create('run-layer')
except:
self.db = couch['run-layer']
print (self.db)
def __init__(self, db_uri):
local.application = self
server = Server(db_uri)
try:
db = server.create("urls")
except Exception:
db = server["urls"]
self.dispatch = SharedDataMiddleware(self.dispatch,
{"/static": STATIC_PATH})
URL.db = db
def __init__(self, db_uri):
local.application = self
server = Server(db_uri)
try:
db = server.create('urls')
except:
db = server['urls']
self.dispatch = SharedDataMiddleware(self.dispatch, {
'/static': STATIC_PATH
})
URL.db = db
def __init__(self, host, db_name, id_generator):
self.server = Server(host)
self.db_name = db_name
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (host, e))
self._db = self.server[self.db_name]
self.sync_views()
self._generate_id = id_generator
def __init__(self):
try:
self.__server = Server()
except:
print 'can not connect to Couchdb:%s' % (settings.c['db_url'])
self.__db = {}
self.__db_name = settings.c['db_name']
DEBUG.p(self.__db_name.items())
for (k, v) in self.__db_name.items():
try:
self.__db[v] = self.__server.create(v)
except:
self.__db[v] = self.__server[v]
def _verify_token_in_couch(self, uuid, token):
"""
Query couchdb to decide if C{token} is valid for C{uuid}.
@param uuid: The user uuid.
@type uuid: str
@param token: The token.
@type token: str
@raise InvalidAuthTokenError: Raised when token received from user is
either missing in the tokens db or is
invalid.
"""
server = Server(url=self._app.state.couch_url)
dbname = self.TOKENS_DB
db = server[dbname]
# lookup key is a hash of the token to prevent timing attacks.
token = db.get(sha512(token).hexdigest())
if token is None:
raise InvalidAuthTokenError()
# we compare uuid hashes to avoid possible timing attacks that
# might exploit python's builtin comparison operator behaviour,
# which fails immediatelly when non-matching bytes are found.
couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest()
req_uuid_hash = sha512(uuid).digest()
if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \
or couch_uuid_hash != req_uuid_hash:
raise InvalidAuthTokenError()
return True
def __init__(self):
"""Initialises a new connection to couch and
creates a new mis databse if nonexistent"""
LOG.info('initialising database')
database_name = CONFIG.get('couchdb', 'database')
host = CONFIG.get('couchdb', 'hostname')
port = CONFIG.get('couchdb', 'port')
database_uri = 'http://' + host + ':' + port + '/'
self.server = Server(database_uri)
try:
# This statement appears to do nothing, but is used to
# make sure the database is reachable.
self.server.version
except AttributeError as error:
if not test_tcp_connection(host, int(port)):
LOG.critical("couchdb cannot be reached at " + database_uri)
exit(1)
else:
LOG.error('unknown AttributeError thrown')
raise error
try:
LOG.debug('opening database')
self.database = self.server[database_name]
except(ResourceNotFound):
LOG.info('creating database')
# The database didn't exist. Lets create it.
self.database = self.server.create(database_name)
self.create_views()
def __init__(self, url, dbname, replica_uid=None, full_commit=True,
session=None):
"""
Create a new Couch data container.
@param url: the url of the couch database
@type url: str
@param dbname: the database name
@type dbname: str
@param replica_uid: an optional unique replica identifier
@type replica_uid: str
@param full_commit: turn on the X-Couch-Full-Commit header
@type full_commit: bool
@param session: an http.Session instance or None for a default session
@type session: http.Session
"""
# save params
self._url = url
self._full_commit = full_commit
self._session = session
# configure couch
self._server = Server(url=self._url,
full_commit=self._full_commit,
session=self._session)
self._dbname = dbname
# this will ensure that transaction and sync logs exist and are
# up-to-date.
try:
self._database = self._server[self._dbname]
except ResourceNotFound:
self._server.create(self._dbname)
self._database = self._server[self._dbname]
ObjectStoreDatabase.__init__(self, replica_uid=replica_uid)
def test_view(self):
# This test does quite a bit. First, create 4 test records.
# Then, create a view that will emit those records and insert that into
# the db. Finally, call our cushion.view object and compare results.
self._save_some_data({'foo': 1, 'bar': 'a'})
self._save_some_data({'foo': 2, 'bar': 'a'})
self._save_some_data({'foo': 3, 'bar': 'b'})
self._save_some_data({'foo': 4, 'bar': 'b'})
fake_map = """ function (doc) { emit(doc['bar'], doc); } """
# we're going to use python-couchdb's dynamic view loader stuff here
from couchdb.design import ViewDefinition
from couchdb.client import Server
global baseurl
cdb = Server(baseurl)
couchdb = cdb[self.dbname]
view_defn = ViewDefinition('test',
'view',
map_fun=fake_map,
language='javascript')
view_defn.sync(couchdb)
self.cushion.view(self.dbname, 'test/view', self.stop, key='b')
records = self.wait()
self.assertTrue(len(records) == 2)
def __init__(self, server, name):
self.server = Server(server)
if name in self.server:
self.db = self.server[name]
else:
self.db = self.server.create(name)
self._load_views()
self.view = self.db.view('_view/update/mtime')
def test(request):
server = Server('http://43.240.96.132:5984/')
documents = server['testdb']
if request.method == 'GET':
return render(request, 'test.html')
if request.method == 'POST':
x = {'haha': 'sdfgdfgf'}
return HttpResponse(json.dumps(x))
def setUp(self):
self.server = Server('http://localhost:5984')
self.db_name = 'test_%s' % uuid4()
self.server.create(self.db_name)
db = self.server[self.db_name]
ViewDefinition.sync_many(db, couchdb_views)
self.db = CouchDBDatabase(db, lambda: six.text_type(uuid4()))
super(TestCouchDBBackend, self).setUp()
def __init__(self, uri=None, database=None):
print '%s|%s' % (uri, database)
self.server = Server(uri)
try:
# python-couchdb will create the database or raise an error if it
# already exists
self.database = self.server.create(database)
except:
self.database = self.server[database]
def __init__(self, config):
settings = config.registry.settings
self.config = config
self.server = Server(settings['backend.db_host'])
self.db_name = os.environ.get('DB_NAME', settings['backend.db_name'])
self.create_db_if_not_exist()
self.sync_views()
self.config.add_subscriber(self.add_db_to_request, NewRequest)
class DB(object):
server = None
sensors = None
readings = None
devices = None
config = {
# TODO Server credentials are currently not used
'server': {
'host': 'localhost',
'port': 5984
},
'schema': {
'sensors_db': 'sensors',
'readings_db': 'readings',
'devices_db': 'devices'
}
}
def __init__(self, config_path=None):
self.server = Server()
if config_path:
self.config = yaml.load(open(config_path, 'r').read())
if not all(db_name in self.server
for db_name in self.config['schema'].values()):
self.setup()
self.connect()
def connect(self):
for db_name, attr_name in [['sensors_db', 'sensors'],
['readings_db', 'readings'],
['devices_db', 'devices']]:
setattr(self, attr_name,
self.server[self.config['schema'][db_name]])
def setup(self):
for db_name in ['sensors_db', 'readings_db', 'devices_db']:
if self.config['schema'][db_name] not in self.server:
self.server.create(self.config['schema'][db_name])
def copy_couch_database_for_test(test, db):
port = str(test.wrapper.port)
couch_url = 'http://localhost:' + port
new_dbname = db._replica_uid + '_copy'
new_db = couch.CouchDatabase.open_database(
urljoin(couch_url, new_dbname),
create=True,
replica_uid=db._replica_uid or 'test')
# copy all docs
session = couch.Session()
old_couch_db = Server(couch_url, session=session)[db._replica_uid]
new_couch_db = Server(couch_url, session=session)[new_dbname]
for doc_id in old_couch_db:
doc = old_couch_db.get(doc_id)
# bypass u1db_config document
if doc_id == 'u1db_config':
pass
# copy design docs
elif doc_id.startswith('_design'):
del doc['_rev']
new_couch_db.save(doc)
# copy u1db docs
elif 'u1db_rev' in doc:
new_doc = {
'_id': doc['_id'],
'u1db_transactions': doc['u1db_transactions'],
'u1db_rev': doc['u1db_rev']
}
attachments = []
if ('u1db_conflicts' in doc):
new_doc['u1db_conflicts'] = doc['u1db_conflicts']
for c_rev in doc['u1db_conflicts']:
attachments.append('u1db_conflict_%s' % c_rev)
new_couch_db.save(new_doc)
# save conflict data
attachments.append('u1db_content')
for att_name in attachments:
att = old_couch_db.get_attachment(doc_id, att_name)
if (att is not None):
new_couch_db.put_attachment(new_doc, att,
filename=att_name)
# cleanup connections to prevent file descriptor leaking
return new_db
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
authn_policy = AuthTktAuthenticationPolicy(
'sosecret', hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings, root_factory=RootFactory)
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
dbserver = Server(url = settings['couchdb.url'])
if settings['couchdb.dbname'] not in dbserver:
dbserver.create(settings['couchdb.dbname'])
config.registry.settings['couchdb.server'] = dbserver
config.add_renderer(".html", "pyramid.mako_templating.renderer_factory")
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_route('home', '/')
config.add_route('item', '/item/*traverse', factory=Container)
config.add_route('tag', '/tag/*traverse', factory=Tags)
config.scan()
return config.make_wsgi_app()
def tearDown(self):
# if current test is `test_close` we have to use saved objects to
# delete the database because the close() method will have removed the
# references needed to do it using the CouchDatabase.
if self.id().endswith('test_couch.CouchTests.test_close(couch)'):
session = couch.Session()
server = Server(url=self._url, session=session)
del (server[self._dbname])
else:
self.db.delete_database()
test_backends.AllDatabaseTests.tearDown(self)
def __init__(self, db, url=None, initialize=False, **kwargs):
self._url = url or DEFAULT_BASE_URL
self._server = Server(self._url)
self.unique_id = "%s_%s" % (self._url, db)
super(CouchStorage, self).__init__(**kwargs)
if db not in self._server:
if initialize:
self._server.create(db)
else:
raise Exception("DB '%s' not found at %s" % (db, url))
self._db = self._server[db]
def main():
couch_uri = 'http://localhost:5984/'
db_name = 'redditron-comments'
server = Server(couch_uri)
db = get_db(server, db_name)
if (len(sys.argv) == 1):
poll_collect(db, 4 * 60)
else:
collect_comments(db,
flatten_comments(fetch_link_comments(sys.argv[1])))
def GetDatabase(
cls,
server = GetConfig('COUCHDB_SERVER'),
database = GetConfig('COUCHDB_DATABASE')
):
if server is None:
return None
from couchdb.client import Server
couchdb_server = Server(server)
return couchdb_server[database]
def __init__(self, host, db_name, id_generator):
self.server = Server(host)
self.db_name = db_name
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (host, e))
self._db = self.server[self.db_name]
self.sync_views()
def db(self):
if hasattr(request, "_couchdb_db"):
db = request._couchdb_db
else:
dburi = config.get("couchdb.dburi")
server = Server(dburi)
username = config.get("couchdb.username")
password = config.get("couchdb.password")
if username:
server.resource.http.add_credentials(username, password)
db = server[config.get("couchdb.database")]
request._couchdb_db = db
return db
def main():
couch_uri = 'http://localhost:5984/'
db_name = 'redditron-comments'
server = Server(couch_uri)
db = server[db_name]
c = build_chain(db)
outputs = []
for i in range(0, 100):
outputs.append(c.generate_text())
print "\n---\n".join(outputs)
def __init__(self,
url,
database,
replica_uid=None,
full_commit=True,
session=None):
"""Create a new Couch data container."""
self._url = url
self._full_commit = full_commit
self._session = session
self._server = Server(url=self._url,
full_commit=self._full_commit,
session=self._session)
self._dbname = database
# this will ensure that transaction and sync logs exist and are
# up-to-date.
try:
self._database = self._server[database]
except ResourceNotFound:
self._server.create(database)
self._database = self._server[database]
super(CouchDatabase, self).__init__(replica_uid=replica_uid,
document_factory=LeapDocument)
def __init__(self):
try:
self.__server = Server()
except:
print 'can not connect to Couchdb:%s'%(settings.c['db_url'])
self.__db = {}
self.__db_name = settings.c['db_name']
DEBUG.p(self.__db_name.items())
for (k, v) in self.__db_name.items():
try:
self.__db[v] = self.__server.create(v)
except:
self.__db[v] = self.__server[v]
def couch_server(url):
"""
Provide a connection to a couch server and cleanup after use.
For database creation and deletion we use an ephemeral connection to the
couch server. That connection has to be properly closed, so we provide it
as a context manager.
:param url: The URL of the Couch server.
:type url: str
"""
session = Session(timeout=COUCH_TIMEOUT)
server = Server(url=url, full_commit=False, session=session)
yield server
def copy_couch_database_for_test(test, db):
port = str(test.wrapper.port)
couch_url = "http://localhost:" + port
new_dbname = db._replica_uid + "_copy"
new_db = couch.CouchDatabase.open_database(
urljoin(couch_url, new_dbname), create=True, replica_uid=db._replica_uid or "test"
)
# copy all docs
session = couch.Session()
old_couch_db = Server(couch_url, session=session)[db._replica_uid]
new_couch_db = Server(couch_url, session=session)[new_dbname]
for doc_id in old_couch_db:
doc = old_couch_db.get(doc_id)
# bypass u1db_config document
if doc_id == "u1db_config":
pass
# copy design docs
elif doc_id.startswith("_design"):
del doc["_rev"]
new_couch_db.save(doc)
# copy u1db docs
elif "u1db_rev" in doc:
new_doc = {"_id": doc["_id"], "u1db_transactions": doc["u1db_transactions"], "u1db_rev": doc["u1db_rev"]}
attachments = []
if "u1db_conflicts" in doc:
new_doc["u1db_conflicts"] = doc["u1db_conflicts"]
for c_rev in doc["u1db_conflicts"]:
attachments.append("u1db_conflict_%s" % c_rev)
new_couch_db.save(new_doc)
# save conflict data
attachments.append("u1db_content")
for att_name in attachments:
att = old_couch_db.get_attachment(doc_id, att_name)
if att is not None:
new_couch_db.put_attachment(new_doc, att, filename=att_name)
# cleanup connections to prevent file descriptor leaking
return new_db
class Connection(object):
"""
Connect to CouchDb according to params in the settings.py file
and store that internally.
Access is made with this class cause it's a singleton.
"""
_inst = None
def __new__(cls, address=None, port=None, db_name=None, *args, **kwargs):
"""
Ensure we have only one instance for the connection.
"""
if not cls._inst:
cls._inst = object.__new__(cls, *args, **kwargs)
cls._inst.connect(address, port, db_name)
return cls._inst
def close(self):
Connection._inst = None
def connect(self, address=None, port=None, db_name=None):
"""
Connect to the CouchDB server and work on the databse mentioned in
the settngs.
"""
address = address or settings.SERVER_ADDRESS
self.url = "%s:%s/" % (address.rstrip("/"), port or settings.SERVER_PORT)
self.server = Server(self.url)
db_name = db_name or settings.DATABASE_NAME
try:
self.db = self.server.create(db_name)
except PreconditionFailed:
self.db = self.server[db_name]
def __unicode__(self):
return u"Connected on %s - working on %s" % (self.url, self.db.name)
def __str__(self):
return unicode(self)
def __repr__(self):
return repr(self.db)
class CouchDBBackend(object):
def db(self):
return Database(self.server[self.db_name], self._generate_id)
def __init__(self, config):
settings = config.registry.settings
self.config = config
self.server = Server(settings['backend.db_host'])
self.db_name = os.environ.get('DB_NAME', settings['backend.db_name'])
# model id generator
generator = config.maybe_dotted(settings['daybed.id_generator'])
self._generate_id = generator(config)
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (
settings['backend.db_host'], e))
self.sync_views()
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.info('Creating and using db "%s"' % self.db_name)
except PreconditionFailed:
logger.info('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.server[self.db_name], docs)
def fire_em_all(skip, limit):
collection = 'pantip'
svr = Server()
if collection not in svr:
src = svr.create(collection)
else:
src = svr[collection]
# Iterate through the collection and fire
n = 0
n_processed = 0
for _id in src:
n += 1
rec = src.get(_id)
if n < skip: continue
if n_processed > limit:
print(colored('Out of ammo!', 'red'))
return
# Fire a single request
print(colored('Firing #{0}'.format(_id)))
fire_request(rec)
n_processed += 1
def connect(self, address=None, port=None, db_name=None):
"""
Connect to the CouchDB server and work on the databse mentioned in
the settngs.
"""
address = address or settings.SERVER_ADDRESS
self.url = "%s:%s/" % (address.rstrip("/"), port or settings.SERVER_PORT)
self.server = Server(self.url)
db_name = db_name or settings.DATABASE_NAME
try:
self.db = self.server.create(db_name)
except PreconditionFailed:
self.db = self.server[db_name]
def open_database(cls, url, create):
"""Open a U1DB database using CouchDB as backend."""
# get database from url
m = re.match('(^https?://[^/]+)/(.+)$', url)
if not m:
raise InvalidURLError
url = m.group(1)
dbname = m.group(2)
server = Server(url=url)
try:
server[dbname]
except ResourceNotFound:
if not create:
raise DatabaseDoesNotExist()
return cls(url, dbname)
class TestCouchDBBackend(BackendTestBase, TestCase):
def setUp(self):
self.server = Server('http://localhost:5984')
self.db_name = 'test_%s' % uuid4()
self.server.create(self.db_name)
db = self.server[self.db_name]
ViewDefinition.sync_many(db, couchdb_views)
self.db = CouchDBDatabase(db, lambda: six.text_type(uuid4()))
super(TestCouchDBBackend, self).setUp()
def tearDown(self):
del self.server[self.db_name]
def test_server_unreachable(self):
config = mock.Mock()
config.registry = mock.Mock()
config.registry.settings = defaultdict(str)
config.registry.settings['backend.db_host'] = 'http://unreachable/'
config.registry.settings['backend.db_name'] = 'daybed'
with self.assertRaises(CouchDBBackendConnectionError):
CouchDBBackend(config)
def __init__(self, url=DEFAULT_STORE_ADMIN_URI, **client_opts):
self.logger = logging.getLogger('ers-store')
self._server = Server(url=url, **client_opts)
self.db_names = {'public': ERS_PUBLIC_DB,
'private': ERS_PRIVATE_DB,
'cache': ERS_CACHE_DB,}
# Add aggregate functions
# for method_name in ('docs_by_entity', 'by_property', 'by_property_value'):
# self.add_aggregate(method_name)
# Check the status of the databases
self._ers_dbs = {}
self._repair()
def fire_em_all(skip,limit):
collection = 'pantip'
svr = Server()
if collection not in svr:
src = svr.create(collection)
else:
src = svr[collection]
# Iterate through the collection and fire
n = 0
n_processed = 0
for _id in src:
n += 1
rec = src.get(_id)
if n<skip: continue
if n_processed>limit:
print(colored('Out of ammo!','red'))
return
# Fire a single request
print(colored('Firing #{0}'.format(_id)))
fire_request(rec)
n_processed += 1
def _verify_token_in_couch(self, uuid, token):
"""
Query couchdb to decide if C{token} is valid for C{uuid}.
@param uuid: The user uuid.
@type uuid: str
@param token: The token.
@type token: str
@raise InvalidAuthTokenError: Raised when token received from user is
either missing in the tokens db or is
invalid.
"""
server = Server(url=self._app.state.couch_url)
# the tokens db rotates every 30 days, and the current db name is
# "tokens_NNN", where NNN is the number of seconds since epoch divided
# by the rotate period in seconds. When rotating, old and new tokens
# db coexist during a certain window of time and valid tokens are
# replicated from the old db to the new one. See:
# https://leap.se/code/issues/6785
dbname = self.TOKENS_DB_PREFIX + \
str(int(time.time() / self.TOKENS_DB_EXPIRE))
db = server[dbname]
# lookup key is a hash of the token to prevent timing attacks.
token = db.get(sha512(token).hexdigest())
if token is None:
raise InvalidAuthTokenError()
# we compare uuid hashes to avoid possible timing attacks that
# might exploit python's builtin comparison operator behaviour,
# which fails immediatelly when non-matching bytes are found.
couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest()
req_uuid_hash = sha512(uuid).digest()
if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \
or couch_uuid_hash != req_uuid_hash:
raise InvalidAuthTokenError()
return True
def __init__(self, basedir, url) :
self.basedir = basedir
self.language = "javascript"
# parse out the design document id, etc
regex = re.compile( r'(.*)/([^\s/]*)/(_design/[^\s/]*)/?$' )
match = regex.match( url )
if match is None:
_usage("no match on url of design document")
(self.couchurl, self.dbname, self.designdoc) = [ match.group(i) for i in [1,2,3] ]
self.server = Server(self.couchurl)
if self.dbname not in self.server:
_usage( "dbname %s doesnt exist in database %s" % (self.dbname, self.couchurl) )
self.db = self.server[self.dbname]
self.designDocFound = False
if self.designdoc in self.db:
self.ddoc = self.db[ self.designdoc ]
self.designDocFound = True
else :
self.ddoc = { "language" : "javascript" }
def __init__(self, app):
manager = CouchDBManager()
# ...add document types and view definitions...
manager.setup(app)
server = Server()
self.db = server['active_server_cp']
def _get_server(url):
resource = Resource(url.geturl(),
Session(retry_delays=[1, 2, 4, 8], timeout=10))
return Server(url=resource)
def __init__(self):
server = Server(config.get('couchdb', 'host'))
self.conn = server[config.get('couchdb', 'db')]
def clear(self):
s = Server()
del s['python-tests']
db = s.create('python-tests')
class Database():
"""The Database class incorperates functions you wish to ask
the database"""
jsdir = CONFIG.get('couchdb', 'javascript_directory')
__DESIGN_VIEWS_PATHS_MAP = \
file(jsdir + '/design_views_paths_map.js').read()
__DESIGN_VIEWS_SHASUMS_MAP = \
file(jsdir + '/design_views_shasums_map.js').read()
__DESIGN_VIEWS_FORMATS_MAP = \
file(jsdir + '/design_views_formats_map.js').read()
__DESIGN_VIEWS_FORMATS_REDUCE = '_sum'
__DESIGN_VIEWS_SOUND_MAP = \
file(jsdir + '/design_views_sound_map.js').read()
__DESIGN_VIEWS_VIDEO_MAP = \
file(jsdir + '/design_views_sound_map.js').read()
__DESIGN_FULLTEXT_ARTIST_INDEX = \
file(jsdir + '/design_fulltext_artist_index.js').read()
__DESIGN_FULLTEXT_EVERYTHING_INDEX = \
file(jsdir + '/design_fulltext_artist_index.js').read()
def create_views(self):
"""creates views and saves them to the database"""
LOG.info('creating views')
views = {
'_id': '_design/views',
'language': 'javascript',
'views': {
'shasums': {
'map': self.__DESIGN_VIEWS_SHASUMS_MAP
},
'paths': {
'map': self.__DESIGN_VIEWS_PATHS_MAP
},
'formats': {
'map': self.__DESIGN_VIEWS_FORMATS_MAP,
'reduce': self.__DESIGN_VIEWS_FORMATS_REDUCE
},
'sound': {
'map': self.__DESIGN_VIEWS_SOUND_MAP
},
'video': {
'map': self.__DESIGN_VIEWS_VIDEO_MAP
}
},
'fulltext': {
'artist': {
'index': self.__DESIGN_FULLTEXT_ARTIST_INDEX
},
'everything': {
'index': self.__DESIGN_FULLTEXT_EVERYTHING_INDEX
}
}
}
self.database.create(views)
def __init__(self):
"""Initialises a new connection to couch and
creates a new mis databse if nonexistent"""
LOG.info('initialising database')
database_name = CONFIG.get('couchdb', 'database')
host = CONFIG.get('couchdb', 'hostname')
port = CONFIG.get('couchdb', 'port')
database_uri = 'http://' + host + ':' + port + '/'
self.server = Server(database_uri)
try:
# This statement appears to do nothing, but is used to
# make sure the database is reachable.
self.server.version
except AttributeError as error:
if not test_tcp_connection(host, int(port)):
LOG.critical("couchdb cannot be reached at " + database_uri)
exit(1)
else:
LOG.error('unknown AttributeError thrown')
raise error
try:
LOG.debug('opening database')
self.database = self.server[database_name]
except (ResourceNotFound):
LOG.info('creating database')
# The database didn't exist. Lets create it.
self.database = self.server.create(database_name)
self.create_views()
def iterate_all_files(self):
"""With a big database, this is probably a bad idea. this
iterates through every single document."""
for entry in self.database:
yield (entry)
def get_document(self, shasum):
"""extracts a (full) document from the database using the
shasum as an identifier"""
assert shasum is not None
assert shasum != ''
LOG.debug('getting document')
result = None
try:
result = self.database[shasum]
# make sure it actually exists
except (ResourceNotFound) as error:
LOG.error("don't have that document, doesn't exist:" + str(error))
return result
def add_userdata(self, shasum, data):
"""Adds userdata to the database shasum"""
LOG.debug('add userdata')
shasum = unicode(shasum)
user = getuser()
node = platform_node()
user_key = node + ':' + user
userdata = {}
if not self.file_exists(shasum):
LOG.error('trying to add userdata to nonexistent file' + shasum)
return None
entry = self.database[shasum]
userdatalist = {}
if 'userdata' in entry:
userdatalist = entry['userdata']
if user_key in userdatalist:
userdata = userdatalist[user_key]
userdata.update(data)
userdatalist[user_key] = userdata
entry['userdata'] = userdatalist
self.database[shasum] = entry
def add_data(self, shasum, name, data):
"""adds data to a record"""
assert shasum is not None
shasum = unicode(shasum)
name = unicode(name)
LOG.debug('adding data')
if self.file_exists(shasum):
mis_file = self.database[shasum]
if name not in mis_file or mis_file[name] != data:
LOG.info(shasum + " info " + name + " has changed, updating")
mis_file[name] = data
self.database[shasum] = mis_file
else: # create when nonexistent
LOG.info(shasum + " info " + name + " added")
entry = {'_id': shasum, name: data}
self.database.create(entry)
def add_path(self, shasum, node, path):
"""Adds a path to the database"""
assert shasum is not None
shasum = unicode(shasum)
node = unicode(node)
path = unicode(path)
LOG.debug('adding path ' + path + " to " + shasum)
path_info = {'node': node, 'path': path}
if self.file_exists(shasum):
mis_file = self.database[shasum]
mis_file['paths'].append(path_info)
self.database[mis_file['_id']] = mis_file
else: # create when nonexistent
entry = {'_id': shasum, 'paths': [path_info]}
self.database.create(entry)
def file_exists(self, shasum):
"""Checks if a file (shasum) exists in the database, and
returns the entry when found"""
assert shasum is not None
shasum = unicode(shasum)
result = None
LOG.debug('checking if file exists: ' + shasum)
try:
# Note: the following line triggers the
# ResourceNotFound if the sha is not known, this is
# the way we catch whether it exists.
self.database[shasum] # pylint: disable-msg=W0104
result = shasum
except ResourceNotFound:
LOG.debug('trying to find nonexistent entry ' + shasum)
return result
def path_exists(self, path, node=None):
"""Checks whether a certain path exists and returns True
or False"""
if node is None:
node = platform_node
node = unicode(node)
path = unicode(path)
LOG.debug('path exists: ' + node + ':' + path)
result = None
key = [node, path]
results = self.database.view('views/paths', key=key)
if (len(results) > 0):
result = results.rows[0]['value']
return result
