class CouchDBBackend(object):
@property
def db(self):
return self.server[self.db_name]
def __init__(self, config):
settings = config.registry.settings
self.config = config
self.server = Server(settings['backend.db_host'])
self.db_name = os.environ.get('DB_NAME', settings['backend.db_name'])
self.create_db_if_not_exist()
self.sync_views()
self.config.add_subscriber(self.add_db_to_request, NewRequest)
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.debug('Creating and using db "%s"' % self.db_name)
except PreconditionFailed:
logger.debug('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.db, docs)
def add_db_to_request(self, event):
event.request.db = Database(self.db)
def _open_couch_db(dbname):
server = Server(url=self._couch_url)
try:
server[dbname]
except ResourceNotFound:
server.create(dbname)
return server[dbname]
class CouchDBBackend(object):
@classmethod
def load_from_config(cls, config):
settings = config.registry.settings
return CouchDBBackend(
host=settings['backend.db_host'],
db_name=os.environ.get('DB_NAME', settings['backend.db_name']),
)
def __init__(self, host, db_name, id_generator):
self.server = Server(host)
self.db_name = db_name
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (host, e))
self._db = self.server[self.db_name]
self.sync_views()
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.info('Creating and using db "%s"' % self.db_name)
except PreconditionFailed:
logger.info('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.server[self.db_name], docs)
def __get_raw_user_token(self, user_id):
try:
return views.usertokens(self._db, key=user_id).rows[0].value
except IndexError:
raise UserIdNotFound(user_id)
def get_user_token(self, user_id):
"""Returns the information associated with a user token"""
usertoken = dict(**self.__get_raw_user_token(user_id))
return usertoken['token']
def add_token(self, user_id, token):
# Check that the token doesn't already exist.
try:
self.__get_raw_user_token(user_id)
raise UserIdAlreadyExist(user_id)
except UserIdNotFound:
pass
doc = dict(token=token, user_id=user_id, type='usertoken')
self._db.save(doc)
def connector(collection):
# Make a server connection
svr = Server()
if collection not in svr:
return svr.create(collection)
else:
return svr[collection]
def couch_connect():
server = Server()
try:
db = server.create('feedme')
except:
db = server['feedme']
return db
def connector(collection):
# Make a server connection
svr = Server()
if collection not in svr:
return svr.create(collection)
else:
return svr[collection]
def get_db(self, db_uri, db_name):
server = Server(db_uri)
try:
db = server[db_name]
except ResourceNotFound:
db = server.create(db_name)
return db
def initDatabase():
server = Server('http://localhost:5984')
del server['lazycrawler']
try:
db = server.create('lazycrawler')
except Exception:
db = server['lazycrawler']
return db
def Create_Couchdb_Instance(dtbs_name):
server = Server("http://127.0.0.1:5984")
try:
db = server.create(dtbs_name)
except Exception:
db = server[dtbs_name]
return db
def saveDocument(self):
server = Server()
server.resource.credentials = ('secret', 'aedca29aed23e103f27f8e29dd6ec473')
# create database
db = server.create("new.job")
# associate NewJob to the db
NewJob.set_db(db)
self.save()
class DB(object):
server = None
sensors = None
readings = None
devices = None
config = {
# TODO Server credentials are currently not used
'server': {
'host': 'localhost',
'port': 5984
},
'schema': {
'sensors_db': 'sensors',
'readings_db': 'readings',
'devices_db': 'devices'
}
}
def __init__(self, config_path=None):
self.server = Server()
if config_path:
self.config = yaml.load(open(config_path, 'r').read())
if not all(db_name in self.server
for db_name in self.config['schema'].values()):
self.setup()
self.connect()
def connect(self):
for db_name, attr_name in [['sensors_db', 'sensors'],
['readings_db', 'readings'],
['devices_db', 'devices']]:
setattr(self, attr_name,
self.server[self.config['schema'][db_name]])
def setup(self):
for db_name in ['sensors_db', 'readings_db', 'devices_db']:
if self.config['schema'][db_name] not in self.server:
self.server.create(self.config['schema'][db_name])
def init_boards():
"""
:rtype : object
"""
server = Server()
try:
db = server.create('boards')
except Exception:
db = server['boards']
return db
def init_boards():
"""
:rtype : object
"""
server = Server()
try:
db = server.create("boards")
except Exception:
db = server["boards"]
return db
def __init__(self, db_uri):
local.application = self
server = Server(db_uri)
try:
db = server.create("urls")
except Exception:
db = server["urls"]
self.dispatch = SharedDataMiddleware(self.dispatch, {"/static": STATIC_PATH})
URL.db = db
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
authn_policy = AuthTktAuthenticationPolicy(
'sosecret', hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings, root_factory=RootFactory)
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
dbserver = Server(url = settings['couchdb.url'])
if settings['couchdb.dbname'] not in dbserver:
dbserver.create(settings['couchdb.dbname'])
config.registry.settings['couchdb.server'] = dbserver
config.add_renderer(".html", "pyramid.mako_templating.renderer_factory")
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_route('home', '/')
config.add_route('item', '/item/*traverse', factory=Container)
config.add_route('tag', '/tag/*traverse', factory=Tags)
config.scan()
return config.make_wsgi_app()
def __init__(self, db_uri):
local.application = self
server = Server(db_uri)
try:
db = server.create("urls")
except Exception:
db = server["urls"]
self.dispatch = SharedDataMiddleware(self.dispatch,
{"/static": STATIC_PATH})
URL.db = db
def __init__(self):
from couchdb.client import Server
couch = Server('http://localhost:5984')
print (couch)
try:
self.db = couch.create('run-layer')
except:
self.db = couch['run-layer']
print (self.db)
def __init__(self, db_uri):
local.application = self
server = Server(db_uri)
try:
db = server.create('urls')
except:
db = server['urls']
self.dispatch = SharedDataMiddleware(self.dispatch, {
'/static': STATIC_PATH
})
URL.db = db
class Connection(object):
"""
Connect to CouchDb according to params in the settings.py file
and store that internally.
Access is made with this class cause it's a singleton.
"""
_inst = None
def __new__(cls, address=None, port=None, db_name=None, *args, **kwargs):
"""
Ensure we have only one instance for the connection.
"""
if not cls._inst:
cls._inst = object.__new__(cls, *args, **kwargs)
cls._inst.connect(address, port, db_name)
return cls._inst
def close(self):
Connection._inst = None
def connect(self, address=None, port=None, db_name=None):
"""
Connect to the CouchDB server and work on the databse mentioned in
the settngs.
"""
address = address or settings.SERVER_ADDRESS
self.url = "%s:%s/" % (address.rstrip("/"), port or settings.SERVER_PORT)
self.server = Server(self.url)
db_name = db_name or settings.DATABASE_NAME
try:
self.db = self.server.create(db_name)
except PreconditionFailed:
self.db = self.server[db_name]
def __unicode__(self):
return u"Connected on %s - working on %s" % (self.url, self.db.name)
def __str__(self):
return unicode(self)
def __repr__(self):
return repr(self.db)
class CouchDBBackend(object):
def db(self):
return Database(self.server[self.db_name], self._generate_id)
def __init__(self, config):
settings = config.registry.settings
self.config = config
self.server = Server(settings['backend.db_host'])
self.db_name = os.environ.get('DB_NAME', settings['backend.db_name'])
# model id generator
generator = config.maybe_dotted(settings['daybed.id_generator'])
self._generate_id = generator(config)
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (
settings['backend.db_host'], e))
self.sync_views()
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.info('Creating and using db "%s"' % self.db_name)
except PreconditionFailed:
logger.info('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.server[self.db_name], docs)
def main(global_config, **settings):
auth_token = MozillaTokenLibAuthenticationPolicy(secret='what_makes_so_secret', hashmod=hashlib.sha256, callback=groupfinder, timeout=86400)
auth_permission = ACLAuthorizationPolicy()
config = Configurator(settings=settings,
root_factory=RootFactory)
config.set_authentication_policy(auth_token)
config.set_authorization_policy(auth_permission)
db_server = Server(url=settings['CouchDB.url'])
es_server = ElasticSearch(settings['ES.url'])
upyun_server = UpYun(settings['upyun.space'], settings['upyun.username'], settings['upyun.password'], timeout=30, endpoint=ED_AUTO)
if settings['CouchDB.db_name'] not in db_server:
db_server.create(settings['CouchDB.db_name'])
config.registry.settings['CouchDB.server'] = db_server
config.registry.settings['ES.server'] = es_server
config.registry.settings['UpYun.server'] = upyun_server
config.add_static_view('static', 'static', cache_max_age=3600)
config.include("cornice")
config.add_subscriber(add_couchdb_to_request, NewRequest)
config.add_route('home', '/')
config.scan("api.views.restapi")
config.scan("api.views.display")
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
return config.make_wsgi_app()
class TestCouchDBBackend(BackendTestBase, TestCase):
def setUp(self):
self.server = Server('http://localhost:5984')
self.db_name = 'test_%s' % uuid4()
self.server.create(self.db_name)
db = self.server[self.db_name]
ViewDefinition.sync_many(db, couchdb_views)
self.db = CouchDBDatabase(db, lambda: six.text_type(uuid4()))
super(TestCouchDBBackend, self).setUp()
def tearDown(self):
del self.server[self.db_name]
def test_server_unreachable(self):
config = mock.Mock()
config.registry = mock.Mock()
config.registry.settings = defaultdict(str)
config.registry.settings['backend.db_host'] = 'http://unreachable/'
config.registry.settings['backend.db_name'] = 'daybed'
with self.assertRaises(CouchDBBackendConnectionError):
CouchDBBackend(config)
class CouchProvider(object):
def __init__(self, uri=None, database=None):
print '%s|%s' % (uri, database)
self.server = Server(uri)
try:
# python-couchdb will create the database or raise an error if it
# already exists
self.database = self.server.create(database)
except:
self.database = self.server[database]
def fetch(self, uuid, type=None):
doc = self.database[uuid]
if doc['type'] == type:
return doc
else:
raise DocumentNotFoundException(uuid)
def fire_em_all(skip, limit):
collection = 'pantip'
svr = Server()
if collection not in svr:
src = svr.create(collection)
else:
src = svr[collection]
# Iterate through the collection and fire
n = 0
n_processed = 0
for _id in src:
n += 1
rec = src.get(_id)
if n < skip: continue
if n_processed > limit:
print(colored('Out of ammo!', 'red'))
return
# Fire a single request
print(colored('Firing #{0}'.format(_id)))
fire_request(rec)
n_processed += 1
def fire_em_all(skip,limit):
collection = 'pantip'
svr = Server()
if collection not in svr:
src = svr.create(collection)
else:
src = svr[collection]
# Iterate through the collection and fire
n = 0
n_processed = 0
for _id in src:
n += 1
rec = src.get(_id)
if n<skip: continue
if n_processed>limit:
print(colored('Out of ammo!','red'))
return
# Fire a single request
print(colored('Firing #{0}'.format(_id)))
fire_request(rec)
n_processed += 1
class CouchDBBackend(object):
@classmethod
def load_from_config(cls, config):
settings = config.registry.settings
generator = config.maybe_dotted(settings['daybed.id_generator'])
return CouchDBBackend(host=settings['backend.db_host'],
db_name=os.environ.get(
'DB_NAME', settings['backend.db_name']),
id_generator=generator(config))
def __init__(self, host, db_name, id_generator):
self.server = Server(host)
self.db_name = db_name
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (host, e))
self._db = self.server[self.db_name]
self.sync_views()
self._generate_id = id_generator
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.info('Creating and using db "%s"' % self.db_name)
except (PreconditionFailed, Unauthorized):
logger.info('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.server[self.db_name], docs)
def get_models(self, principals):
principals = list(set(principals))
models = {}
for result in views.models(self._db, keys=principals).rows:
doc = result.value
_id = doc["_id"]
models[_id] = {
"id": _id,
"title": doc["definition"].get("title", _id),
"description": doc["definition"].get("description", "")
}
return list(models.values())
def __get_raw_model(self, model_id):
try:
doc = views.model_definitions(self._db, key=model_id).rows[0]
return doc.value
except IndexError:
raise backend_exceptions.ModelNotFound(model_id)
def get_model_definition(self, model_id):
return self.__get_raw_model(model_id)['definition']
def __get_raw_records(self, model_id):
# Make sure the model exists.
self.__get_raw_model(model_id)
return views.records(self._db, key=model_id).rows
def get_records(self, model_id, raw_records=None):
return [
r["record"]
for r in self.get_records_with_authors(model_id, raw_records)
]
def get_records_with_authors(self, model_id, raw_records=None):
if raw_records is None:
raw_records = self.__get_raw_records(model_id)
records = []
for item in raw_records:
item.value['record']['id'] = item.value['_id'].split('-')[1]
records.append({
"authors": item.value['authors'],
"record": item.value['record']
})
return records
def __get_raw_record(self, model_id, record_id):
key = u'-'.join((model_id, record_id))
try:
return views.records_all(self._db, key=key).rows[0].value
except IndexError:
raise backend_exceptions.RecordNotFound(u'(%s, %s)' %
(model_id, record_id))
def _model_exists(self, model_id):
try:
self.__get_raw_model(model_id)
return True
except backend_exceptions.ModelNotFound:
return False
def _record_exists(self, model_id, record_id):
try:
self.__get_raw_record(model_id, record_id)
return True
except backend_exceptions.RecordNotFound:
return False
def get_record(self, model_id, record_id):
doc = self.__get_raw_record(model_id, record_id)
record = doc['record']
record['id'] = record_id
return record
def get_record_authors(self, model_id, record_id):
doc = self.__get_raw_record(model_id, record_id)
return doc['authors']
def put_model(self, definition, permissions, model_id=None):
if model_id is None:
model_id = self._generate_id(key_exist=self._model_exists)
try:
doc = self.__get_raw_model(model_id)
except backend_exceptions.ModelNotFound:
doc = {'_id': model_id, 'type': 'definition'}
doc['definition'] = definition
doc['permissions'] = permissions
definition_id, _ = self._db.save(doc)
return definition_id
def put_record(self, model_id, record, authors, record_id=None):
doc = {
'type': 'record',
'authors': authors,
'model_id': model_id,
'record': record
}
if record_id is not None:
try:
old_doc = self.__get_raw_record(model_id, record_id)
except backend_exceptions.RecordNotFound:
doc['_id'] = '-'.join((model_id, record_id))
else:
authors = list(set(authors) | set(old_doc['authors']))
doc['authors'] = authors
old_doc.update(doc)
doc = old_doc
else:
key_exist = functools.partial(self._record_exists, model_id)
record_id = self._generate_id(key_exist=key_exist)
doc['_id'] = '-'.join((model_id, record_id))
self._db.save(doc)
return record_id
def delete_record(self, model_id, record_id):
doc = self.__get_raw_record(model_id, record_id)
if doc:
self._db.delete(doc)
return doc
def delete_records(self, model_id):
results = self.__get_raw_records(model_id)
for result in results:
self._db.delete(result.value)
return self.get_records(model_id, raw_records=results)
def delete_model(self, model_id):
"""DELETE ALL THE THINGS"""
# Delete the associated data if any.
records = self.delete_records(model_id)
try:
doc = views.model_definitions(self._db, key=model_id).rows[0].value
except IndexError:
raise backend_exceptions.ModelNotFound(model_id)
# Delete the model definition if it exists.
self._db.delete(doc)
return {
"definition": doc["definition"],
"permissions": doc["permissions"],
"records": records
}
def __get_raw_token(self, credentials_id):
try:
return views.tokens(self._db, key=credentials_id).rows[0].value
except IndexError:
raise backend_exceptions.CredentialsNotFound(credentials_id)
def get_token(self, credentials_id):
"""Returns the information associated with a credentials_id"""
credentials = dict(**self.__get_raw_token(credentials_id))
return credentials['token']
def get_credentials_key(self, credentials_id):
"""Retrieves a token by its id"""
credentials = dict(**self.__get_raw_token(credentials_id))
return credentials['credentials']['key']
def store_credentials(self, token, credentials):
# Check that the token doesn't already exist.
assert 'id' in credentials and 'key' in credentials
try:
self.__get_raw_token(credentials['id'])
raise backend_exceptions.CredentialsAlreadyExist(credentials['id'])
except backend_exceptions.CredentialsNotFound:
pass
doc = dict(token=token, credentials=credentials, type='token')
self._db.save(doc)
def get_model_permissions(self, model_id):
doc = self.__get_raw_model(model_id)
return doc['permissions']
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
##
import couchdb
from couchdb.client import Server
#This connects to the couchdb server
server = Server('http://127.0.0.1:9999/')
print 'Connected Established'
try:
#This creates a database in couchdb called gamedb
#Database names must be lowercase, but table/document names do not need to be
db = server.create('gamedb')
print 'database created'
except:
#In the case the database already exists we will delete it and start a new one
del server['gamedb']
db = server.create('gamedb')
print 'database deleted and created'
#Creating JSON object data, db is your database, while 'character#' is the table name you will be assigning.
#The dictionary holds a variable you are adding and it's value. IE: uid is the variable and 1 is the value.
#Python will automatically change this to a json object for you when it writes to the database
db['character1'] = dict(uid=1, level=12, name='py')
db['character2'] = dict(uid=2, level=25, name='thon')
db['character3'] = dict(uid=3, level=30, name='three')
#this is how you build a query, the query is exported javascript
class Database():
"""The Database class incorperates functions you wish to ask
the database"""
jsdir = CONFIG.get('couchdb', 'javascript_directory')
__DESIGN_VIEWS_PATHS_MAP = \
file(jsdir + '/design_views_paths_map.js').read()
__DESIGN_VIEWS_SHASUMS_MAP = \
file(jsdir + '/design_views_shasums_map.js').read()
__DESIGN_VIEWS_FORMATS_MAP = \
file(jsdir + '/design_views_formats_map.js').read()
__DESIGN_VIEWS_FORMATS_REDUCE = '_sum'
__DESIGN_VIEWS_SOUND_MAP = \
file(jsdir + '/design_views_sound_map.js').read()
__DESIGN_VIEWS_VIDEO_MAP = \
file(jsdir + '/design_views_sound_map.js').read()
__DESIGN_FULLTEXT_ARTIST_INDEX = \
file(jsdir + '/design_fulltext_artist_index.js').read()
__DESIGN_FULLTEXT_EVERYTHING_INDEX = \
file(jsdir + '/design_fulltext_artist_index.js').read()
def create_views(self):
"""creates views and saves them to the database"""
LOG.info('creating views')
views = {
'_id': '_design/views',
'language': 'javascript',
'views': {
'shasums': {
'map': self.__DESIGN_VIEWS_SHASUMS_MAP
},
'paths': {
'map': self.__DESIGN_VIEWS_PATHS_MAP
},
'formats': {
'map': self.__DESIGN_VIEWS_FORMATS_MAP,
'reduce': self.__DESIGN_VIEWS_FORMATS_REDUCE
},
'sound': {
'map': self.__DESIGN_VIEWS_SOUND_MAP
},
'video': {
'map': self.__DESIGN_VIEWS_VIDEO_MAP
}
},
'fulltext': {
'artist': {
'index': self.__DESIGN_FULLTEXT_ARTIST_INDEX
},
'everything': {
'index': self.__DESIGN_FULLTEXT_EVERYTHING_INDEX
}
}
}
self.database.create(views)
def __init__(self):
"""Initialises a new connection to couch and
creates a new mis databse if nonexistent"""
LOG.info('initialising database')
database_name = CONFIG.get('couchdb', 'database')
host = CONFIG.get('couchdb', 'hostname')
port = CONFIG.get('couchdb', 'port')
database_uri = 'http://' + host + ':' + port + '/'
self.server = Server(database_uri)
try:
# This statement appears to do nothing, but is used to
# make sure the database is reachable.
self.server.version
except AttributeError as error:
if not test_tcp_connection(host, int(port)):
LOG.critical("couchdb cannot be reached at " + database_uri)
exit(1)
else:
LOG.error('unknown AttributeError thrown')
raise error
try:
LOG.debug('opening database')
self.database = self.server[database_name]
except (ResourceNotFound):
LOG.info('creating database')
# The database didn't exist. Lets create it.
self.database = self.server.create(database_name)
self.create_views()
def iterate_all_files(self):
"""With a big database, this is probably a bad idea. this
iterates through every single document."""
for entry in self.database:
yield (entry)
def get_document(self, shasum):
"""extracts a (full) document from the database using the
shasum as an identifier"""
assert shasum is not None
assert shasum != ''
LOG.debug('getting document')
result = None
try:
result = self.database[shasum]
# make sure it actually exists
except (ResourceNotFound) as error:
LOG.error("don't have that document, doesn't exist:" + str(error))
return result
def add_userdata(self, shasum, data):
"""Adds userdata to the database shasum"""
LOG.debug('add userdata')
shasum = unicode(shasum)
user = getuser()
node = platform_node()
user_key = node + ':' + user
userdata = {}
if not self.file_exists(shasum):
LOG.error('trying to add userdata to nonexistent file' + shasum)
return None
entry = self.database[shasum]
userdatalist = {}
if 'userdata' in entry:
userdatalist = entry['userdata']
if user_key in userdatalist:
userdata = userdatalist[user_key]
userdata.update(data)
userdatalist[user_key] = userdata
entry['userdata'] = userdatalist
self.database[shasum] = entry
def add_data(self, shasum, name, data):
"""adds data to a record"""
assert shasum is not None
shasum = unicode(shasum)
name = unicode(name)
LOG.debug('adding data')
if self.file_exists(shasum):
mis_file = self.database[shasum]
if name not in mis_file or mis_file[name] != data:
LOG.info(shasum + " info " + name + " has changed, updating")
mis_file[name] = data
self.database[shasum] = mis_file
else: # create when nonexistent
LOG.info(shasum + " info " + name + " added")
entry = {'_id': shasum, name: data}
self.database.create(entry)
def add_path(self, shasum, node, path):
"""Adds a path to the database"""
assert shasum is not None
shasum = unicode(shasum)
node = unicode(node)
path = unicode(path)
LOG.debug('adding path ' + path + " to " + shasum)
path_info = {'node': node, 'path': path}
if self.file_exists(shasum):
mis_file = self.database[shasum]
mis_file['paths'].append(path_info)
self.database[mis_file['_id']] = mis_file
else: # create when nonexistent
entry = {'_id': shasum, 'paths': [path_info]}
self.database.create(entry)
def file_exists(self, shasum):
"""Checks if a file (shasum) exists in the database, and
returns the entry when found"""
assert shasum is not None
shasum = unicode(shasum)
result = None
LOG.debug('checking if file exists: ' + shasum)
try:
# Note: the following line triggers the
# ResourceNotFound if the sha is not known, this is
# the way we catch whether it exists.
self.database[shasum] # pylint: disable-msg=W0104
result = shasum
except ResourceNotFound:
LOG.debug('trying to find nonexistent entry ' + shasum)
return result
def path_exists(self, path, node=None):
"""Checks whether a certain path exists and returns True
or False"""
if node is None:
node = platform_node
node = unicode(node)
path = unicode(path)
LOG.debug('path exists: ' + node + ':' + path)
result = None
key = [node, path]
results = self.database.view('views/paths', key=key)
if (len(results) > 0):
result = results.rows[0]['value']
return result
#!/usr/bin/python
import urllib2
import uuid
import feedparser
from couchdb.client import Server
server = Server()
try:
db = server.create('cuke')
except:
db = server['cuke']
feeds = ['http://freemusicarchive.org/interesting.atom']
# for more music, include the 'recent' feed
#feeds.append('http://freemusicarchive.org/recent.atom')
for f in feeds:
feed = feedparser.parse(f)
for track in feed.entries:
track_doc = {}
track_doc['type'] = 'track'
if track.enclosures:
track_link = track.enclosures[0].href
track_doc['stream'] = track_link
track_uuid = uuid.uuid5(uuid.NAMESPACE_URL, str(track_link)).hex
track_doc['_id'] = track_uuid
track_doc['artist_name'] = track.author
title_split = track.title.split(":")
track_doc['album_name'] = title_split[1].strip()
track_doc['name'] = title_split[2].strip()
def clear(self):
s = Server()
del s['python-tests']
db = s.create('python-tests')
class Store(object):
"""
ERS store
"""
def __init__(self, url=DEFAULT_STORE_ADMIN_URI, **client_opts):
self.logger = logging.getLogger('ers-store')
self._server = Server(url=url, **client_opts)
self.db_names = {'public': ERS_PUBLIC_DB,
'private': ERS_PRIVATE_DB,
'cache': ERS_CACHE_DB,}
# Add aggregate functions
# for method_name in ('docs_by_entity', 'by_property', 'by_property_value'):
# self.add_aggregate(method_name)
# Check the status of the databases
self._ers_dbs = {}
self._repair()
def __getitem__(self, dbname):
return self._ers_dbs[dbname]
# return ERSDatabase(self._db_uri(dbname), server=self)
def __iter__(self):
for dbname in self.all_dbs():
yield self._ers_dbs[dbname]
# yield ERSDatabase(self._db_uri(dbname), server=self)
@classmethod
def add_aggregate(cls, method_name):
"""
"""
def aggregate(self, *args, **kwargs):
return chain(*[getitem(self[db_name], method_name)(*args, **kwargs).iterator()
for db_name in ALL_DBS])
aggregate.__doc__ = """Calls method {}() of all databases in the store and returns an iterator over combined results""".format(method_name)
aggregate.__name__ = method_name
setattr(cls, method_name, aggregate)
def reset(self):
for db_name in ALL_DBS:
try:
del self._server[db_name]
except http.ResourceNotFound:
pass
self._repair()
def by_property_value(self, property, value=None):
results = []
for db in self._ers_dbs.itervalues():
for res in db.by_property_value(property, value):
results.append(res)
return results
def get_ers_db(self, dbname, **params):
"""
Try to return an ERSDatabase object for dbname.
"""
return self._ers_dbs[dbname]
def info(self):
return self._server.config()['couchdb']
def _repair(self):
# Authenticate with the local store
# user, password = auth
try:
state_db = self._server[ERS_STATE_DB]
except http.ResourceNotFound:
state_db = self._server.create(ERS_STATE_DB)
if not '_local/state' in state_db:
state_db.save(state_doc())
if not '_design/index' in state_db:
state_db.save(index_doc())
self._ers_dbs[ERS_STATE_DB] = ERSDatabase(state_db)
for dbname in ALL_DBS:
# Recreate database if needed
try:
db = self._server[dbname]
except http.ResourceNotFound:
db = self._server.create(dbname)
# Create index design doc if needed
if not '_design/index' in db:
db.save(index_doc())
## Create state doc in the public database if needed
#if dbname == ERS_PUBLIC_DB:
# if not '_local/state' in db:
# db.save(state_doc())
# Save the ERSDatabase object
self._ers_dbs[dbname] = ERSDatabase(db)
class Model:
def __init__(self):
try:
self.__server = Server()
except:
print 'can not connect to Couchdb:%s' % (settings.c['db_url'])
self.__db = {}
self.__db_name = settings.c['db_name']
DEBUG.p(self.__db_name.items())
for (k, v) in self.__db_name.items():
try:
self.__db[v] = self.__server.create(v)
except:
self.__db[v] = self.__server[v]
def create(self, dbname):
return self.__server.create(dbname)
def delete(self, dbname):
try:
self.__server.delete(dbname)
except:
print 'database %s doeso not exist!!!' % (dbname)
def clearDB(self, dbname):
try:
self.__server.delete(dbname)
except:
print 'database %s doeso not exist!!!' % (dbname)
self.__db[v] = self.__server.create(dbname)
def getAllDoc(self):
return self.__db[self.__db_name['cover']]
def addOneDoc(self, doc):
keyStr = '%s:%s' % (doc['artist'], doc['album_name'])
doc['_id'] = hashlib.md5(keyStr.encode('ascii', 'ignore')).hexdigest()
print '_id:%s' % (doc['_id'])
doc_id, doc_rev = self.__db[self.__db_name['cover']].save(doc)
print 'addOneDoc done'
def delOneDoc(self, doc):
self.__db[self.__db_name['cover']].delete(doc)
def getById(self, artist, albumName):
keyStr = '%s:%s' % (artist, albumName)
_id = hashlib.md5(keyStr.encode('ascii', 'ignore')).hexdigest()
print _id
album = self.__db[self.__db_name['cover']].get(_id)
if album:
return dict(album)
return album
#map_fun = '''function(doc) {
# if (doc._id == '%s')
# emit(doc, null);
# }''' % (_id)
#albums = self.__db[self.__db_name['cover']].query(map_fun)
#for album in albums:
# return album.key
#return None
def getById2(self, id):
print id
album = self.__db[self.__db_name['cover']].get(id)
if album:
return dict(album)
return album
def getByKeyValue(self, key, value):
map_fun = '''function(doc) {
if (doc.%s == '%s')
emit(doc, null);
}''' % (key, value)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getByArtist(self, artist):
map_fun = '''function(doc) {
if (doc.artist == '%s')
emit(doc, null);
}''' % (artist)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getByAbbumName(self, albumName):
map_fun = '''function(doc) {
if (doc.album_name == '%s')
emit(doc, null);
}''' % (albumName)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getByCopyRight(self, copyRight):
map_fun = '''function(doc) {
if (doc.copy_right == '%s')
emit(doc, null);
}''' % (copyRight)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getRandom(self, seed=11111):
db = self.getAllDoc()
tenNews = []
for id in db:
rd = random.randint(0, seed)
if rd != 1:
continue
if len(tenNews) < 10:
#print db[id]
tenNews.append(db[id])
return tenNews
class CouchDBBackend(object):
@classmethod
def load_from_config(cls, config):
settings = config.registry.settings
return CouchDBBackend(
host=settings['backend.db_host'],
db_name=os.environ.get('DB_NAME', settings['backend.db_name']),
)
def __init__(self, host, db_name, id_generator):
self.server = Server(host)
self.db_name = db_name
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (host, e))
self._db = self.server[self.db_name]
self.sync_views()
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.info('Creating and using db "%s"' % self.db_name)
except PreconditionFailed:
logger.info('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.server[self.db_name], docs)
def __get_raw_user_token(self, user_id):
try:
return views.usertokens(self._db, key=user_id).rows[0].value
except IndexError:
raise UserTokenNotFound(user_id)
def get_user_token(self, user_id):
"""Returns the information associated with a user token"""
usertoken = dict(**self.__get_raw_user_token(user_id))
return usertoken['token']
def add_token(self, user_id, token):
# Check that the token doesn't already exist.
try:
self.__get_raw_user_token(user_id)
raise UserTokenAlreadyExist(user_id)
except UserTokenNotFound:
pass
doc = dict(token=token, user_id=user_id, type='usertoken')
self._db.save(doc)
def __get_raw_state(self, session_id):
try:
return views.states(self._db, key=session_id).rows[0].value
except IndexError:
raise StateNotFound(session_id)
def get_state(self, session_id):
"""Retrives the session_id state."""
state_doc = dict(**self.__get_raw_state(session_id))
return state_doc['state']
def set_state(self, session_id):
"""Set a session_id state."""
try:
doc = self.__get_raw_state(session_id)
except StateNotFound:
doc = {"type": "fxa_oauth_states", "session_id": session_id}
doc['state'] = uuid.uuid4().hex
self._db.save(doc)
return doc['state']
def get_or_set_state(self, session_id):
"""Retrieves or creates a state for the session_id"""
try:
return self.get_state(session_id)
except StateNotFound:
return self.set_state(session_id)
def __get_raw_redirect_uri(self, state):
try:
return views.redirect_uris(self._db, key=state).rows[0].value
except IndexError:
raise RedirectURINotFound(session_id)
def get_state(self, state):
"""Retrives the session_id state."""
redirect_uri_doc = dict(**self.__get_raw_state(state))
return redirect_uri_doc['redirect_uri']
def set_state(self, state, redirect_uri):
"""Set a redirect_uri."""
try:
doc = self.__get_raw_state(state)
except RedirectURINotFound:
doc = {"type": "fxa_oauth_redirect_uri", "state": state}
doc['redirect_uri'] = redirect_uri
self._db.save(doc)
def __get_raw_oauth_access_token(self, session_id):
try:
return views.access_tokens(self._db, key=session_id).rows[0].value
except IndexError:
raise OAuthAccessTokenNotFound(session_id)
def get_oauth_access_token(self, session_id):
"""Retrives the session_id oauth_access_token."""
token_doc = dict(**self.__get_raw_oauth_access_tokens(session_id))
return token_doc['access_token']
def set_oauth_access_token(self, session_id, access_token):
"""Set the session_id oauth_access_token."""
try:
doc = self.__get_raw_oauth_access_tokens(session_id)
except OAuthAccessTokenNotFound:
doc = {"type": "fxa_oauth_access_tokens", "session_id": session_id}
doc['access_token'] = access_token
self._db.save(doc)
class Database:
def __init__(self, server, name):
self.server = Server(server)
if name in self.server:
self.db = self.server[name]
else:
self.db = self.server.create(name)
self._load_views()
self.view = self.db.view('_view/update/mtime')
def add(self, path):
song = self._song_for(os.path.realpath(path))
if song is None: return
song = self._dict_for(song)
doc = self._doc_for(song)
if doc: song["_rev"] = doc.value["_rev"]
self.db[song["_id"]] = song
def add_many(self, paths):
paths = map(os.path.realpath, paths)
def updated_file(path):
doc = self._doc_for(path)
if not doc: return True
return util.mtime(path) > doc.value["mtime"]
songs = filter(None, map(self._song_for, filter(updated_file, paths)))
if not songs: return
songs = map(self._dict_for, songs)
for song in songs:
doc = self._doc_for(song)
if not doc: continue
song["_rev"] = doc.value["_rev"]
self.db.update(songs)
def remove(self, path):
del self.db[_id(path)]
def remove_docs(self, songs):
for song in songs:
song["_deleted"] = True
self.db.update(songs)
def update(self, doc, path):
song = self._song_for(path)
if song is None: return
song = self._dict_for(song)
song["_rev"] = doc["_rev"]
self.db[song["_id"]] = song
def docs_beneath(self, path):
path = util.qdecode(path).split(os.path.sep)
return [row.value for row in self.db.view('_view/tree/by-path', startkey=path, endkey=path + [{}])]
def _song_for(self, path):
try: return MusicFile(path)
except IOError: return None
def _dict_for(self, song):
d = {}
for tag in SAVED_METATAGS + song.realkeys():
val = song(tag)
if val:
if isinstance(val, basestring):
if tag in QUOTED_TAGS:
val = util.qdecode(val)
else:
val = util.fsdecode(val)
if not tag in SINGLETON_TAGS:
val = val.split("\n")
d[tag] = val
for tag, default in DEFAULTS.items():
if not tag in song: song[tag] = default
d["~path"] = d["~filename"].split(os.path.sep)
# CouchDB doesn't like apostrophes in keys for some reason...
d["_id"] = _id(song.key)
return d
def _doc_for(self, song):
docs = list(self.view[_id(song) if isinstance(song, basestring) else song["_id"]])
if not docs: return None
return docs[0]
def _load_views(self):
for view in glob.glob(util.data_path('views', '*.json')):
name = os.path.basename(view)[0:-5]
f = open(view)
# Can't use __setitem__ 'cause it assumes content is a hash
self.db.resource.put('_design/' + name, content=f.read())
f.close()
f = feedparser.parse(
'http://crondia.blogspot.com/feeds/posts/default?alt=rss')
for entry in f.entries:
if entry.title in as_cronicas:
unicode('"cronica velha:", entry.title', 'UTF-8', 'REPLACE')
continue
unicode('"cronica NOVA:", entry.title', 'UTF-8', 'REPLACE')
uma_cronica = Cronica(titulo=entry.title, texto=entry.description)
uma_cronica.store(cronica)
server = Server('http://localhost:5984')
if not 'manchete' in server:
manchete = server.create('manchete')
if not 'noticia' in server:
noticia = server.create('noticia')
if not 'receita' in server:
receita = server.create('receita')
if not 'artigo' in server:
artigo = server.create('artigo')
if not 'resenha' in server:
resenha = server.create('resenha')
if not 'horoscopo' in server:
horoscopo = server.create('horoscopo')
if not 'cronica' in server:
cronica = server.create('cronica')
else:
manchete = server['manchete']
noticia = server['noticia']
from couchdb.client import Server
import json
import fnmatch
server = Server('http://*****:*****@172.26.133.237:5984')
if not 'parties_data' in server:
db = server.create('parties_data')
else:
db = server['parties_data']
with open('../../../../../grouped_election_data(1).json') as jsonfile:
data = json.load(jsonfile)
jsonfile.close()
result = {}
feature_list = data['features']
for row in feature_list:
data = json.loads(json.dumps(row))
coordinate = str(
data['geometry']['coordinates']) # _id has to be a string
property_dict = {
i: j
for i, j in data['properties'].items() if j != None
} # take out null
pattern = '*_percent'
doc = {}
for item in property_dict:
if fnmatch.fnmatch(item, pattern):
party_name = item.replace('_percent', '')
class CouchDBBackend(object):
@classmethod
def load_from_config(cls, config):
settings = config.registry.settings
generator = config.maybe_dotted(settings['daybed.id_generator'])
return CouchDBBackend(
host=settings['backend.db_host'],
db_name=os.environ.get('DB_NAME', settings['backend.db_name']),
id_generator=generator(config)
)
def __init__(self, host, db_name, id_generator):
self.server = Server(host)
self.db_name = db_name
try:
self.create_db_if_not_exist()
except socket.error as e:
raise CouchDBBackendConnectionError(
"Unable to connect to the CouchDB server: %s - %s" % (host, e))
self._db = self.server[self.db_name]
self.sync_views()
self._generate_id = id_generator
def delete_db(self):
del self.server[self.db_name]
def create_db_if_not_exist(self):
try:
self.server.create(self.db_name)
logger.info('Creating and using db "%s"' % self.db_name)
except (PreconditionFailed, Unauthorized):
logger.info('Using db "%s".' % self.db_name)
def sync_views(self):
ViewDefinition.sync_many(self.server[self.db_name], docs)
def get_models(self, principals):
principals = list(set(principals))
models = {}
for result in views.models(self._db, keys=principals).rows:
doc = result.value
_id = doc["_id"]
models[_id] = {
"id": _id,
"title": doc["definition"].get("title", _id),
"description": doc["definition"].get("description", "")
}
return list(models.values())
def __get_raw_model(self, model_id):
try:
doc = views.model_definitions(self._db, key=model_id).rows[0]
return doc.value
except IndexError:
raise backend_exceptions.ModelNotFound(model_id)
def get_model_definition(self, model_id):
return self.__get_raw_model(model_id)['definition']
def __get_raw_records(self, model_id):
# Make sure the model exists.
self.__get_raw_model(model_id)
return views.records(self._db, key=model_id).rows
def get_records(self, model_id, raw_records=None):
return [r["record"] for r in
self.get_records_with_authors(model_id, raw_records)]
def get_records_with_authors(self, model_id, raw_records=None):
if raw_records is None:
raw_records = self.__get_raw_records(model_id)
records = []
for item in raw_records:
item.value['record']['id'] = item.value['_id'].split('-')[1]
records.append({"authors": item.value['authors'],
"record": item.value['record']})
return records
def __get_raw_record(self, model_id, record_id):
key = u'-'.join((model_id, record_id))
try:
return views.records_all(self._db, key=key).rows[0].value
except IndexError:
raise backend_exceptions.RecordNotFound(
u'(%s, %s)' % (model_id, record_id)
)
def _model_exists(self, model_id):
try:
self.__get_raw_model(model_id)
return True
except backend_exceptions.ModelNotFound:
return False
def _record_exists(self, model_id, record_id):
try:
self.__get_raw_record(model_id, record_id)
return True
except backend_exceptions.RecordNotFound:
return False
def get_record(self, model_id, record_id):
doc = self.__get_raw_record(model_id, record_id)
record = doc['record']
record['id'] = record_id
return record
def get_record_authors(self, model_id, record_id):
doc = self.__get_raw_record(model_id, record_id)
return doc['authors']
def put_model(self, definition, permissions, model_id=None):
if model_id is None:
model_id = self._generate_id(key_exist=self._model_exists)
try:
doc = self.__get_raw_model(model_id)
except backend_exceptions.ModelNotFound:
doc = {'_id': model_id,
'type': 'definition'}
doc['definition'] = definition
doc['permissions'] = permissions
definition_id, _ = self._db.save(doc)
return definition_id
def put_record(self, model_id, record, authors, record_id=None):
doc = {
'type': 'record',
'authors': authors,
'model_id': model_id,
'record': record}
if record_id is not None:
try:
old_doc = self.__get_raw_record(model_id, record_id)
except backend_exceptions.RecordNotFound:
doc['_id'] = '-'.join((model_id, record_id))
else:
authors = list(set(authors) | set(old_doc['authors']))
doc['authors'] = authors
old_doc.update(doc)
doc = old_doc
else:
key_exist = functools.partial(self._record_exists, model_id)
record_id = self._generate_id(key_exist=key_exist)
doc['_id'] = '-'.join((model_id, record_id))
self._db.save(doc)
return record_id
def delete_record(self, model_id, record_id):
doc = self.__get_raw_record(model_id, record_id)
if doc:
self._db.delete(doc)
return doc
def delete_records(self, model_id):
results = self.__get_raw_records(model_id)
for result in results:
self._db.delete(result.value)
return self.get_records(model_id, raw_records=results)
def delete_model(self, model_id):
"""DELETE ALL THE THINGS"""
# Delete the associated data if any.
records = self.delete_records(model_id)
try:
doc = views.model_definitions(self._db, key=model_id).rows[0].value
except IndexError:
raise backend_exceptions.ModelNotFound(model_id)
# Delete the model definition if it exists.
self._db.delete(doc)
return {"definition": doc["definition"],
"permissions": doc["permissions"],
"records": records}
def __get_raw_token(self, credentials_id):
try:
return views.tokens(self._db, key=credentials_id).rows[0].value
except IndexError:
raise backend_exceptions.CredentialsNotFound(credentials_id)
def get_token(self, credentials_id):
"""Returns the information associated with a credentials_id"""
credentials = dict(**self.__get_raw_token(credentials_id))
return credentials['token']
def get_credentials_key(self, credentials_id):
"""Retrieves a token by its id"""
credentials = dict(**self.__get_raw_token(credentials_id))
return credentials['credentials']['key']
def store_credentials(self, token, credentials):
# Check that the token doesn't already exist.
assert 'id' in credentials and 'key' in credentials
try:
self.__get_raw_token(credentials['id'])
raise backend_exceptions.CredentialsAlreadyExist(credentials['id'])
except backend_exceptions.CredentialsNotFound:
pass
doc = dict(token=token, credentials=credentials, type='token')
self._db.save(doc)
def get_model_permissions(self, model_id):
doc = self.__get_raw_model(model_id)
return doc['permissions']
parser.add_argument('--year', type=int, default=2018)
parser.add_argument('--monthlength', type=int, default=12)
parser.add_argument('--end', type=str, default='2018,12,31')
args = parser.parse_args()
# argsparser
url = 'http://couchdb.socmedia.bigtwitter.cloud.edu.au/twitter/_design/twitter/_view/summary'
BATCHSIZE = args.batch
#该地区在sydney那里
start_key = '[\"' + args.location + '\",' + args.start + ']'
end_key = '[\"' + args.location + '\",' + args.end + ']'
month = args.monthlength
serverName = args.location + str(args.year)
try:
db = secure_remote_server.create(serverName)
except:
print('database already exist!!')
params = {
'include_docs': 'true',
'reduce': 'false',
'start_key': start_key,
'end_key': end_key,
"skip": "0",
"limit": str(BATCHSIZE)
}
TOTALSIZE = args.total_forMonth
count = 1
tweetlist = []
while count <= month:
class Model:
def __init__(self):
try:
self.__server = Server()
except:
print 'can not connect to Couchdb:%s'%(settings.c['db_url'])
self.__db = {}
self.__db_name = settings.c['db_name']
DEBUG.p(self.__db_name.items())
for (k, v) in self.__db_name.items():
try:
self.__db[v] = self.__server.create(v)
except:
self.__db[v] = self.__server[v]
def create(self, dbname):
return self.__server.create(dbname)
def delete(self, dbname):
try:
self.__server.delete(dbname)
except:
print 'database %s doeso not exist!!!'%(dbname)
def clearDB(self, dbname):
try:
self.__server.delete(dbname)
except:
print 'database %s doeso not exist!!!'%(dbname)
self.__db[v] = self.__server.create(dbname)
def getAllDoc(self):
return self.__db[self.__db_name['cover']]
def addOneDoc(self, doc):
keyStr = '%s:%s'%(doc['artist'], doc['album_name'])
doc['_id'] = hashlib.md5(keyStr.encode('ascii', 'ignore')).hexdigest()
print '_id:%s'%(doc['_id'])
doc_id, doc_rev = self.__db[self.__db_name['cover']].save(doc)
print 'addOneDoc done'
def delOneDoc(self, doc):
self.__db[self.__db_name['cover']].delete(doc)
def getById(self, artist, albumName):
keyStr = '%s:%s'%(artist, albumName)
_id = hashlib.md5(keyStr.encode('ascii', 'ignore')).hexdigest()
print _id
album = self.__db[self.__db_name['cover']].get(_id)
if album:
return dict(album)
return album
#map_fun = '''function(doc) {
# if (doc._id == '%s')
# emit(doc, null);
# }''' % (_id)
#albums = self.__db[self.__db_name['cover']].query(map_fun)
#for album in albums:
# return album.key
#return None
def getById2(self, id):
print id
album = self.__db[self.__db_name['cover']].get(id)
if album:
return dict(album)
return album
def getByKeyValue(self, key, value):
map_fun = '''function(doc) {
if (doc.%s == '%s')
emit(doc, null);
}''' % (key, value)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getByArtist(self, artist):
map_fun = '''function(doc) {
if (doc.artist == '%s')
emit(doc, null);
}''' % (artist)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getByAbbumName(self, albumName):
map_fun = '''function(doc) {
if (doc.album_name == '%s')
emit(doc, null);
}''' % (albumName)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getByCopyRight(self, copyRight):
map_fun = '''function(doc) {
if (doc.copy_right == '%s')
emit(doc, null);
}''' % (copyRight)
albums = self.__db[self.__db_name['cover']].query(map_fun)
for album in albums:
return album.key
return None
def getRandom(self, seed = 11111):
db = self.getAllDoc()
tenNews = []
for id in db:
rd = random.randint(0, seed)
if rd != 1:
continue
if len(tenNews) < 10:
#print db[id]
tenNews.append(db[id])
return tenNews
class CouchDatabase(ObjectStoreDatabase):
"""
A U1DB backend that uses Couch as its persistence layer.
"""
U1DB_TRANSACTION_LOG_KEY = '_transaction_log'
U1DB_CONFLICTS_KEY = '_conflicts'
U1DB_OTHER_GENERATIONS_KEY = '_other_generations'
U1DB_INDEXES_KEY = '_indexes'
U1DB_REPLICA_UID_KEY = '_replica_uid'
U1DB_DATA_KEYS = [
U1DB_TRANSACTION_LOG_KEY,
U1DB_CONFLICTS_KEY,
U1DB_OTHER_GENERATIONS_KEY,
U1DB_INDEXES_KEY,
U1DB_REPLICA_UID_KEY,
]
COUCH_ID_KEY = '_id'
COUCH_REV_KEY = '_rev'
COUCH_U1DB_ATTACHMENT_KEY = 'u1db_json'
COUCH_U1DB_REV_KEY = 'u1db_rev'
# the following map describes information about methods usage of
# properties that have to persist on the underlying database. The format
# of the map is assumed to be:
#
# {
# 'property_name': [
# ('property_load_method_name', 'property_dump_method_name'),
# [('method_1_name', bool),
# ...
# ('method_N_name', bool)]],
# ...
# }
#
# where the booleans indicate if the property should be stored after
# each method execution (i.e. if the method alters the property). Property
# load/dump methods will be run after/before properties are read/written
# to the underlying db.
PERSISTENCY_MAP = {
U1DB_TRANSACTION_LOG_KEY: [
('_load_transaction_log_from_json', None),
[('_get_transaction_log', False),
('_get_generation', False),
('_get_generation_info', False),
('_get_trans_id_for_gen', False),
('whats_changed', False),
('_put_and_update_indexes', True)]],
U1DB_CONFLICTS_KEY: [
(None, None),
[('_has_conflicts', False),
('get_doc_conflicts', False),
('_prune_conflicts', False),
('resolve_doc', False),
('_replace_conflicts', True),
('_force_doc_sync_conflict', True)]],
U1DB_OTHER_GENERATIONS_KEY: [
('_load_other_generations_from_json', None),
[('_get_replica_gen_and_trans_id', False),
('_do_set_replica_gen_and_trans_id', True)]],
U1DB_INDEXES_KEY: [
('_load_indexes_from_json', '_dump_indexes_as_json'),
[('list_indexes', False),
('get_from_index', False),
('get_range_from_index', False),
('get_index_keys', False),
('_put_and_update_indexes', True),
('create_index', True),
('delete_index', True)]],
U1DB_REPLICA_UID_KEY: [
(None, None),
[('_allocate_doc_rev', False),
('_put_doc_if_newer', False),
('_ensure_maximal_rev', False),
('_prune_conflicts', False),
('_set_replica_uid', True)]]}
@classmethod
def open_database(cls, url, create):
"""
Open a U1DB database using CouchDB as backend.
@param url: the url of the database replica
@type url: str
@param create: should the replica be created if it does not exist?
@type create: bool
@return: the database instance
@rtype: CouchDatabase
"""
# get database from url
m = re.match('(^https?://[^/]+)/(.+)$', url)
if not m:
raise InvalidURLError
url = m.group(1)
dbname = m.group(2)
server = Server(url=url)
try:
server[dbname]
except ResourceNotFound:
if not create:
raise DatabaseDoesNotExist()
return cls(url, dbname)
def __init__(self, url, dbname, replica_uid=None, full_commit=True,
session=None):
"""
Create a new Couch data container.
@param url: the url of the couch database
@type url: str
@param dbname: the database name
@type dbname: str
@param replica_uid: an optional unique replica identifier
@type replica_uid: str
@param full_commit: turn on the X-Couch-Full-Commit header
@type full_commit: bool
@param session: an http.Session instance or None for a default session
@type session: http.Session
"""
# save params
self._url = url
self._full_commit = full_commit
self._session = session
# configure couch
self._server = Server(url=self._url,
full_commit=self._full_commit,
session=self._session)
self._dbname = dbname
# this will ensure that transaction and sync logs exist and are
# up-to-date.
try:
self._database = self._server[self._dbname]
except ResourceNotFound:
self._server.create(self._dbname)
self._database = self._server[self._dbname]
ObjectStoreDatabase.__init__(self, replica_uid=replica_uid)
#-------------------------------------------------------------------------
# methods from Database
#-------------------------------------------------------------------------
def _get_doc(self, doc_id, check_for_conflicts=False):
"""
Get just the document content, without fancy handling.
@param doc_id: The unique document identifier
@type doc_id: str
@param include_deleted: If set to True, deleted documents will be
returned with empty content. Otherwise asking for a deleted
document will return None.
@type include_deleted: bool
@return: a Document object.
@type: u1db.Document
"""
cdoc = self._database.get(doc_id)
if cdoc is None:
return None
has_conflicts = False
if check_for_conflicts:
has_conflicts = self._has_conflicts(doc_id)
doc = self._factory(
doc_id=doc_id,
rev=cdoc[self.COUCH_U1DB_REV_KEY],
has_conflicts=has_conflicts)
contents = self._database.get_attachment(
cdoc,
self.COUCH_U1DB_ATTACHMENT_KEY)
if contents:
doc.content = json.loads(contents.read())
else:
doc.make_tombstone()
return doc
def get_all_docs(self, include_deleted=False):
"""
Get the JSON content for all documents in the database.
@param include_deleted: If set to True, deleted documents will be
returned with empty content. Otherwise deleted documents will not
be included in the results.
@type include_deleted: bool
@return: (generation, [Document])
The current generation of the database, followed by a list of all
the documents in the database.
@rtype: tuple
"""
generation = self._get_generation()
results = []
for doc_id in self._database:
if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
continue
doc = self._get_doc(doc_id, check_for_conflicts=True)
if doc.content is None and not include_deleted:
continue
results.append(doc)
return (generation, results)
def _put_doc(self, doc):
"""
Update a document.
This is called everytime we just want to do a raw put on the db (i.e.
without index updates, document constraint checks, and conflict
checks).
@param doc: The document to update.
@type doc: u1db.Document
@return: The new revision identifier for the document.
@rtype: str
"""
# prepare couch's Document
cdoc = CouchDocument()
cdoc[self.COUCH_ID_KEY] = doc.doc_id
# we have to guarantee that couch's _rev is consistent
old_cdoc = self._database.get(doc.doc_id)
if old_cdoc is not None:
cdoc[self.COUCH_REV_KEY] = old_cdoc[self.COUCH_REV_KEY]
# store u1db's rev
cdoc[self.COUCH_U1DB_REV_KEY] = doc.rev
# save doc in db
self._database.save(cdoc)
# store u1db's content as json string
if not doc.is_tombstone():
self._database.put_attachment(
cdoc, doc.get_json(),
filename=self.COUCH_U1DB_ATTACHMENT_KEY)
else:
self._database.delete_attachment(
cdoc,
self.COUCH_U1DB_ATTACHMENT_KEY)
def get_sync_target(self):
"""
Return a SyncTarget object, for another u1db to synchronize with.
@return: The sync target.
@rtype: CouchSyncTarget
"""
return CouchSyncTarget(self)
def create_index(self, index_name, *index_expressions):
"""
Create a named index, which can then be queried for future lookups.
@param index_name: A unique name which can be used as a key prefix.
@param index_expressions: Index expressions defining the index
information.
"""
if index_name in self._indexes:
if self._indexes[index_name]._definition == list(
index_expressions):
return
raise errors.IndexNameTakenError
index = InMemoryIndex(index_name, list(index_expressions))
for doc_id in self._database:
if doc_id.startswith(self.U1DB_DATA_DOC_ID_PREFIX):
continue # skip special files
doc = self._get_doc(doc_id)
if doc.content is not None:
index.add_json(doc_id, doc.get_json())
self._indexes[index_name] = index
def close(self):
"""
Release any resources associated with this database.
@return: True if db was succesfully closed.
@rtype: bool
"""
# TODO: fix this method so the connection is properly closed and
# test_close (+tearDown, which deletes the db) works without problems.
self._url = None
self._full_commit = None
self._session = None
#self._server = None
self._database = None
return True
def sync(self, url, creds=None, autocreate=True):
"""
Synchronize documents with remote replica exposed at url.
@param url: The url of the target replica to sync with.
@type url: str
@param creds: optional dictionary giving credentials.
to authorize the operation with the server.
@type creds: dict
@param autocreate: Ask the target to create the db if non-existent.
@type autocreate: bool
@return: The local generation before the synchronisation was performed.
@rtype: int
"""
return Synchronizer(self, CouchSyncTarget(url, creds=creds)).sync(
autocreate=autocreate)
#-------------------------------------------------------------------------
# methods from ObjectStoreDatabase
#-------------------------------------------------------------------------
def _init_u1db_data(self):
"""
Initialize u1db configuration data on backend storage.
A U1DB database needs to keep track of all database transactions,
document conflicts, the generation of other replicas it has seen,
indexes created by users and so on.
In this implementation, all this information is stored in special
documents stored in the underlying with doc_id prefix equal to
U1DB_DATA_DOC_ID_PREFIX. Those documents ids are reserved: put_doc(),
get_doc() and delete_doc() will not allow documents with a doc_id with
that prefix to be accessed or modified.
"""
for key in self.U1DB_DATA_KEYS:
doc_id = '%s%s' % (self.U1DB_DATA_DOC_ID_PREFIX, key)
doc = self._get_doc(doc_id)
if doc is None:
doc = self._factory(doc_id)
doc.content = {'content': getattr(self, key)}
self._put_doc(doc)
#-------------------------------------------------------------------------
# Couch specific methods
#-------------------------------------------------------------------------
INDEX_NAME_KEY = 'name'
INDEX_DEFINITION_KEY = 'definition'
INDEX_VALUES_KEY = 'values'
def delete_database(self):
"""
Delete a U1DB CouchDB database.
"""
del(self._server[self._dbname])
def _dump_indexes_as_json(self):
"""
Dump index definitions as JSON.
"""
indexes = {}
for name, idx in self._indexes.iteritems():
indexes[name] = {}
for attr in [self.INDEX_NAME_KEY, self.INDEX_DEFINITION_KEY,
self.INDEX_VALUES_KEY]:
indexes[name][attr] = getattr(idx, '_' + attr)
return indexes
def _load_indexes_from_json(self, indexes):
"""
Load index definitions from stored JSON.
@param indexes: A JSON representation of indexes as
[('index-name', ['field', 'field2', ...]), ...].
@type indexes: str
"""
self._indexes = {}
for name, idx_dict in indexes.iteritems():
idx = InMemoryIndex(name, idx_dict[self.INDEX_DEFINITION_KEY])
idx._values = idx_dict[self.INDEX_VALUES_KEY]
self._indexes[name] = idx
def _load_transaction_log_from_json(self, transaction_log):
"""
Load transaction log from stored JSON.
@param transaction_log: A JSON representation of transaction_log as
[('generation', 'transaction_id'), ...].
@type transaction_log: list
"""
self._transaction_log = []
for gen, trans_id in transaction_log:
self._transaction_log.append((gen, trans_id))
def _load_other_generations_from_json(self, other_generations):
"""
Load other generations from stored JSON.
@param other_generations: A JSON representation of other_generations
as {'replica_uid': ('generation', 'transaction_id'), ...}.
@type other_generations: dict
"""
self._other_generations = {}
for replica_uid, [gen, trans_id] in other_generations.iteritems():
self._other_generations[replica_uid] = (gen, trans_id)
class CouchDatabase(ObjectStoreDatabase):
"""A U1DB backend that uses Couch as its persistence layer."""
@classmethod
def open_database(cls, url, create):
"""Open a U1DB database using CouchDB as backend."""
# get database from url
m = re.match('(^https?://[^/]+)/(.+)$', url)
if not m:
raise InvalidURLError
url = m.group(1)
dbname = m.group(2)
server = Server(url=url)
try:
server[dbname]
except ResourceNotFound:
if not create:
raise DatabaseDoesNotExist()
return cls(url, dbname)
def __init__(self,
url,
database,
replica_uid=None,
full_commit=True,
session=None):
"""Create a new Couch data container."""
self._url = url
self._full_commit = full_commit
self._session = session
self._server = Server(url=self._url,
full_commit=self._full_commit,
session=self._session)
self._dbname = database
# this will ensure that transaction and sync logs exist and are
# up-to-date.
try:
self._database = self._server[database]
except ResourceNotFound:
self._server.create(database)
self._database = self._server[database]
super(CouchDatabase, self).__init__(replica_uid=replica_uid,
document_factory=LeapDocument)
#-------------------------------------------------------------------------
# methods from Database
#-------------------------------------------------------------------------
def _get_doc(self, doc_id, check_for_conflicts=False):
"""Get just the document content, without fancy handling."""
cdoc = self._database.get(doc_id)
if cdoc is None:
return None
has_conflicts = False
if check_for_conflicts:
has_conflicts = self._has_conflicts(doc_id)
doc = self._factory(doc_id=doc_id,
rev=cdoc['u1db_rev'],
has_conflicts=has_conflicts)
contents = self._database.get_attachment(cdoc, 'u1db_json')
if contents:
doc.content = json.loads(contents.getvalue())
else:
doc.make_tombstone()
return doc
def get_all_docs(self, include_deleted=False):
"""Get the JSON content for all documents in the database."""
generation = self._get_generation()
results = []
for doc_id in self._database:
if doc_id == self.U1DB_DATA_DOC_ID:
continue
doc = self._get_doc(doc_id, check_for_conflicts=True)
if doc.content is None and not include_deleted:
continue
results.append(doc)
return (generation, results)
def _put_doc(self, doc):
"""Store document in database."""
# prepare couch's Document
cdoc = CouchDocument()
cdoc['_id'] = doc.doc_id
# we have to guarantee that couch's _rev is cosistent
old_cdoc = self._database.get(doc.doc_id)
if old_cdoc is not None:
cdoc['_rev'] = old_cdoc['_rev']
# store u1db's rev
cdoc['u1db_rev'] = doc.rev
# save doc in db
self._database.save(cdoc)
# store u1db's content as json string
if not doc.is_tombstone():
self._database.put_attachment(cdoc,
doc.get_json(),
filename='u1db_json')
else:
self._database.delete_attachment(cdoc, 'u1db_json')
def get_sync_target(self):
"""
Return a SyncTarget object, for another u1db to synchronize with.
"""
return CouchSyncTarget(self)
def create_index(self, index_name, *index_expressions):
"""
Create a named index, which can then be queried for future lookups.
"""
if index_name in self._indexes:
if self._indexes[index_name]._definition == list(
index_expressions):
return
raise errors.IndexNameTakenError
index = InMemoryIndex(index_name, list(index_expressions))
for doc_id in self._database:
if doc_id == self.U1DB_DATA_DOC_ID:
continue
doc = self._get_doc(doc_id)
if doc.content is not None:
index.add_json(doc_id, doc.get_json())
self._indexes[index_name] = index
# save data in object store
self._store_u1db_data()
def close(self):
"""Release any resources associated with this database."""
# TODO: fix this method so the connection is properly closed and
# test_close (+tearDown, which deletes the db) works without problems.
self._url = None
self._full_commit = None
self._session = None
#self._server = None
self._database = None
return True
def sync(self, url, creds=None, autocreate=True):
"""Synchronize documents with remote replica exposed at url."""
from u1db.sync import Synchronizer
return Synchronizer(self, CouchSyncTarget(
url, creds=creds)).sync(autocreate=autocreate)
#-------------------------------------------------------------------------
# methods from ObjectStoreDatabase
#-------------------------------------------------------------------------
def _init_u1db_data(self):
if self._replica_uid is None:
self._replica_uid = uuid.uuid4().hex
doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
doc.content = {
'transaction_log': [],
'conflicts': b64encode(json.dumps({})),
'other_generations': {},
'indexes': b64encode(json.dumps({})),
'replica_uid': self._replica_uid
}
self._put_doc(doc)
def _fetch_u1db_data(self):
# retrieve u1db data from couch db
cdoc = self._database.get(self.U1DB_DATA_DOC_ID)
jsonstr = self._database.get_attachment(cdoc, 'u1db_json').getvalue()
content = json.loads(jsonstr)
# set u1db database info
#self._sync_log = content['sync_log']
self._transaction_log = content['transaction_log']
self._conflicts = json.loads(b64decode(content['conflicts']))
self._other_generations = content['other_generations']
self._indexes = self._load_indexes_from_json(
b64decode(content['indexes']))
self._replica_uid = content['replica_uid']
# save couch _rev
self._couch_rev = cdoc['_rev']
def _store_u1db_data(self):
doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
doc.content = {
'transaction_log': self._transaction_log,
# Here, the b64 encode ensures that document content
# does not cause strange behaviour in couchdb because
# of encoding.
'conflicts': b64encode(json.dumps(self._conflicts)),
'other_generations': self._other_generations,
'indexes': b64encode(self._dump_indexes_as_json()),
'replica_uid': self._replica_uid,
'_rev': self._couch_rev
}
self._put_doc(doc)
#-------------------------------------------------------------------------
# Couch specific methods
#-------------------------------------------------------------------------
def delete_database(self):
"""Delete a U1DB CouchDB database."""
del (self._server[self._dbname])
def _dump_indexes_as_json(self):
indexes = {}
for name, idx in self._indexes.iteritems():
indexes[name] = {}
for attr in ['name', 'definition', 'values']:
indexes[name][attr] = getattr(idx, '_' + attr)
return json.dumps(indexes)
def _load_indexes_from_json(self, indexes):
dict = {}
for name, idx_dict in json.loads(indexes).iteritems():
idx = InMemoryIndex(name, idx_dict['definition'])
idx._values = idx_dict['values']
dict[name] = idx
return dict
import json
import time
# chat with the couchdb server
class Tweet(Document):
id = IntegerField()
from_user = TextField()
text = TextField()
# TODO optimize for speed... very verbose
j = open('geotagged_tweets_from_haiti.json')
# open json objects
jsonobjects = [i for i in j]
l = [json.loads(j) for j in jsonobjects]
server = Server('http://127.0.0.1:5984')
try:
db = server.create('tweetgeneral')
except:
del server['tweetgeneral']
db = server.create('tweetgeneral')
# for each_tweet in l:
# tweet = Tweet(id=each_tweet['id'],from_user=each_tweet['from_user'],text=each_tweet['text'])
# tweet.store(db)
#
class Database():
"""The Database class incorperates functions you wish to ask
the database"""
jsdir = CONFIG.get('couchdb', 'javascript_directory')
__DESIGN_VIEWS_PATHS_MAP = \
file(jsdir + '/design_views_paths_map.js').read()
__DESIGN_VIEWS_SHASUMS_MAP = \
file(jsdir + '/design_views_shasums_map.js').read()
__DESIGN_VIEWS_FORMATS_MAP = \
file(jsdir + '/design_views_formats_map.js').read()
__DESIGN_VIEWS_FORMATS_REDUCE = '_sum'
__DESIGN_VIEWS_SOUND_MAP = \
file(jsdir + '/design_views_sound_map.js').read()
__DESIGN_VIEWS_VIDEO_MAP = \
file(jsdir + '/design_views_sound_map.js').read()
__DESIGN_FULLTEXT_ARTIST_INDEX = \
file(jsdir + '/design_fulltext_artist_index.js').read()
__DESIGN_FULLTEXT_EVERYTHING_INDEX = \
file(jsdir + '/design_fulltext_artist_index.js').read()
def create_views(self):
"""creates views and saves them to the database"""
LOG.info('creating views')
views = {'_id': '_design/views',
'language': 'javascript',
'views': {
'shasums': {'map': self.__DESIGN_VIEWS_SHASUMS_MAP},
'paths': {'map': self.__DESIGN_VIEWS_PATHS_MAP},
'formats': {
'map': self.__DESIGN_VIEWS_FORMATS_MAP,
'reduce': self.__DESIGN_VIEWS_FORMATS_REDUCE},
'sound': {'map': self.__DESIGN_VIEWS_SOUND_MAP},
'video': {'map': self.__DESIGN_VIEWS_VIDEO_MAP}
},
'fulltext': {
'artist': {'index': self.__DESIGN_FULLTEXT_ARTIST_INDEX},
'everything': {'index':
self.__DESIGN_FULLTEXT_EVERYTHING_INDEX}
}}
self.database.create(views)
def __init__(self):
"""Initialises a new connection to couch and
creates a new mis databse if nonexistent"""
LOG.info('initialising database')
database_name = CONFIG.get('couchdb', 'database')
host = CONFIG.get('couchdb', 'hostname')
port = CONFIG.get('couchdb', 'port')
database_uri = 'http://' + host + ':' + port + '/'
self.server = Server(database_uri)
try:
# This statement appears to do nothing, but is used to
# make sure the database is reachable.
self.server.version
except AttributeError as error:
if not test_tcp_connection(host, int(port)):
LOG.critical("couchdb cannot be reached at " + database_uri)
exit(1)
else:
LOG.error('unknown AttributeError thrown')
raise error
try:
LOG.debug('opening database')
self.database = self.server[database_name]
except(ResourceNotFound):
LOG.info('creating database')
# The database didn't exist. Lets create it.
self.database = self.server.create(database_name)
self.create_views()
def iterate_all_files(self):
"""With a big database, this is probably a bad idea. this
iterates through every single document."""
for entry in self.database:
yield(entry)
def get_document(self, shasum):
"""extracts a (full) document from the database using the
shasum as an identifier"""
assert shasum is not None
assert shasum != ''
LOG.debug('getting document')
result = None
try:
result = self.database[shasum]
# make sure it actually exists
except (ResourceNotFound) as error:
LOG.error("don't have that document, doesn't exist:" +
str(error))
return result
def add_userdata(self, shasum, data):
"""Adds userdata to the database shasum"""
LOG.debug('add userdata')
shasum = unicode(shasum)
user = getuser()
node = platform_node()
user_key = node + ':' + user
userdata = {}
if not self.file_exists(shasum):
LOG.error('trying to add userdata to nonexistent file' + shasum)
return None
entry = self.database[shasum]
userdatalist = {}
if 'userdata' in entry:
userdatalist = entry['userdata']
if user_key in userdatalist:
userdata = userdatalist[user_key]
userdata.update(data)
userdatalist[user_key] = userdata
entry['userdata'] = userdatalist
self.database[shasum] = entry
def add_data(self, shasum, name, data):
"""adds data to a record"""
assert shasum is not None
shasum = unicode(shasum)
name = unicode(name)
LOG.debug('adding data')
if self.file_exists(shasum):
mis_file = self.database[shasum]
if name not in mis_file or mis_file[name] != data:
LOG.info(shasum + " info " + name + " has changed, updating")
mis_file[name] = data
self.database[shasum] = mis_file
else: # create when nonexistent
LOG.info(shasum + " info " + name + " added")
entry = {'_id': shasum, name: data}
self.database.create(entry)
def add_path(self, shasum, node, path):
"""Adds a path to the database"""
assert shasum is not None
shasum = unicode(shasum)
node = unicode(node)
path = unicode(path)
LOG.debug('adding path ' + path + " to " + shasum)
path_info = {'node': node, 'path': path}
if self.file_exists(shasum):
mis_file = self.database[shasum]
mis_file['paths'].append(path_info)
self.database[mis_file['_id']] = mis_file
else: # create when nonexistent
entry = {'_id': shasum, 'paths': [path_info]}
self.database.create(entry)
def file_exists(self, shasum):
"""Checks if a file (shasum) exists in the database, and
returns the entry when found"""
assert shasum is not None
shasum = unicode(shasum)
result = None
LOG.debug('checking if file exists: ' + shasum)
try:
# Note: the following line triggers the
# ResourceNotFound if the sha is not known, this is
# the way we catch whether it exists.
self.database[shasum] # pylint: disable-msg=W0104
result = shasum
except ResourceNotFound:
LOG.debug('trying to find nonexistent entry ' + shasum)
return result
def path_exists(self, path, node=None):
"""Checks whether a certain path exists and returns True
or False"""
if node is None:
node = platform_node
node = unicode(node)
path = unicode(path)
LOG.debug('path exists: ' + node + ':' + path)
result = None
key = [node, path]
results = self.database.view('views/paths', key=key)
if(len(results) > 0):
result = results.rows[0]['value']
return result
