def testSetupConfigWithCustomPort(self):
"""
L{setupConfig} can override the port defined in the configuration
file.
"""
config = setupConfig(self.path, port=9002)
self.assertEqual('9002', config.get('service', 'port'))
def run(self,
database_uri,
username,
password,
fullname,
email,
role=None):
username = username.decode('utf-8')
password = password.decode('utf-8')
fullname = '' if fullname is None else fullname.decode('utf-8')
email = '' if email is None else email.decode('utf-8')
roles = {
'anonymous': Role.ANONYMOUS,
'superuser': Role.SUPERUSER,
'user': Role.USER,
'usermanager': Role.USER_MANAGER
}
role = roles.get(role.lower() if role else 'user')
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
config.set('index', 'url', '')
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
createUser(username, password, fullname, email, role)
def testSetupConfig(self):
"""L{setupConfig} loads data from a configuration file."""
config = setupConfig(self.path)
self.assertEqual('var/run', config.get('service', 'temp-path'))
self.assertEqual('var/log', config.get('service', 'trace-path'))
self.assertEqual('2', config.get('service', 'max-threads'))
self.assertEqual('9000', config.get('service', 'port'))
self.assertEqual('False', config.get('service', 'development'))
self.assertEqual('http://*****:*****@localhost/fluiddb-test',
config.get('store', 'main-uri'))
self.assertEqual('127.0.0.1', config.get('cache', 'host'))
self.assertEqual(6379, config.getint('cache', 'port'))
self.assertEqual(0, config.getint('cache', 'db'))
self.assertEqual(3600, config.getint('cache', 'expire-timeout'))
self.assertEqual('testaccesssecret',
config.get('oauth', 'access-secret'))
self.assertEqual('testrenewalsecrt',
config.get('oauth', 'renewal-secret'))
self.assertEqual('24', config.get('oauth', 'renewal-token-duration'))
self.assertTrue(config.getboolean('comments', 'extract-atnames'))
self.assertTrue(config.getboolean('comments', 'extract-hashtags'))
self.assertTrue(config.getboolean('comments', 'extract-plustags'))
self.assertTrue(config.getboolean('comments', 'extract-urls'))
self.assertTrue(config.getboolean('comments', 'extract-files'))
self.assertEqual(':files:', config.get('comments', 'file-object'))
def testSetupConfig(self):
"""L{setupConfig} loads data from a configuration file."""
config = setupConfig(self.path)
self.assertEqual('var/run', config.get('service', 'temp-path'))
self.assertEqual('var/log', config.get('service', 'trace-path'))
self.assertEqual('2', config.get('service', 'max-threads'))
self.assertEqual('9000', config.get('service', 'port'))
self.assertEqual('False', config.get('service', 'development'))
self.assertEqual('http://*****:*****@localhost/fluiddb-test',
config.get('store', 'main-uri'))
self.assertEqual('127.0.0.1', config.get('cache', 'host'))
self.assertEqual(6379, config.getint('cache', 'port'))
self.assertEqual(0, config.getint('cache', 'db'))
self.assertEqual(3600, config.getint('cache', 'expire-timeout'))
self.assertEqual('testaccesssecret',
config.get('oauth', 'access-secret'))
self.assertEqual('testrenewalsecrt',
config.get('oauth', 'renewal-secret'))
self.assertEqual('24', config.get('oauth', 'renewal-token-duration'))
self.assertTrue(config.getboolean('comments', 'extract-atnames'))
self.assertTrue(config.getboolean('comments', 'extract-hashtags'))
self.assertTrue(config.getboolean('comments', 'extract-plustags'))
self.assertTrue(config.getboolean('comments', 'extract-urls'))
self.assertTrue(config.getboolean('comments', 'extract-files'))
self.assertEqual(':files:', config.get('comments', 'file-object'))
def testSetupConfigWithCustomPort(self):
"""
L{setupConfig} can override the port defined in the configuration
file.
"""
config = setupConfig(self.path, port=9002)
self.assertEqual('9002', config.get('service', 'port'))
def run(self,
database_uri,
username,
email=None,
fullname=None,
password=None,
role=None):
if not any([email, fullname, password, role]):
print(
'You must provide an email, name, password, or role '
'to update.')
return
if role is not None:
try:
role = Role.fromName(role)
except LookupError:
print 'Invalid role'
return
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
updateUser(username, password, fullname, email, role)
def run(self, comment_url, database_uri=None):
"""Delete a comment, given its loveme.do URL.
@param comment_url: A loveme.do comment URL, in the form
http://loveme.do/comment/fluidinfo.com/terrycojones/
2012-08-03T22:04:13.698896
"""
database_uri = database_uri or 'postgres:///fluidinfo'
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
try:
importer, username, timestamp = parseCommentURL(comment_url)
except ValueError as error:
self._error(str(error))
# Delete the comment & report status.
count = CommentAPI(None).delete(importer, username, timestamp)
if count == 0:
self._error('Comment not found.')
elif count == 1:
transaction.commit()
print >> self.outf, 'Comment deleted.'
sys.exit(0)
else:
self._error('Unexpected return result (%r) trying to delete '
'comment.' % count)
def run(self, comment_url, database_uri=None):
"""Delete a comment, given its loveme.do URL.
@param comment_url: A loveme.do comment URL, in the form
http://loveme.do/comment/fluidinfo.com/terrycojones/
2012-08-03T22:04:13.698896
"""
database_uri = database_uri or 'postgres:///fluidinfo'
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
try:
importer, username, timestamp = parseCommentURL(comment_url)
except ValueError as error:
self._error(str(error))
# Delete the comment & report status.
count = CommentAPI(None).delete(importer, username, timestamp)
if count == 0:
self._error('Comment not found.')
elif count == 1:
transaction.commit()
print >> self.outf, 'Comment deleted.'
sys.exit(0)
else:
self._error('Unexpected return result (%r) trying to delete '
'comment.' % count)
def testSetupCacheSetsGlobalValue(self):
"""
L{setupCache} makes the new L{redis.ConnectionPool} available via the
global L{fluiddb.application.getCacheConnectionPool} function.
"""
config = setupConfig(None)
connectionPool = setupCache(config)
self.assertIdentical(connectionPool, getCacheConnectionPool())
def testSetupCacheSetsGlobalValue(self):
"""
L{setupCache} makes the new L{redis.ConnectionPool} available via the
global L{fluiddb.application.getCacheConnectionPool} function.
"""
config = setupConfig(None)
connectionPool = setupCache(config)
self.assertIdentical(connectionPool, getCacheConnectionPool())
def run(self, database_uri, version):
version = version.decode('utf-8')
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
setVersionTag(version)
def run(self, database_uri, version):
version = version.decode('utf-8')
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
setVersionTag(version)
def make(self, dependency_resources):
"""Setup the configuration instance."""
from fluiddb.application import setupConfig
config = setupConfig(path=None, port=None, development=True)
self._originalConfig = getConfig()
setConfig(config)
return config
def run(self, database_uri, max_rows_per_query=None):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
logging.info('Checking database:')
if max_rows_per_query:
checkIntegrity(max_rows_per_query)
else:
checkIntegrity()
def run(self, db=None):
if db is None:
db = 0
config = setupConfig(None)
config.set('cache', 'db', db)
setConfig(config)
setupCache(config)
redisClient = getCacheClient()
redisClient.flushdb()
def run(self, database_uri, index_url, username):
username = username.decode('utf-8')
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
config.set('index', 'url', str(index_url))
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
deleteUser(username)
def run(self, database_uri, max_rows_per_query=None):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
logging.info('Checking database:')
if max_rows_per_query:
checkIntegrity(max_rows_per_query)
else:
checkIntegrity()
def run(self, database_uri, index_url, username):
username = username.decode('utf-8')
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
config.set('index', 'url', str(index_url))
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
deleteUser(username)
def run(self, db=None):
if db is None:
db = 0
config = setupConfig(None)
config.set('cache', 'db', db)
setConfig(config)
setupCache(config)
redisClient = getCacheClient()
redisClient.flushdb()
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
store = setupStore(database_uri, 'main')
schema = main.createSchema()
logging.info('Creating schema.')
patchDatabase(store, schema)
logging.info('Creating system data.')
bootstrapSystemData(store)
logging.info('Creating web admin data.')
bootstrapWebAdminData()
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
store = setupStore(database_uri, 'main')
schema = main.createSchema()
logging.info('Creating schema.')
patchDatabase(store, schema)
logging.info('Creating system data.')
bootstrapSystemData(store)
logging.info('Creating web admin data.')
bootstrapWebAdminData()
def testSetupCache(self):
"""
L{setupCache} creates a new L{redis.ConnectionPool} with the defined
configuration.
"""
config = setupConfig(None)
connectionPool = setupCache(config)
expected = {
'host': config.get('cache', 'host'),
'port': config.getint('cache', 'port'),
'db': config.getint('cache', 'db')
}
self.assertEqual(expected, connectionPool.connection_kwargs)
def testSetupCache(self):
"""
L{setupCache} creates a new L{redis.ConnectionPool} with the defined
configuration.
"""
config = setupConfig(None)
connectionPool = setupCache(config)
expected = {
'host': config.get('cache', 'host'),
'port': config.getint('cache', 'port'),
'db': config.getint('cache', 'db')
}
self.assertEqual(expected, connectionPool.connection_kwargs)
def run(self, path_list, username=None, database_uri=None,
batch_size=None):
if database_uri is None:
database_uri = 'postgres:///fluidinfo'
if batch_size is None:
batch_size = 100
if username is None:
print >> self.outf, 'You must provide a username.'
sys.exit(1)
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
client = DatasetImporter(batch_size)
for filename in path_list:
with open(filename, 'r') as file:
data = load(file)
client.upload(username, data['objects'])
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
store = setupStore(database_uri, 'main')
schema = main.createSchema()
status = getPatchStatus(store, schema)
if status.unknownPatches:
unknownPatches = ', '.join(
'patch_%d' % version for version in status.unknownPatches)
logging.critical('Database has unknown patches: %s',
unknownPatches)
return 1
if status.unappliedPatches:
unappliedPatches = ', '.join(
'patch_%d' % version for version in status.unappliedPatches)
logging.info('Applying patches: %s', unappliedPatches)
patchDatabase(store, schema)
else:
logging.info('Database is up-to-date.')
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
store = setupStore(database_uri, 'main')
schema = main.createSchema()
status = getPatchStatus(store, schema)
if status.unknownPatches:
unknownPatches = ', '.join('patch_%d' % version
for version in status.unknownPatches)
logging.critical('Database has unknown patches: %s',
unknownPatches)
return 1
if status.unappliedPatches:
unappliedPatches = ', '.join(
'patch_%d' % version for version in status.unappliedPatches)
logging.info('Applying patches: %s', unappliedPatches)
patchDatabase(store, schema)
else:
logging.info('Database is up-to-date.')
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
store = setupStore(database_uri, 'main')
schema = main.createSchema()
status = getPatchStatus(store, schema)
if status.unappliedPatches:
patches = ', '.join('patch_%d' % version
for version in status.unappliedPatches)
logging.info('Unapplied patches: %s' % patches)
sys.exit(1)
if status.unknownPatches:
patches = ', '.join('patch_%d' % version
for version in status.unknownPatches)
logging.info('Unknown patches: %s' % patches)
sys.exit(2)
if not status.unappliedPatches and not status.unknownPatches:
logging.info('Database is up-to-date.')
sys.exit(0)
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
store = setupStore(database_uri, 'main')
schema = main.createSchema()
status = getPatchStatus(store, schema)
if status.unappliedPatches:
patches = ', '.join('patch_%d' % version
for version in status.unappliedPatches)
logging.info('Unapplied patches: %s' % patches)
sys.exit(1)
if status.unknownPatches:
patches = ', '.join('patch_%d' % version
for version in status.unknownPatches)
logging.info('Unknown patches: %s' % patches)
sys.exit(2)
if not status.unappliedPatches and not status.unknownPatches:
logging.info('Database is up-to-date.')
sys.exit(0)
def run(self, database_uri, username, password, fullname,
email, role=None):
username = username.decode('utf-8')
password = password.decode('utf-8')
fullname = '' if fullname is None else fullname.decode('utf-8')
email = '' if email is None else email.decode('utf-8')
roles = {'anonymous': Role.ANONYMOUS,
'superuser': Role.SUPERUSER,
'user': Role.USER,
'usermanager': Role.USER_MANAGER}
role = roles.get(role.lower() if role else 'user')
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
config.set('index', 'url', '')
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
createUser(username, password, fullname, email, role)
def run(self, database_uri, username, email=None, fullname=None,
password=None, role=None):
if not any([email, fullname, password, role]):
print ('You must provide an email, name, password, or role '
'to update.')
return
if role is not None:
try:
role = Role.fromName(role)
except LookupError:
print 'Invalid role'
return
config = setupConfig(None)
config.set('store', 'main-uri', database_uri)
setConfig(config)
setupLogging(self.outf)
setupStore(database_uri, 'main')
updateUser(username, password, fullname, email, role)
def run(self,
path_list,
username=None,
database_uri=None,
batch_size=None):
if database_uri is None:
database_uri = 'postgres:///fluidinfo'
if batch_size is None:
batch_size = 100
if username is None:
print >> self.outf, 'You must provide a username.'
sys.exit(1)
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
client = DatasetImporter(batch_size)
for filename in path_list:
with open(filename, 'r') as file:
data = load(file)
client.upload(username, data['objects'])
@param store: The store to use when fetching data.
@param tagIDs: A sequence of L{TagValue.tagID}s to match values against.
@param objectIDs: A sequence of L{TagValue.objectID}s to match values
against.
"""
result = store.find(And(TagValue.tagID.is_in(tagIDs),
TagValue.objectID.is_in(objectIDs))).remove()
if result:
touchObjects(objectIDs)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)8s %(message)s')
logging.info(__doc__)
store = setupStore('postgres:///fluidinfo', 'main')
config = setupConfig(None)
setConfig(config)
setupCache(config)
logging.info('Loading musicbrainz.org tag paths.')
tags = getTags(store)
totalObjects = getMusicbrainzObjectCount(store)
logging.info('Found %d objects to remove musicbrainz.org tag values from.',
totalObjects)
deletedObjects = 0
try:
for i in itertools.count(1):
objectIDs = getObjectIDs(store, tags)
if not objectIDs:
break
def run(self, database_uri, index_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
return buildIndex(str(index_uri))
def testSetupConfigWithCustomDevelopmentMode(self):
"""L{setupConfig} can override the development mode flag."""
config = setupConfig(self.path, development=True)
self.assertEqual('True', config.get('service', 'development'))
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
removeTestingData()
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
prepareForTesting()
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
removeTestingData()
def testSetupConfigWithCustomDevelopmentMode(self):
"""L{setupConfig} can override the development mode flag."""
config = setupConfig(self.path, development=True)
self.assertEqual('True', config.get('service', 'development'))
def run(self, index_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
return deleteIndex(str(index_uri))
def run(self, database_uri, filename, interval, max_objects):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
return batchIndex(filename, int(interval), int(max_objects))
@param objectIDs: A sequence of L{TagValue.objectID}s to match values
against.
"""
result = store.find(
And(TagValue.tagID.is_in(tagIDs),
TagValue.objectID.is_in(objectIDs))).remove()
if result:
touchObjects(objectIDs)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)8s %(message)s')
logging.info(__doc__)
store = setupStore('postgres:///fluidinfo', 'main')
config = setupConfig(None)
setConfig(config)
setupCache(config)
logging.info('Loading musicbrainz.org tag paths.')
tags = getTags(store)
totalObjects = getMusicbrainzObjectCount(store)
logging.info('Found %d objects to remove musicbrainz.org tag values from.',
totalObjects)
deletedObjects = 0
try:
for i in itertools.count(1):
objectIDs = getObjectIDs(store, tags)
if not objectIDs:
break
def run(self, database_uri, index_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
return buildIndex(str(index_uri))
def run(self, database_uri, index_uri, modified_since):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
modified_since = datetime.strptime(modified_since, '%Y-%m-%d')
return updateIndex(str(index_uri), modified_since)
def run(self, database_uri, filename, interval, max_objects):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
return batchIndex(filename, int(interval), int(max_objects))
def run(self, database_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
prepareForTesting()
def run(self, database_uri, index_uri, modified_since):
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
modified_since = datetime.strptime(modified_since, '%Y-%m-%d')
return updateIndex(str(index_uri), modified_since)
"""Creates fluiddb/users/role for all the users."""
from fluiddb.application import setConfig, setupConfig
from fluiddb.scripts.commands import setupStore
from fluiddb.data.user import getUsers
from fluiddb.model.user import getUser
from fluiddb.model.value import TagValueAPI
if __name__ == '__main__':
store = setupStore('postgres:///fluidinfo', 'main')
setConfig(setupConfig(None))
print __doc__
tagValues = TagValueAPI(getUser(u'fluiddb'))
for user in list(getUsers()):
print 'Adding role for', user.username
values = {user.objectID: {u'fluiddb/users/role': unicode(user.role)}}
tagValues.set(values)
store.commit()
def run(self, index_uri):
setConfig(setupConfig(None))
setupLogging(self.outf)
return deleteIndex(str(index_uri))
