class TestNamespaces(unittest.TestCase): def setUp(self): endpoint = os.environ.get('FLUIDDB_ENDPOINT', 'http://localhost:9000') username = os.environ.get('FLUIDDB_ADMIN_USERNAME', 'fluiddb') password = os.environ.get('FLUIDDB_ADMIN_PASSWORD', 'secret') self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password) self.fluiddb.bind() def testPOSTNewNamespaceUnderUnicodeNamespace(self): """ There shouldn't be a problem creating a namespace under a parent whose name is expressed in non-ascii characters. See the following bug: https://bugs.edge.launchpad.net/fluiddb/+bug/762779 """ # Use the testuser1 root namespace. ns = Namespace('testuser1') # Create an interestingly named namespace quran = u'\ufe8e\ufee0\ufed7\ufead\ufe81' quranNS = ns.create_namespace(quran, 'For the purposes of testing') # Attempt to create a new namespace underneath newNS = False try: # just check it can be created newNS = quranNS.create_namespace('foo', 'This is a test') expectedPath = u'testuser1/' + quran + '/foo' self.assertEqual(expectedPath, newNS.path) finally: if newNS: newNS.delete() quranNS.delete()
def setUp(self): endpoint = os.environ.get('FLUIDDB_ENDPOINT', 'http://localhost:9000') username = os.environ.get('FLUIDDB_ADMIN_USERNAME', 'fluiddb') password = os.environ.get('FLUIDDB_ADMIN_PASSWORD', 'secret') self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password) self.fluiddb.bind()
class TestTags(unittest.TestCase): def setUp(self): endpoint = os.environ.get('FLUIDDB_ENDPOINT', 'http://localhost:9000') username = os.environ.get('FLUIDDB_ADMIN_USERNAME', 'fluiddb') password = os.environ.get('FLUIDDB_ADMIN_PASSWORD', 'secret') self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password) self.fluiddb.bind() def testPOSTNewTagUnderUnicodeNamespace(self): """ There shouldn't be a problem creating a tag under a namespace whose name is expressed in non-ascii characters. See the following bug: https://bugs.edge.launchpad.net/fluiddb/+bug/762779 """ # Use the testuser1 root namespace. ns = Namespace('testuser1') # Create an interestingly named namespace quran = u'\ufe8e\ufee0\ufed7\ufead\ufe81' quranNS = ns.create_namespace(quran, 'For the purposes of testing') # Attempt to create a new tag underneath newTag = False try: # just check it can be created newTag = quranNS.create_tag('foo', 'This is a test', False) expectedPath = u'testuser1/' + quran + '/foo' self.assertEqual(expectedPath, newTag.path) finally: if newTag: newTag.delete() quranNS.delete() def testPOSTNewUnicodeTagUnderUnicodeNamespace(self): """ There shouldn't be a problem creating a tag under a namespace whose name is expressed in non-ascii characters. See the following bug: https://bugs.edge.launchpad.net/fluiddb/+bug/762779 """ # Use the testuser1 root namespace. ns = Namespace('testuser1') # Create an interestingly named namespace quran = u'\ufe8e\ufee0\ufed7\ufead\ufe81' quranNS = ns.create_namespace(quran, 'For the purposes of testing') # Attempt to create a new tag underneath newTag = False try: # just check it can be created sura = u'\ufeb1\ufeed\ufead' newTag = quranNS.create_tag(sura, 'This is a test', False) expectedPath = u'testuser1/' + quran + '/' + sura self.assertEqual(expectedPath, newTag.path) finally: if newTag: newTag.delete() quranNS.delete()
def __init__(self): endpoint = os.environ['FLUIDDB_ENDPOINT'] username = os.environ['FLUIDDB_ADMIN_USERNAME'] password = os.environ['FLUIDDB_ADMIN_PASSWORD'] self.solrURL = os.environ['FLUIDDB_INDEXING_SERVER_URL'] self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password)
class TestRunner: def __init__(self): endpoint = os.environ['FLUIDDB_ENDPOINT'] username = os.environ['FLUIDDB_ADMIN_USERNAME'] password = os.environ['FLUIDDB_ADMIN_PASSWORD'] self.solrURL = os.environ['FLUIDDB_INDEXING_SERVER_URL'] self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password) def solrCommit(self): request = HTTPRequest() url = urljoin(self.solrURL, 'update') request.POST(url, '<commit />') def __call__(self): fluiddb = self.fluiddb testNamespace = 'fluiddb/testing' # CREATE objectId = objects.createRandomObject(fluiddb) aboutValue = about.createRandomObject(fluiddb) namespace = namespaces.createRandomNamespace(fluiddb, testNamespace) tag = tags.createRandomTag(fluiddb, testNamespace) self.solrCommit() #UPDATE objects.setRandomTagValueString(fluiddb, objectId, tag) about.setRandomTagValueString(fluiddb, aboutValue, tag) namespaces.updateNamespaceWithRandomDescription(fluiddb, namespace) tags.UpdateTagWithRandomDescription(fluiddb, tag) # READ objects.simpleEqualsQuery(fluiddb, aboutValue) objects.simpleMatchesQuery(fluiddb, aboutValue) objects.getObjectInfo(fluiddb, objectId) objects.getTagValue(fluiddb, objectId, tag) objects.hasTagValue(fluiddb, objectId, tag) about.getObjectInfo(fluiddb, aboutValue) about.getTagValue(fluiddb, aboutValue, tag) about.hasTagValue(fluiddb, aboutValue, tag) namespaces.getNamespaces(fluiddb, namespace) tags.getTagInfo(fluiddb, tag) #DELETE objects.deleteTagValue(fluiddb, objectId, tag) about.deleteTagValue(fluiddb, aboutValue, tag) namespaces.deleteNamespace(fluiddb, namespace) tags.deleteTag(fluiddb, tag)
def test_user(self): "Check if fluiddb user exists" fdb = Fluid(self.url) try: ret = fdb.__call__('GET', '/users/fluiddb') if ret[0] == 200: return (True, '%s instance is now reachable' % self.shortname.capitalize(), ret[1]['id']) except socket.error: return (False, '%s instance is unreachable' % self.shortname.capitalize()) except: pass return (False, 'Wrong thing happends on %s instance' % self.shortname)
def Login(username, password): instance = session.get('instance', 'fluiddb') flogin = Fluid(get_instance_url(instance, ssl=False)) flogin.login(username, password) try: flogin.namespaces[username].get() session['logged'] = True session['username'] = username session['password'] = password return {'success': True} except: return {'success': False}
def generateData(username, password, endpoint): """Worker function creates random data. Requests to create namespaces, tags and values are continuously generated. This function never returns. @param username: The username to connect as. @param password: The password to use. @param endpoint: The Fluidinfo API endpoint to make requests against. """ fluidinfo = Fluid(endpoint) fluidinfo.login(username, password) fluidinfo.bind() while True: try: generateNamespaceData(username) except StandardError, e: logQueue.put(('ERROR %s' % str(e), ()))
class TestPUT(unittest.TestCase): def setUp(self): endpoint = os.environ.get('FLUIDDB_ENDPOINT', 'http://localhost:9000') self.username = os.environ.get('FLUIDDB_ADMIN_USERNAME', 'fluiddb') password = os.environ.get('FLUIDDB_ADMIN_PASSWORD', 'secret') self.fluiddb = Fluid(endpoint) self.fluiddb.login(self.username, password) self.fluiddb.bind() def testChangesTakeEffect(self): """ When PUTting updated values for a user's password, email or full-name make sure the changes propogate to the database for later retrieval. """ newName = 'Kathpakalaxmikanthan' body = {'name': newName} response = self.fluiddb.db('PUT', ['users', 'testuser1'], body) userID = self.fluiddb.users['testuser1'].get().value['id'] response = self.fluiddb.objects[userID]['fluiddb/users/name'].get() self.assertEqual(newName, response.value)
class Clipboard(object): """ Get/set/clear a clipboard tag in Fluidinfo. """ _username = '******' # Your Fluidinfo username. _password = '******' # Your Fluidinfo password. _tag = _username + '/clipboard' def __init__(self): self._fdb = Fluid() self._fdb.login(self._username, self._password) def get(self): """ Return the value of the user's clipboard tag if any, else C{None}. """ try: result = self._fdb.values.get('has ' + self._tag, [self._tag]) except FluidError, e: print 'Error:', e.args[0].response raise else:
def __init__(self): self._fdb = Fluid() self._fdb.login(self._username, self._password)
def __init__(self): Fluid.__init__(self, SANDBOX_URL)
# optionally create a logger - you don't need to do this but it's good to be # able to see what's been going on since flimp logs pretty much everything # it's doing. logger = logging.getLogger("flimp") logger.setLevel(logging.DEBUG) logfile_handler = logging.FileHandler('flimp.log') logfile_handler.setLevel(logging.DEBUG) log_format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - "\ "%(message)s") logfile_handler.setFormatter(log_format) logger.addHandler(logfile_handler) # Use fom to create a session with FluidDB - remember flimp uses fom for # connecting to FluidDB fdb = Fluid('https://sandbox.fluidinfo.com') # we'll use the sandbox fdb.login('test', 'test') # replace these with something that works fdb.bind() # Values to pass into the "process" function data_list = [ { 'foo': 'a', 'bar': 'b', 'baz': 'c' }, { 'foo': 'x', 'bar': 'y', 'baz': 'z' },
default=False, help='If True, open the current URL (default %default).') args, opt = opts.parse_args() # Check that any custom URL suffix is legal as a Fluidinfo tag name. if args.custom: if not re.match('^[\:\.\-\w/]+$', args.custom, re.UNICODE): print >>sys.stderr, ('Custom suffixes can only contain letters, ' 'digits, dot, hyphen, colon, and slash.') sys.exit(1) args.custom = '-' + args.custom.replace('/', '-') # Get an instance of fom.session.Fluid and provide it with credentials, # if we have been given a password to use. fdb = Fluid() if args.password: if not args.user: print >>sys.stderr, 'Please use --user USERNAME.' sys.exit(1) if not args.password: print >>sys.stderr, 'Please use --password PASSWORD.' sys.exit(1) fdb.login(args.user, args.password) # The Fluidinfo tag that will be acted on. tag = '%s/lastpage%s' % (args.user, args.custom) # Prepend 'http://' if it looks like we can be helpful. if args.url and not args.url.startswith('http'): args.url = 'http://' + args.url
from fom.session import Fluid from json import loads import os import sys if len(sys.argv) != 1: print >> sys.stderr, 'Usage: %s < manifest.json' % sys.argv[0] sys.exit(1) APP_ID = 'nknjamdijihneiclbpmcmfklakjkgdpf' codebase = ( 'https://fluiddb.fluidinfo.com/about/tabcast/fluidinfo.com/chrome.crx') version = loads(sys.stdin.read())['version'] fdb = Fluid('https://fluiddb.fluidinfo.com') password = os.environ['FLUIDDB_FLUIDINFO_DOT_COM_PASSWORD'] fdb.login('fluidinfo.com', password) about = 'tabcast' tag = 'fluidinfo.com/chrome-update.xml' data = '''<?xml version='1.0' encoding='UTF-8'?> <gupdate xmlns='http://www.google.com/update2/response' protocol='2.0'> <app appid='%(id)s'> <updatecheck codebase='%(codebase)s' version='%(version)s' /> </app> </gupdate> ''' % { 'codebase': codebase, 'id': APP_ID,
def setUp(self): # sort out FluidDB fdb = Fluid('https://sandbox.fluidinfo.com') fdb.bind() fdb.login('test', 'test')
#!/usr/bin/env python from json import dumps import os from fom.session import Fluid fdb = Fluid() fdb.login('telegraph.co.uk', os.environ['FLUDINFO_TELEGRAPH_PASSWORD']) datasets = [ { 'about': [ 'abu qatada', 'http://www.guardian.co.uk/commentisfree/2012/apr/17/abu-qatada-taking-the-righton-line', 'http://www.guardian.co.uk/commentisfree/2012/apr/17/abu-qatada-taking-the-righton-line?INTCMP=SRCH', 'http://www.guardian.co.uk/commentisfree/cartoon/2012/apr/17/abu-qatada-uk-security-cartoon', 'http://www.guardian.co.uk/commentisfree/cartoon/2012/apr/17/abu-qatada-uk-security-cartoon?INTCMP=SRCH', 'http://www.guardian.co.uk/commentisfree/cartoon/2012/apr/19/steve-bell-may-abu-qatada', 'http://www.guardian.co.uk/commentisfree/cartoon/2012/apr/19/steve-bell-may-abu-qatada?INTCMP=SRCH', 'http://www.guardian.co.uk/law/2012/apr/19/abu-qatada-deadline-appeal-strasbourg', 'http://www.guardian.co.uk/law/2012/apr/19/abu-qatada-deadline-appeal-strasbourg?INTCMP=SRCH', 'http://www.guardian.co.uk/law/2012/apr/19/brighton-abu-qatada-strasbourg', 'http://www.guardian.co.uk/law/2012/apr/19/brighton-abu-qatada-strasbourg?INTCMP=SRCH', 'http://www.guardian.co.uk/politics/2012/apr/19/theresa-may-sketch-abu-qatada', 'http://www.guardian.co.uk/politics/2012/apr/19/theresa-may-sketch-abu-qatada?INTCMP=SRCH', 'http://www.guardian.co.uk/politics/2012/apr/23/lords-reform-cameron-downplays-referendum', 'http://www.guardian.co.uk/politics/2012/apr/23/lords-reform-cameron-downplays-referendum?INTCMP=SRCH', 'http://www.guardian.co.uk/world/2009/feb/19/abu-qatada-profile', 'http://www.guardian.co.uk/world/2009/feb/19/abu-qatada-profile?INTCMP=SRCH', 'http://www.guardian.co.uk/world/2012/apr/17/abu-qatada-to-be-deported',
def execute(): """ Grab a bunch of args from the command line, verify and get the show on the road """ parser = OptionParser(version="%prog " + flimp.VERSION) parser.add_option('-f', '--file', dest='filename', help='The FILE to process (valid filetypes: %s)' % ', '.join(VALID_FILETYPES.keys()), metavar="FILE") parser.add_option('-d', '--dir', dest='directory', help="The root directory for a filesystem import into"\ " FluidDB") parser.add_option('-u', '--uuid', dest="uuid", default="", help="The uuid of the object to which the filesystem"\ " import is to attach its tags") parser.add_option('-a', '--about', dest="about", default="", help="The about value of the object to which the"\ " filesystem import is to attach its tags") parser.add_option('-p', '--preview', action="store_true", dest="preview", help="Show a preview of what will happen, don't import"\ " anything", default=False) parser.add_option('-i', '--instance', dest='instance', default="https://fluiddb.fluidinfo.com", help="The URI for the instance of FluidDB to use") parser.add_option('-l', '--log', dest='log', default="flimp.log", help="The log file to write to (defaults to flimp.log)") parser.add_option('-v', '--verbose', dest='verbose', default=False, action="store_true", help="Display status messages to"\ " console") parser.add_option('-c', '--check', dest='check', default=False, action="store_true", help="Validate the data file"\ " containing the data to import into FluidDB - don't"\ " import anything") options, args = parser.parse_args() # Some options validation if not (options.filename or options.directory): parser.error("You must supply either a source file or root directory"\ " to import.") if options.filename and options.directory: parser.error("You may only supply either a source file OR root"\ " directory to import (not both).") if options.uuid and options.about: parser.error("You may only supply either an object's uuid OR its"\ " about tag value (not both).") # Setup logging properly logger = logging.getLogger("flimp") logger.setLevel(logging.DEBUG) logfile_handler = logging.FileHandler(options.log) logfile_handler.setLevel(logging.DEBUG) log_format = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s") logfile_handler.setFormatter(log_format) logger.addHandler(logfile_handler) # verbose..? if options.verbose: # console handler ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(log_format) logger.addHandler(ch) if options.check: # No need to get information from the user if we're just validating # the file. A bit hacky! username = password = root_path = name = desc = about = "" else: # In the same way that sphinx interrogates the user using q&a we need to # assemble some more data that is probably not so easy to grab from the # arguments of the command username = get_argument('FluidDB username') password = get_argument('FluidDB password', password=True) root_path = get_argument('Absolute Namespace path (under which imported'\ ' namespaces and tags will be created)') if options.filename: name = get_argument( 'Name of dataset (defaults to filename)', os.path.basename(options.filename).split('.')[0]) about = get_argument('Key field for about tag value (if none given,'\ ' will use anonymous objects)', required=False) else: name = get_argument('Name of dataset') desc = get_argument('Description of the dataset') # Dump the recently collected information into the log file logger.info('FluidDB instance: %r' % options.instance) logger.info('Username: %r' % username) logger.info('Absolute Namespace path: %r' % root_path) logger.info('Dataset name: %r' % name) logger.info('Dataset description: %r' % desc) # Log into FluidDB fdb = Fluid(options.instance) fdb.bind() fdb.login(username, password) # Process the file or directory try: print "Working... (this might take some time, why not: tail -f the"\ " log?)" if options.filename: msg = process_file(options.filename, root_path, name, desc, about, options.preview, options.check) logger.info(msg) print msg else: obj = process_directory(options.directory, root_path, name, desc, options.uuid, options.about, options.preview) if obj: msg = 'Tags added to object with uuid: %s' % obj.uid logger.info(msg) print msg except Exception, e: # We want to catch all exceptions so we can log them nicely ex_type, ex_val, ex_trace = sys.exc_info() for msg in format_exception(ex_type, ex_val, ex_trace): logger.critical(msg) # this will be handled nicely by the try of last resort in the # flimp command line tool raise e
from fom.session import Fluid from json import loads import os import sys if len(sys.argv) != 1: print >>sys.stderr, 'Usage: %s < manifest.json' % sys.argv[0] sys.exit(1) APP_ID = 'kaagoiiakipocenbkhmcmbnldkbphcbn' codebase = ( 'https://fluiddb.fluidinfo.com/about/open-it-later/fluidinfo.com/chrome.crx') version = loads(sys.stdin.read())['version'] fdb = Fluid('https://fluiddb.fluidinfo.com') password = os.environ['FLUIDDB_FLUIDINFO_DOT_COM_PASSWORD'] fdb.login('fluidinfo.com', password) about = 'open-it-later' tag = 'fluidinfo.com/chrome-update.xml' data = '''<?xml version='1.0' encoding='UTF-8'?> <gupdate xmlns='http://www.google.com/update2/response' protocol='2.0'> <app appid='%(id)s'> <updatecheck codebase='%(codebase)s' version='%(version)s' /> </app> </gupdate> ''' % { 'codebase': codebase, 'id': APP_ID,
class TestGET(unittest.TestCase): def setUp(self): endpoint = os.environ.get('FLUIDDB_ENDPOINT', 'http://localhost:9000') username = os.environ.get('FLUIDDB_ADMIN_USERNAME', 'fluiddb') password = os.environ.get('FLUIDDB_ADMIN_PASSWORD', 'secret') self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password) self.fluiddb.bind() def testQueryWithUnicodeTagName(self): """ Make sure that a query using a tag whose name contains non-ASCII unicode characters works correctly. See https://bugs.edge.launchpad.net/fluiddb/+bug/681354 for the reason this is tested. """ # Use the testuser1 root namespace. ns = Namespace('testuser1') # Umlauts FTW! tag_name = u'C\xfc\xe4h' # Create the tag. tag = ns.create_tag(tag_name, 'Used for testing purposes', False) try: # Run a query that uses the tag. If we haven't fixed the bug, # FluidDB will hang at this point. result = Object.filter('has testuser1/%s' % tag_name) # Check the result is an empty list (i.e., no results found). self.assertEqual([], result) finally: # Clean up the tag. tag.delete() def testTagValueTypeHeaderInt(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the int type. """ # Create and object and add one tag. id = self.fluiddb.objects.post().value['id'] self.fluiddb.objects[id]['fluiddb/testing/test1'].put(1) try: response = self.fluiddb.objects[id]['fluiddb/testing/test1'].get() self.assertEqual('int', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.objects[id]['fluiddb/testing/test1'].head() self.assertEqual('int', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.objects[id]['fluiddb/testing/test1'].delete() def testTagValueTypeHeaderFloat(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the float type. """ # Create and object and add one tag. id = self.fluiddb.objects.post().value['id'] self.fluiddb.objects[id]['fluiddb/testing/test1'].put(1.1) try: response = self.fluiddb.objects[id]['fluiddb/testing/test1'].get() self.assertEqual('float', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.objects[id]['fluiddb/testing/test1'].head() self.assertEqual('float', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.objects[id]['fluiddb/testing/test1'].delete() def testTagValueTypeHeaderString(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the string type. """ # Create and object and add one tag. id = self.fluiddb.objects.post().value['id'] self.fluiddb.objects[id]['fluiddb/testing/test1'].put('hello') try: response = self.fluiddb.objects[id]['fluiddb/testing/test1'].get() self.assertEqual('string', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.objects[id]['fluiddb/testing/test1'].head() self.assertEqual('string', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.objects[id]['fluiddb/testing/test1'].delete() def testTagValueTypeHeaderBool(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the bool type. """ # Create and object and add one tag. id = self.fluiddb.objects.post().value['id'] self.fluiddb.objects[id]['fluiddb/testing/test1'].put(True) try: response = self.fluiddb.objects[id]['fluiddb/testing/test1'].get() self.assertEqual('boolean', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.objects[id]['fluiddb/testing/test1'].head() self.assertEqual('boolean', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.objects[id]['fluiddb/testing/test1'].delete() def testTagValueTypeHeaderNull(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the none type. """ # Create and object and add one tag. id = self.fluiddb.objects.post().value['id'] self.fluiddb.objects[id]['fluiddb/testing/test1'].put(None) try: response = self.fluiddb.objects[id]['fluiddb/testing/test1'].get() self.assertEqual('null', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.objects[id]['fluiddb/testing/test1'].head() self.assertEqual('null', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.objects[id]['fluiddb/testing/test1'].delete() def testTagValueTypeHeaderList(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the set type. """ # Create and object and add one tag. id = self.fluiddb.objects.post().value['id'] self.fluiddb.objects[id]['fluiddb/testing/test1'].put(['one', 'two']) try: response = self.fluiddb.objects[id]['fluiddb/testing/test1'].get() self.assertEqual('list-of-strings', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.objects[id]['fluiddb/testing/test1'].head() self.assertEqual('list-of-strings', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.objects[id]['fluiddb/testing/test1'].delete()
def execute(): """ Grab a bunch of args from the command line, verify and get the show on the road """ parser = OptionParser(version="%prog " + flimp.VERSION) parser.add_option('-f', '--file', dest='filename', help='The FILE to process (valid filetypes: %s)' % ', '.join(VALID_FILETYPES.keys()), metavar="FILE") parser.add_option('-d', '--dir', dest='directory', help="The root directory for a filesystem import into"\ " FluidDB") parser.add_option('-u', '--uuid', dest="uuid", default="", help="The uuid of the object to which the filesystem"\ " import is to attach its tags") parser.add_option('-a', '--about', dest="about", default="", help="The about value of the object to which the"\ " filesystem import is to attach its tags") parser.add_option('-p', '--preview', action="store_true", dest="preview", help="Show a preview of what will happen, don't import"\ " anything", default=False) parser.add_option('-i', '--instance', dest='instance', default="https://fluiddb.fluidinfo.com", help="The URI for the instance of FluidDB to use") parser.add_option('-l', '--log', dest='log', default="flimp.log", help="The log file to write to (defaults to flimp.log)") parser.add_option('-v', '--verbose', dest='verbose', default=False, action="store_true", help="Display status messages to"\ " console") parser.add_option('-c', '--check', dest='check', default=False, action="store_true", help="Validate the data file"\ " containing the data to import into FluidDB - don't"\ " import anything") options, args = parser.parse_args() # Some options validation if not (options.filename or options.directory): parser.error("You must supply either a source file or root directory"\ " to import.") if options.filename and options.directory: parser.error("You may only supply either a source file OR root"\ " directory to import (not both).") if options.uuid and options.about: parser.error("You may only supply either an object's uuid OR its"\ " about tag value (not both).") # Setup logging properly logger = logging.getLogger("flimp") logger.setLevel(logging.DEBUG) logfile_handler = logging.FileHandler(options.log) logfile_handler.setLevel(logging.DEBUG) log_format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") logfile_handler.setFormatter(log_format) logger.addHandler(logfile_handler) # verbose..? if options.verbose: # console handler ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(log_format) logger.addHandler(ch) if options.check: # No need to get information from the user if we're just validating # the file. A bit hacky! username = password = root_path = name = desc = about = "" else: # In the same way that sphinx interrogates the user using q&a we need to # assemble some more data that is probably not so easy to grab from the # arguments of the command username = get_argument('FluidDB username') password = get_argument('FluidDB password', password=True) root_path = get_argument('Absolute Namespace path (under which imported'\ ' namespaces and tags will be created)') if options.filename: name = get_argument('Name of dataset (defaults to filename)', os.path.basename(options.filename).split('.')[0]) about = get_argument('Key field for about tag value (if none given,'\ ' will use anonymous objects)', required=False) else: name = get_argument('Name of dataset') desc = get_argument('Description of the dataset') # Dump the recently collected information into the log file logger.info('FluidDB instance: %r' % options.instance) logger.info('Username: %r' % username) logger.info('Absolute Namespace path: %r' % root_path) logger.info('Dataset name: %r' % name) logger.info('Dataset description: %r' % desc) # Log into FluidDB fdb = Fluid(options.instance) fdb.bind() fdb.login(username, password) # Process the file or directory try: print "Working... (this might take some time, why not: tail -f the"\ " log?)" if options.filename: msg = process_file(options.filename, root_path, name, desc, about, options.preview, options.check) logger.info(msg) print msg else: obj = process_directory(options.directory, root_path, name, desc, options.uuid, options.about, options.preview) if obj: msg = 'Tags added to object with uuid: %s' % obj.uid logger.info(msg) print msg except Exception, e: # We want to catch all exceptions so we can log them nicely ex_type, ex_val, ex_trace = sys.exc_info() for msg in format_exception(ex_type, ex_val, ex_trace): logger.critical(msg) # this will be handled nicely by the try of last resort in the # flimp command line tool raise e
# optionally create a logger - you don't need to do this but it's good to be # able to see what's been going on since flimp logs pretty much everything # it's doing. logger = logging.getLogger("flimp") logger.setLevel(logging.DEBUG) logfile_handler = logging.FileHandler('flimp.log') logfile_handler.setLevel(logging.DEBUG) log_format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - "\ "%(message)s") logfile_handler.setFormatter(log_format) logger.addHandler(logfile_handler) # Use fom to create a session with FluidDB - remember flimp uses fom for # connecting to FluidDB fdb = Fluid('https://sandbox.fluidinfo.com') # we'll use the sandbox fdb.login('test', 'test') # replace these with something that works fdb.bind() # Values to pass into the "process" function root_dir = 'test_dir' # the path to the directory you want to import root_path = 'test/foo' # the namespace in FluidDB that maps to root_dir name = 'dataset_name' # the name of the filesystem data to be imported desc = 'Plain English dataset description' # exactly what it says uuid = None # the uuid of the object to tag about = None # the about value of the object to tag preview = False # True will cause flimp to print out the preview # Make magic happen... obj = process(root_dir, root_path, name, desc, uuid, about, preview) # obj is a fom.mapping.Object instance representing the FluidDB object that
for sAbout, dictTags in dictObjects.items(): print "Commiting:", sAbout fdb.values.put(query='fluiddb/about = "' + sAbout + '"', values=dictTags) if __name__ == "__main__": ############################# # Bind to FluidInfo instance fileCredentials = open(os.path.expanduser('~/.fluidDBcredentials'), 'r') username = fileCredentials.readline().strip() password = fileCredentials.readline().strip() fileCredentials.close() # fdb = Fluid('https://sandbox.fluidinfo.com') # The sandbox instance fdb = Fluid() # The main instance fdb.login(username, password) fdb.bind() nsUser = Namespace(username) sUserNS = nsUser.path # Ugly use of a global, I know. :-) dictObjects = dict() fRequest = urllib2.urlopen(urlISOcodeListing) # Get enconding of raw listing. # This should usually be UTF-8, but it could change and cause a big headache! # sEncoding = fRequest.headers['content-type'].split('charset=')[-1] # ^^^^This is not working, since http://loc.org is not specifying the encoding. Dammit! # fRequest.headers['content-type'] = 'text/plain'
#!/usr/bin/env python from fom.session import Fluid import os import sys if len(sys.argv) != 2: print >> sys.stderr, "Usage: %s extension-file.crx" % sys.argv[0] sys.exit(1) crxFile = sys.argv[1] data = open(crxFile).read() fdb = Fluid("https://fluiddb.fluidinfo.com") password = os.environ["FLUIDDB_FLUIDINFO_DOT_COM_PASSWORD"] fdb.login("fluidinfo.com", password) about = "open-it-later" tag = "fluidinfo.com/chrome.crx" fdb.about[about][tag].put(data, "application/x-chrome-extension") print "Uploaded %s to https://fluiddb.fluidinfo.com/about/%s/%s" % (crxFile, about, tag)
""" for sAbout, dictTags in dictObjects.items(): print "Commiting:",sAbout fdb.values.put( query='fluiddb/about = "'+sAbout+'"',values=dictTags) if __name__ == "__main__": ############################# # Bind to FluidInfo instance fileCredentials = open(os.path.expanduser('~/.fluidDBcredentials'), 'r') username = fileCredentials.readline().strip() password = fileCredentials.readline().strip() fileCredentials.close() # fdb = Fluid('https://sandbox.fluidinfo.com') # The sandbox instance fdb = Fluid() # The main instance fdb.login(username, password) fdb.bind() nsUser = Namespace(username) sUserNS = nsUser.path # Ugly use of a global, I know. :-) dictObjects = dict() fRequest = urllib2.urlopen(urlISOcodeListing) # Get enconding of raw listing. # This should usually be UTF-8, but it could change and cause a big headache! # sEncoding = fRequest.headers['content-type'].split('charset=')[-1] # ^^^^This is not working, since http://loc.org is not specifying the encoding. Dammit! # fRequest.headers['content-type'] = 'text/plain'
def run(self, host, username=None, password=None): if username is None: username = u'fluiddb' if password is None: password = u'secret' if host.find('://') == -1: host = 'http://' + host print 'Logging to', host fluiddb = Fluid(host) fluiddb.login(username, password) print 'Creating an object...', try: result = fluiddb.objects.post(u'test-object') objectID = result.value[u'id'] assert result.status == 201 except: print >>sys.stderr, 'FAILED' print_exc() else: print 'OK' tagname = 'test' + str(datetime.now().toordinal()) print 'Creating a tag...', try: result = fluiddb.tags[username].post(tagname, None, True) assert result.status == 201 except: print >>sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Creating a tag value...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.about['test-object'][path].put(200) assert result.status == 204 except: print >>sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Giving time to the DIH (2 minutes)...' time.sleep(120) print 'Making a solr query...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.objects.get('{0} = 200'.format(path)) print result.value[u'ids'] assert result.value[u'ids'] == [objectID] except: print >>sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Removing a tag value...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.about['test-object'][path].delete() assert result.status == 204 except: print >>sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Removing a tag...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.tags[path].delete() assert result.status == 204 except: print >>sys.stderr, 'FAILED' print_exc() else: print 'OK'
class TestGETHEAD(unittest.TestCase): def setUp(self): endpoint = os.environ.get('FLUIDDB_ENDPOINT', 'http://localhost:9000') username = os.environ.get('FLUIDDB_ADMIN_USERNAME', 'fluiddb') password = os.environ.get('FLUIDDB_ADMIN_PASSWORD', 'secret') self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password) self.fluiddb.bind() def testTagValueTypeHeaderFloat(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the float type. """ # Create and object and add one tag. self.fluiddb.about.post('about').value['id'] path = 'fluiddb/testing/test1' self.fluiddb.about['about'][path].put(1.1) try: response = self.fluiddb.about['about'][path].get() self.assertEqual('float', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.about['about'][path].head() self.assertEqual('float', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.about['about'][path].delete() def testTagValueTypeHeaderString(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the string type. """ # Create and object and add one tag. self.fluiddb.about.post('about').value['id'] path = 'fluiddb/testing/test1' self.fluiddb.about['about'][path].put('hello') try: response = self.fluiddb.about['about'][path].get() self.assertEqual('string', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.about['about'][path].head() self.assertEqual('string', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.about['about'][path].delete() def testTagValueTypeHeaderBool(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the bool type. """ # Create and object and add one tag. self.fluiddb.about.post('about').value['id'] path = 'fluiddb/testing/test1' self.fluiddb.about['about'][path].put(True) try: response = self.fluiddb.about['about'][path].get() self.assertEqual('boolean', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.about['about'][path].head() self.assertEqual('boolean', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.about['about'][path].delete() def testTagValueTypeHeaderNull(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the none type. """ # Create and object and add one tag. self.fluiddb.about.post('about').value['id'] path = 'fluiddb/testing/test1' self.fluiddb.about['about'][path].put(None) try: response = self.fluiddb.about['about'][path].get() self.assertEqual('null', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.about['about'][path].head() self.assertEqual('null', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.about['about'][path].delete() def testTagValueTypeHeaderList(self): """ When requesting a primitive tag value using a GET or HEAD request to /objects/id/namespace/tag, the response should put an X-FluidDB-Type header indicating the type of the response. This particular test checks header values for the set type. """ # Create and object and add one tag. self.fluiddb.about.post('about').value['id'] path = 'fluiddb/testing/test1' self.fluiddb.about['about'][path].put(['one', 'two']) try: response = self.fluiddb.about['about'][path].get() self.assertEqual('list-of-strings', response.response.headers.get('x-fluiddb-type')) response = self.fluiddb.about['about'][path].head() self.assertEqual('list-of-strings', response.response.headers.get('x-fluiddb-type')) finally: self.fluiddb.about['about'][path].delete()
from fom.errors import FluidError import csv import sys import time def checkCompatible(header): """Check that the input is compatible with expectations.""" assert header[1] == "ScreenName" assert header[8] == "RawTunkRankScore" assert header[9] == "TunkRankScore" assert header[10] == "TunkRankComputedAt" if __name__ == "__main__": fdb = Fluid() password = getpass("Enter the tunkrank.com Fluidinfo user password: "******"tunkrank.com", password) reader = csv.reader(sys.stdin) # Check that the first (header) line of input looks good. checkCompatible(reader.next()) totalTime = 0.0 for i, row in enumerate(reader): screenName = row[1] rawTunkRankScore = float(row[8]) tunkRankScore = float(row[9]) tunkRankComputedAt = time.mktime(parse(row[10]).utctimetuple())
class TestGET(unittest.TestCase): def setUp(self): endpoint = os.environ.get('FLUIDDB_ENDPOINT', 'http://localhost:9000') username = os.environ.get('FLUIDDB_ADMIN_USERNAME', 'fluiddb') password = os.environ.get('FLUIDDB_ADMIN_PASSWORD', 'secret') self.fluiddb = Fluid(endpoint) self.fluiddb.login(username, password) def test_resultsStructure(self): """ Verify the structure of the results object in the response of a GET request to /values. """ # Create a new object. response = self.fluiddb.objects.post('about') objectId = response.value['id'] path = 'fluiddb/testing/test1' value = 'value' try: # Add a new set tag to the object. self.fluiddb.objects[objectId][path].put(value) # Get the tag value using the values API. response = self.fluiddb.values.get('fluiddb/about = "about"', [path]) self.assertTrue('results' in response.value) results = response.value['results'] self.assertTrue('id' in results) ids = results['id'] self.assertTrue(objectId in ids) paths = ids[objectId] self.assertTrue(path in paths) tagInfo = paths[path] self.assertTrue("value" in tagInfo) finally: # Clean created tags. self.fluiddb.objects[objectId][path].delete() def test_setOfStrings(self): """ Check if the respose is correct when requesting a tag of type set using GET requests to /values. Bug #677215. """ # Create a new object. response = self.fluiddb.objects.post('about') objectId = response.value['id'] path = 'fluiddb/testing/test1' value = ['one', 'two', 'three'] try: # Add a new set tag to the object. self.fluiddb.objects[objectId][path].put(value) # Get the tag value using the values API. response = self.fluiddb.values.get('fluiddb/about = "about"', [path]) expected = response.value['results']['id'][objectId][path]['value'] self.assertEqual(set(expected), set(value)) finally: # Clean created tags. self.fluiddb.objects[objectId][path].delete()
#!/usr/bin/env python from json import dumps import os import sys from urllib import quote from fom.session import Fluid fdb = Fluid() fdb.login('lexisnexis.com', os.environ['FLUDINFO_LEXISNEXIS_PASSWORD']) TAG = 'lexisnexis.com/posts' datasets = [ { 'about': [ 'Audit', 'Confidentiality', 'Audit Commission Act 1998, s 15(1)', 'European Convention for the Protection of Human Rights and Fundamental Freedoms 1950', 'European Convention for the Protection of Human Rights and Fundamental Freedoms 1950, First Protocol, art 1.', 'RIX LJ', 'RIX LJJ', 'ETHERTON LJ', 'ETHERTON LJJ', 'JACKSON LJ', 'JACKSON LJJ', 'Philip Coppel QC', 'Clive Lewis QC', 'Timothy Pitt-Payne QC',
# See README.markdown for usage instructions. from datetime import datetime import json import os import sys import time from fom.session import Fluid from fom.errors import FluidError if __name__ == '__main__': password = os.environ['FLUIDINFO_PEERINDEX_PASSWORD'] assert password, 'Please set FLUIDINFO_PEERINDEX_PASSWORD in your env.' fdb = Fluid() fdb.login('peerindex.com', password) totalTime = 0.0 for i, line in enumerate(sys.stdin.readlines()): start = time.time() info = json.loads(line[:-1]) screenname = info['twitter'] updatedAt = time.mktime(datetime.utcnow().timetuple()) about = '@%s' % screenname.lower() values = { 'peerindex.com/updated-at': {'value': updatedAt}, } for var in ('activity', 'audience', 'authority', 'peerindex', 'realness', 'name', 'slug', 'url', 'topics'): try:
def run(self, host, username=None, password=None): if username is None: username = u'fluiddb' if password is None: password = u'secret' if host.find('://') == -1: host = 'http://' + host print 'Logging to', host fluiddb = Fluid(host) fluiddb.login(username, password) print 'Creating an object...', try: result = fluiddb.objects.post(u'test-object') objectID = result.value[u'id'] assert result.status == 201 except: print >> sys.stderr, 'FAILED' print_exc() else: print 'OK' tagname = 'test' + str(datetime.now().toordinal()) print 'Creating a tag...', try: result = fluiddb.tags[username].post(tagname, None, True) assert result.status == 201 except: print >> sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Creating a tag value...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.about['test-object'][path].put(200) assert result.status == 204 except: print >> sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Giving time to the DIH (2 minutes)...' time.sleep(120) print 'Making a solr query...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.objects.get('{0} = 200'.format(path)) print result.value[u'ids'] assert result.value[u'ids'] == [objectID] except: print >> sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Removing a tag value...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.about['test-object'][path].delete() assert result.status == 204 except: print >> sys.stderr, 'FAILED' print_exc() else: print 'OK' print 'Removing a tag...', try: path = '{0}/{1}'.format(username, tagname) result = fluiddb.tags[path].delete() assert result.status == 204 except: print >> sys.stderr, 'FAILED' print_exc() else: print 'OK'