def processDirectoryIntoNamespace(directory, last_update, credentials, object):
"""
Given a path to a directory on the local filesystem this method will
find/create an associated namespace on FluidDB and populate it with child
tags that map to the local files
"""
directory_path = directory[0]
logging.info('Checking directory %s'%directory_path)
# We ignore hidden directories
if directory_path.find('.DS_Store') >= 0 or directory_path.find('.git') >= 0:
logging.info('Ignoring GIT related directory')
return
elif directory_path == "python":
logging.info('Ignoring python directory')
return
elif directory_path.startswith('./') or directory_path == '.':
logging.info('OK')
else:
logging.info('Ignoring hidden directory')
return
# Lets build the path as it would appear in FluidDB
fluid_namespace_path = '/'.join((
'/namespaces', # namespaces root
credentials['username'], # user's root namespace
'fluidapp', # the namespace underneath which we store fluid applications
credentials['root_namespace'] # the "application" namespace
))
if len(directory_path)>2:
fluid_namespace_path = '/'.join((fluid_namespace_path, directory_path[2:]) # the path of the directory with ./ knocked off the front
)
logging.info('FluidDB namespace: %s'%fluid_namespace_path)
# Lets see what FluidDB responds with
state_of_namespace = fd.call('GET', fluid_namespace_path)
logging.info('State of namespace:')
logging.info(state_of_namespace)
if state_of_namespace[0]['status'] == '404':
# The namespace that represents this directory doesn't exist so lets
# create it...
logging.info('Creating new namespace')
parent_namespace = '/'.join(fluid_namespace_path.split('/')[:-1])
logging.info('Parent namespace: %s'%parent_namespace)
new_namespace_name = fluid_namespace_path.split('/')[-1:][0]
logging.info('New namespace name: %s'%new_namespace_name)
logging.info(fd.call('POST', parent_namespace,
{'description': u'A directory in the %s FluidApp'%credentials['root_namespace'],
'name': new_namespace_name}))
if state_of_namespace[0]['status'] == '404' or state_of_namespace[0]['status'] == '200':
# process the files
files = directory[2]
logging.info('Files in this directory:')
logging.info(files)
for file in files:
processFileIntoTag(file, directory_path,
fluid_namespace_path.replace('/namespaces','/tags'),
last_update, object, credentials)
else:
logging.error('Barfed! -----------------------------------')
return
logging.info('Finished! ----------------------------------')
def test_call_HEAD(self):
fluiddb.login(USERNAME, PASSWORD)
# Grab an object ID for a user for us to use in the HEAD path
result = fluiddb.call("GET", "/users/test")
obj_id = result[1]["id"]
path = "/objects/%s/fluiddb/users/username" % obj_id
result = fluiddb.call("HEAD", path)
self.assertEqual("200", result[0]["status"])
self.assertFalse(result[1]) # no response body with HEAD call
def processDirectoryIntoNamespace(directory, last_update, credentials, object):
"""
Given a path to a directory on the local filesystem this method will
find/create an associated namespace on FluidDB and populate it with child
tags that map to the local files
"""
directory_path = directory[0]
logging.info('Checking directory %s'%directory_path)
# We ignore hidden directories
if directory_path.find('.DS_Store') >= 0 or directory_path.find('.git') >= 0:
logging.info('Ignoring GIT related directory')
return
elif directory_path.startswith('./') or directory_path == '.':
logging.info('OK')
else:
logging.info('Ignoring hidden directory')
return
# Lets build the path as it would appear in FluidDB
fluid_namespace_path = '/'.join((
'/namespaces', # namespaces root
credentials['username'], # user's root namespace
'fluidapp', # the namespace underneath which we store fluid applications
credentials['root_namespace'] # the "application" namespace
))
if len(directory_path)>2:
fluid_namespace_path = '/'.join((fluid_namespace_path, directory_path[2:]) # the path of the directory with ./ knocked off the front
)
logging.info('FluidDB namespace: %s'%fluid_namespace_path)
# Lets see what FluidDB responds with
state_of_namespace = fd.call('GET', fluid_namespace_path)
logging.info('State of namespace:')
logging.info(state_of_namespace)
if state_of_namespace[0]['status'] == '404':
# The namespace that represents this directory doesn't exist so lets
# create it...
logging.info('Creating new namespace')
parent_namespace = '/'.join(fluid_namespace_path.split('/')[:-1])
logging.info('Parent namespace: %s'%parent_namespace)
new_namespace_name = fluid_namespace_path.split('/')[-1:][0]
logging.info('New namespace name: %s'%new_namespace_name)
logging.info(fd.call('POST', parent_namespace,
{'description': u'A directory in the %s FluidApp'%credentials['root_namespace'],
'name': new_namespace_name}))
if state_of_namespace[0]['status'] == '404' or state_of_namespace[0]['status'] == '200':
# process the files
files = directory[2]
logging.info('Files in this directory:')
logging.info(files)
for file in files:
processFileIntoTag(file, directory_path,
fluid_namespace_path.replace('/namespaces','/tags'),
last_update, object, credentials)
else:
logging.error('Barfed! -----------------------------------')
return
logging.info('Finished! ----------------------------------')
def test_call_POST(self):
fluiddb.login(USERNAME, PASSWORD)
new_namespace = str(uuid.uuid4())
ns_body = {"description": "a test namespace", "name": new_namespace}
# Make sure that if the body is a dict it gets translated to json
result = fluiddb.call("POST", "/namespaces/test", ns_body)
self.assertEqual("201", result[0]["status"])
self.assertTrue(result[1].has_key("id"))
# Housekeeping
fluiddb.call("DELETE", "/namespaces/test/" + new_namespace)
def test_call_DELETE(self):
fluiddb.login(USERNAME, PASSWORD)
# Simply create a new namespace and then delete it
new_namespace = str(uuid.uuid4())
body = {"description": "a test namespace", "name": new_namespace}
result = fluiddb.call("POST", "/namespaces/test", body)
self.assertEqual("201", result[0]["status"])
self.assertTrue(result[1].has_key("id"))
result = fluiddb.call("DELETE", "/namespaces/test/" + new_namespace)
self.assertEqual("204", result[0]["status"])
def test_logout(self):
# Lets first log in and check we're good to go
fluiddb.login(USERNAME, PASSWORD)
result = fluiddb.call("GET", "/users/test")
self.assertEqual("200", result[0]["status"])
# Log out (this should clear the Authorization header)
fluiddb.logout()
# We should still be able to do anonymous calls
result = fluiddb.call("GET", "/users/test")
self.assertEqual("200", result[0]["status"])
# but we can't do anything that requires us to be authenticated
new_namespace = str(uuid.uuid4())
result = fluiddb.call("POST", "/namespaces/test", {"description": "will fail", "name": new_namespace})
self.assertEqual("401", result[0]["status"])
def processFileIntoTag(file, directory_path, fluid_namespace_path, last_update,
object, credentials):
logging.info('Checking file: %s'%file)
if file.startswith('.') or file.endswith('.log') or file.endswith('.pyc') or file.endswith('.swp') or file == 'credentials.json':
logging.info('Ignoring log/hidden file...')
return
# do we need to process it (compare timestamps)?
file_stat = os.stat('/'.join((directory_path, file)))
if not file_stat.st_mtime > last_update:
logging.info('Unchanged since last push so ignoring...')
return
logging.info('Changed since last push')
# does the tag exist..?
tag_path = "/".join((fluid_namespace_path, file))
logging.info('Tag path in FluidDB: %s'%tag_path)
state_of_tag = fd.call('GET', tag_path)
logging.info(state_of_tag)
if state_of_tag[0]['status'] == '404':
# NO! so we need to create it
logging.info(fd.call('POST', fluid_namespace_path, {
'name': file,
'description': 'A tag in the %s FluidApp'%credentials['root_namespace'],
'indexed': True
}))
# Lets create tag value on the appropriate object. To start we need to work
# out the appropriate mime type (very hacky)
mime = 'application/octet-stream'
if file.endswith('.css'):
mime = 'text/css'
elif file.endswith('.js'):
mime = 'application/javascript'
elif file.endswith('.html'):
mime = 'text/html'
elif file.endswith('.png'):
mime = 'image/png'
elif file.endswith('.jpg'):
mime = 'image/jpg'
elif file.endswith('.gif'):
mime = 'image/gif'
elif file.endswith('.pdf'):
mime = 'application/pdf'
elif file.endswith('.ppt'):
mime = 'application/vnd.ms-powerpoint'
logging.info('Pushing file %s as a value for tag: %s with mime: %s'%(file, tag_path, mime))
body = open('/'.join((directory_path, file)), 'r')
logging.info(fd.call('PUT', ''.join([object, tag_path[5:]]), body.read(), mime))
body.close()
def processFileIntoTag(file, directory_path, fluid_namespace_path, last_update,
object, credentials):
logging.info('Checking file: %s'%file)
if file.startswith('.') or file.endswith('.log') or file.endswith('.pyc') or file.endswith('.swp') or file == 'credentials.json':
logging.info('Ignoring log/hidden file...')
return
# do we need to process it (compare timestamps)?
file_stat = os.stat('/'.join((directory_path, file)))
if not file_stat.st_mtime > last_update:
logging.info('Unchanged since last push so ignoring...')
return
logging.info('Changed since last push')
# does the tag exist..?
tag_path = "/".join((fluid_namespace_path, file))
logging.info('Tag path in FluidDB: %s'%tag_path)
state_of_tag = fd.call('GET', tag_path)
logging.info(state_of_tag)
if state_of_tag[0]['status'] == '404':
# NO! so we need to create it
logging.info(fd.call('POST', fluid_namespace_path, {
'name': file,
'description': 'A tag in the %s FluidApp'%credentials['root_namespace'],
'indexed': True
}))
# Lets create tag value on the appropriate object. To start we need to work
# out the appropriate mime type (very hacky)
mime = 'text/plain'
if file.endswith('.css'):
mime = 'text/css'
elif file.endswith('.js'):
mime = 'application/javascript'
elif file.endswith('.html'):
mime = 'text/html'
elif file.endswith('.png'):
mime = 'image/png'
elif file.endswith('.jpg'):
mime = 'image/jpg'
elif file.endswith('.gif'):
mime = 'image/gif'
elif file.endswith('.pdf'):
mime = 'application/pdf'
elif file.endswith('.ppt'):
mime = 'application/vnd.ms-powerpoint'
logging.info('Pushing file %s as a value for tag: %s with mime: %s'%(file, tag_path, mime))
body = open('/'.join((directory_path, file)), 'r')
logging.info(fd.call('PUT', ''.join([object, tag_path[5:]]), body.read(), mime))
body.close()
def test_login(self):
# we're not logged in but able to do anonymous calls
result = fluiddb.call("GET", "/users/test")
self.assertEqual("200", result[0]["status"])
new_namespace = str(uuid.uuid4())
# and we can't do anything that requires us to be authenticated
result = fluiddb.call("POST", "/namespaces/test", {"description": "will fail", "name": new_namespace})
self.assertEqual("401", result[0]["status"])
# Now lets log in with *bad* credentials
fluiddb.login(USERNAME, PASSWORD + "bad_password")
result = fluiddb.call("GET", "/users/test")
# Unauthorised due to bad credentials
self.assertEqual("401", result[0]["status"])
# Try again with the good case
fluiddb.login(USERNAME, PASSWORD)
result = fluiddb.call("GET", "/users/test")
self.assertEqual("200", result[0]["status"])
def test_put_about_type_header(self):
"""
There was a bug where the fluiddb.py wasn't creating the correct
content-type header when PUTting to an about tag value, this test
re-creates it.
"""
# ensures we have an object about foo
headers, response = fluiddb.call("GET", "/about/foo")
# create a one off tag to use for the purposes of testing
fluiddb.login(USERNAME, PASSWORD)
new_tag = str(uuid.uuid4())
tag_body = {"description": "a test tag", "name": new_tag, "indexed": False}
# create a tag to use in a bit
result = fluiddb.call("POST", "/tags/test", tag_body)
self.assertEqual("201", result[0]["status"])
self.assertTrue(result[1].has_key("id"))
# make sure we can PUT using the about API
try:
header, content = fluiddb.call("PUT", "/about/foo/test/" + new_tag, "this is a test")
# check that it worked
self.assertEqual("204", header["status"])
finally:
# Housekeeping
fluiddb.call("DELETE", "/tags/test/" + new_tag)
def test_fluiddb_call(self):
"""this test will test fluiddb call"""
status, result = fluiddb.call('GET', '/objects', query='fluiddb/users/username = "******"')
assert status == 200
assert result == {u'ids': [u'71d5aa6f-d5fa-4578-9301-411fd92b1727']}
def test_call_GET(self):
fluiddb.login(USERNAME, PASSWORD)
# No query string args to append
result = fluiddb.call("GET", "/namespaces/test")
self.assertEqual("200", result[0]["status"])
# make sure the resulting json is turned into a Python dictionary
self.assertTrue(isinstance(result[1], dict))
# ...and we have the expected id
self.assertTrue(result[1].has_key("id"))
# The same call WITH query string args to append to the URL
# eg we'll get /namespaces/test?returnDescription=True as the path
result = fluiddb.call("GET", "/namespaces/test", None, None, returnDescription=True)
self.assertEqual("200", result[0]["status"])
# make sure the result has the expected description field
self.assertTrue(result[1].has_key("description"))
# finally we need to make sure that primitive values returned from
# fluidDB are turned from their json representation to their
# Pythonic form
new_namespace = str(uuid.uuid4())
new_tag = str(uuid.uuid4())
ns_body = {"description": "a test namespace", "name": new_namespace}
tag_body = {"description": "a test tag", "name": new_tag, "indexed": False}
# create a namespace and tag to use in a bit
result = fluiddb.call("POST", "/namespaces/test", ns_body)
self.assertEqual("201", result[0]["status"])
self.assertTrue(result[1].has_key("id"))
ns_id = result[1]["id"] # for later use
result = fluiddb.call("POST", "/tags/test/" + new_namespace, tag_body)
self.assertEqual("201", result[0]["status"])
self.assertTrue(result[1].has_key("id"))
path = "/" + "/".join(["objects", ns_id, "test", new_namespace, new_tag])
primitives = [1, 1.1, u"foo", ["a", "b", u"c"], True, None]
for primitive in primitives:
result = fluiddb.call("PUT", path, primitive)
self.assertEqual("204", result[0]["status"])
# GET the new tag value and check it gets translated back to
# the correct type
result = fluiddb.call("GET", path)
self.assertEqual("application/vnd.fluiddb.value+json", result[0]["content-type"])
self.assertTrue(isinstance(result[1], type(primitive)))
# check the new /values GET works
result = fluiddb.call(
"GET",
"/values",
tags=["fluiddb/about", "test/%s/%s" % (new_namespace, new_tag)],
query="has test/%s/%s" % (new_namespace, new_tag),
)
self.assertEqual("200", result[0]["status"])
self.assertTrue(result[1].has_key("results"))
# Housekeeping
fluiddb.call("DELETE", "/tags/test/" + new_namespace + "/" + new_tag)
fluiddb.call("DELETE", "/namespaces/test/" + new_namespace)
def test_custom_headers(self):
custom_headers = {"Origin": "http://foo.com"}
result = fluiddb.call("GET", "/users/test", custom_headers=custom_headers)
self.assertEqual("200", result[0]["status"])
self.assertEqual("http://foo.com", result[0]["access-control-allow-origin"])
def test_call_PUT(self):
fluiddb.login(USERNAME, PASSWORD)
new_namespace = str(uuid.uuid4())
new_tag = str(uuid.uuid4())
ns_body = {"description": "a test namespace", "name": new_namespace}
tag_body = {"description": "a test tag", "name": new_tag, "indexed": False}
# create a namespace and tag to use in a bit
result = fluiddb.call("POST", "/namespaces/test", ns_body)
self.assertEqual("201", result[0]["status"])
self.assertTrue(result[1].has_key("id"))
ns_id = result[1]["id"] # for later use
result = fluiddb.call("POST", "/tags/test/" + new_namespace, tag_body)
self.assertEqual("201", result[0]["status"])
self.assertTrue(result[1].has_key("id"))
path = "/" + "/".join(["objects", ns_id, "test", new_namespace, new_tag])
# Make sure that primitive types are json encoded properly with
# the correct mime-type, dicts are translated to json, the
# mime-type argument for opaque types is used properly and if
# no mime-type is supplied and the previous checks are not met
# an appropriate exception is raised.
primitives = [1, 1.1, "foo", u"foo", True, None, ["a", "b", u"c"]]
for primitive in primitives:
result = fluiddb.call("PUT", path, primitive)
self.assertEqual("204", result[0]["status"])
# call HEAD verb on that tag value to get the mime-type from
# FluidDB
result = fluiddb.call("HEAD", path)
self.assertEqual("application/vnd.fluiddb.value+json", result[0]["content-type"])
# dicts are json encoded
result = fluiddb.call("PUT", path, {"foo": "bar"})
# check again with HEAD verb
result = fluiddb.call("HEAD", path)
self.assertEqual("application/json", result[0]["content-type"])
# Make sure that the body and mime args work as expected (mime
# overrides the primitive string type making the value opaque)
result = fluiddb.call("PUT", path, "<html><body><h1>Hello," "World!</h1></body></html>", "text/html")
result = fluiddb.call("HEAD", path)
self.assertEqual("text/html", result[0]["content-type"])
# unspecified mime-type on a non-primitive value results in an
# exception
self.assertRaises(TypeError, fluiddb.call, "PUT", path, object())
# make sure it's possible to PUT a tag value using a list based path
pathAsList = ["objects", ns_id, "test", new_namespace, new_tag]
result = fluiddb.call("PUT", pathAsList, "foo")
self.assertEqual("204", result[0]["status"])
# Housekeeping
fluiddb.call("DELETE", "/tags/test/" + new_namespace + "/" + new_tag)
fluiddb.call("DELETE", "/namespaces/test/" + new_namespace)
def main(args):
"""
Set stuff up, read credentials, start the ball rolling
"""
logging.info("++++++++++++++++++++++++++++++++++++++++++++++")
logging.info("++++++++++++++++++++++++++++++++++++++++++++++")
logging.info("Starting push at %s"%datetime.datetime.today().ctime())
try:
credentials_file = open('credentials.json', 'r')
credentials = json.loads(credentials_file.read())
except Exception as e:
logging.error("Could not process credentials...")
logging.error(e)
sys.exit()
finally:
credentials_file.close()
if 'sandbox' in args:
fd.prefix = fd.SANDBOX
else:
fd.prefix = fd.MAIN
logging.info("FluidDB instance: %s"%fd.prefix)
fd.login(credentials['username'], credentials['password'])
new_object_flag = False
last_update = False
if credentials.has_key('object_id'):
object = '/objects/'+credentials['object_id']
# lets just check we've got the expected object (in case of Sandbox reset)
logging.info('Checking object in FluidDB: %s'%object)
check_response = fd.call('GET', object)
logging.info(check_response)
if check_response[0]['status'] == '200':
new_object_flag = not check_response[1]['tagPaths']
else:
new_object_flag = True
else:
new_object_flag = True
if new_object_flag:
# ok, so the expected object doesn't exist so create a new one and store
# back in the credentials file...
new_about = raw_input("Please describe your new FluidDB application: ")
new_object_response = fd.call('POST', '/objects', {"about": new_about})
if new_object_response[0]['status'] == '201':
new_id = new_object_response[1]['id']
credentials['object_id'] = new_id
object = '/objects/'+credentials['object_id']
try:
credentials_out = open('credentials.json', 'w')
credentials_out.write(json.dumps(credentials, indent=2))
finally:
credentials_out.close()
logging.warning('New object created: %s'%object)
print 'New object created: %s'%object
last_update = 1.0
logging.info("Object id: %s"%object)
# Change out of the python directory to the root of the project
os.chdir('..')
# timestamp allows us to check for updated files
if not last_update:
try:
timestamp_file = open('.timestamp', 'r')
last_update = float(timestamp_file.read())
timestamp_file.close()
except:
last_update = 1.0
# Update the timestamp to now
try:
timestamp_file = open('.timestamp', 'w')
timestamp_file.write(str(time.mktime(datetime.datetime.today().timetuple())))
finally:
timestamp_file.close()
# lets check we have a fluidApp namespace
# Lets build the path as it would appear in FluidDB
fluid_namespace_path = '/'.join((
'/namespaces', # namespaces root
credentials['username'], # user's root namespace
'fluidapp', # the namespace underneath which we store fluid applications
))
state_of_fluidapp_namespace = fd.call('GET', fluid_namespace_path)
logging.info(state_of_fluidapp_namespace)
if state_of_fluidapp_namespace[0]['status'] == '404':
# The namespace that represents the fluidapp namespace
logging.info('Creating new namespace for fluidapps')
parent_namespace = '/'.join(fluid_namespace_path.split('/')[:-1])
logging.info('Parent namespace: %s'%parent_namespace)
new_namespace_name = fluid_namespace_path.split('/')[-1:][0]
logging.info('New namespace name: %s'%new_namespace_name)
logging.info(fd.call('POST', parent_namespace,
{'description': u'A namespace for storing fluidapps created by %s'%credentials['username'],
'name': new_namespace_name}))
# navigate the directory tree and work out what are the new/updated files
# and namespaces that need pushing to FluidDB
for directory in os.walk('.'):
processDirectoryIntoNamespace(directory, last_update, credentials, object)
def main(args):
"""
Set stuff up, read credentials, start the ball rolling
"""
logging.info("++++++++++++++++++++++++++++++++++++++++++++++")
logging.info("++++++++++++++++++++++++++++++++++++++++++++++")
logging.info("Starting push at %s"%datetime.datetime.today().ctime())
try:
homedir = os.path.expanduser('~')
logging.info("Attempting to load credentials.json from %s"%homedir)
credentials_file = open('credentials.json', 'r')
credentials = json.loads(credentials_file.read())
except Exception as e:
logging.error("Could not process credentials...")
logging.error(e)
sys.exit()
finally:
credentials_file.close()
if 'sandbox' in args:
fd.prefix = fd.SANDBOX
else:
fd.prefix = fd.MAIN
logging.info("FluidDB instance: %s"%fd.prefix)
fd.login(credentials['username'], credentials['password'])
new_object_flag = False
last_update = False
if credentials.has_key('object_id'):
object = '/objects/'+credentials['object_id']
# lets just check we've got the expected object (in case of Sandbox reset)
check_response = fd.call('GET', object)
if check_response[0]['status'] == '200':
new_object_flag = not check_response[1]['tagPaths']
else:
new_object_flag = True
else:
new_object_flag = True
if new_object_flag:
# ok, so the expected object doesn't exist so create a new one and store
# back in the credentials file...
new_about = raw_input("Please describe your new FluidDB application: ")
new_object_response = fd.call('POST', '/objects', {"about": new_about})
if new_object_response[0]['status'] == '201':
new_id = new_object_response[1]['id']
credentials['object_id'] = new_id
object = '/objects/'+credentials['object_id']
try:
credentials_out = open('credentials.json', 'w')
credentials_out.write(json.dumps(credentials, indent=2))
finally:
credentials_out.close()
logging.warning('New object created: %s'%object)
print 'New object created: %s'%object
last_update = 1.0
logging.info("Object id: %s"%object)
os.chdir('..')
if not last_update:
try:
timestamp_file = open('.timestamp', 'r')
last_update = float(timestamp_file.read())
timestamp_file.close()
except:
last_update = 1.0
# Update the timestamp to now
try:
timestamp_file = open('.timestamp', 'w')
timestamp_file.write(str(time.mktime(datetime.datetime.today().timetuple())))
finally:
timestamp_file.close()
# lets check we have a fluidApp namespace
# Lets build the path as it would appear in FluidDB
fluid_namespace_path = '/'.join((
'/namespaces', # namespaces root
credentials['username'], # user's root namespace
'fluidapp', # the namespace underneath which we store fluid applications
))
state_of_fluidapp_namespace = fd.call('GET', fluid_namespace_path)
logging.info(state_of_fluidapp_namespace)
if state_of_fluidapp_namespace[0]['status'] == '404':
# The namespace that represents the fluidapp namespace
logging.info('Creating new namespace for fluidapps')
parent_namespace = '/'.join(fluid_namespace_path.split('/')[:-1])
logging.info('Parent namespace: %s'%parent_namespace)
new_namespace_name = fluid_namespace_path.split('/')[-1:][0]
logging.info('New namespace name: %s'%new_namespace_name)
logging.info(fd.call('POST', parent_namespace,
{'description': u'A namespace for storing fluidapps created by %s'%credentials['username'],
'name': new_namespace_name}))
# navigate the directory tree and work out what are the new/updated files
# and namespaces that need pushing to FluidDB
for directory in os.walk('.'):
processDirectoryIntoNamespace(directory, last_update, credentials, object)