def handle_object(self, obj):
if type(obj) == dict and len(obj.keys()) == 1 and obj.has_key('_ref'):
L.info("ReferenceFetcher: Attempting to fetch %r" % obj)
url = obj['_ref']
if url.strip() == '':
return None
data = None
try:
data = internal_find(url, follow=False, lang=self.token.lang)
if data != None:
return data
except Exception:
L.exception("ReferenceFetcher: Failed to fetch %r" % obj)
elif type(obj) == dict:
return dict([(k, self.handle_object(v))
for k, v in obj.iteritems()])
elif type(obj) == list:
return [self.handle_object(el) for el in obj]
return obj
def noslug(path):
L.info("dbserver::noslug::%s:%s, %s" % (request.method, path, request.url))
response, content_type, headers = process(path, None)
L.info("%s %s %s" % (snip(response), content_type, headers))
return Response(response=response,
content_type=content_type,
headers=headers)
def handle_object(self,obj):
if type(obj) == dict and len(obj.keys())==1 and obj.has_key('_ref'):
L.info("ReferenceFetcher: Attempting to fetch %r" % obj)
url = obj['_ref']
if url.strip() == '':
return None
data = None
try:
data = internal_find(url,follow=False,lang=self.token.lang)
if data != None:
return data
except Exception:
L.exception("ReferenceFetcher: Failed to fetch %r" % obj)
elif type(obj) == dict:
return dict([ (k,self.handle_object(v)) for k,v in obj.iteritems()])
elif type(obj) == list:
return [ self.handle_object(el) for el in obj]
return obj
def get(self):
def get_data(rec):
x = rec.get(self.DATA_EL,{})
x[self.SOURCE_EL] = "%s/%s" % (rec[self.META_EL][self.PATH_EL], rec[self.META_EL][self.ID_EL])
x[self.SOURCE_SLUG_EL] = x[self.SOURCE_EL].replace('/','__')
return x
if self.token.slug != None:
rec = g.db.find_one( {self.META_PATH : self.token.path,
self.META_ID : self.token.slug } )
if rec != None:
ret = get_data(rec)
else:
ret = None
self.token.response = ret
else:
if self.query != None:
self.query.update({self.META_PATH : self.token.path })
else:
self.query = {self.META_PATH : self.token.path }
L.debug("DBOperation:: find using query=%r" % self.query)
if self.count == 0:
recs = g.db.find(self.query,fields=self.fields,limit=self.limit,start=self.start)
recs = [ get_data(rec) for rec in recs ]
ret = recs
else:
ret = g.db.count(self.query,fields=self.fields,limit=self.limit,start=self.start)
self.token.response = ret
def get(self):
def get_data(rec):
x = rec.get(self.DATA_EL, {})
x[self.SOURCE_EL] = "%s/%s" % (rec[self.META_EL][self.PATH_EL], rec[self.META_EL][self.ID_EL])
x[self.SOURCE_SLUG_EL] = x[self.SOURCE_EL].replace("/", "__")
return x
if self.token.slug != None:
rec = g.db.find_one({self.META_PATH: self.token.path, self.META_ID: self.token.slug})
if rec != None:
ret = get_data(rec)
else:
ret = None
self.token.response = ret
else:
if self.query != None:
self.query.update({self.META_PATH: self.token.path})
else:
self.query = {self.META_PATH: self.token.path}
L.debug("DBOperation:: find using query=%r" % self.query)
if self.count == 0:
recs = g.db.find(self.query, fields=self.fields, limit=self.limit, start=self.start)
recs = [get_data(rec) for rec in recs]
ret = recs
else:
ret = g.db.count(self.query, fields=self.fields, limit=self.limit, start=self.start)
self.token.response = ret
def process(self):
data = json.dumps(self.token.response)
if self.token.request.method == "GET":
if data != 'null':
L.info("CacheSetter: storing in %s <== %s" %
(self.token.cache_key, snip(str(data))))
store_in_cache(self.token.cache_key, data)
def slug(path, slug):
L.info("dbserver::slug::%s:%s, %s, %s" %
(request.method, path, slug, request.url))
response, content_type, headers = process(path, slug)
return Response(response=response,
content_type=content_type,
headers=headers)
def process(self):
data = json.dumps(self.token.response)
if data != 'null':
L.info("CacheSetter: storing in %s <== %s" %
(self.token.cache_key, snip(str(data))))
store_in_cache(
self.token.cache_key.encode('utf8').encode('hex'),
data.encode('utf8').encode('hex'))
def process(self):
if self.token.request.method in ['PUT', 'POST' ]:
try:
self.token.data = json.loads(self.token.data)
except:
self.token.data = dict( [ (k,v) for k,v in self.token.request.form.iteritems() ] )
L.debug("DataLoader:: data=%r" % self.token.data)
def process(self):
self.token.cache_key = json.dumps(self.token.path) + '|' + json.dumps(self.token.slug) + '|' + '|'.join([ self.token.request.args.get(arg,'') for arg in self.ARGS_FOR_KEY ])
if self.token.request.args.get('hitcache',"1") == "0":
L.info("Skipping cache as requested for %s/%s" % (self.token.path,self.token.slug))
return
data = hit_cache(self.token.cache_key.encode('utf8').encode('hex'))
L.info("Data for key %s == %s" % (self.token.cache_key,snip(repr(data))))
if data != None:
self.token.response = json.loads(data.decode('hex').decode('utf8'))
self.should_stop = True
self.skip_to = "DataFormatter"
def create_or_login(resp):
session['openid'] = resp.identity_url
openid_key = urllib.quote(urllib.quote(session['openid'],safe=''))
user = internal_find('/data/admin/users/%s' % openid_key)
if user != None:
L.info(u'Successfully signed in fullname=%s, email=%s (%r)' % (resp.fullname, resp.email, resp.__dict__))
else:
data = { "fullname" : resp.fullname,
"email" : resp.email,
"key" : openid_key }
user = internal_save('/data/admin/users/%s' % openid_key, data)
L.info(u'Successfully created fullname=%s, email=%s (%r)' % (resp.fullname, resp.email, resp.__dict__))
g.user = user
return redirect(oid.get_next_url())
def validate_api(self,api_key):
try:
L.info('PermissionChecker:: api_key = %s' % api_key)
api_key = api_key.decode('base64')
api_key = json.loads(api_key)
app = api_key['a']
referrer = api_key['r']
request_referrer = self.token.request.referrer
request_referrer = urlparse.urlparse(request_referrer).netloc
L.info('PermissionChecker:: app = %s, referrer = %s, req.referrer = %s' % (app, referrer, request_referrer))
assert( referrer == request_referrer )
secret = api_key['s']
assert( secret == md5().update("%s:%s:p4tpp" % (app,referrer)).hexdigest()[:8] )
self.app = "%s@%s" % (app,referrer)
except:
return False
def internal_find(url,follow=True,lang=None,query=None,apikey=None,fields=None):
params = { 'o' : 'json',
'limit' : 0 }
params['follow'] = "yes" if follow else "no"
if lang: params['lang'] = lang
if query: params['query'] = json.dumps(query)
if apikey: params['apikey'] = apikey
if fields: params['fields'] = json.dumps(fields)
L.info("internal_find: Attempting to fetch %s with %s" % (url,params))
params = urlencode(params)
url += "?" + params
data = g.app.test_client().get(url).data
data = json.loads(data)
return data
def create_or_login(resp):
session['openid'] = resp.identity_url
openid_key = urllib.quote(urllib.quote(session['openid'], safe=''))
user = internal_find('/data/admin/users/%s' % openid_key)
if user != None:
L.info(u'Successfully signed in fullname=%s, email=%s (%r)' %
(resp.fullname, resp.email, resp.__dict__))
else:
data = {
"fullname": resp.fullname,
"email": resp.email,
"key": openid_key
}
user = internal_save('/data/admin/users/%s' % openid_key, data)
L.info(u'Successfully created fullname=%s, email=%s (%r)' %
(resp.fullname, resp.email, resp.__dict__))
g.user = user
return redirect(oid.get_next_url())
def process(self):
self.token.cache_key = json.dumps(self.token.path) + '|' + json.dumps(
self.token.slug) + '|' + '|'.join([
self.token.request.args.get(arg, '')
for arg in self.ARGS_FOR_KEY
])
if self.token.request.args.get('hitcache', "1") == "0":
L.info("Skipping cache as requested for %s/%s" %
(self.token.path, self.token.slug))
return
data = hit_cache(self.token.cache_key.encode('utf8').encode('hex'))
L.info("Data for key %s == %s" %
(self.token.cache_key, snip(repr(data))))
if data != None:
self.token.response = json.loads(data.decode('hex').decode('utf8'))
self.should_stop = True
self.skip_to = "DataFormatter"
def validate_api(self, api_key):
try:
L.info('PermissionChecker:: api_key = %s' % api_key)
api_key = api_key.decode('base64')
api_key = json.loads(api_key)
app = api_key['a']
referrer = api_key['r']
request_referrer = self.token.request.referrer
request_referrer = urlparse.urlparse(request_referrer).netloc
L.info(
'PermissionChecker:: app = %s, referrer = %s, req.referrer = %s'
% (app, referrer, request_referrer))
assert (referrer == request_referrer)
secret = api_key['s']
assert (secret == md5().update("%s:%s:p4tpp" %
(app, referrer)).hexdigest()[:8])
self.app = "%s@%s" % (app, referrer)
except:
return False
def post(self):
try:
rec = g.db.find_one({self.META_PATH: self.token.path, self.META_ID: self.token.slug})
if rec == None:
g.db.save(
{
self.META_EL: {self.PATH_EL: self.token.path, self.ID_EL: self.token.slug},
self.DATA_EL: self.token.data,
}
)
else:
rec[self.DATA_EL] = self.token.data
g.db.save(rec)
self.token.response = True
except:
L.exception("db_op::post")
self.token.response = False
def post(self):
if self.token.slug == None:
self.token.slug = uuid.uuid4()
try:
rec = g.db.find_one({self.META_PATH : self.token.path,
self.META_ID : self.token.slug })
if rec == None:
g.db.save( { self.META_EL : { self.PATH_EL : self.token.path,
self.ID_EL : self.token.slug },
self.DATA_EL : self.token.data })
else:
rec[self.DATA_EL] = self.token.data
g.db.save(rec)
self.token.response = True, self.token.path, self.token.slug
except:
L.exception('db_op::post')
self.token.response = False
def internal_find(url,
follow=True,
lang=None,
query=None,
apikey=None,
fields=None):
params = {'o': 'json', 'limit': 0}
params['follow'] = "yes" if follow else "no"
if lang: params['lang'] = lang
if query: params['query'] = json.dumps(query)
if apikey: params['apikey'] = apikey
if fields: params['fields'] = json.dumps(fields)
L.info("internal_find: Attempting to fetch %s with %s" % (url, params))
params = urlencode(params)
url += "?" + params
data = g.app.test_client().get(url).data
data = json.loads(data)
return data
def process(self):
self.token.cache_key = json.dumps(self.token.path) + '|' + json.dumps(
self.token.slug) + '|' + '|'.join([
self.token.request.args.get(arg, '')
for arg in self.ARGS_FOR_KEY
])
if self.token.request.args.get('hitcache', "1") == "0":
L.info("Skipping cache as requested for %s/%s" %
(self.token.path, self.token.slug))
return
if self.token.request.method == "GET":
data = hit_cache(self.token.cache_key)
L.info("Data for key %s == %s" %
(self.token.cache_key, snip(repr(data))))
if data != None:
self.token.response = json.loads(data)
self.should_stop = True
self.skip_to = "DataFormatter"
else:
clear_cache(self.token.cache_key)
clear_cache_for_path(self.token.path)
def process(self):
self.token.cache_key = (
json.dumps(self.token.path)
+ "|"
+ json.dumps(self.token.slug)
+ "|"
+ "|".join([self.token.request.args.get(arg, "") for arg in self.ARGS_FOR_KEY])
)
if self.token.request.args.get("hitcache", "1") == "0":
L.info("Skipping cache as requested for %s/%s" % (self.token.path, self.token.slug))
return
if self.token.request.method == "GET":
data = hit_cache(self.token.cache_key)
L.info("Data for key %s == %s" % (self.token.cache_key, snip(repr(data))))
if data != None:
self.token.response = json.loads(data)
self.should_stop = True
self.skip_to = "DataFormatter"
else:
clear_cache(self.token.cache_key)
clear_cache_for_path(self.token.path)
def process(self):
method = self.token.request.method
self.query = self.token.request.args.get("query", "{}")
self.query = json.loads(self.query)
self.query = dict([("%s.%s" % (self.DATA_EL, k), v) for k, v in self.query.iteritems()])
self.order = self.token.request.args.get("order", "[]")
self.order = json.loads(self.order)
self.order = [("%s.%s" % (self.DATA_EL, k), v) for k, v in self.order]
self.start = self.token.request.args.get("start", 0)
self.start = int(self.start)
self.limit = self.token.request.args.get("limit", 1000)
self.limit = int(self.limit)
if self.limit == 0:
self.limit = None
self.fields = self.token.request.args.get("fields")
if self.fields != None:
self.fields = json.loads(self.fields)
self.fields = ["%s.%s" % (self.DATA_EL, k) for k in self.fields]
self.fields.append(self.META_EL)
self.count = self.token.request.args.get("count", 0)
self.count = int(self.count)
L.debug("DBOperation:: method=%r" % method)
L.debug("DBOperation:: path=%r slug=%r data=%r" % (self.token.path, self.token.slug, self.token.data))
if method in ["GET", "PUT", "POST", "DELETE"]:
getattr(self, method.lower())()
L.debug("DBOperation:: result=%s" % snip(repr(self.token.response)))
def process(self):
method = self.token.request.method
self.query = self.token.request.args.get('query',"{}")
self.query = json.loads(self.query)
self.query=dict([("%s.%s" % (self.DATA_EL,k),v) for k,v in self.query.iteritems()])
self.order = self.token.request.args.get('order',"[]")
self.order = json.loads(self.order)
self.order=[("%s.%s" % (self.DATA_EL,k),v) for k,v in self.order]
self.start = self.token.request.args.get('start',0)
self.start = int(self.start)
self.limit = self.token.request.args.get('limit',1000)
self.limit = int(self.limit)
if self.limit == 0:
self.limit = None
self.fields = self.token.request.args.get('fields')
if self.fields != None:
self.fields = json.loads(self.fields)
self.fields =[ "%s.%s" % (self.DATA_EL,k) for k in self.fields]
self.fields.append(self.META_EL)
self.count = self.token.request.args.get('count',0)
self.count = int(self.count)
L.debug("DBOperation:: method=%r" % method)
L.debug("DBOperation:: path=%r slug=%r data=%r" % (self.token.path, self.token.slug, self.token.data))
if method in ['GET', 'PUT', 'POST', 'DELETE']:
getattr(self,method.lower())()
L.debug("DBOperation:: result=%s" % snip(repr(self.token.response)))
def process(self):
method = self.token.request.method
self.should_stop = False
self.app = self.token.request.args.get('apikey', None)
if self.app != None:
if self.app == "admin":
return
if not self.validate_api(self.app):
return
self.user = None ## TODO: '''<<getuser>>'''
if self.token.slug != None:
fullpath = os.path.join(self.token.path, self.token.slug)
else:
fullpath = self.token.path
L.debug("PermissionChecker: full_path=%r" % fullpath)
fullpath = fullpath.split('/')
L.debug("PermissionChecker: full_path=%r" % fullpath)
perms = set()
for i in range(len(fullpath)):
partial_path = fullpath[0:i + 1]
L.debug("PermissionChecker: partial_path=%r" % partial_path)
spec = json.dumps({"ref": "/" + "/".join(partial_path)})
data = internal_find('/data/admin/permissions/',
query=spec,
apikey='admin')
for rec in data:
auth = rec.get('auth')
if self.match_auth(auth):
perms.update(set(rec.get('perms', set())))
L.info("PermissionChecker: rule %s, perms=%r" %
(rec.get('_src'), set(rec.get('perms', set()))))
if ((method == "POST" and "new" in perms)
or (method == "DELETE" and "delete" in perms)
or (method == "PUT" and "edit" in perms)
or (method == "GET" and "read" in perms)):
pass
else:
self.should_stop = True
def process(self):
method = self.token.request.method
self.should_stop = False
self.app = self.token.request.args.get('apikey',None)
if self.app != None:
if self.app == "admin":
return
if not self.validate_api(self.app):
return
self.user = None ## TODO: '''<<getuser>>'''
if self.token.slug != None:
fullpath = os.path.join(self.token.path, self.token.slug)
else:
fullpath = self.token.path
L.debug("PermissionChecker: full_path=%r" % fullpath)
fullpath = fullpath.split('/')
L.debug("PermissionChecker: full_path=%r" % fullpath)
perms = set()
for i in range(len(fullpath)):
partial_path = fullpath[0:i+1]
L.debug("PermissionChecker: partial_path=%r" % partial_path)
spec = json.dumps({ "ref" : "/" + "/".join(partial_path) })
data = internal_find('/data/admin/permissions/',query=spec,apikey='admin')
for rec in data:
auth = rec.get('auth')
if self.match_auth(auth):
perms.update(set(rec.get('perms',set())))
L.info("PermissionChecker: rule %s, perms=%r" % (rec.get('_src'),set(rec.get('perms',set()))))
if ( (method == "POST" and "new" in perms) or
(method == "DELETE" and "delete" in perms) or
(method == "PUT" and "edit" in perms) or
(method == "GET" and "read" in perms) ):
pass
else:
self.should_stop = True
def process(self):
for emitter in EMITTERS:
E = Emitter.get_emitter(emitter)
e = E(self.token)
if e.condition():
L.debug("DataFormatter:: using emitter %s" % emitter)
e.format()
break
else:
emitter = "HtmlFormatter"
E = Emitter.get_emitter(emitter)
e = E(self.token)
L.debug("DataFormatter:: using default emitter %s" % emitter)
e.format()
L.debug("DataFormatter:: token.response=%s (%s)" % (snip(repr(self.token.response)),self.token.content_type))
def process(self):
for emitter in EMITTERS:
E = Emitter.get_emitter(emitter)
e = E(self.token)
if e.condition():
L.debug("DataFormatter:: using emitter %s" % emitter)
e.format()
break
else:
emitter = "HtmlFormatter"
E = Emitter.get_emitter(emitter)
e = E(self.token)
L.debug("DataFormatter:: using default emitter %s" % emitter)
e.format()
L.debug("DataFormatter:: token.response=%s (%s)" %
(snip(repr(self.token.response)), self.token.content_type))
def process(self):
data = json.dumps(self.token.response)
if self.token.request.method == "GET":
if data != 'null':
L.info("CacheSetter: storing in %s <== %s" % (self.token.cache_key,snip(str(data))))
store_in_cache(self.token.cache_key,data)
def logout():
session.pop('openid', None)
L.info(u'You have been signed out')
return redirect(oid.get_next_url())
def logout():
session.pop('openid', None)
L.info(u'You have been signed out')
return redirect(oid.get_next_url())
def slug(path,slug):
L.info("dbserver::slug::%s:%s, %s, %s" % (request.method,path,slug,request.url))
response, content_type, headers = process(path,slug)
return Response(response=response, content_type=content_type, headers=headers)
def noslug(path):
L.info("dbserver::noslug::%s:%s, %s" % (request.method,path,request.url))
response, content_type, headers = process(path,None)
L.info("%s %s %s" % (snip(response), content_type, headers))
return Response(response=response, content_type=content_type, headers=headers)
import os
os.environ["DJANGO_SETTINGS_MODULE"] = "openid_db.settings"
from gevent import monkey
monkey.patch_all()
from gevent.pywsgi import WSGIServer
from dbserver import app
from log import L
L.info("Listening on 5555")
http_server = WSGIServer(('', 5555), app)
http_server.serve_forever()
import os
os.environ["DJANGO_SETTINGS_MODULE"]="openid_db.settings"
from gevent import monkey; monkey.patch_all()
from gevent.pywsgi import WSGIServer
from dbserver import app
from log import L
L.info("Listening on 5555")
http_server = WSGIServer(('', 5555), app)
http_server.serve_forever()
def process(self):
data = json.dumps(self.token.response)
if data != 'null':
L.info("CacheSetter: storing in %s <== %s" % (self.token.cache_key,snip(str(data))))
store_in_cache(self.token.cache_key.encode('utf8').encode('hex'),data.encode('utf8').encode('hex'))
