def __init__(self, _id, addr=None, blocking=True, channels=True):
self._id = _id
self.queue = RedisPool(addr)
self.blocking = blocking
self.channels = channels
log.debug('queue %s connected using %s' % (_id, addr))
def _route(user, jid, msg):
session = env.user.session()
try:
return session(msg['body'])
except SessionCallError:
pass
message = env.user.resolve_aliases(msg['body'])
args = {}
for r in self.route:
m = re.search(r['resource'], msg['resource'])
if m:
args = m.groupdict()
route = r['route']
break
for regex, view in route:
match = re.search(regex, message)
if match:
for g, val in match.groupdict().iteritems():
args[g] = val
log.debug(">>> %s routed to %s(%s) via '%s'" % \
(jid, view.__name__, str(args), regex.pattern))
return view(**args)
def get(table, param):
try:
return self.profile[table][param]
except KeyError:
try:
# FIXME: profile models
db.perform("INSERT INTO %s (id) VALUES (%%s);" % \
table, [self.id])
except IntegrityError:
pass
res = cache_get('profile:%s:%s' % (table, self.id))
if res:
self.profile[table] = res
else:
res = db.fetchone("SELECT * FROM %s WHERE id=%%s;" % \
table, [self.id])
log.debug('RES %s %s' % (table, res))
if res:
self.profile[table] = dict(res)
cache_store('profile:%s:%s' % (table, self.id),
self.profile[table])
else:
#try:
# return cls._profile[param]['default']
#except KeyError:
# return None
return None
# FIXME: remove recursive call
try:
return self.profile[table][param]
except KeyError:
cache_del('profile:%s:%s' % (table, self.id))
return get(table, param)
def get(table, param):
try:
return self.profile[table][param]
except KeyError:
try:
# FIXME: profile models
db.perform("INSERT INTO %s (id) VALUES (%%s);" % \
table, [self.id])
except IntegrityError:
pass
res = cache_get('profile:%s:%s' % (table, self.id))
if res:
self.profile[table] = res
else:
res = db.fetchone("SELECT * FROM %s WHERE id=%%s;" % \
table, [self.id])
log.debug('RES %s %s' % (table, res))
if res:
self.profile[table] = dict(res)
cache_store('profile:%s:%s' % (table, self.id),
self.profile[table])
else:
#try:
# return cls._profile[param]['default']
#except KeyError:
# return None
return None
# FIXME: remove recursive call
try:
return self.profile[table][param]
except KeyError:
cache_del('profile:%s:%s' % (table, self.id))
return get(table, param)
def xmpp_template(tmpl_name, _lang=None, _type=None, **context):
if not _lang:
if env.user and env.user.id:
_lang = env.user.get_profile('lang')
else:
_lang = settings.lang
if not _type:
try:
if env.user and env.user.get_profile('xhtml'):
_type = 'html'
except KeyError:
pass
tmpl_dict = {}
if _type == 'html':
try:
tmpl_path = os.path.join(_lang, 'xhtml', tmpl_name+'.tmpl')
tmpl = jinja_env.get_template(tmpl_path)
log.debug('Template %s' % tmpl_path)
tmpl_dict['html'] = tmpl.render(context, settings=settings)
except TemplateNotFound:
tmpl_dict['html'] = None
log.error('Template %s not found' % tmpl_path)
finally:
pass
tmpl_path = os.path.join(_lang, 'text', tmpl_name+'.tmpl')
tmpl = jinja_env.get_template(tmpl_path)
tmpl_dict['body'] = tmpl.render(context, settings=settings)
return tmpl_dict
def __init__(self, _id, addr=None, blocking=True, channels=True):
self._id = _id
self.queue = RedisPool(addr)
self.blocking = blocking
self.channels = channels
log.debug('queue %s connected using %s' % (_id, addr))
def _route(user, jid, msg):
session = env.user.session()
try:
return session(msg['body'])
except SessionCallError:
pass
message = env.user.resolve_aliases(msg['body'])
args = {}
for r in self.route:
m = re.search(r['resource'], msg['resource'])
if m:
args = m.groupdict()
route = r['route']
break
for regex, view in route:
match = re.search(regex, message)
if match:
for g, val in match.groupdict().iteritems():
args[g] = val
log.debug(">>> %s routed to %s(%s) via '%s'" % \
(jid, view.__name__, str(args), regex.pattern))
return view(**args)
def xmpp_template(tmpl_name, _lang=None, _type=None, **context):
if not _lang:
if env.user and env.user.id:
_lang = env.user.get_profile('lang')
else:
_lang = settings.lang
if not _type:
try:
if env.user and env.user.get_profile('xhtml'):
_type = 'html'
except KeyError:
pass
tmpl_dict = {}
if _type == 'html':
try:
tmpl_path = os.path.join(_lang, 'xhtml', tmpl_name + '.tmpl')
tmpl = jinja_env.get_template(tmpl_path)
log.debug('Template %s' % tmpl_path)
tmpl_dict['html'] = tmpl.render(context, settings=settings)
except TemplateNotFound:
tmpl_dict['html'] = None
log.error('Template %s not found' % tmpl_path)
finally:
pass
tmpl_path = os.path.join(_lang, 'text', tmpl_name + '.tmpl')
tmpl = jinja_env.get_template(tmpl_path)
tmpl_dict['body'] = tmpl.render(context, settings=settings)
return tmpl_dict
def save(self, data):
dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime) else None
self.redis.set(self.key, json.dumps(data, default=dthandler))
try:
self.redis.expire(self.key, settings.session_expires * 86400)
except AttributeError:
pass
log.debug("RedisBackend.save: %s" % self.key)
def save(self, data):
dthandler = lambda obj: obj.isoformat() \
if isinstance(obj, datetime) else None
self.redis.set(self.key, json.dumps(data, default=dthandler))
try:
self.redis.expire(self.key, settings.session_expires * 86400)
except AttributeError:
pass
log.debug('RedisBackend.save: %s' % self.key)
def cache_store(key, data, expire=None):
log.debug('cache_store %s' % key)
redis = RedisPool(settings.cache_socket)
redis.set(key, json.dumps(data))
if not expire:
try:
expire = settings.cache_expire_max
except AttributeError:
expire = 3600 * 24 # day
redis.expire(key, expire)
def cache_store(key, data, expire=None):
log.debug("cache_store %s" % key)
redis = RedisPool(settings.cache_socket)
redis.set(key, json.dumps(data))
if not expire:
try:
expire = settings.cache_expire_max
except AttributeError:
expire = 3600 * 24 # day
redis.expire(key, expire)
def executemany(self, *args, **kwargs):
with self.cursor() as cursor:
t = ''
if self.debug:
t1 = time.time()
cursor.executemany(*args, **kwargs)
if self.debug:
t = '%.3f' % (time.time() - t1)
try:
log.debug(u'executemany %s %s' % \
(t, cursor.mogrify(*args, **kwargs)))
except:
pass
def __init__(self, sessid):
self._loaded = False
try:
path = settings.session_dir
except AttributeError:
path = '/tmp'
try:
prefix = settings.session_prefix
except ArrtibuteError:
prefix = 'geweb-session-'
self.filename = os.path.join(path, '%s%s' % (prefix, sessid))
log.debug('FileBackend: %s' % self.filename)
def __init__(self, sessid):
self._loaded = False
try:
path = settings.session_dir
except AttributeError:
path = '/tmp'
try:
prefix = settings.session_prefix
except ArrtibuteError:
prefix='geweb-session-'
self.filename = os.path.join(path, '%s%s' % (prefix, sessid))
log.debug('FileBackend: %s' % self.filename)
def resolve(self, request):
if self.methods and request.method.upper() not in self.methods:
raise R.NotMatched
if self.host and not fnmatch(request.host, self.host):
raise R.NotMatched
m = re.match(self.pattern, request.path)
if not m:
raise R.NotMatched
log.debug('%s resolved to %s' % (request, self.view))
return self.view(**m.groupdict())
def resolve(self, request):
if self.methods and request.method.upper() not in self.methods:
raise R.NotMatched
if self.host and not fnmatch(request.host, self.host):
raise R.NotMatched
m = re.match(self.pattern, request.path)
if not m:
raise R.NotMatched
log.debug('%s resolved to %s' % (request, self.view))
return self.view(**m.groupdict())
def fetchall(self, *args, **kwargs):
with self.cursor() as cursor:
t = ''
if self.debug:
t1 = time.time()
cursor.execute(*args, **kwargs)
if self.debug:
t = ' %.3f' % (time.time() - t1)
try:
log.debug(u'fetchall %s %s' %
(t, cursor.mogrify(*args, **kwargs)))
except:
pass
return cursor.fetchall()
def pop(self):
if self.blocking:
data = self.queue.brpop(self._id, settings.queue_timeout)
else:
data = self.queue.rpop(self._id)
if not data:
return None
try:
if self.channels:
data = data[1]
log.debug('POP %s: %s' % (self._id, data))
return json.loads(data)
except ValueError:
return None
def pop(self):
if self.blocking:
data = self.queue.brpop(self._id, settings.queue_timeout)
else:
data = self.queue.rpop(self._id)
if not data:
return None
try:
if self.channels:
data = data[1]
log.debug('POP %s: %s' % (self._id, data))
return json.loads(data)
except ValueError:
return None
def send_message(uid, message):
try:
wslist = wss[uid]
except KeyError:
return
dead = []
for i, ws in enumerate(wslist):
try:
ws.send(message)
except WebSocketError:
log.debug('WebSocket %s (uid=%s) id dead.' % (ws, uid))
dead.append(i)
wss[uid] = [i for j, i in enumerate(wss[uid]) if j not in dead]
def send_message(uid, message):
try:
wslist = wss[uid]
except KeyError:
return
dead = []
for i, ws in enumerate(wslist):
try:
ws.send(message)
except WebSocketError:
log.debug('WebSocket %s (uid=%s) id dead.' % (ws, uid))
dead.append(i)
wss[uid] = [i for j, i in enumerate(wss[uid]) if j not in dead]
def register_middleware(middleware):
try:
if _middleware[middleware]:
return
except KeyError:
pass
log.debug('Importing %s middleware', middleware)
if isinstance(middleware, (str, unicode)):
module, cls = middleware.rsplit('.', 1)
try:
module = __import__(module, globals(), locals(), cls, -1)
cls = getattr(module, cls)
except (ImportError, AttributeError), e:
print e
raise MiddlewareError('%s: %s' % (middleware, e))
def register_middleware(middleware):
try:
if _middleware[middleware]:
return
except KeyError:
pass
log.debug('Importing %s middleware', middleware)
if isinstance(middleware, (str, unicode)):
module, cls = middleware.rsplit('.', 1)
try:
module = __import__(module, globals(), locals(), cls, -1)
cls = getattr(module, cls)
except (ImportError, AttributeError), e:
print e
raise MiddlewareError('%s: %s' % (middleware, e))
def attach(path, dest, filename, remove=True):
if isinstance(path, str):
path = path.decode('utf-8')
if isinstance(dest, str):
dest = dest.decode('utf-8')
if isinstance(filename, str):
filename = filename.decode('utf-8')
dirname = os.path.join(settings.media_path, dest)
error = False
try:
os.makedirs(dirname)
except OSError, e:
if e.errno == 17:
log.debug('OSError %s: %s %s' % (e.errno, e.strerror, dirname))
else:
log.warn('OSError %s: %s %s' % (e.errno, e.strerror, dirname))
error = True
def thumbnail(url):
log.debug('- URL %s %s' % (type(url), url))
hash = md5(url).hexdigest()
path = os.path.join(settings.thumbnail_path, hash[:2], hash)
if os.path.isfile(path) and os.stat(path) > 0:
log.debug('%s: thumbnail exists' % path)
return
log.info('Making thumbnail %s %s' % (path, url))
if cache_get('thumbnail:%s' % hash):
return
cache_store('thumbnail:%s' % hash, 1, 60)
try:
dirname = os.path.join(settings.thumbnail_path, hash[:2])
try:
os.mkdir(dirname)
except OSError, e:
if e.errno == 17:
log.debug('OSError %s: %s %s' % (e.errno, e.strerror, dirname))
else:
log.warn('OSError %s: %s %s' % (e.errno, e.strerror, dirname))
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
resp = opener.open(url)
#resp = urllib2.urlopen(url)
buf = StringIO(resp.read())
img = Image.open(buf)
if img.size[0] * img.size[1] > settings.max_image_size:
log.error('too big: %sx%s %s' % (img.size[0], img.size[1], url))
return
img.load()
fmt = img.format
if fmt == 'JPEG':
img = _rotate(img)
elif fmt == 'GIF':
img.seek(0)
#img = img.copy()
img.thumbnail(settings.thumbnail_size, Image.ANTIALIAS)
img.save(path, fmt, **img.info)
def thumbnail(url):
log.debug('- URL %s %s' % (type(url), url))
hash = md5(url).hexdigest()
dirname = os.path.join(settings.thumbnail_path, hash[:2])
path = os.path.join(dirname, hash)
if os.path.isfile(path) and os.stat(path) > 0:
log.debug('%s: thumbnail exists' % path)
return
log.info('Making thumbnail %s %s' % (path, url))
if cache_get('thumbnail:%s' % hash):
return
cache_store('thumbnail:%s' % hash, 1, 60)
try:
try:
os.mkdir(dirname)
except OSError, e:
if e.errno == 17:
log.debug('OSError %s: %s %s' % (e.errno, e.strerror, dirname))
else:
log.warn('OSError %s: %s %s' % (e.errno, e.strerror, dirname))
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
resp = opener.open(url)
#resp = urllib2.urlopen(url)
buf = StringIO(resp.read())
img = Image.open(buf)
if img.size[0] * img.size[1] > settings.max_image_size:
log.error('too big: %sx%s %s' % (img.size[0], img.size[1], url))
return
img.load()
fmt = img.format
if fmt == 'JPEG':
img = _rotate(img)
elif fmt == 'GIF':
img.seek(0)
#img = img.copy()
img.thumbnail(settings.thumbnail_size, Image.ANTIALIAS)
img.save(path, fmt, **img.info)
"%s.%s" % (filename, randint(1000, 9999)))
fd = open(tmp_path, 'w')
fd.write(resp.read())
fd.close()
resp.close()
path = tmp_path
except (urllib2.URLError, OSError), e:
return {'status': 'fail', 'message': e.msg}
source = Image.open(path)
if source.format == 'GIF':
source.seek(0)
log.debug('%s opened: %s' % (path, source))
w, h = source.size
if w > h:
box = ((w-h)/2, 0, (w+h)/2, h)
elif w < h:
box = (0, (h-w)/2, w, (h+w)/2)
else:
box = None
if box:
crop = source.crop(box)
else:
crop = source
class Request(object):
def __init__(self, environ):
self._headers = { h[5:].lower().replace('_', '-'): val \
for h, val in environ.iteritems() \
if h.startswith('HTTP_') }
self.protocol = self.header('x-forwarded-proto') or \
environ['wsgi.url_scheme'] or 'http'
self.host = environ['HTTP_HOST']
self.method = environ['REQUEST_METHOD'].upper()
self.path = environ['PATH_INFO']
self.remote_host = self.header('X-Forwarded-For') or \
environ['REMOTE_ADDR']
try:
self.remote_port = int(environ['REMOTE_PORT'])
except (TypeError, ValueError):
self.remote_port = None
self.user_agent = environ['HTTP_USER_AGENT']
self.referer = environ['HTTP_REFERER'] if 'HTTP_REFERER' in environ else ''
self.is_xhr = self.header('X-Requested-With') == 'XMLHttpRequest'
self._args = {}
self._files = {}
self.query_string = environ['QUERY_STRING']
self.uri = '%s?%s' % (self.path, self.query_string) \
if self.query_string else self.path
if self.method in ('GET', 'HEAD'):
self._args = urlparse.parse_qs(environ['QUERY_STRING'])
elif self.method in ('POST', 'PUT', 'DELETE'):
ctype = self.header('Content-Type')
if not ctype or ctype.startswith('application/x-www-form-urlencoded'):
_buf = environ['wsgi.input'].read()
self._args = urlparse.parse_qs(_buf)
elif ctype.startswith('multipart/form-data'):
form = FieldStorage(fp=environ['wsgi.input'],
environ=environ,
keep_blank_values=True)
for field in form.list:
try:
if field.filename:
pos = field.filename.rfind('/')
if pos == -1:
pos = field.filename.rfind('\\')
filename = field.filename[pos+1:]
try:
if not isinstance(self._args[field.name], (list, tuple)):
self._args[field.name] = [self._args[field.name]]
self._args[field.name].append(filename)
except KeyError:
self._args[field.name] = filename
tmpfile = md5("%s%s" % (filename, datetime.now().isoformat())).hexdigest()
try:
upload_dir = settings.upload_dir
except AttributeError:
upload_dir = '/tmp' # FIXME: get from environment
tmpfile_path = os.path.join(upload_dir, tmpfile)
try:
if not isinstance(self._files[field.name], (list, tuple)):
self._files[field.name] = [self._files[field.name]]
self._files[field.name].append(tmpfile_path)
except KeyError:
self._files[field.name] = tmpfile_path
fd = open(tmpfile_path, 'w')
while True:
b = field.file.read(4096)
if b == '':
break
fd.write(b)
fd.close()
log.info('Upload %s: %s' % (field.name, field.filename))
else:
if not field.value:
continue
try:
if not isinstance(self._args[field.name], (list, tuple)):
self._args[field.name] = [self._args[field.name]]
self._args[field.name].append(field.value)
except KeyError:
self._args[field.name] = field.value
except IOError, e:
log.error('Cannot write %s: %s' % \
(self._files[field.name], e.strerror))
del form
self._cookies = Cookie(self.header('Cookie'))
for c, v in self._cookies.iteritems():
log.debug('Cookie: %s=%s' % (c, v.value))
def process(url):
for regex, ProcClass in processors:
if re.search(regex, url):
log.debug('%s was chosen for %s' % (ProcClass.__name__, url))
return ProcClass(url)
return FeedProcessor(url)
def cache_get(key):
log.debug("cache_get %s" % key)
redis = RedisPool(settings.cache_socket)
data = redis.get(key)
if data:
return json.loads(data)
def push(self, data):
log.debug('PUSH %s: %s' % (self._id, json.dumps(data)))
self.queue.lpush(self._id, json.dumps(data))
def cache_del(key):
log.debug('cache_del %s' % key)
redis = RedisPool(settings.cache_socket)
redis.delete(key)
def cache_del(key):
log.debug("cache_del %s" % key)
redis = RedisPool(settings.cache_socket)
redis.delete(key)
def cache_get(key):
log.debug('cache_get %s' % key)
redis = RedisPool(settings.cache_socket)
data = redis.get(key)
if data:
return json.loads(data)
def push(self, data):
log.debug('PUSH %s: %s' % (self._id, json.dumps(data)))
self.queue.lpush(self._id, json.dumps(data))