def run_server(host=None, port=None, workers=None, debug=None, logfile=None, stdout=None, loglevel=None): if not host: host = settings.server_host if not port: port = settings.server_port if workers is not None: settings.workers = workers if debug: settings.debug = True try: proctitle = settings.proctitle except AttributeError: proctitle = 'geweb' setproctitle(proctitle) log.info('Starting HTTP server at %s:%d' % (host, port)) pool = Pool(10000) server = WSGIServer("%s:%s" % (host, port), handle, spawn=pool) server.init_socket() for i in xrange(settings.workers - 1): pid = gevent.fork() if pid == 0: break server.serve_forever()
def add_post(): text = env.request.args('text', '').strip() tags = env.request.args('tags', '').strip(' \t*,;') if isinstance(tags, str): tags = tags.decode('utf-8') tags = [t.replace(u"\xa0", " ") for t in re.split(r'\s*[,;*]\s*', tags)] private = bool(env.request.args('private')) m = re.search(r'^\s*(?P<to>(?:@[a-zA-Z0-9_-]+[,\s]*)+)', text) to = parse_logins(m.group('to')) if m else [] files = _files([]) sess = Session() sess['clear_post_input'] = True sess.save() try: id = posts.add_post(text, tags=tags, to=to, private=private, files=files) except PostTextError: return render('/post-error.html') log.info('add_post: #%s %s %s' % (id, env.user.login, env.request.remote_host)) return Response(redirect='%s://%s.%s/%s' % \ (env.request.protocol, env.user.login, settings.domain, id))
def fetch(self): if self._posts: return if not self.get_url(): raise InvalidFeedUrl log.info('Feed #%s fetch from %s' % (self.id, self.get_url())) proc = process(self.get_url()) if proc.get_error(): log.error("Feed #%s fetch error: %s %s" % \ (self.id, self.get_url(), proc.get_error())) if not proc.entries(): #if 'status' in d and d['status'] < 400: # raise InvalidFeedType #else: raise FeedFetchError info = proc.get_info() for param in ['name', 'about', 'homepage']: if info[param] and info[param] != self.get_info(param): self.set_info(param, info[param]) if self.id and self.info_changed(): self.save() self._posts = [Post(None, self, **p) for p in proc.entries()] log.info("Feed #%s: %s entries fetched from %s" % \ (self.id, len(self._posts), self.get_url()))
def __init__(self): log.info('ws: pubsub init') pool = RedisPool(settings.pubsub_socket) pubsub = pool.pubsub() pubsub.subscribe(['msg', 'msg.self', 'sub', 'rec', 'confirm']) for msg in pubsub.listen(): self.handle_message(msg)
def put_tasks(self): dstart = datetime.now() qname = 'fqueue:%02d%02d' % (dstart.hour, dstart.minute) queue = Queue(qname, settings.feed_queue_socket, blocking=False, channels=False) ids = [] cnt = 0 while True: data = queue.pop() if not data: break self.fqueue.push(data) if 'id' in data: ids.append(data['id']) cnt += 1 dend = datetime.now() td = dend - dstart later = settings.feed_queue_update_timeout - td.seconds if later < 0: later = 0 if cnt: log.info('put_tasks: %d tasks in %d sec for IDs: %s' % \ (cnt, td.seconds, str(ids))) #gevent.spawn_later(later, self.put_tasks) return later
def __init__(self): proctitle('bot') log.info('bot started with PID=%d' % os.getpid()) self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource) sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password) self.register_plugin('xep_0184') self.register_plugin('xep_0163') self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune') self.plugin['xep_0060'].map_node_event( 'http://jabber.org/protocol/tune', 'user_tune') self.add_event_handler("session_start", self.session_start) self.add_event_handler("message", self.handle_message) self.add_event_handler("presence_subscribed", self.handle_subscription) self.add_event_handler("user_tune_publish", self.handle_tune) self.add_event_handler("got_offline", self.handle_disconnection) self.add_event_handler("receipt_received", self.handle_receipt) self.xin = Queue('xin', addr=settings.queue_socket) self.xout = Queue('xout', addr=settings.queue_socket) self.auto_authorize = True self.auto_subscribe = True spawn(self.listen_queue)
def __init__(self): proctitle('bot') log.info('bot started with PID=%d' % os.getpid()) self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource) sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password) self.register_plugin('xep_0184') self.register_plugin('xep_0163') self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune') self.plugin['xep_0060'].map_node_event('http://jabber.org/protocol/tune', 'user_tune') self.add_event_handler("session_start", self.session_start) self.add_event_handler("message", self.handle_message) self.add_event_handler("presence_subscribed", self.handle_subscription) self.add_event_handler("user_tune_publish", self.handle_tune) self.add_event_handler("got_offline", self.handle_disconnection) self.add_event_handler("receipt_received", self.handle_receipt) self.xin = Queue('xin', addr=settings.queue_socket) self.xout = Queue('xout', addr=settings.queue_socket) self.auto_authorize = True self.auto_subscribe = True spawn(self.listen_queue)
def update_at(self, dt): if not self.id: return log.info('Feed #%s update_at %s' % (self.id, dt)) qname = 'fqueue:%02d%02d' % (dt.hour, dt.minute) fqueue = Queue(qname, settings.feed_queue_socket, channels=False) fqueue.push({'id': self.id}) cache_store('feed:next_update:%s' % self.id, dt.isoformat())
def __init__(self): proctitle('feed-manager') log.info('FeedManager started with PID=%d' % os.getpid()) self.fqueue = Queue('fqueue', settings.feed_queue_socket, channels=False) #gevent.spawn(self.put_tasks) gevent.joinall([gevent.spawn(self.run)])
def __init__(self): proctitle('sender-queue') log.info('sender queue started with PID=%s' % os.getpid()) self.xsq = Queue('xsq', settings.queue_socket) pool = RedisPool(settings.pubsub_socket) pubsub = pool.pubsub() pubsub.subscribe(['msg', 'sub', 'rec', 'confirm', 'remember']) for msg in pubsub.listen(): self.xsq.push("%s %s" % (msg['channel'], msg['data']))
def __init__(self): self.queue = Queue('imgq', settings.imgproc_socket) log.info('imgproc worker started') while True: data = self.queue.pop() if data and isinstance(data, dict): fn = data['fn'] del data['fn'] try: handlers[fn](**data) except Exception, e: log.error(traceback.format_exc())
def __init__(self): proctitle('support-worker') log.info('support worker started with PID=%s' % os.getpid()) pool = RedisPool(settings.pubsub_socket) pubsub = pool.pubsub() pubsub.subscribe(['msg']) for msg in pubsub.listen(): try: data = json.loads(msg['data']) except TypeError: continue if data['a'] in ('post', 'post_edited'): gevent.spawn(self.handle_post, data)
def __init__(self): proctitle('sender') log.info('sender started with PID=%s' % os.getpid()) self.xsq = Queue('xsq', settings.queue_socket) self.xout = Queue('xout', settings.queue_socket) while True: msg = self.xsq.pop() if msg: channel, msg = msg.split(' ', 1) data = json.loads(msg) if isinstance(data, int): continue gevent.spawn(self.handle_message, channel, data)
def __init__(self): proctitle('feed-worker') log.info('FeedWorker started with PID=%d' % os.getpid()) self.fqueue = Queue('fqueue', settings.feed_queue_socket) while True: data = self.fqueue.pop() if not data or 'id' not in data or not data['id']: continue try: gevent.spawn(self.update_feed, int(data['id'])) except ValueError: continue
def thumbnail(url): log.debug('- URL %s %s' % (type(url), url)) hash = md5(url).hexdigest() path = os.path.join(settings.thumbnail_path, hash[:2], hash) if os.path.isfile(path) and os.stat(path) > 0: log.debug('%s: thumbnail exists' % path) return log.info('Making thumbnail %s %s' % (path, url)) if cache_get('thumbnail:%s' % hash): return cache_store('thumbnail:%s' % hash, 1, 60) try: dirname = os.path.join(settings.thumbnail_path, hash[:2]) try: os.mkdir(dirname) except OSError, e: if e.errno == 17: log.debug('OSError %s: %s %s' % (e.errno, e.strerror, dirname)) else: log.warn('OSError %s: %s %s' % (e.errno, e.strerror, dirname)) opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] resp = opener.open(url) #resp = urllib2.urlopen(url) buf = StringIO(resp.read()) img = Image.open(buf) if img.size[0] * img.size[1] > settings.max_image_size: log.error('too big: %sx%s %s' % (img.size[0], img.size[1], url)) return img.load() fmt = img.format if fmt == 'JPEG': img = _rotate(img) elif fmt == 'GIF': img.seek(0) #img = img.copy() img.thumbnail(settings.thumbnail_size, Image.ANTIALIAS) img.save(path, fmt, **img.info)
def thumbnail(url): log.debug('- URL %s %s' % (type(url), url)) hash = md5(url).hexdigest() dirname = os.path.join(settings.thumbnail_path, hash[:2]) path = os.path.join(dirname, hash) if os.path.isfile(path) and os.stat(path) > 0: log.debug('%s: thumbnail exists' % path) return log.info('Making thumbnail %s %s' % (path, url)) if cache_get('thumbnail:%s' % hash): return cache_store('thumbnail:%s' % hash, 1, 60) try: try: os.mkdir(dirname) except OSError, e: if e.errno == 17: log.debug('OSError %s: %s %s' % (e.errno, e.strerror, dirname)) else: log.warn('OSError %s: %s %s' % (e.errno, e.strerror, dirname)) opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] resp = opener.open(url) #resp = urllib2.urlopen(url) buf = StringIO(resp.read()) img = Image.open(buf) if img.size[0] * img.size[1] > settings.max_image_size: log.error('too big: %sx%s %s' % (img.size[0], img.size[1], url)) return img.load() fmt = img.format if fmt == 'JPEG': img = _rotate(img) elif fmt == 'GIF': img.seek(0) #img = img.copy() img.thumbnail(settings.thumbnail_size, Image.ANTIALIAS) img.save(path, fmt, **img.info)
def __init__(self): proctitle('worker') log.info('worker started with PID=%s' % os.getpid()) self.route = prepare_route(route) self.qin = Queue('xin', settings.queue_socket) self.qout = Queue('xout', settings.queue_socket) while True: data = self.qin.pop() if data: try: data = json.loads(data) gevent.spawn(self.handle_message, data) except ValueError, err: log.error("%s: %s" % (err.__class__.__name__, err.message))
def subscribe(login): """Subscribe to user returns True if subscribed, and False if subscription request is sent """ if isinstance(login, User): user = login else: user = User('login', login) if env.user.id == user.id: raise SubscribeError if user.add_subscription_request(env.user): publish('sub', {'to': user.id, 'a': 'sub', 'from': env.user.login}) log.info('sub %d %s' % (user.id, env.user.login)) return True else: publish('sub', {'to': user.id, 'a': 'req', 'from': env.user.login}) log.info('sub request %d %s' % (user.id, env.user.login)) return False
def subscribe(login): """Subscribe to user returns True if subscribed, and False if subscription request is sent """ if isinstance(login, User): user = login else: user = User('login', login) if env.user.id == user.id: raise SubscribeError if user.add_subscription_request(env.user): publish('sub', {'to':user.id, 'a':'sub', 'from':env.user.login}) log.info('sub %d %s' % (user.id, env.user.login)) return True else: publish('sub', {'to':user.id, 'a':'req', 'from':env.user.login}) log.info('sub request %d %s' % (user.id, env.user.login)) return False
def update_feed(self, id): try: feed = Feed(id) except FeedNotFound: log.error('Feed #%s does not exist. Skipped.' % id) return redis = RedisPool(settings.storage_socket) try: feed.fetch() redis.delete('feed:retries:%s' % feed.id) for p in feed.posts(): if not p.id: if p.tags: p.tags.insert(0, 'news') else: p.tags = ['news'] add_post(p) log.info('Feed #%s: %s new entries saved' % \ (feed.id, len(feed.posts()))) feed.update_task() except FeedFetchError: retries = redis.incr('feed:retries:%s' % feed.id) log.error('Feed #%s: %s retries failed' % (feed.id, retries)) if retries > settings.feed_retries: redis.delete('feed:retries:%s' % feed.id) return timeout = settings.feed_retry_timeout * retries feed.update_at(datetime.now() + timedelta(seconds=timeout)) except InvalidFeedType: redis.delete('feed:retries:%s' % feed.id) feed.update_at(datetime.now() + \ timedelta(seconds=settings.feed_max_update_timeout))
def __init__(self): proctitle('bot') log.info('bot started with PID=%d' % os.getpid()) self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource) sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password) self.register_plugin('xep_0184') self.add_event_handler("session_start", self.session_start) self.add_event_handler("message", self.handle_message) self.add_event_handler("presence_subscribed", self.handle_subscription) self.add_event_handler("receipt_received", self.handle_receipt) self.xin = Queue('xin', addr=settings.queue_socket) self.xout = Queue('xout', addr=settings.queue_socket) self.auto_authorize = True self.auto_subscribe = True spawn(self.listen_queue)
def __init__(self): proctitle("worker") log.info("worker started with PID=%s" % os.getpid()) self.route = prepare_route(route) self.qin = Queue("xin", settings.queue_socket) self.qout = Queue("xout", settings.queue_socket) while True: data = self.qin.pop() if data: try: data = json.loads(data) data_type = data.get("type", "msg") if data_type == "msg": method = self.handle_message elif data_type == "tune": method = self.handle_tune gevent.spawn(method, data) except ValueError, err: log.error("%s: %s" % (err.__class__.__name__, err.message))
def add_comment(id): to_comment_id = env.request.args('comment_id') text = env.request.args('text', '').strip() files = _files([]) try: comment_id = posts.add_comment(id, to_comment_id, text, files=files) except PostTextError: return render('/comment-error.html') if env.owner and env.owner.login: login = env.owner.login.lower() else: post = Post(id) login = post.author.login.lower() log.info('add_comment: #%s/%s %s %s' % (id, comment_id, env.user.login, env.request.remote_host)) return Response(redirect='%s://%s.%s/%s#%s' % \ (env.request.protocol, login, settings.domain, id, comment_id))
def avatar(path, filename, remove=True, old=None): if old: remove_avatar(old) if isinstance(path, str): path = path.decode('utf-8') if isinstance(filename, str): filename = filename.decode('utf-8') log.info('Making avatars %s %s' % (path, filename)) if path.startswith('http://') or path.startswith('https://'): try: resp = urllib2.urlopen(path) tmp_path = os.path.join(settings.upload_dir, "%s.%s" % (filename, randint(1000, 9999))) fd = open(tmp_path, 'w') fd.write(resp.read()) fd.close() resp.close() path = tmp_path except (urllib2.URLError, OSError), e: return {'status': 'fail', 'message': e.msg}
while True: gevent.sleep(settings.websocket_timeout) for uid in wss: gevent.spawn(send_message, uid, 'ping') def send_message(uid, message): try: wslist = wss[uid] except KeyError: return dead = [] for i, ws in enumerate(wslist): try: ws.send(message) except WebSocketError: log.debug('WebSocket %s (uid=%s) id dead.' % (ws, uid)) dead.append(i) wss[uid] = [i for j, i in enumerate(wss[uid]) if j not in dead] gevent.spawn(ping_sockets) log.info('ws server') WebSocketServer((settings.websocket_host, settings.websocket_port), Resource({settings.websocket_url: WsApplication})).serve_forever()
def __init__(self, environ): self._headers = { h[5:].lower().replace('_', '-'): val \ for h, val in environ.iteritems() \ if h.startswith('HTTP_') } self.protocol = self.header('x-forwarded-proto') or \ environ['wsgi.url_scheme'] or 'http' self.host = environ['HTTP_HOST'] self.method = environ['REQUEST_METHOD'].upper() self.path = environ['PATH_INFO'] self.remote_host = self.header('X-Forwarded-For') or \ environ['REMOTE_ADDR'] try: self.remote_port = int(environ['REMOTE_PORT']) except (TypeError, ValueError): self.remote_port = None self.user_agent = environ['HTTP_USER_AGENT'] self.referer = environ['HTTP_REFERER'] if 'HTTP_REFERER' in environ else '' self.is_xhr = self.header('X-Requested-With') == 'XMLHttpRequest' self._args = {} self._files = {} self.query_string = environ['QUERY_STRING'] self.uri = '%s?%s' % (self.path, self.query_string) \ if self.query_string else self.path if self.method in ('GET', 'HEAD'): self._args = urlparse.parse_qs(environ['QUERY_STRING']) elif self.method in ('POST', 'PUT', 'DELETE'): ctype = self.header('Content-Type') if not ctype or ctype.startswith('application/x-www-form-urlencoded'): _buf = environ['wsgi.input'].read() self._args = urlparse.parse_qs(_buf) elif ctype.startswith('multipart/form-data'): form = FieldStorage(fp=environ['wsgi.input'], environ=environ, keep_blank_values=True) for field in form.list: try: if field.filename: pos = field.filename.rfind('/') if pos == -1: pos = field.filename.rfind('\\') filename = field.filename[pos+1:] try: if not isinstance(self._args[field.name], (list, tuple)): self._args[field.name] = [self._args[field.name]] self._args[field.name].append(filename) except KeyError: self._args[field.name] = filename tmpfile = md5("%s%s" % (filename, datetime.now().isoformat())).hexdigest() try: upload_dir = settings.upload_dir except AttributeError: upload_dir = '/tmp' # FIXME: get from environment tmpfile_path = os.path.join(upload_dir, tmpfile) try: if not isinstance(self._files[field.name], (list, tuple)): self._files[field.name] = [self._files[field.name]] self._files[field.name].append(tmpfile_path) except KeyError: self._files[field.name] = tmpfile_path fd = open(tmpfile_path, 'w') while True: b = field.file.read(4096) if b == '': break fd.write(b) fd.close() log.info('Upload %s: %s' % (field.name, field.filename)) else: if not field.value: continue try: if not isinstance(self._args[field.name], (list, tuple)): self._args[field.name] = [self._args[field.name]] self._args[field.name].append(field.value) except KeyError: self._args[field.name] = field.value except IOError, e: log.error('Cannot write %s: %s' % \ (self._files[field.name], e.strerror)) del form
def ping_sockets(): while True: gevent.sleep(settings.websocket_timeout) for uid in wss: gevent.spawn(send_message, uid, 'ping') def send_message(uid, message): try: wslist = wss[uid] except KeyError: return dead = [] for i, ws in enumerate(wslist): try: ws.send(message) except WebSocketError: log.debug('WebSocket %s (uid=%s) id dead.' % (ws, uid)) dead.append(i) wss[uid] = [i for j, i in enumerate(wss[uid]) if j not in dead] gevent.spawn(ping_sockets) log.info('ws server') WebSocketServer((settings.websocket_host, settings.websocket_port), Resource({settings.websocket_url: WsApplication}) ).serve_forever()
code = response.code message = response.message process_response(response) status, headers = response.render_headers() body = response.render() if isinstance(body, unicode): body = body.encode('utf-8') if settings.debug: tm = round(time() - tm, 4) else: tm = '' log.info('[%s] %d %s' % (tm, code, env.request)) start_response(status, headers) return [body] def run_server(host=None, port=None, workers=None, debug=None, logfile=None, stdout=None, loglevel=None): if not host: host = settings.server_host if not port: port = settings.server_port if workers is not None: settings.workers = workers if debug:
code = response.code message = response.message process_response(response) status, headers = response.render_headers() body = response.render() if isinstance(body, unicode): body = body.encode('utf-8') if settings.debug: tm = round(time() - tm, 4) else: tm = '' log.info('[%s] %d %s' % (tm, code, env.request)) start_response(status, headers) return [body] def run_server(host=None, port=None, workers=None, debug=None, logfile=None, stdout=None, loglevel=None): if not host: host = settings.server_host if not port: