def webdav_delete(self, url): "Downloads given URL and returns it" if url is None: return if type(url) == unicode: url = url.encode("utf-8") url_obj = urlparse.urlparse(url, "http", False) if url_obj.scheme != "http": self.error("Scheme '%s' is not supported", url_obj.scheme) elif url_obj.hostname is None: self.error("Empty hostname: %s", url) else: cnn = HTTPConnection() try: with Timeout.push(50): port = url_obj.port if port is None: port = 80 cnn.connect((url_obj.hostname, port)) request = HTTPRequest() request.method = "DELETE" request.path = url_obj.path + url_obj.query request.host = url_obj.hostname request.add_header("Connection", "close") cnn.perform(request) except TimeoutError: self.error("Timeout deleting %s", url) except Exception as e: self.error("Error deleting %s: %s", url, str(e)) finally: try: cnn.close() except Exception: pass
def do_query_service(self, service_id, svc, url, timeout, verbose=False, *args, **kwargs): try: with Timeout.push(timeout): cnn = HTTPConnection() addr = (svc.get("addr").encode("utf-8"), svc.get("port")) try: cnn.connect(addr) except IOError as e: raise ClusterError("Error connecting to service %s(%s:%s): %s" % (service_id, addr[0], addr[1], e)) params = { "args": json.dumps(args), "kwargs": json.dumps(kwargs) } try: uri = utf2str("/service/call/%s%s" % (service_id, url)) request = cnn.post(uri, urlencode(params)) request.add_header("Content-type", "application/x-www-form-urlencoded") request.add_header("Connection", "close") if verbose: self.debug("Query http://%s:%s%s", addr[0], addr[1], uri) response = cnn.perform(request) if response.status_code != 200: raise ClusterError("Service %s (%s:%d) returned status %d for URL %s" % (service_id, addr[0], addr[1], response.status_code, uri)) res = json.loads(response.body) if res.get("error"): raise ClusterError(u"Service %s returned error: %s" % (service_id, res["error"])) return res.get("retval") finally: cnn.close() except TimeoutError: raise ClusterError("Timeout querying %s of service %s" % (url, service_id))
def query(host, port, uri, params, timeout=20): try: with Timeout.push(timeout): cnn = HTTPConnection() cnn.connect((str(host), int(port))) try: request = cnn.post(str(uri), urlencode(params)) request.add_header("Content-type", "application/x-www-form-urlencoded") request.add_header("Connection", "close") response = cnn.perform(request) if response.status_code != 200: raise HTTPError("Error downloading http://%s:%s%s: %s" % (host, port, uri, response.status)) body = response.body if response.get_header("Content-type") == "application/json": body = json.loads(body) return body finally: cnn.close() except IOError as e: if e.errno == 111: raise HTTPConnectionRefused("Connection refused during downloading http://%s:%s%s" % (host, port, uri)) else: raise HTTPError("Error downloading http://%s:%s%s: %s" % (host, port, uri, e)) except TimeoutError: raise HTTPError("Timeout downloading http://%s:%s%s" % (host, port, uri))
def query(host, port, uri, params, timeout=20): try: with Timeout.push(timeout): cnn = HTTPConnection() cnn.connect((str(host), int(port))) try: request = cnn.post(str(uri), urlencode(params)) request.add_header("Content-type", "application/x-www-form-urlencoded") request.add_header("Connection", "close") response = cnn.perform(request) if response.status_code != 200: raise HTTPError("Error downloading http://%s:%s%s: %s" % (host, port, uri, response.status)) body = response.body if response.get_header("Content-type") == "application/json": body = json.loads(body) return body finally: cnn.close() except IOError as e: if e.errno == 111: raise HTTPConnectionRefused( "Connection refused during downloading http://%s:%s%s" % (host, port, uri)) else: raise HTTPError("Error downloading http://%s:%s%s: %s" % (host, port, uri, e)) except TimeoutError: raise HTTPError("Timeout downloading http://%s:%s%s" % (host, port, uri))
def appconfig_changed(self): tag = None try: tag = self.app().tag except AttributeError: pass if tag is not None: try: with Timeout.push(3): self.call("cluster.query-services", "int", "/core/appconfig/%s" % tag) except TimeoutError: pass
def test_01_turn_order(self): combat = SimulationCombat(self.app) service = DebugCombatService(combat) # joining member 1 member1 = CombatMember(combat) ai1 = ManualDebugController(member1) member1.add_controller(ai1) member1.set_team(1) combat.join(member1) # joining member 2 member2 = CombatMember(combat) ai2 = ManualDebugController(member2) member2.add_controller(ai2) member2.set_team(2) combat.join(member2) # checking list of members and ensuring # that nobody has right to turn before # combat started self.assertEqual(combat.members, [member1, member2]) self.assertFalse(member1.may_turn) self.assertFalse(member2.may_turn) service.run_combat() # member1 must have right of turn self.assertTrue(member1.may_turn) self.assertFalse(member2.may_turn) # waiting for turn timeout with Timeout.push(3): self.assertRaises(TurnTimeout, service.run) # member2 must have right of turn self.assertFalse(member1.may_turn) self.assertTrue(member2.may_turn) # waiting for turn timeout with Timeout.push(3): self.assertRaises(TurnTimeout, service.run) # member1 must have right of turn self.assertTrue(member1.may_turn) self.assertFalse(member2.may_turn)
def testTimeout(self): mc = Memcache() mc.set_servers([((MEMCACHE_IP, 11212), 100)]) self.memcachedPause() def clientTimeout(): with Timeout.push(0.5): self.assertEquals(MemcacheResult.TIMEOUT, mc.set('blaat', 'aap')) def clientError(): with Timeout.push(0.5): self.assertEquals(MemcacheResult.ERROR, mc.set('blaat', 'aap')) # Send some requests for i in xrange(0, 1000): Tasklet.new(clientTimeout)() Tasklet.join_children() with Timeout.push(0.5): self.assertEquals(MemcacheResult.TIMEOUT, mc.set('foo', 'bar')) print 'done (timeout)' self.memcachedResume() self.assertEquals(mc.get('blaat'), 'aap') self.assertEquals(mc.get('foo'), 'bar') self.memcachedPause() # Send some requests for i in xrange(0, 1000): Tasklet.new(clientTimeout)() Tasklet.join_children() self.memcachedKill() # Send some requests expected for i in xrange(0, 1000): Tasklet.new(clientError)() Tasklet.join_children() self.memcachedRun() self.assertEquals(MemcacheResult.STORED, mc.set("bar", "baz")) self.assertEquals(None, mc.get("blaat")) self.assertEquals(None, mc.get("foo")) self.assertEquals("baz", mc.get("bar"))
def do_query_service(self, service_id, svc, url, timeout, verbose=False, *args, **kwargs): try: with Timeout.push(timeout): cnn = HTTPConnection() addr = (svc.get("addr").encode("utf-8"), svc.get("port")) try: cnn.connect(addr) except IOError as e: raise ClusterError( "Error connecting to service %s(%s:%s): %s" % (service_id, addr[0], addr[1], e)) params = { "args": json.dumps(args), "kwargs": json.dumps(kwargs) } try: uri = utf2str("/service/call/%s%s" % (service_id, url)) request = cnn.post(uri, urlencode(params)) request.add_header("Content-type", "application/x-www-form-urlencoded") request.add_header("Connection", "close") if verbose: self.debug("Query http://%s:%s%s", addr[0], addr[1], uri) response = cnn.perform(request) if response.status_code != 200: raise ClusterError( "Service %s (%s:%d) returned status %d for URL %s" % (service_id, addr[0], addr[1], response.status_code, uri)) res = json.loads(response.body) if res.get("error"): raise ClusterError(u"Service %s returned error: %s" % (service_id, res["error"])) return res.get("retval") finally: cnn.close() except TimeoutError: raise ClusterError("Timeout querying %s of service %s" % (url, service_id))
def testTimeout(self): mc = Memcache() mc.set_servers([((MEMCACHE_IP, 11211), 100)]) def callback(socket, count, event, args, kwargs): print count, event, Tasklet.current() if (count, event) == (1, "write"): pass elif (count, event) == (2, "read"): Tasklet.sleep(1.0) return "OK\r\n" unittest.TestSocket.install((MEMCACHE_IP, 11211), callback) with Timeout.push(0.5): self.assertEquals(MemcacheResult.TIMEOUT, mc.set('blaat', 'aap')) print 'done (timeout)' Tasklet.sleep(4.0)
def download(self, url): "Downloads given URL and returns it" if url is None: raise DownloadError() if type(url) == unicode: url = url.encode("utf-8") url_obj = urlparse.urlparse(url, "http", False) if url_obj.scheme != "http": self.error("Scheme '%s' is not supported", url_obj.scheme) elif url_obj.hostname is None: self.error("Empty hostname: %s", url) else: cnn = HTTPConnection() try: with Timeout.push(50): cnn.set_limit(20000000) port = url_obj.port if port is None: port = 80 cnn.connect((url_obj.hostname, port)) request = cnn.get(url_obj.path + url_obj.query) request.add_header("Connection", "close") response = cnn.perform(request) if response.status_code != 200: self.error("Error downloading %s: %s %s", url, response.status_code, response.status) return "" return response.body except TimeoutError: self.error("Timeout downloading %s", url) except Exception as e: self.error("Error downloading %s: %s", url, str(e)) finally: try: cnn.close() except Exception: pass raise DownloadError()
def clientTimeout(): with Timeout.push(0.5): self.assertEquals(MemcacheResult.TIMEOUT, mc.set('blaat', 'aap'))
def clientError(): with Timeout.push(0.5): self.assertEquals(MemcacheResult.ERROR, mc.set('blaat', 'aap'))
def parts(self, params, errors): tokens = re_image.split(params["content"]) image_num = 0 parts = [] if len(tokens) > 1: content = u"" for token in tokens: m = re_image_src.match(token) if m: before, src, after = m.group(1, 2, 3) if type(src) == unicode: src = src.encode("utf-8") url_obj = urlparse.urlparse(src, "http", False) if url_obj.scheme != "http" or url_obj.hostname is None: errors["v_mode"] = self._( "Image URL %s is incorrect") % src else: cnn = HTTPConnection() try: with Timeout.push(50): cnn.set_limit(20000000) port = url_obj.port if port is None: port = 80 cnn.connect((url_obj.hostname, port)) request = cnn.get(url_obj.path + url_obj.query) request.add_header("Connection", "close") response = cnn.perform(request) if response.status_code != 200: if response.status_code == 404: errors["v_mode"] = self._( "%s Resource not found") % src elif response.status_code == 403: errors["v_mode"] = self._( "%s: Access denied") % src elif response.status_code == 500: errors["v_mode"] = self._( "%s: Internal server error") % src else: errors["v_mode"] = "%s: %s" % ( src, htmlescape(response.status)) else: content_type = "" for header in response.headers: if header[0].lower() == "content-type": content_type = header[1] break m = re_image_type.match(content_type) if m: subtype = m.group(1) image_num += 1 part = MIMEImage( response.body, subtype) part.add_header( "Content-ID", "<image%d>" % image_num) filename = src.split("/")[-1] part.add_header( "Content-Disposition", "attachment; filename=%s" % filename) parts.append(part) content += u'%scid:image%d%s' % ( before, image_num, after) else: errors["v_mode"] = self._( "URL %s is not an image") % src except TimeoutError as e: errors["v_mode"] = self._( "Timeout on downloading %s. Time limit - 30 sec" ) % src except Exception as e: errors["v_mode"] = "%s: %s" % (src, htmlescape(str(e))) finally: try: cnn.close() except Exception: pass else: content += token params["content"] = content params["parts"] = parts
def image(self): self.call("web.upload_handler") req = self.req() url = req.param("url") image_field = "image" errors = {} image = req.param_raw("image") if not image and url: url_obj = urlparse.urlparse(url.encode("utf-8"), "http", False) if url_obj.scheme != "http": errors["url"] = self._("Scheme '%s' is not supported" ) % htmlescape(url_obj.scheme) elif url_obj.hostname is None: errors["url"] = self._("Enter correct URL") else: cnn = HTTPConnection() try: with Timeout.push(50): cnn.set_limit(20000000) port = url_obj.port if port is None: port = 80 cnn.connect((url_obj.hostname, port)) request = cnn.get(url_obj.path + url_obj.query) request.add_header("Connection", "close") response = cnn.perform(request) if response.status_code != 200: if response.status_code == 404: errors["url"] = self._( "Remote server response: Resource not found" ) elif response.status_code == 403: errors["url"] = self._( "Remote server response: Access denied") elif response.status_code == 500: errors["url"] = self._( "Remote server response: Internal server error" ) else: errors["url"] = self._( "Download error: %s") % htmlescape( response.status) else: image = response.body image_field = "url" except TimeoutError as e: errors["url"] = self._( "Timeout on downloading image. Time limit - 30 sec") except Exception as e: errors["url"] = self._("Download error: %s") % htmlescape( str(e)) finally: try: cnn.close() except Exception: pass if image: try: image_obj = Image.open(cStringIO.StringIO(image)) except IOError: errors[image_field] = self._("Image format not recognized") if not errors: format = image_obj.format if format == "GIF": ext = "gif" content_type = "image/gif" target_format = "GIF" elif format == "PNG": ext = "png" content_type = "image/png" target_format = "PNG" else: target_format = "JPEG" ext = "jpg" content_type = "image/jpeg" if target_format != format: im_data = cStringIO.StringIO() image_obj.save(im_data, target_format) im_data = im_data.getvalue() else: im_data = image uri = self.call("cluster.static_upload", "socio", ext, content_type, im_data) self.call("web.response_json_html", { "success": True, "uri": uri }) elif not errors: errors["image"] = self._("Upload an image") if errors: self.call("web.response_json_html", { "success": False, "errors": errors }) else: self.call( "web.response_json_html", { "success": False, "errmsg": self._("Unknown error uploading image") })
def parts(self, params, errors): tokens = re_image.split(params["content"]) image_num = 0 parts = [] if len(tokens) > 1: content = u"" for token in tokens: m = re_image_src.match(token) if m: before, src, after = m.group(1, 2, 3) if type(src) == unicode: src = src.encode("utf-8") url_obj = urlparse.urlparse(src, "http", False) if url_obj.scheme != "http" or url_obj.hostname is None: errors["v_mode"] = self._("Image URL %s is incorrect") % src else: cnn = HTTPConnection() try: with Timeout.push(50): cnn.set_limit(20000000) port = url_obj.port if port is None: port = 80 cnn.connect((url_obj.hostname, port)) request = cnn.get(url_obj.path + url_obj.query) request.add_header("Connection", "close") response = cnn.perform(request) if response.status_code != 200: if response.status_code == 404: errors["v_mode"] = self._("%s Resource not found") % src elif response.status_code == 403: errors["v_mode"] = self._("%s: Access denied") % src elif response.status_code == 500: errors["v_mode"] = self._("%s: Internal server error") % src else: errors["v_mode"] = "%s: %s" % (src, htmlescape(response.status)) else: content_type = "" for header in response.headers: if header[0].lower() == "content-type": content_type = header[1] break m = re_image_type.match(content_type) if m: subtype = m.group(1) image_num += 1 part = MIMEImage(response.body, subtype) part.add_header("Content-ID", "<image%d>" % image_num) filename = src.split("/")[-1] part.add_header("Content-Disposition", "attachment; filename=%s" % filename) parts.append(part) content += u'%scid:image%d%s' % (before, image_num, after) else: errors["v_mode"] = self._("URL %s is not an image") % src except TimeoutError as e: errors["v_mode"] = self._("Timeout on downloading %s. Time limit - 30 sec") % src except Exception as e: errors["v_mode"] = "%s: %s" % (src, htmlescape(str(e))) finally: try: cnn.close() except Exception: pass else: content += token params["content"] = content params["parts"] = parts
def image(self): self.call("web.upload_handler") req = self.req() url = req.param("url") image_field = "image" errors = {} image = req.param_raw("image") if not image and url: url_obj = urlparse.urlparse(url.encode("utf-8"), "http", False) if url_obj.scheme != "http": errors["url"] = self._("Scheme '%s' is not supported") % htmlescape(url_obj.scheme) elif url_obj.hostname is None: errors["url"] = self._("Enter correct URL") else: cnn = HTTPConnection() try: with Timeout.push(50): cnn.set_limit(20000000) port = url_obj.port if port is None: port = 80 cnn.connect((url_obj.hostname, port)) request = cnn.get(url_obj.path + url_obj.query) request.add_header("Connection", "close") response = cnn.perform(request) if response.status_code != 200: if response.status_code == 404: errors["url"] = self._("Remote server response: Resource not found") elif response.status_code == 403: errors["url"] = self._("Remote server response: Access denied") elif response.status_code == 500: errors["url"] = self._("Remote server response: Internal server error") else: errors["url"] = self._("Download error: %s") % htmlescape(response.status) else: image = response.body image_field = "url" except TimeoutError as e: errors["url"] = self._("Timeout on downloading image. Time limit - 30 sec") except Exception as e: errors["url"] = self._("Download error: %s") % htmlescape(str(e)) finally: try: cnn.close() except Exception: pass if image: try: image_obj = Image.open(cStringIO.StringIO(image)) except IOError: errors[image_field] = self._("Image format not recognized") if not errors: format = image_obj.format if format == "GIF": ext = "gif" content_type = "image/gif" target_format = "GIF" elif format == "PNG": ext = "png" content_type = "image/png" target_format = "PNG" else: target_format = "JPEG" ext = "jpg" content_type = "image/jpeg" if target_format != format: im_data = cStringIO.StringIO() image_obj.save(im_data, target_format) im_data = im_data.getvalue() else: im_data = image uri = self.call("cluster.static_upload", "socio", ext, content_type, im_data) self.call("web.response_json_html", {"success": True, "uri": uri}) elif not errors: errors["image"] = self._("Upload an image") if errors: self.call("web.response_json_html", {"success": False, "errors": errors}) else: self.call("web.response_json_html", {"success": False, "errmsg": self._("Unknown error uploading image")})
def email_send(self, to_email, to_name, subject, content, from_email=None, from_name=None, immediately=False, subtype="plain", signature=True, headers={}): with Timeout.push(30): fingerprint = utf2str(from_email) + "/" + utf2str(to_email) + "/" + utf2str(subject) m = hashlib.md5() m.update(fingerprint) fingerprint = m.hexdigest() mcid = "email-sent-%s" % fingerprint sent = intz(self.app().mc.get(mcid)) if sent < 0: return if sent >= MAX_SIMILAR_EMAILS: self.warning("Blocked email flood to %s: %s", to_email, subject) self.app().mc.set(mcid, -1, 3600) return self.app().mc.set(mcid, sent + 1, 600) if not immediately: return self.call("queue.add", "email.send", { "to_email": to_email, "to_name": to_name, "subject": subject, "content": content, "from_email": from_email, "from_name": from_name, "immediately": True, "subtype": subtype, "signature": signature, "headers": headers, }) params = { "email": "robot@%s" % self.main_host, "name": "Metagam Robot", "prefix": "[mg] ", } self.call("email.sender", params) if from_email is None or from_name is None: from_email = params["email"] from_name = params["name"] self.info("%s: To %s <%s>: %s", mcid, utf2str(to_name), utf2str(to_email), utf2str(subject)) s = SMTP(self.clconf("smtp_server", "127.0.0.1")) try: if type(content) == unicode: content = content.encode("utf-8") if type(from_email) == unicode: from_email = from_email.encode("utf-8") if type(to_email) == unicode: to_email = to_email.encode("utf-8") if signature and subtype == "plain": sig = params.get("signature") or "" sig = str2unicode(sig) # unsubscribe if sig: sig += "\n" if to_name: sig += self._('Your name is {user_name}').format( user_name=to_name ) if getattr(self.app(), "canonical_domain", None): sig += "\n" sig += self._("Remind password - {href}").format( href="{protocol}://{domain}/auth/remind".format( protocol=self.app().protocol, domain=self.app().canonical_domain, ), ) sig += "\n" sig += self.call("email.unsubscribe-text", to_email) if sig: content += "\n\n--\n%s" % utf2str(sig) if subtype == "raw": body = content else: msg = MIMEText(content, _subtype=subtype, _charset="utf-8") msg["Subject"] = "%s%s" % (params["prefix"], Header(subject, "utf-8")) msg["From"] = "%s <%s>" % (Header(from_name, "utf-8"), from_email) msg["To"] = "%s <%s>" % (Header(to_name, "utf-8"), to_email) now = datetime.datetime.now() stamp = time.mktime(now.timetuple()) msg["Date"] = formatdate(timeval=stamp, localtime=False, usegmt=True) try: msg["X-Metagam-Project"] = self.app().tag except AttributeError: pass for key, val in headers.iteritems(): msg[key] = val body = msg.as_string() s.sendmail("<%s>" % from_email, ["<%s>" % to_email], body) except SMTPRecipientsRefused as e: self.warning(e) except SMTPException as e: self.error(e) self.call("web.service_unavailable") finally: s.quit()
def email_send(self, to_email, to_name, subject, content, from_email=None, from_name=None, immediately=False, subtype="plain", signature=True, headers={}): with Timeout.push(30): fingerprint = utf2str(from_email) + "/" + utf2str( to_email) + "/" + utf2str(subject) m = hashlib.md5() m.update(fingerprint) fingerprint = m.hexdigest() mcid = "email-sent-%s" % fingerprint sent = intz(self.app().mc.get(mcid)) if sent < 0: return if sent >= MAX_SIMILAR_EMAILS: self.warning("Blocked email flood to %s: %s", to_email, subject) self.app().mc.set(mcid, -1, 3600) return self.app().mc.set(mcid, sent + 1, 600) if not immediately: return self.call( "queue.add", "email.send", { "to_email": to_email, "to_name": to_name, "subject": subject, "content": content, "from_email": from_email, "from_name": from_name, "immediately": True, "subtype": subtype, "signature": signature, "headers": headers, }) params = { "email": "robot@%s" % self.main_host, "name": "Metagam Robot", "prefix": "[mg] ", } self.call("email.sender", params) if from_email is None or from_name is None: from_email = params["email"] from_name = params["name"] self.info("%s: To %s <%s>: %s", mcid, utf2str(to_name), utf2str(to_email), utf2str(subject)) s = SMTP(self.clconf("smtp_server", "127.0.0.1")) try: if type(content) == unicode: content = content.encode("utf-8") if type(from_email) == unicode: from_email = from_email.encode("utf-8") if type(to_email) == unicode: to_email = to_email.encode("utf-8") if signature and subtype == "plain": sig = params.get("signature") or "" sig = str2unicode(sig) # unsubscribe if sig: sig += "\n" if to_name: sig += self._('Your name is {user_name}').format( user_name=to_name) if getattr(self.app(), "canonical_domain", None): sig += "\n" sig += self._("Remind password - {href}").format( href="{protocol}://{domain}/auth/remind".format( protocol=self.app().protocol, domain=self.app().canonical_domain, ), ) sig += "\n" sig += self.call("email.unsubscribe-text", to_email) if sig: content += "\n\n--\n%s" % utf2str(sig) if subtype == "raw": body = content else: msg = MIMEText(content, _subtype=subtype, _charset="utf-8") msg["Subject"] = "%s%s" % (params["prefix"], Header(subject, "utf-8")) msg["From"] = "%s <%s>" % (Header(from_name, "utf-8"), from_email) msg["To"] = "%s <%s>" % (Header(to_name, "utf-8"), to_email) now = datetime.datetime.now() stamp = time.mktime(now.timetuple()) msg["Date"] = formatdate(timeval=stamp, localtime=False, usegmt=True) try: msg["X-Metagam-Project"] = self.app().tag except AttributeError: pass for key, val in headers.iteritems(): msg[key] = val body = msg.as_string() s.sendmail("<%s>" % from_email, ["<%s>" % to_email], body) except SMTPRecipientsRefused as e: self.warning(e) except SMTPException as e: self.error(e) self.call("web.service_unavailable") finally: s.quit()