def __init__(self, status, headers=Headers(), content=None, message=None, request=None): """ Construct a new ``Response`` object. :param status: HTTP status code for the response :type status: integer :param headers: HTTP headers :type status: a list of tuples or a class:`Headers` object :param content: content :param message: HTTP message for the response :param request: origin Request object used :type request: class:`Request` .. attribute:: redirects List of redirections """ self._status = status self.message = message self.redirects = list() if (not isinstance(headers, Headers)): headers = Headers(headers) self._headers = headers self._content = content self._request = request
def __init__(self, method, url, headers=None, content=None): """ Construct a new ``Request`` object :param method: HTTP method :type method: string :param url: URL of the request :type url: string or class:`Url` :param headers: Headers for the request :type headers: list of tuples or class:`Headers` :param content: body """ self.method = method self.content = content if not isinstance(url, Url): url = Url(url) self.url = url if headers is None: headers = Headers() elif not isinstance(headers, Headers): headers = Headers(headers) self._headers = headers
def __init__(self, status, headers=Headers(), content=None, message=None, request=None): self.status = status self.message = message self.redirects = list() if (not isinstance(headers, Headers)): headers = Headers(headers) self._headers = headers methods_from_headers = [ 'last_modified', 'date', 'expires', 'content_length', 'content_is_text', 'content_is_xml', 'content_is_xhtml' ] for m in methods_from_headers: setattr(self.__class__, m, getattr(headers, m)) self._content = content self._request = request
def __init__(self, case_root, infile): fullpath = os.path.join(case_root, infile) EntryID.__init__(self, fullpath) if (not os.path.isfile(fullpath)): headerobj = Headers() headernode = headerobj.get_header_node(os.path.basename(fullpath)) self.root.append(headernode)
def get_page(url): retries = 1 for i in range(10): print("Try : ", i, url) try: userAgentDict = Headers.get_user_agent() proxyDict = Headers.get_proxy() print(url) response = requests.get(url, proxies=proxyDict, headers=userAgentDict, timeout=10) status = response.status_code print(status) if status == 200: return response.text print("Got the response code:", status) except Exception as e: if (i == 9): send_email( "Exception occurred for this url while getting the html for archive :" + url + "the error: " + e) print( "Exception occurred for this url while getting the html for archive :", url, "the error: ", e) wait = retries * 2 time.sleep(wait) retries += 1 return ""
def from_socket(cls, sock: socket.socket) -> "Request": """Read and parse the request from a socket object. Raises: ValueError: When the request cannot be parsed. """ lines = iter_lines(sock) try: request_line = next(lines).decode("ascii") except StopIteration: raise ValueError("Request line missing") try: method, path, __ = request_line.split(" ") except ValueError: raise ValueError(f"Malformed request line {request_line!r}.") headers = Headers() while True: try: line = next(lines) except StopIteration as e: buff = e.value break try: name, __, value = line.decode("ascii").partition(":") headers.add(name.lower(), value.lstrip()) except ValueError: raise ValueError(f"Malformed header line {line!r}.") body = BodyReader(sock, buff=buff) return cls(method=method.upper(), path=path, headers=headers, body=body)
def __init__(self, request): self.request = request self.code = 200 self.code_text = self.CODES[self.code] self.headers = Headers() if self.has_body(): self.body = ""
def login(username, password, activation_token=None, expecting=200): payload = {"username": username, "password": password} if activation_token: payload['activation_token'] = activation_token r = client.put('/user/login', json=payload, headers=Headers.set()) assert r.status_code == expecting if r.status_code == 200: Headers.update(authorization='Bearer %s' % r.json) return
def test_put_user(standard, init_ac_general): print() login('fx', 'lost') u = User.read(username='******') r = client.get('api/user/%d' % u.id, headers=Headers.get()) assert r.status_code == 200 u.fname = 'Big' u.lname = 'Bird' r = client.put('api/user', headers=Headers.get(), json=u.get_dict()) assert r.status_code == 200 return
def test_get_ugrs_by_id(standard, init_ac_general): login('fx', 'lost') r = client.get('api/user', headers=Headers.get()) assert r.status_code == 200 r = client.get('api/ugrs/user/%d' % r.json['id'], headers=Headers.get()) assert r.status_code == 200 print(json.dumps(r.json, indent=3)) #print(r.json) # WP_734 ugrs_dict = r.json UGRS = "ugrs" assert (UGRS in ugrs_dict) ugrs_dle = ugrs_dict[UGRS] ugr_dict_list_elem1 = ugrs_dle[0] ugr_dl_elem = { "_access_rights": { "permissions": ["read", "write", "add", "delete"], "read_only": [], "read_write": ["uuid", "id", "role_tag", "user_id", "group_id"] }, "_group_name": "Foundation", "_obj_type": "user_group_role", "group_id": 1, "id": 1, "role_tag": "admin", "user_id": 1, "uuid": "e11d5af1-0326-4f24-aad0-f3fe437dd990" } for ugrs_key in ugr_dl_elem: assert (ugrs_key in ugr_dict_list_elem1) acc_rights_dict = ugr_dict_list_elem1["_access_rights"] assert ("permissions" in acc_rights_dict) assert ("read_only" in acc_rights_dict) assert ("read_write" in acc_rights_dict) arp_list = acc_rights_dict["permissions"] perm_list = ["read", "write", "add", "delete"] for _elem in perm_list: assert (_elem in arp_list) arrw_list = acc_rights_dict["read_write"] rw_list = ["uuid", "id", "role_tag", "user_id", "group_id"] for _elem in rw_list: assert (_elem in arrw_list) arro_list = acc_rights_dict["read_only"] assert (len(arro_list) == 0) # WP_734 return
def test_get_groups(standard, init_ac_general): print() login('fx', 'lost') r = client.get('api/groups/summary', headers=Headers.get()) assert r.status_code == 200 print(json.dumps(r.json, indent=3)) # TODO: verify data r = client.get('api/groups/standard', headers=Headers.get()) assert r.status_code == 200 print(json.dumps(r.json, indent=3)) # TODO: verify data return
def getHeaders(self, info): """Extract the headers from an IRequestInfo or IResponseInfo object. Returns a Headers object with the headers. Args: * info (IRequestInfo or IResponseInfo): Request info. Use the output from getInfo or getInfoFromBytes. """ from headers import Headers hdr = Headers() # this is IRequestInfo.getHeaders() or IResponseInfo.getHeaders() from Burp rawHdr = info.getHeaders() hdr.importRaw(rawHdr) return hdr
def _fetch_url(url, is_threaded, timeout=None, crawl_type=None): """ Crawls the html content of the parameter url and saves the html in _results :param url: :param is_threaded: If True, results will be stored for later processing by the fetch_urls method. Else not. :param timeout: in seconds, if None, the urllib default is used :return: html of the url """ # print("----------------------------------------Entered _fetch_urls", url) html = "" response = None for i in range(10): retries = 1 print("Try to fetch the html:", i, "url:", url) try: userAgentDict = Headers.get_user_agent() if crawl_type == "Archives": # using proxy only for Archives # print("It is inside") proxyDict = Headers.get_proxy() response = requests.get( url, proxies=proxyDict, headers=userAgentDict, timeout=10) else: response = requests.get( url, headers=userAgentDict, timeout=10) status = response.status_code print(url, status) if status == 200: print("[Successful]", url) html = response.text break except Exception as e: print( "Exception occurred for this url while getting the html for this archive article", url) print("the error: ", e) wait = retries*2 time.sleep(wait) retries += 1 if html == "": if response == None: raise Exception( "[Error]Could not get the html file for this article url: "+url) else: raise Exception("[Error]Got the status code : ", str(status) + "\n" + "[Error]Could not get the html file for this article url: "+url, response.text) if is_threaded: SimpleCrawler._results[url] = html # print(html) return html
def test_get_commands(standard, init_ac_general): print() login('fx', 'lost') r = client.get('api/asset/H5V2JiWiwCi0/commands', headers=Headers.get()) assert r.status_code == 200 print(json.dumps(r.json, indent=3)) return
def test_get_assets(standard, init_ac_general): print() login('brad', 'lost') r = client.get('api/assets', headers=Headers.get()) assert r.status_code == 200 print(json.dumps(r.json, indent=3)) return
class Response: CODES = { 200: "OK", 404: "Not Found", 500: "Internal Server Error" } def __init__(self, request): self.request = request self.code = 200 self.code_text = self.CODES[self.code] self.headers = Headers() if self.has_body(): self.body = "" def set_code(self, code): self.code = code self.code_text = self.CODES[code] # The HEAD method doesn't have a body def has_body(self): return self.request["method"] != "HEAD" def __str__(self): s = "{0} {1} {2}\r\n".format(self.request["version"], self.code, self.code_text) s += self.headers.to_headers() if self.has_body(): s += self.body return s
def getContent(self, url, category_id): headers = Headers.getHeaders() sleep_time = 1 while True: try: req = requests.get(url, headers=headers, timeout=30) break except: print('sleep10') time.sleep(10 * sleep_time) sleep_time = sleep_time + 1 if req.status_code == 200: html = req.content.decode('gb2312', 'ignore') selector = etree.HTML(html) root_path = selector.xpath('//div[contains(@id,"Zoom")]') names = selector.xpath( '//div[contains(@class,"title_all")]/h1/font/text()') if len(root_path) > 0: contents = simplejson.dumps( root_path[0].xpath('descendant::text()')) imgs = simplejson.dumps( root_path[0].xpath('descendant::img/@src')) if len(names) > 0: name = names[0] else: name = '' print(name) created_at = time.strftime('%Y-%m-%d %H:%M:%S') sql_values = (category_id, name, contents, imgs, created_at, url) return sql_values else: pass
def test_get_user_current(standard, prime): login('fx', 'lost') r = client.options('api/user') assert r.status_code == 200 r = client.get('api/user', headers=Headers.get()) assert r.status_code == 200 return
def __init__(self, method, url, headers=Headers(), content=None): # XXX no content on GET / DELETE ? self.method = method self.content = content if not isinstance(url, Url): url = Url(url) self.url = url if not isinstance(headers, Headers): headers = Headers(headers) self._headers = headers methods_from_headers = ['if_modified_since', 'if_unmodified_since'] for m in methods_from_headers: setattr(self.__class__, m, getattr(headers, m))
def Questions_ID(word, topicName): topic_base_url = 'https://www.zhihu.com/topic' topic_url = [] question_id = [] for i in range(1): #I_want函数提供子话题ID topic_url.append(topic_base_url + '/' + I_want(word, topicName)[i]) get_questiontitle_data = requests.get(topic_url[i], headers=Headers('zhihu')) questiontitle_data_text = get_questiontitle_data.text re_question_id = r'content="https://www.zhihu.com/question/(.*?)"' # re_question_name = r'content="(.*?)"' question_id_list = re.findall(re_question_id, questiontitle_data_text) # question_name_list = re.findall(re_question_name, questiontitle_data_text) # print(question_name_list) #去重 for i in range(len(question_id_list)): if 'answer' in question_id_list[i]: pass else: question_id.append(question_id_list[i]) return question_id
def test_get_user_by_id(standard, init_ac_general): login('fx', 'lost') print() u = User.read(username='******') r = client.get('api/user/%d' % u.id, headers=Headers.get()) assert r.status_code == 200 return
def test_register_asset(standard, init_ac_general, load_asset_dbs): print() login('fx', 'lost') r = client.put('api/asset/C4wjiHHMu0gJ/register', headers=Headers.get(), json={ 'name': 'purple haze', 'vendor': 'ACME' }) assert r.status_code == 200 #print(r.json) dvc = Device() dvc.setDeviceId("C4wjiHHMu0gJ") dvc.getRegInfo() # check sentinels/asset data assert (dvc.dvcNm == 'purple haze') assert (dvc.custCode == 'ACME') # check command list dvc.getCommands() cmds = ["initscan", "register"] list_num = 0 for cmd in cmds: cmd_list = dvc.commands[list_num] assert (cmd_list[2] == cmd) list_num = list_num + 1 return
def Topics_Data(type, key): #构建话题请求链接 topic_base_url = 'https://www.zhihu.com/topic' topics_url = topic_base_url + 's' #构建话题请求 topics_data = requests.get(topics_url, headers=Headers('zhihu')) #提取父话题文本 topics_text = topics_data.text #正则提取父话题Name&ID re_topic_id = r'data-id="(.*?)"' re_topic_name = r'<li.*?">.*?">(.*?)<' topics_id = re.findall(re_topic_id, topics_text) topics_name = re.findall(re_topic_name, topics_text) #根据需要,返回父话题名称、ID、字典组合 if type == 'name': return topics_name elif type == 'id': return topics_id elif type == dict: topicsDict = dict(zip(topics_id, topics_name)) return topicsDict
def test_get_user_by_id_bad(standard, init_ac_general): Ac.set_user_id() u = User.read(username='******') login('grimly', 'lost') r = client.get('api/user/%d' % u.id, headers=Headers.get()) assert r.status_code == 404 return
def __init__(self, agent=None, timeout=10, keep_alive=1, default_headers={}, max_redirect=7): self.timeout = 60 self.max_redirect = max_redirect self._handlers = Handlers() if agent is None: self.agent = 'python-fluffyhttp' else: self.agent = agent if len(default_headers) == 0: default_headers = { 'Connection': 'keep-alive', } if 'User-Agent' not in default_headers: default_headers['User-Agent'] = self.agent self._default_headers = Headers(default_headers) self._poolmanager = PoolManager(maxsize=keep_alive)
class Response: CODES = {200: "OK", 404: "Not Found", 500: "Internal Server Error"} def __init__(self, request): self.request = request self.code = 200 self.code_text = self.CODES[self.code] self.headers = Headers() if self.has_body(): self.body = "" def set_code(self, code): self.code = code self.code_text = self.CODES[code] # The HEAD method doesn't have a body def has_body(self): return self.request["method"] != "HEAD" def __str__(self): s = "{0} {1} {2}\r\n".format(self.request["version"], self.code, self.code_text) s += self.headers.to_headers() if self.has_body(): s += self.body return s
def __init__(self): """Instantiate the Coordinator object. Automatically creates & links the required modules.""" if CONFIG['settings']['debug']: print("coordinator.__init__: starting instantiation") self.overall_config = CONFIG self.settings = copy.deepcopy(CONFIG['settings']) self.contents = copy.deepcopy(CONFIG['contents']) self.callbacks = {} self.register_callbacks() self.active_guis = {} self.metrics = { 'sending-rate': None, 'sending-time': None, 'remaining': None, 'sent': None, 'etc': None, 'etr': None, 'no-active-connections': None, } self.email = Email(self, None) self.sender = EmailSendHandler(self) self.gui = EmailGUI(self) self.headers = Headers(self, self.email) self.email.headers = self.headers self.last_exc = None self.ready_to_send = True if self.settings['debug']: print("coordinator.__init__: instantiation complete")
def getUrl(url, page): url = url + str(page) + '.html' print(url) mysqlDao = MysqlDao() try: n = 1 while True: try: headers = Headers.getHeaders() req = requests.get(url, headers=headers, timeout=10) break except Exception, e: print Exception, ":", e print('sleep') time.sleep(n * 10) n = n + 1 if req.status_code == 200: html = req.content selector = etree.HTML(html) url_contents = selector.xpath('//div[@class="box3"]/descendant::a/@href') for url_content in url_contents: sql = 'insert ignore into loldytt_url (`category_id`,`url`,`status`,created_at) VALUES (%s,%s,%s,%s)' created_at = time.strftime('%Y-%m-%d %H:%M:%S') values = (category_id, url_content, 0, created_at) print(values) mysqlDao.executeValues(sql, values) mysqlDao.close()
def test_rename_host(standard, fixture): print() login('fx', 'lost') r = client.patch('/api/asset/H5V2JiWiwCi0/host/3', headers=Headers.get(), json={'name': 'The Third Host'}) assert r.status_code == 200 print(r.json) return
def test_get_users_known(standard, init_ac_general): print() login('fx', 'lost') r = client.get('api/users', headers=Headers.get()) assert r.status_code == 200 assert users_in_response(r) == ['fx', 'bill', 'brad', 'george', 'grimly'] login('bill', 'lost') r = client.get('api/users', headers=Headers.get()) assert r.status_code == 200 assert users_in_response(r) == ['bill', 'brad', 'george', 'grimly'] login('grimly', 'lost') r = client.get('api/users', headers=Headers.get()) assert r.status_code == 200 assert users_in_response(r) == [] return
def api_put(path): print('%s =>' % path) r = client.put( '%s' % path, headers=Headers.get(), ) assert r.status_code == 200 print(r.json) return
def clear(self): self.done = False self.close = False if self.request.server: server_version = self.request.server.version else: server_version = SERVER_VERSION self.headers = Headers([("Server", server_version), ("Date", strftime("%a, %d %b %Y %H:%M:%S %Z")), ("X-Powered-By", server_version)]) self.cookie = self.request.cookie self.stream = False self.body = None self.time = time() self.status = "200 OK"
def start_response(self, status, headers, exc_info=None): # 根据PEP333说明 # 在提供了exc_info变量之后 # 如果已经发送了header,就直接raise error # 如果没有发送header,就可以先修改status和header再发送 # 并且要在最后将exc_info置空 if exc_info: try: if self.headers_sent: raise exc_info[0], exc_info[1], exc_info[2] finally: exc_info = None elif self.headers is not None: raise AssertionError("headers already sent") self.status = status self.headers = Headers(headers) return self.write
def clear(self): self.done = False self.close = False if self.request.server: server_version = self.request.server.version else: server_version = SERVER_VERSION self.headers = Headers([ ("Date", strftime("%a, %d %b %Y %H:%M:%S %Z")), ("X-Powered-By", server_version)]) if self.request.server is not None: self.headers.add_header("Server", server_version) self.cookie = self.request.cookie self.stream = False self._body = [] self.time = time() self.status = "200 OK" self.protocol = "HTTP/%d.%d" % self.request.server_protocol
class Response(object): """Response(sock, request) -> new Response object A Response object that holds the response to send back to the client. This ensure that the correct data is sent in the correct order. """ chunked = False body = Body() def __init__(self, sock, request): "initializes x; see x.__class__.__doc__ for signature" self.sock = sock self.request = request self.clear() def __repr__(self): return "<Response %s %s (%d)>" % ( self.status, self.headers["Content-Type"], (len(self.body) if type(self.body) == str else 0)) def __str__(self): self.prepare() protocol = self.protocol status = self.status headers = self.headers return "%s %s\r\n%s" % (protocol, status, headers) def clear(self): self.done = False self.close = False if self.request.server: server_version = self.request.server.version else: server_version = SERVER_VERSION self.headers = Headers([ ("Date", strftime("%a, %d %b %Y %H:%M:%S %Z")), ("X-Powered-By", server_version)]) if self.request.server is not None: self.headers.add_header("Server", server_version) self.cookie = self.request.cookie self.stream = False self._body = [] self.time = time() self.status = "200 OK" self.protocol = "HTTP/%d.%d" % self.request.server_protocol def prepare(self): if self.body and type(self.body) is ListType: if unicode in map(type, self.body): cLength = sum(map(lambda s: len(s.encode("utf-8")), self.body)) else: cLength = sum(map(len, self.body)) self.headers.setdefault("Content-Type", "text/html") self.headers["Content-Length"] = str(cLength) if self.stream: self.headers.setdefault("Content-Type", "application/octet-stream") for k, v in self.cookie.iteritems(): self.headers.add_header("Set-Cookie", v.OutputString()) status = int(self.status.split(" ", 1)[0]) if status == 413: self.close = True elif "Content-Length" not in self.headers: if status < 200 or status in (204, 205, 304): pass else: if self.protocol == "HTTP/1.1" \ and self.request.method != "HEAD" \ and self.request.server is not None: self.chunked = True self.headers.add_header("Transfer-Encoding", "chunked") else: self.close = True if self.request.server is not None and "Connection" not in self.headers: if self.protocol == "HTTP/1.1": if self.close: self.headers.add_header("Connection", "close") else: if not self.close: self.headers.add_header("Connection", "Keep-Alive") if self.headers.get("Transfer-Encoding", "") == "chunked": self.chunked = True
class EasyHandler(object): # 用于出错时候 error_status = "500 Dude, this is whack!" error_headers = [('Content-Type', 'text/plain')] error_body = "Server Error, please debug !!!" # 控制变量 status = result = None headers_sent = False headers = None bytes_sent = 0 def __init__(self, stdin, stdout, stderr, environ, multithread=True, multiprocess=False): self.stdin = stdin self.stdout = stdout self.stderr = stderr self.environ = environ self.multithread = multithread self.multiprocess = multiprocess def run(self, application): try: self.setup() self.result = application(self.environ, self.start_response) self.finish() except: try: self.handle_error() except: self.close() raise def setup(self): # 根据PEP333 # 设置必要的环境变量 env = self.environ env['wsgi.input'] = self.get_stdin() env['wsgi.errors'] = self.get_stderr() env['wsgi.version'] = (0, 1) env['wsgi.run_once'] = False env['wsgi.url_scheme'] = self.http_s_() env['wsgi.multithread'] = True env['wsgi.multiprocess'] = True def start_response(self, status, headers, exc_info=None): # 根据PEP333说明 # 在提供了exc_info变量之后 # 如果已经发送了header,就直接raise error # 如果没有发送header,就可以先修改status和header再发送 # 并且要在最后将exc_info置空 if exc_info: try: if self.headers_sent: raise exc_info[0], exc_info[1], exc_info[2] finally: exc_info = None elif self.headers is not None: raise AssertionError("headers already sent") self.status = status self.headers = Headers(headers) return self.write def finish(self): for data in self.result: self.write(data) self.finish_content() self.close() def handle_error(self): if not self.headers_sent: self.result = self.error_output(self.environ, self.start_response) self.finish() def error_output(self, environ, start_response): start_response(self.error_status, self.error_headers, sys.exc_info()) return [self.error_body] def finish_content(self): if not self.headers_sent: self.headers['Content-Length'] = 0 self.send_headers() else: pass def send_headers(self): # 根据PEP333 # 如果应用程序没有提供content-length参数 # 我们就需要自己计算出他的长度 # 如果返回的可迭代对象长度是1,就直接使用第一个yield的字符串的长度当成最终值 # 或者可以在连接结束时关闭客户端 if not self.headers.haskey('Content-Length'): try: blocks = len(self.result) except: pass else: if blocks == 1: self.headers['Content-Length'] = str(self.bytes_sent) else: pass # todo self.headers_sent = True self._write('Status %s\r\n' % self.status) self._write(str(self.headers)) def write(self, data): # 根据PEP333 # 如果在发送数据之前 assert type(data) is types.StringType if not self.status: raise AssertionError('write() before start_response()') if not self.headers_sent: # 根据PEP333 # 在第一次发送数据之前,需要先把header发送出去 # 并且一定要预先知道我们发送的数据的长度 self.bytes_sent = len(data) self.send_headers() else: self.bytes_sent += len(data) # 这里使用一次性发送数据 # 每次发送之后应该清空缓冲区,不要影响下次请求 self._write(data) self._flush() def close(self): try: # 根据PEP333 # 如果迭代对象返回了close方法,不管这个请求如何,在请求完成时都必须调用这个方法 if hasattr(self.result, 'close'): self.result.close() finally: # 每次处理完一个请求之后,需要重置各种变量 # 因为下次的请求环境变量是不一样的 self.reset() def _write(self, data): self.stdout.write(data) def _flash(self): self.stdout.flush() def get_stdin(self): return self.stdin def get_stdout(self): return self.stdout def get_stderr(self): return self.stderr def reset(self): self.result = None self.headers = None self.status = None self.environ = None self.bytes_sent = 0 self.headers_sent = False def http_s_(self): if self.environ.get("HTTPS") in ('yes', 'on', '1'): return 'https' return 'http'