def HTTPProxy(self, domain, method, path, headers, post_data = None): conn = HTTPConnection(domain) conn.request(method, path, post_data, headers) response = conn.getresponse() response_headers = response.getheaders() data = response.read() if 'Content-Length' not in dict(response_headers).keys() and 'content-length' not in dict(response_headers).keys(): response_headers.append(('Content-Length', len(data))) for item in response_headers: if item[0].lower() == 'transfer-encoding' and item[1].lower() == 'chunked': response_headers[response_headers.index(item)] = ('Transfer-Encoding', 'text/html; charset=utf-8') self.push(HTTP_RESPONSE(response.version, response.status, response.reason)) for item in response_headers: self.push(HTTP_HEADER(item)) self.push(HTTP_RN) self.push(data) self.push(HTTP_RN) print("%s %s %s %d %s %d" % (self.version, method, path, response.status, response.reason, len(data))) for item in response_headers: if item[0].lower() == 'connection' and item[1].lower() == 'keep-alive': self.push(HTTP_0RN + HTTP_RN) return self.close_when_done()
def run(info): if (not isinstance(info, Info)): print(__name__ + "::run() info is not instance Info, please check") sys.exit(1) url = info.GetServiceUrl(Constant.ClientPort) print("Use endpoint to detect service: " + url) try: # Found error:timed out while health check, continue loop... need to create every time. conn = HTTPConnection(url, timeout=Constant.ConnectTimeout) # check health info, etcd case sensitive conn.request("GET", "/health") resp = conn.getresponse() con = resp.read().decode("UTF-8").strip("") # json need to docode too try: health = json.loads(con) except Exception as err: health = "" # str(resp.headers) print("Get health response:" + str(resp.status) + " " + str(resp.reason) + " " + str(health)) if (con != "" and isinstance(health, object) and health["health"] == "true"): print("Node " + info.GetNodename() + " health status check passing") else: print("Cluster health checks failed, /health info check failed.") sys.exit(1) except Exception as err: print("Found error:" + str(err) + " while health check, continue loop...") print(traceback.format_exc()) sys.exit(1)
def _send_notify_event(self, uuid, notify_url, notify_message): """ Send out notify event to subscriber and return a response. """ # Remove <> that surround the real unicode url if they exist... notify_url = notify_url.translate({ord(k): u"" for k in "<>"}) parsed_url = urlparse(notify_url) headers = { 'HOST': parsed_url.netloc, 'Content-Type': 'text/xml', 'SID': 'uuid:' + uuid, 'Content-Length': len(notify_message), 'NT': 'upnp:event', 'NTS': 'upnp:propchange' } http_handler = HTTPConnection(parsed_url.netloc) http_handler.request('NOTIFY', parsed_url.path, notify_message, headers) http_response = http_handler.getresponse() current_app.logger.info('_send_notify_event: status:{0} reason:{1} headers:{2}'.format(http_response.status,http_response.reason,headers)) if http_response.status != 200: error_msg = 'UPNPPush Notification failed: ({0}: {1})'.format(http_response.status, http_response.read()) current_app.logger.warning(error_msg) raise Exception(error_msg)
def connect(self): """ Override the connect() function to intercept calls to certain host/ports. If no app at host/port has been registered for interception then a normal HTTPConnection is made. """ if debuglevel: sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,)) try: (app, script_name) = self.get_app(self.host, self.port) if app: if debuglevel: sys.stderr.write('INTERCEPTING call to %s:%s\n' % (self.host, self.port,)) self.sock = wsgi_fake_socket(app, self.host, self.port, script_name, self.environ) else: HTTPConnection.connect(self) except Exception: if debuglevel: # intercept & print out tracebacks traceback.print_exc() raise
def get_topic(use_offline): class TopicParser(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.pattern = re.compile('http://www.zhihu.com/topic/\d{8}/questions') self.links = {} self.href = None def handle_starttag(self, tag, attrs): attrs = dict(attrs) if (tag == 'a' and attrs.get('href') and re.match(self.pattern, attrs.get('href'))): self.href = attrs.get('href') def handle_data(self, data): if self.href != None: self.links[data] = self.href self.href = None text = None if use_offline: with open(spider['topic']['offline'], 'r', encoding = 'utf-8') as fp: text = fp.read() else: conn = HTTPConnection(spider['host']) conn.request('GET', spider['topic']['online']) text = conn.getresponse().read().decode('utf-8') hp = TopicParser() hp.feed(text) hp.close() return hp.links
class YQLQuery(object): connection = None def __enter__(self): self.connection = None def __init__(self): while True: self.connection = HTTPConnection("query.yahooapis.com") if self.connection != None: break else: time.sleep(1) def execute(self, yql, token=None): while True: self.connection.request( "GET", PUBLIC_API_URL + "?" + urlencode({"q": yql, "format": "json", "env": DATATABLES_URL}) ) try: return simplejson.loads(self.connection.getresponse().read()) except Exception, e: error_log = open("/usr/lib/cgi-bin/kchang_cdlcapital/logs/yql_log.txt", "a") error_log.write(str(e)) print str(e) error_log.close() time.sleep(1)
def resume(self): if self.next_token: verb = "ListRecords" print(self.next_token) command = "&resumptionToken=" + self.next_token url = urlparse(self.OAIbaseURL + "?verb=" + verb + command) response = False try: conn = HTTPConnection(url.netloc) conn.request("GET", url.path + "?" + url.query) response = conn.getresponse() except: print(self.OAIbaseURL + "?verb=" + verb + command) print("No 200 OK !!, sleeping 1 min.") time.sleep(50) if response: if response.status == 200: data = response.read().decode("utf-8", "ignore") self.data = fromstring(data) token = self.data.findall(".//{http://www.openarchives.org/OAI/2.0/}resumptionToken")[0].text if token and not self.token == token: self.token = self.next_token self.next_token = token elif token: print("Token = Next_token") os._exit(-1) else: print("No 200 OK !!, sleeping 1 min.") time.sleep(50)
def connect_to_tester(): """ Connect to the tester Returns: HTTPConnection: A connection to the tester """ while True: try: # Get the host host = input("Please provide the IP of the host (input nothing for localhost): ").strip() if not host: host = "localhost" # Connect print(info_message("Connecting to tester...")) connection = HTTPConnection(host, PORT) # Inform the tester about the connection connection.request('POST', ROUTE_CONNECT_INFO, socket.gethostbyname(socket.gethostname())) print(info_message(connection.getresponse().read().decode())) break except socket.gaierror: print(info_message("Invalid host '{}'".format(host))) return connection
def request(self, method, endpoint, qargs={}, data=None): path = self.format_path(endpoint, qargs) conn = HTTPConnection("%s.myinsales.ru:80" % self.account, timeout=self.response_timeout) auth = b64encode("{0}:{1}".format(self.api_key, self.password).encode("utf-8")).decode("utf-8") headers = {"Authorization": "Basic {0}".format(auth), "Content-Type": "application/xml"} done = False while not done: try: conn.request(method, path, headers=headers, body=data) except (socket.gaierror, socket.timeout): if self.retry_on_socket_error: time.sleep(self.retry_timeout) continue else: raise resp = conn.getresponse() body = resp.read() if resp.status == 503 and self.retry_on_503: time.sleep(self.retry_timeout) else: done = True if 200 <= resp.status < 300: return body else: raise ApiError("%s request to %s returned: %s\n%s" % (method, path, resp.status, body), resp.status)
def queryState(self): """Execute a request for retrieving the associated host's state Make an HTTP query to determine if the server is reachable or not @return[int] : the server status """ c_retry = 0 c_success = 0 # Max retry is defined by config while c_retry < self.max_retry: # if the query success the try block will continue # if not the except block will be run h = HTTPConnection(self.address, self.port, timeout=self.tcp_timeout) try: # try to query status = h.request(self.query_method, '/') # parsing the result res = h.getresponse() c_success += 1 sys_log.debug('[' + self.getName() + '] get server code : %d', res.status) # if we have sufficient number of success if c_success >= self.min_retry: return self.AVAILABLE except: sys_log.debug('[' + self.getName() + '] unable to reach the host') finally: c_retry += 1 # if we have reach the max retry amount return self.UNAVAILABLE
def test_requesting_basebackup(self, pghoard): nonexistent_basebackup = "/{}/archive/basebackup".format(pghoard.test_site) conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"]) conn.request("PUT", nonexistent_basebackup) status = conn.getresponse().status assert status == 201 assert pghoard.requested_basebackup_sites == {"test_requesting_basebackup"}
def _get_content(self, url): con = HTTPConnection(self.site_url) encoded_url = quote(unquote(url), "/=?&") print(encoded_url) con.request('GET', encoded_url) response = con.getresponse() return response.read()
def __init__(self, host, port=None, *args, **kwargs): if (port is None): port = 80 HTTPConnection.__init__(self, host, port, *args, **kwargs) self.req_string = b'' self.__host = host self.__port = port
def main(argv): utils.drop_privileges() socket.setdefaulttimeout(DEFAULT_TIMEOUT) servers = [] if json is None: utils.err("This collector requires the `json' Python module.") return 1 for conf in elasticsearch_conf.get_servers(): server = HTTPConnection( *conf ) try: server.connect() except socket.error as exc: if exc.errno == errno.ECONNREFUSED: continue raise servers.append( server ) if len( servers ) == 0: return 13 # No ES running, ask tcollector to not respawn us. lock = threading.Lock() while True: threads = [] for server in servers: status = node_status(server) version = status["version"]["number"] t = threading.Thread(target = _collect_server, args = (server, version, lock)) t.start() threads.append(t) for thread in threads: thread.join() time.sleep(COLLECTION_INTERVAL)
def download_regionals(year): conn = HTTPConnection('www.thebluealliance.com') conn.request('GET', '/api/v1/events/list?year=' + year, headers=DEF_HEADERS) r = conn.getresponse() answer = r.read().decode('utf-8') return answer
def download_match(m, quiet=False): conn = HTTPConnection('www.thebluealliance.com') conn.request('GET', '/api/v1/match/details?match=' + m, headers=DEF_HEADERS) r = conn.getresponse() answer = r.read().decode('utf-8') return answer
def get_contents(user): ''' 得到目录的方法:首先,根据首页的信息可以知道目录一共有多少页,然后构造页数 大于实际页数的URL,便可以直接得到所有博客的目录列表,省去了很多麻烦。 此处直接取一个较大的值,比如 100000. DO NOT need the cookie in headers. ''' class ContentsParser(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.contents = {} self.flag = 0 self.link = '' def handle_starttag(self, tag, attrs): if tag == 'span' and dict(attrs).get('class') == 'link_title': self.flag = 1 if self.flag > 0: self.flag += 1 if tag == 'a': self.link = dict(attrs).get('href') def handle_endtag(self, tag): self.flag -= 1 def handle_data(self, data): if self.flag == 3: self.contents[self.link] = data.strip() conn = HTTPConnection(spider['blog']['host']) conn.request('GET', '/'+user+spider['blog']['path']+'10000', headers = spider['headers']) text = gzip.decompress(conn.getresponse().read()).decode('utf-8') hp = ContentsParser() hp.feed(text) hp.close() return hp.contents
def do_remote(self, path, body=None, headers={}): """和远程主机通讯,同时处理501错误(不是代理请求)""" if self.handleHeaders: del self.headers["Proxy-Connection"] del self.headers["Keep-Alive"] del self.headers["Proxy-Authorization"] self.headers["Connection"] = "close" if not path.scheme: self.send_response(501) self.send_header("Server", self.server_version) self.send_header("Content-Type", "text/html; charset=utf-8") if self.command in ("GET", "POST"): content = directError.format( server_version=self.server_version, domain=self.server.server_address[0], port=self.server.server_address[1], ).encode("utf-8") else: content = "Unknown Error" self.send_header("Content-Length", str(len(content))) self.end_headers() self.wfile.write(content) self.remoteResponse = None return client = HTTPConnection(path.netloc) headers = dict(headers) # 有些网站,比如 WebQQ,在 cookie 中插入了非 ASCII 字符 # XXX:如果 UTF-8 不适合怎么办? for i in headers: headers[i] = headers[i].encode("utf-8") client.request(self.command, path.getpath(), body, headers) self.remoteResponse = client.getresponse() self.getdata = self.remoteResponse.read()
def render(self, formula): # Prepare the formula formula = formula.replace("$", "") encoded_formula = formula.replace("%", "[comment]").replace("+", "%2B") display_formula = formula.replace("\n", "") print('Rendering: %s ...' % display_formula) # Prepare POST request to QuickLaTeX via ProblemSetMarmoset # (for added processing) params = urlencode({ 'engine': 'quicklatex', 'input': encoded_formula, }) headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} conn = HTTPConnection("www.problemsetmarmoset.com") # Make the request conn.request("POST", "/latex/render.php", params, headers) response = conn.getresponse() img_url = response.read() # Display as Markdown image rendered_tex = '![{0}]({1} "{0}")\n'.format(display_formula, img_url.decode('utf-8')) return rendered_tex
def lookup_unspent_outputs(bitcoin_addresses): conn = HTTPConnection('blockchain.info') conn.request('GET', '/unspent?active={}'.format('|'.join(bitcoin_addresses))) result = json.loads(conn.getresponse().read().decode('utf8')) unspent = defaultdict(list) for u in result['unspent_outputs']: program_bytes = Bitcoin.hexstring_to_bytes(u['script'], reverse=False) scriptPubKey, _ = script.Script.unserialize(program_bytes, len(program_bytes)) address = None # Try to extract the address from the scriptpubkey program if len(scriptPubKey.program) == 6 and \ scriptPubKey.program[0][0] == script.OP_DUP and scriptPubKey.program[1][0] == script.OP_HASH160 and \ scriptPubKey.program[2][0] == 20 and \ scriptPubKey.program[4][0] == script.OP_EQUALVERIFY and scriptPubKey.program[5][0] == script.OP_CHECKSIG: address = scriptPubKey.program[3] elif len(scriptPubKey.program) == 3 and scriptPubKey.program[2][0] == script.OP_CHECKSIG: if scriptPubKey.program[2][0] in (0x04, 0x03, 0x02): address = base58.decode_to_bytes(addressgen.generate_address(scriptPubKey.program[1], version=0))[-24:-4] if address is not None: i = 0 while address[i] == 0: i += 1 address = '1' + ('1' * i) + addressgen.base58_check(address, version=0) unspent[address].append(u) return unspent
class Client: #構構子 def __init__(self): self.conn = None #對 mops service 送出 POST def requestServer(self, ajaxService, form_body): self.conn = HTTPConnection("61.57.47.131", 80) headers = {"Accept":"*/*", "Accept-Encoding":"gzip, deflate", "Accept-Language":"zh-TW,zh;q=0.8,en-US;q=0.6,en;q=0.4", "Connection":"keep-alive", "Content-Type":"application/x-www-form-urlencoded", "Host":"mops.twse.com.tw", "Origin":"http://mops.twse.com.tw", "Referer":"http://mops.twse.com.tw/mops/web/" + ajaxService, "User-Agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36"} body = form_body self.conn.request("POST", "/mops/web/" + ajaxService, body, headers) res = self.conn.getresponse() res_raw = res.read() res_data = res_raw.decode("utf-8") return res_data #關閉連線 def closeConnection(self): self.conn.close()
def httpStatus(host, path, status): try: conn = HTTPConnection(host) conn.request("HEAD", path) return conn.getresponse().status == status except: return False
def manage_network(nwid, action="join"): """ Join or leave a ZeroTier network. nwid: ZeroTier network ID, e.g. "e5cd7a9e1c8a5e83" action: either "join" or "leave" """ if action == "join": method = "POST" elif action == "leave": method = "DELETE" else: raise Exception("Unsupported action: {}".format(action)) conn = HTTPConnection("localhost", 9993) path = "/network/{}".format(nwid) body = "{}" headers = { "Content-Type": "application/json", "X-ZT1-Auth": get_auth_token() } conn.request(method, path, body, headers) res = conn.getresponse() data = json.loads(res.read()) return data
def getHeadResponse(url,response_cache = {}): """ İlk argüman olarak verilen adrese HEAD istemi yaparak, aldığı yanıtı döndürür. Yaptığı istemleri önbellekleme yapar. O yüzden eğer binlerce farklı adres için kullanılacaksa, fazla ram kullanmaya başlayabilir. Eğer önbelleği boşaltmak gerekirse, getHeadResponse("",{}) komutuyla, önbellek silinebilir. Ancak, çok gerekli değilse, bunun yapılması tavsiye olunmaz. --------------------------------------------------------------------------- Takes an url as string, tries to make an HEAD request and returns the response of the request. Caches responses, so only one request per page made in a particular runtime. This function may return None if an error occurs. """ try: return response_cache[url] except KeyError: url = urlparse(url) if url.scheme == "http": conn = HTTPConnection(url.netloc, timeout=2) elif url.scheme == "https" and ssl: conn = HTTPSConnection(url.netloc, timeout=2) else: return None try: conn.request("HEAD", url.path) response = conn.getresponse() except: response = None if response: response_cache[url.geturl()] = response return response
def test1(): s = HelloServer() import threading import time import sys s.stdout = sys.stdout t = threading.Thread(None, s.serve_forever) t.daemon = True print("Serving forever") t.start() time.sleep(1) from http.client import HTTPConnection c = HTTPConnection("localhost", 12345) print("Requesting") c.request("GET", "/webservice/interface/?&call=getMainValues") print("Getting response") stuff = c.getresponse().read() out = "C:\\.replcache\\helloserver.xml" with open(out, 'wb') as f: f.write(stuff) import webbrowser webbrowser.open_new_tab(out) from urllib.request import urlopen print(urlopen("http://localhost:12345/webservice/interface/?&call=getMainValues").read().decode())
def get_content_type(url): parsed = urlparse(url) connection = HTTPConnection(parsed.netloc) connection.request("HEAD", parsed.path + "?" + parsed.query) response = connection.getresponse() return response.getheader("content-type")
def main(): answer_data = [] url = 'www.asahi-net.or.jp' path = '/~kc7k-nd/onlispjhtml/' http_client = HTTPConnection(url) # print(http_client) http_client.request('GET', path) response = http_client.getresponse() response_html = response.readall().decode('UTF-8') # print(response_html) parser = MyHTMLParser() parser.feed(response_html) # parser = HTMLParser(strict=True) # parser.feed(response_html) hrefs = parser.get_hrefs() for h in hrefs: # print(h) # find_word(h[1], url, 'mappend') ret = find_word(path + h[1], url, 'mappend') for r in ret: # print(r) if r: # for a in r: print(r) # print(ret) # print('path: ', path + hrefs[2][1], ', url: ', url) print()
def createConnection(self): if self.debug: conn = HTTPConnection("127.0.0.1", Connection.debugPort) conn.set_tunnel(self.server) else: conn = HTTPConnection(self.server) return conn
class YQLQuery(object): RETRY_MAX = 3 DELAY_TIME = timedelta(seconds = 3) def __init__(self): self.connection = HTTPConnection('query.yahooapis.com') self.retry_count = 0 self.cool_down = None def execute(self, yql, token = None): if self.cool_down is not None: delta = datetime.now() - self.cool_down if delta < YQLQuery.DELAY_TIME: logging.debug("waiting for next request") time.sleep(YQLQuery.DELAY_TIME - delta) self.cool_down = datetime.now() logging.debug("sending request, %s" % yql) self.connection.request('GET', PUBLIC_API_URL + '?' + urlencode({ 'q': yql, 'format': 'json', 'env': DATATABLES_URL })) resp_content = self.connection.getresponse().read() try: return simplejson.loads(resp_content) except simplejson.JSONDecodeError as ex: logging.debug(ex) if self.retry_count < YQLQuery.RETRY_MAX: logging.debug("Retrying...") time.sleep(1) self.retry_count += 1 return self.execute(yql, token) else: logging.debug("Max retry_count is reached") return [] def __del__(self): self.connection.close()
def get_image(self): conn = HTTPConnection(self.url) conn.request("GET", self.request) fh = conn.sock.makefile(mode="rb") # Read in HTTP headers: line = fh.readline().decode() while line.strip() != '': parts = line.split(':') if len(parts) > 1 and parts[0].lower() == 'content-type': # Extract boundary string from content-type content_type = parts[1].strip() boundary = content_type.split(';')[1].split('=')[1] line = fh.readline().decode() if not boundary: raise Exception("Can't find content-type") logger.debug("boundary=%s" % boundary) # Seek ahead to the first chunk while line.strip() != "--%s" % boundary: line = fh.readline().decode() # Read in chunk headers while line.strip() != '': parts = line.split(':') if len(parts) > 1 and parts[0].lower() == 'content-length': # Grab chunk length length = int(parts[1].strip()) line = fh.readline().decode() image = fh.read(length) logger.debug("image has been received - size: %d" % len(image)) return image
def checkoab(host, port, mode, domain, email, data): OABID = input("Input the OABID:") OAB_url = "/OAB/" + OABID + "/oab.xml" tmp = email.split('@') user = tmp[0] if port ==443: try: uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) session = HTTPSConnection(host, port, context=uv_context) except AttributeError: session = HTTPSConnection(host, port) else: session = HTTPConnection(host, port) # Use impacket for NTLM ntlm_nego = ntlm.getNTLMSSPType1(host, domain) #Negotiate auth negotiate = base64.b64encode(ntlm_nego.getData()) # Headers headers = { "Authorization": 'NTLM %s' % negotiate.decode('utf-8'), "Content-type": "text/xml; charset=utf-8", "Accept": "text/xml", "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" } session.request("GET", OAB_url, "", headers) res = session.getresponse() res.read() if res.status != 401: print('Status code returned: %d. Authentication does not seem required for URL'%(res.status)) return False try: if 'NTLM' not in res.getheader('WWW-Authenticate'): print('NTLM Auth not offered by URL, offered protocols: %s'%(res.getheader('WWW-Authenticate'))) return False except (KeyError, TypeError): print('No authentication requested by the server for url %s'%(OAB_url)) return False print('[*] Got 401, performing NTLM authentication') # Get negotiate data try: ntlm_challenge_b64 = re.search('NTLM ([a-zA-Z0-9+/]+={0,2})', res.getheader('WWW-Authenticate')).group(1) ntlm_challenge = base64.b64decode(ntlm_challenge_b64) except (IndexError, KeyError, AttributeError): print('No NTLM challenge returned from server') return False if mode =='plaintext': password1 = data; nt_hash = '' elif mode =='ntlmhash': password1 = '' nt_hash = binascii.unhexlify(data) else: print('[!] Wrong parameter') return False lm_hash = '' ntlm_auth, _ = ntlm.getNTLMSSPType3(ntlm_nego, ntlm_challenge, user, password1, domain, lm_hash, nt_hash) auth = base64.b64encode(ntlm_auth.getData()) headers = { "Authorization": 'NTLM %s' % auth.decode('utf-8'), "Content-type": "text/xml; charset=utf-8", "Accept": "text/xml", "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" } session.request("GET", OAB_url, "", headers) res = session.getresponse() body = res.read() if res.status == 401: print('[!] Server returned HTTP status 401 - authentication failed') return False else: filename = "checkOAB.xml" print('[+] Save response file to %s'%(filename)) with open(filename, 'w+', encoding='utf-8') as file_object: file_object.write(bytes.decode(body)) pattern_name = re.compile(r">(.+lzx)<") name = pattern_name.findall(bytes.decode(body)) if name: print('[+] Default Global Address:%s'%(name[0])) return True
def request(url, method="GET", data=None, headers={}, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, files={}, randua=False, auth=None, length_limit=None, **kwargs): ''' request an URL :param url: URL to be fetched. :type url: string :param method: HTTP method, one of ``GET``, ``DELETE``, ``HEAD``, ``OPTIONS``, ``PUT``, ``POST``, ``TRACE``, ``PATCH``. ``GET`` by default. :type method: string, optional :param headers: HTTP request headers :type headers: dict, optional :param timeout: timeout in seconds, socket._GLOBAL_DEFAULT_TIMEOUT by default :type timeout: integer or float, optional :param files: files to be sended :type files: dict, optional :param randua: if ``True`` or ``path string``, use a random user-agent in headers, instead of ``'urlfetch/' + __version__`` :type randua: bool or string, default is ``False`` :param auth: (username, password) for basic authentication :type auth: tuple, optional :param length_limit: if ``None``, no limits on content length, if the limit reached raised exception 'Content length is more than ...' :type length_limit: integer or None, default is ``none`` :rtype: A :class:`~urlfetch.Response` object ''' method = method.upper() if method not in _allowed_methods: raise UrlfetchException("Method should be one of " + ", ".join(_allowed_methods)) parsed_url = parse_url(url) if parsed_url['scheme'] == 'https': h = HTTPSConnection(parsed_url['host'], port=parsed_url['port'], timeout=timeout) elif parsed_url['scheme'] == 'http': h = HTTPConnection(parsed_url['host'], port=parsed_url['port'], timeout=timeout) else: raise UrlfetchException('Unsupported protocol %s' % scheme) # is randua bool or path if randua and isinstance(randua, basestring) and \ os.path.isfile(path): randua = True randua_file = randua else: randua = bool(randua) randua_file = None # default request headers reqheaders = { 'Accept': '*/*', 'User-Agent': random_useragent(randua_file) if randua else \ 'urlfetch/' + __version__, 'Host': parsed_url['host'], } if parsed_url['username'] is not None and parsed_url['password'] is not \ None and auth is None: auth = (parsed_url['username'], parsed_url['password']) if auth is not None: if isinstance(auth, (list, tuple)): auth = '%s:%s' % tuple(auth) auth = base64.b64encode(auth.encode('utf-8')) reqheaders['Authorization'] = 'Basic ' + auth.decode('utf-8') if files: content_type, data = _encode_multipart(data, files) reqheaders['Content-Type'] = content_type elif isinstance(data, dict): data = urlencode(data, 1) if isinstance(data, basestring) and not files: # httplib will set 'Content-Length', also you can set it by yourself reqheaders["Content-Type"] = "application/x-www-form-urlencoded" # what if the method is GET, HEAD or DELETE # just do not make so much decisions for users for k, v in headers.items(): reqheaders[k.title()] = v h.request(method, parsed_url['uri'], data, reqheaders) response = h.getresponse() return Response.from_httplib(response, reqheaders=reqheaders, connection=h, length_limit=length_limit)
def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) authpair = authpair.encode('utf-8') self.authhdr = b"Basic " + base64.b64encode(authpair) self.conn = HTTPConnection(host, port=port, timeout=30)
def setUp(self): db.reset() self.client = HTTPConnection("localhost", env["PORT"])
class TestServer(unittest.TestCase): def setUp(self): db.reset() self.client = HTTPConnection("localhost", env["PORT"]) def tearDown(self): self.client.close() def request(self, *args, **kwargs): self.client.request(*args, **kwargs) response = self.client.getresponse() return response.status, json.loads(response.read()) def testStatic(self): self.client.request("GET", "/") response = self.client.getresponse() self.assertEqual(200, response.status) self.assertTrue(b'this comment helps us' in response.read()) self.client.request("GET", "/index.html") response = self.client.getresponse() self.assertEqual(200, response.status) self.assertTrue(b'this comment helps us' in response.read()) def testListFavorites(self): status, body = self.request("GET", "/favorites") self.assertEqual(200, status), self.assertEqual({"favorites": db.fixtures}, body) def testAddFavorite(self): request_body = json.dumps({ "artist": "Seal", "song": "Crazy" }).encode("utf-8") status, body = self.request("POST", "/favorites", body=request_body) self.assertEqual(201, status), status, body = self.request("GET", "/favorites") self.assertEqual(200, status), self.assertEqual( { "favorites": db.fixtures + [{ "artist": "Seal", "id": 5, "song": "Crazy" }] }, body, ) def test404(self): status, body = self.request("GET", "/no-such-route") self.assertEqual(404, status), self.assertEqual({"error": "Not found"}, body) def testInvalidJson(self): status, body = self.request("POST", "/favorites", body=b"bogus json") self.assertEqual(400, status), self.assertEqual({"error": "Invalid JSON in request body"}, body) def testInvalidArtist(self): status, body = self.request("POST", "/favorites", body=b'{"song": "Something"}') self.assertEqual(400, status), self.assertEqual({"error": "'artist' is a required parameter"}, body) def testInvalidSong(self): status, body = self.request("POST", "/favorites", body=b'{"artist": "Something"}') self.assertEqual(400, status), self.assertEqual({"error": "'song' is a required parameter"}, body)
def start_test_web_servers(): Fail = 0 Up = 0 while True: last_check_ping = check_ping() if (last_check_ping == "Network Error") and (Fail == 0): Fail = 1 Up = 0 bot_check_ping(last_check_ping) elif (last_check_ping == "Network Active") and (Up == 0): Up = 1 Fail = 0 bot_check_ping(last_check_ping) if last_check_ping == "Network Active": for number in servers.keys(): try: server = HTTPConnection('%s.open.by' % (number), timeout=60) #server.request('POST', '/', data, headers=headers) server.request("GET", "/", headers=headers) check_result = server.getresponse() code = check_result.status reason = check_result.status d[number] = code err[number] = 0 server.close() except Exception as e: err[number] = str(e) d[number] = 1 #print(d) for number in servers: db_worker = SQLighter(config.database) base_result = db_worker.status_code(number) error_result = db_worker.status_error(number) ts = calendar.timegm(time.gmtime()) ts_base = int(float(base_result[0][1])) td = ts - ts_base if (error_result[0][0]) != (err.get(number)): st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') error = err.get(number) db_worker = SQLighter(config.database) db_worker.update_status_error(err.get(number), ts, number) print(number, servers.get(number), err.get(number), st) bot_err_sent(number, servers.get(number), error, st) #err_mail_send(number,error,st) elif ((err.get(number) != 0) and (td > 300)): st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') error = err.get(number) db_worker = SQLighter(config.database) db_worker.update_status_error(err.get(number), ts, number) print(number, servers.get(number), err.get(number), st) bot_err_sent(number, servers.get(number), error, st) #err_mail_send(number,error,st) elif ((int(base_result[0][0])) != (d.get(number))) and (d.get(number) != "1"): st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') code = d.get(number) db_worker = SQLighter(config.database) db_worker.update_status_code(d.get(number), ts, number) print(number, d.get(number), st) bot_code_sent(number, servers.get(number), code, st) #mail_send(number,code,st) elif ((d.get(number) != 200) and (td > 300)): #elif(d.get(number) != 200): st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') code = d.get(number) db_worker = SQLighter(config.database) db_worker.update_status_code(d.get(number), ts, number) print(number, d.get(number), st) bot_code_sent(number, servers.get(number), code, st) #mail_send(number,code,st) db_worker = SQLighter(config.database) get_all_status(db_worker.all_status_code()) db_worker.close() time.sleep(5)
def test_005_get_10(self): client = HTTPConnection("127.0.0.1:%s" % self.http_listener10_port, timeout=TIMEOUT) self._do_request(client, self.TESTS_10["GET"]) client.close()
def _prepare_request( method, url, *, enc_params="", timeout=DEFAULT_TIMEOUT, source_address=None, unix_socket=None, verify=True, ssl_context=None, ): """Parses the URL, returns the path and the right HTTPConnection subclass.""" parsed_url = urllib.parse.urlparse(url) is_unix = unix_socket is not None scheme = parsed_url.scheme.lower() if scheme.endswith("+unix"): scheme = scheme[:-5] is_unix = True if scheme == "https": raise ValueError("https+unix is not implemented") if scheme not in ("http", "https"): raise ValueError("unrecognized scheme", scheme) is_https = scheme == "https" host = parsed_url.hostname port = 443 if is_https else 80 if parsed_url.port: port = parsed_url.port if is_unix and unix_socket is None: unix_socket = urllib.parse.unquote(parsed_url.netloc) path = parsed_url.path if parsed_url.query: if enc_params: path = f"{path}?{parsed_url.query}&{enc_params}" else: path = f"{path}?{parsed_url.query}" else: if enc_params: path = f"{path}?{enc_params}" else: pass # just parsed_url.path in this case if isinstance(source_address, str): source_address = (source_address, 0) if is_unix: conn = UnixHTTPConnection(unix_socket, timeout=timeout) elif is_https: if ssl_context is None: ssl_context = ssl.create_default_context() if not verify: ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE conn = HTTPSConnection( host, port, source_address=source_address, timeout=timeout, context=ssl_context, ) else: conn = HTTPConnection(host, port, source_address=source_address, timeout=timeout) munged_url = urllib.parse.urlunparse(( parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, "", parsed_url.fragment, )) return munged_url, conn, path
def getUsersetting(host, port, mode, domain, email, data): autodiscover_url = "/autodiscover/autodiscover.svc" tmp = email.split('@') user = tmp[0] POST_BODY = '''<?xml version="1.0" encoding="utf-8"?> <soap:Envelope xmlns:a="http://schemas.microsoft.com/exchange/2010/Autodiscover" xmlns:wsa="http://www.w3.org/2005/08/addressing" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"> <soap:Header> <a:RequestedServerVersion>Exchange2013_SP1</a:RequestedServerVersion> <wsa:Action>http://schemas.microsoft.com/exchange/2010/Autodiscover/Autodiscover/GetUserSettings</wsa:Action> <wsa:To>https://{domain}/autodiscover/autodiscover.svc</wsa:To> </soap:Header> <soap:Body> <a:GetUserSettingsRequestMessage xmlns:a="http://schemas.microsoft.com/exchange/2010/Autodiscover"> <a:Request> <a:Users> <a:User> <a:Mailbox>{mail}</a:Mailbox> </a:User> </a:Users> <a:RequestedSettings> <a:Setting>UserDisplayName</a:Setting> <a:Setting>UserDN</a:Setting> <a:Setting>UserDeploymentId</a:Setting> <a:Setting>InternalMailboxServer</a:Setting> <a:Setting>MailboxDN</a:Setting> <a:Setting>PublicFolderServer</a:Setting> <a:Setting>ActiveDirectoryServer</a:Setting> <a:Setting>ExternalMailboxServer</a:Setting> <a:Setting>EcpDeliveryReportUrlFragment</a:Setting> <a:Setting>EcpPublishingUrlFragment</a:Setting> <a:Setting>EcpTextMessagingUrlFragment</a:Setting> <a:Setting>ExternalEwsUrl</a:Setting> <a:Setting>CasVersion</a:Setting> <a:Setting>EwsSupportedSchemas</a:Setting> <a:Setting>GroupingInformation</a:Setting> </a:RequestedSettings> </a:Request> </a:GetUserSettingsRequestMessage> </soap:Body> </soap:Envelope> ''' DomainName = input("Input the domain name of the exchange server(not ip):") POST_BODY = POST_BODY.format(domain=DomainName, mail=email) if port ==443: try: uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) session = HTTPSConnection(host, port, context=uv_context) except AttributeError: session = HTTPSConnection(host, port) else: session = HTTPConnection(host, port) # Use impacket for NTLM ntlm_nego = ntlm.getNTLMSSPType1(host, domain) #Negotiate auth negotiate = base64.b64encode(ntlm_nego.getData()) # Headers headers = { "Authorization": 'NTLM %s' % negotiate.decode('utf-8'), "Content-type": "text/xml; charset=utf-8", "Accept": "text/xml", "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" } session.request("POST", autodiscover_url, POST_BODY, headers) res = session.getresponse() res.read() if res.status != 401: print('Status code returned: %d. Authentication does not seem required for URL'%(res.status)) return False try: if 'NTLM' not in res.getheader('WWW-Authenticate'): print('NTLM Auth not offered by URL, offered protocols: %s'%(res.getheader('WWW-Authenticate'))) return False except (KeyError, TypeError): print('No authentication requested by the server for url %s'%(autodiscover_url)) return False print('[*] Got 401, performing NTLM authentication') # Get negotiate data try: ntlm_challenge_b64 = re.search('NTLM ([a-zA-Z0-9+/]+={0,2})', res.getheader('WWW-Authenticate')).group(1) ntlm_challenge = base64.b64decode(ntlm_challenge_b64) except (IndexError, KeyError, AttributeError): print('No NTLM challenge returned from server') return False if mode =='plaintext': password1 = data; nt_hash = '' elif mode =='ntlmhash': password1 = '' nt_hash = binascii.unhexlify(data) else: print('[!] Wrong parameter') return False lm_hash = '' ntlm_auth, _ = ntlm.getNTLMSSPType3(ntlm_nego, ntlm_challenge, user, password1, domain, lm_hash, nt_hash) auth = base64.b64encode(ntlm_auth.getData()) headers = { "Authorization": 'NTLM %s' % auth.decode('utf-8'), "Content-type": "text/xml; charset=utf-8", "Accept": "text/xml", "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" } session.request("POST", autodiscover_url, POST_BODY, headers) res = session.getresponse() body = res.read() filename = "getUsersetting.xml" if res.status == 401: print('[!] Server returned HTTP status 401 - authentication failed') return False else: print('[+] Valid:%s %s'%(user,data)) #print(body) print('[+] Save response file to %s'%(filename)) with open(filename, 'w+', encoding='utf-8') as file_object: file_object.write(bytes.decode(body)) return True
def downloadlzx(host, port, mode, domain, email, data): OABID = input("Input the OABID:") lzxID = input("Input the lzx ID:(Eg: xx.lzx)") lzxURL = "/OAB/" + OABID + "/" + lzxID tmp = email.split('@') user = tmp[0] if port ==443: try: uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) session = HTTPSConnection(host, port, context=uv_context) except AttributeError: session = HTTPSConnection(host, port) else: session = HTTPConnection(host, port) # Use impacket for NTLM ntlm_nego = ntlm.getNTLMSSPType1(host, domain) #Negotiate auth negotiate = base64.b64encode(ntlm_nego.getData()) # Headers headers = { "Authorization": 'NTLM %s' % negotiate.decode('utf-8'), "Content-type": "text/xml; charset=utf-8", "Accept": "text/xml", "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" } session.request("GET", lzxURL, "", headers) res = session.getresponse() res.read() if res.status != 401: print('Status code returned: %d. Authentication does not seem required for URL'%(res.status)) return False try: if 'NTLM' not in res.getheader('WWW-Authenticate'): print('NTLM Auth not offered by URL, offered protocols: %s'%(res.getheader('WWW-Authenticate'))) return False except (KeyError, TypeError): print('No authentication requested by the server for url %s'%(lzxURL)) return False print('[*] Got 401, performing NTLM authentication') # Get negotiate data try: ntlm_challenge_b64 = re.search('NTLM ([a-zA-Z0-9+/]+={0,2})', res.getheader('WWW-Authenticate')).group(1) ntlm_challenge = base64.b64decode(ntlm_challenge_b64) except (IndexError, KeyError, AttributeError): print('No NTLM challenge returned from server') return False if mode =='plaintext': password1 = data; nt_hash = '' elif mode =='ntlmhash': password1 = '' nt_hash = binascii.unhexlify(data) else: print('[!] Wrong parameter') return False lm_hash = '' ntlm_auth, _ = ntlm.getNTLMSSPType3(ntlm_nego, ntlm_challenge, user, password1, domain, lm_hash, nt_hash) auth = base64.b64encode(ntlm_auth.getData()) headers = { "Authorization": 'NTLM %s' % auth.decode('utf-8'), "Content-type": "text/xml; charset=utf-8", "Accept-Encoding": "gzip, deflate, br", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "******", "Sec-Fetch-Dest": "document", "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" } session.request("GET", lzxURL, "", headers) res = session.getresponse() body = res.read() filedata = gzip.decompress(body) if res.status == 401: print('[!] Server returned HTTP status 401 - authentication failed') return False else: if res.status == 200: filename = lzxID print('[+] Save lzx file to %s'%(filename)) print('\r\n[*] Then you can use oabextract to decrype the lzx file in Kali Linux.') print('Eg.') print('oabextract 4667c322-5c08-4cda-844a-253ff36b4a6a-data-5.lzx gal.oab') print('strings gal.oab|grep SMTP') with open(filename, 'wb+') as file_object: file_object.write(filedata) return True
def smoke_test_release(release, files, expected_hash, plugins): for release_file in files: if not os.path.isfile(release_file): raise RuntimeError('Smoketest failed missing file %s' % (release_file)) tmp_dir = tempfile.mkdtemp() if release_file.endswith('tar.gz'): run('tar -xzf %s -C %s' % (release_file, tmp_dir)) elif release_file.endswith('zip'): run('unzip %s -d %s' % (release_file, tmp_dir)) else: print(' Skip SmokeTest for [%s]' % release_file) continue # nothing to do here es_run_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/elasticsearch') print(' Smoke testing package [%s]' % release_file) es_plugin_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/elasticsearch-plugin') plugin_names = {} for plugin in plugins: print(' Install plugin [%s]' % (plugin)) run('%s; %s -Des.plugins.staging=true %s %s' % (java_exe(), es_plugin_path, 'install', plugin)) plugin_names[plugin] = True if 'shield' in plugin_names: headers = { 'Authorization': 'Basic %s' % base64.b64encode(b"es_admin:foobar").decode("UTF-8") } es_shield_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/shield/esusers') print(" Install dummy shield user") run('%s; %s useradd es_admin -r admin -p foobar' % (java_exe(), es_shield_path)) else: headers = {} print(' Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release)) try: run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.script.inline=true -Des.script.indexed=true -Des.repositories.url.allowed_urls=http://snapshot.test* %s -Des.pidfile=%s' % (java_exe(), es_run_path, '-d', os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid'))) conn = HTTPConnection(host='127.0.0.1', port=9200, timeout=20) if not wait_for_node_startup(header=headers): print("elasticsearch logs:") print('*' * 80) logs = read_fully( os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'logs/prepare_release.log')) print(logs) print('*' * 80) raise RuntimeError('server didn\'t start up') try: # we now get / and /_nodes to fetch basic infos like hashes etc and the installed plugins conn.request('GET', '', headers=headers) res = conn.getresponse() if res.status == 200: version = json.loads(res.read().decode("utf-8"))['version'] if release != version['number']: raise RuntimeError( 'Expected version [%s] but was [%s]' % (release, version['number'])) if version['build_snapshot']: raise RuntimeError('Expected non snapshot version') if expected_hash.startswith(version['build_hash'].strip()): raise RuntimeError( 'HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash'])) print(' Verify if plugins are listed in _nodes') conn.request('GET', '/_nodes?plugin=true&pretty=true', headers=headers) res = conn.getresponse() if res.status == 200: nodes = json.loads(res.read().decode("utf-8"))['nodes'] for _, node in nodes.items(): node_plugins = node['plugins'] for node_plugin in node_plugins: if not plugin_names.get( node_plugin['name'].strip(), False): raise RuntimeError('Unexpected plugin %s' % node_plugin['name']) del plugin_names[node_plugin['name']] if plugin_names: raise RuntimeError('Plugins not loaded %s' % list(plugin_names.keys())) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) finally: conn.close() finally: pid_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid') if os.path.exists( pid_path): # try reading the pid and kill the node pid = int(read_fully(pid_path)) os.kill(pid, signal.SIGKILL) shutil.rmtree(tmp_dir) print(' ' + '*' * 80) print()
def checkAutodiscover(host, port, mode, domain, email, data): autodiscover_url = "/autodiscover/autodiscover.xml" tmp = email.split('@') user = tmp[0] if port ==443: try: uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) session = HTTPSConnection(host, port, context=uv_context) except AttributeError: session = HTTPSConnection(host, port) else: session = HTTPConnection(host, port) # Use impacket for NTLM ntlm_nego = ntlm.getNTLMSSPType1(host, domain) #Negotiate auth negotiate = base64.b64encode(ntlm_nego.getData()) # Headers headers = { "Authorization": 'NTLM %s' % negotiate.decode('utf-8'), "Accept-Encoding": "gzip", "User-Agent": "Microsoft Office/16.0 (Windows NT 6.1; Microsoft Outlook 16.0.4266; Pro)" } session.request("GET", autodiscover_url, "", headers) res = session.getresponse() res.read() if res.status != 401: print('Status code returned: %d. Authentication does not seem required for URL'%(res.status)) return False try: if 'NTLM' not in res.getheader('WWW-Authenticate'): print('NTLM Auth not offered by URL, offered protocols: %s'%(res.getheader('WWW-Authenticate'))) return False except (KeyError, TypeError): print('No authentication requested by the server for url %s'%(autodiscover_url)) return False print('[*] Got 401, performing NTLM authentication') # Get negotiate data try: ntlm_challenge_b64 = re.search('NTLM ([a-zA-Z0-9+/]+={0,2})', res.getheader('WWW-Authenticate')).group(1) ntlm_challenge = base64.b64decode(ntlm_challenge_b64) except (IndexError, KeyError, AttributeError): print('No NTLM challenge returned from server') return False if mode =='plaintext': password1 = data; nt_hash = '' elif mode =='ntlmhash': password1 = '' nt_hash = binascii.unhexlify(data) else: print('[!] Wrong parameter') return False lm_hash = '' ntlm_auth, _ = ntlm.getNTLMSSPType3(ntlm_nego, ntlm_challenge, user, password1, domain, lm_hash, nt_hash) auth = base64.b64encode(ntlm_auth.getData()) headers = { "Authorization": 'NTLM %s' % auth.decode('utf-8'), "Content-type": "text/xml", "X-Anchormailbox": '%s' % email, "X-Mapihttpcapability": '1', "Accept-Encoding": 'gzip' } POST_BODY = '''<?xml version="1.0" encoding="utf-8"?><Autodiscover xmlns="http://schemas.microsoft.com/exchange/autodiscover/outlook/requestschema/2006"> <Request><EMailAddress>{EMailAddress}</EMailAddress> <AcceptableResponseSchema>http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a</AcceptableResponseSchema> </Request></Autodiscover> ''' POST_BODY = POST_BODY.format(EMailAddress=email) session.request("POST", autodiscover_url, POST_BODY, headers) res = session.getresponse() body = res.read() filedata = gzip.decompress(body).decode("utf-8") if res.status == 401: print('[!] Server returned HTTP status 401 - authentication failed') return False else: if 'ErrorCode' in filedata : pattern_name = re.compile(r"<ErrorCode>(.*?)</ErrorCode>") name = pattern_name.findall(filedata) print('[!] ErrorCode:%s'%(name[0])) pattern_name = re.compile(r"<Message>(.*?)</Message>") name = pattern_name.findall(filedata) print('[!] Message:%s'%(name[0])) else: print('[+] Valid:%s %s'%(user,data)) pattern_name = re.compile(r"<LegacyDN>(.*?)</LegacyDN>") name = pattern_name.findall(filedata) print('[+] LegacyDN:%s'%(name[0])) pattern_name = re.compile(r"<OABUrl>(.*?)</OABUrl>") name = pattern_name.findall(filedata) print('[+] OABUrl:%s'%(name[0])) if 'InternalUrl' in filedata: pattern_name = re.compile(r"<InternalUrl>(.*?)</InternalUrl>") name = pattern_name.findall(filedata) print('[+] InternalUrl:%s'%(name[0])) if '<AD>' in filedata: pattern_name = re.compile(r"<AD>(.*?)</AD>") name = pattern_name.findall(filedata) print('[+] AD:%s'%(name[0])) filename = "checkAutodiscover.xml" print('[+] Save response file to %s'%(filename)) with open(filename, 'w+') as file_object: file_object.write(filedata) return True
def __init__(self, path, *args, **kwargs): HTTPConnection.__init__(self, *args, **kwargs) self.path = path
print("\n########## Client HTTP ##########") # Ciclo principal url = "." while url != "": # Obtengo url url = input( "\nIngrese la url del sitio web a obtener mediante peticiones HTTP: ") print() # Si la url es distinta de vacio intenta mostrar el sitio if url != "": try: # Creo conexion connection = HTTPConnection(url) # Realizo peticion connection.request("GET", "/") # Obtengo encabezados y cuerpo de respuesta response = connection.getresponse() headers = "HTTP/1.0" if response.version == 10 else "HTTP/1.1" \ + " " + str(response.status) \ + " " + str(response.reason) \ + "\n" for x in response.getheaders(): headers += x[0] + ": " + x[1] + "\n" body = response.read().decode() # Muestro respuesta por consola
def request(method, urlString, body='', headers={}, auth=None, verbose=False, https_proxy=os.getenv('https_proxy', None), timeout=60): url = urlparse(urlString) if url.scheme == 'http': conn = HTTPConnection(url.netloc, timeout=timeout) else: if httpRequestProps['secure'] or not hasattr( ssl, '_create_unverified_context'): conn = HTTPSConnection( url.netloc if https_proxy is None else https_proxy, timeout=timeout) else: conn = HTTPSConnection( url.netloc if https_proxy is None else https_proxy, context=ssl._create_unverified_context(), timeout=timeout) if https_proxy: conn.set_tunnel(url.netloc) if auth is not None: auth = base64.b64encode(auth.encode()).decode() headers['Authorization'] = 'Basic %s' % auth if verbose: print('========') print('REQUEST:') print('%s %s' % (method, urlString)) print('Headers sent:') print(getPrettyJson(headers)) if body != '': print('Body sent:') print(body) try: conn.request(method, urlString, body, headers) res = conn.getresponse() body = '' try: body = res.read() except IncompleteRead as e: body = e.partial # patch the read to return just the body since the normal read # can only be done once res.read = lambda: body if verbose: print('--------') print('RESPONSE:') print('Got response with code %s' % res.status) print('Body received:') print(res.read()) print('========') return res except socket.timeout: return ErrorResponse(status=500, error='request timed out at %d seconds' % timeout) except Exception as e: return ErrorResponse(status=500, error=str(e))
def ssrf_test(): unsafe_host = request.args["host"] unsafe_path = request.args["path"] user_input = request.args['untrusted_input'] conn = HTTPConnection(unsafe_host) conn.request("GET", unsafe_path) # NOT OK -- user has full control # Full SSRF variant, where there is ALSO made a request with fixed URL on the same # connection later on. This should not change anything on the overall SSRF alerts. conn = HTTPConnection(unsafe_host) conn.request("GET", unsafe_path) # NOT OK -- user has full control # partial SSRF on SAME connection conn.request("GET", "/foo") # NOT OK -- user has control of host # the rest are partial SSRF conn = HTTPConnection(unsafe_host) conn.request("GET", "/foo") # NOT OK -- user controlled domain conn = HTTPConnection("example.com") conn.request("GET", unsafe_path) # NOT OK -- user controlled path path = "foo?" + user_input conn = HTTPConnection("example.com") conn.request("GET", path) # NOT OK -- user controlled query parameters path = "foo#" + user_input conn = HTTPConnection("example.com") conn.request("GET", path) # NOT OK -- user controlled fragment
def get_connection(self): return HTTPConnection('localhost', 8001)
def __init__(self, path, timeout=60): HTTPConnection.__init__(self, 'localhost') self.path = path self.timeout = timeout
class Pipeline(list): "Gawd why am I being so elaborate?" def __init__(self, threshold=10): "threshold is how many requests in parallel to pipeline" self.threshold = threshold self.sent = True def __enter__(self): self.reopen() return self def __exit__(self, typ, exn, trace): self.send() self.drain() def reopen(self): self.c = HTTPConnection(server) self.send() def append(self, url, recv, diemessage): self.sent = False super().append(Penguin(url, recv, diemessage)) if len(self) > self.threshold: self.send() self.drain() def trydrain(self): for penguin in self: print('drain', penguin.url) try: penguin.response.begin() penguin.recv(penguin.response) except BadStatusLine as e: print('derped requesting', penguin.url) return False except HTTPException as e: die(penguin.diemessage + ' ' + repr(e) + ' (url=' + penguin.url + ')') self.clear() return True def drain(self): print('draining pipeline...', len(self)) assert self.sent, "Can't drain without sending the requests!" self.sent = False while self.trydrain() is not True: self.c.close() print('drain failed, trying again') time.sleep(1) self.reopen() def trysend(self): for penguin in pipeline: print('fill', penguin.url) try: self.c.request("GET", penguin.url) self.c._HTTPConnection__state = _CS_IDLE penguin.response = self.c.response_class(self.c.sock, method="GET") # begin LATER so we can send multiple requests w/out response headers except BadStatusLine: return False except HTTPException as e: die(diemessage + ' because of a ' + repr(e)) return True def send(self): if self.sent: return print('filling pipeline...', len(self)) while self.trysend() is not True: self.c.close() print('derped resending') time.sleep(1) self.reopen() self.sent = True
from http.client import HTTPConnection conn = HTTPConnection('www.example.com') conn.request('HEAD', '/') resp = conn.getresponse() print(resp.status, resp.reason) data = resp.read() print(len(data)) print(data == b'')
os.environ['AWS_LAMBDA_FUNCTION_VERSION'] = FUNCTION_VERSION os.environ['AWS_REGION'] = REGION os.environ['AWS_DEFAULT_REGION'] = REGION os.environ['_HANDLER'] = HANDLER MOCKSERVER_ENV = os.environ.copy() MOCKSERVER_ENV['DOCKER_LAMBDA_NO_BOOTSTRAP'] = '1' MOCKSERVER_ENV['DOCKER_LAMBDA_USE_STDIN'] = '1' MOCKSERVER_PROCESS = subprocess.Popen('/var/runtime/mockserver', stdin=subprocess.PIPE, env=MOCKSERVER_ENV) MOCKSERVER_PROCESS.stdin.write(EVENT_BODY.encode()) MOCKSERVER_PROCESS.stdin.close() MOCKSERVER_CONN = HTTPConnection("127.0.0.1", 9001) def eprint(*args, **kwargs): print(*args, file=ORIG_STDERR, **kwargs) def report_user_init_start(): return def report_user_init_end(): global INIT_END INIT_END = time.time()
def reopen(self): self.c = HTTPConnection(server) self.send()
# url = 'http://httpbin.org/post' # files = { 'file': ('data.csv', open('data.csv', 'rb')) } # # r = requests.post(url, files=files) # 对于真的很简单HTTP客户端代码,用内置的 urllib 模块通常就足够了。但是,如果你要做的不仅仅只是简单的GET或POST请求, # 那就真的不能再依赖它的功能了。这时候就是第三方模块比如 requests 大显身手的时候了。 # # 例如,如果你决定坚持使用标准的程序库而不考虑像 requests 这样的第三方库,那么也许就不得 # 不使用底层的 http.client 模块来实现自己的代码。比方说,下面的代码展示了如何执行一个 # HEAD请求: from http.client import HTTPConnection from urllib import parse c = HTTPConnection("www.python.org", 80) c.request("HEAD", "/index.html") resp = c.getresponse() print("Status", resp.status) for name, value in resp.getheaders(): print(name, value) # 同样地,如果必须编写涉及代理、认证、cookies以及其他一些细节方面的代码,那么使用 urllib 就显得特别别扭和啰嗦。 # 比方说,下面这个示例实现在Python包索引上的认证: import urllib.request auth = urllib.request.HTTPBasicAuthHandler() auth.add_password("pypi", "http://pypi.python.org", "username", "password") opener = urllib.request.build_opener(auth)
return True def send(self): if self.sent: return print('filling pipeline...', len(self)) while self.trysend() is not True: self.c.close() print('derped resending') time.sleep(1) self.reopen() self.sent = True with Pipeline() as pipeline: # two connections is okay, right? one for json, one for preview images c = HTTPConnection(server) def addpage(page): global curskin, pages print("Page: " + str(page)) r = 0 try: c.request( "GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64") r = c.getresponse() except Exception: if r != 0: if r.status != 200: die("Error", r.status) return
from http.client import HTTPConnection from urllib.parse import urlencode host = '127.0.0.1:8000' params = urlencode({ 'language': 'python', 'name': '김석훈', 'email': '*****@*****.**', }) headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'text/plain', } conn = HTTPConnection(host) conn.request('PUT', '', params, headers) resp = conn.getresponse() print(resp.status, resp.reason) data = resp.read(300) print(data.decode('utf-8')) conn.close()
""" Making a raw http request to Google maps geocoding API v3 """ import json try: from http.client import HTTPConnection except ImportError: from httplib import HTTPConnection path = u'/maps/api/geocode/json?address=207+N.+Defiance+St%2C+Archbold%2C+OH' connection = HTTPConnection('maps.googleapis.com') connection.request('GET', path) rawreply = connection.getresponse().read() unicode_reply = rawreply.decode("utf-8") reply = json.loads(unicode_reply) location = reply["results"][0]["geometry"]["location"] print(list((location.values())))
def make_connection(self, host): self.log.debug("Connecting to {} through {}".format(host, self.proxy)) self.realhost = host h = HTTPConnection(self.proxy) return h
def socket_http_factory(cls, host): http = HTTPConnection(host, 80) http.sock = cls.socket_conn(cls.server) return http
class RelayQuery: """Call the back-end service and send the response downstream to the client This object is a small wrapper around the stdlib http.client API. The major reason for wrapping that API is so we can ensure the TCP socket to the back-end service gets closed after responding to the down-stream client. :param host: The IP/FQDN/DNS shortname of the back-end service to call. :type host: String :param uri: The API end point to envoke on the back-end service. :type uri: String :param method: The HTTP method to envoke (i.e. GET, POST, etc...) :type method: String :param headers: The HTTP headers to send to the back-end service. :type headers: Dictionary :param body: The HTTP body to send to the back-end service :type body: Bytes :param tls: Set to True to use HTTPS, False for HTTP. Default False. :type tls: Boolean """ def __init__(self, host, uri, method, headers, body, port, tls=False): self._conn = None self._resp = None self._headers = None self._status = None if host is None: logger.error('No host found for {} on {}'.format(method, uri)) self._handle_no_host(host, uri) else: self._call_upstream(host, uri, method, headers, body, port, tls) def _call_upstream(self, host, uri, method, headers, body, port, tls): if tls: self._conn = HTTPSConnection(host=host, port=port, context=const.VLAB_SSL_CONTEXT) else: self._conn = HTTPConnection(host=host, port=port) try: self._conn.request(method=method, url=uri, body=body, headers=headers) except gaierror: logger.error('failed to resolve DNS host {} for URI {}'.format( host, uri)) self._handle_no_host(host, uri) except ConnectionRefusedError: logger.error( 'Connection refused by host - URL {}:{}{}, TLS={}'.format( host, port, uri, tls)) self._handle_no_host(host, uri, status='502 Bad Gateway') else: self._resp = self._conn.getresponse() self._headers = self._resp.getheaders() self._status = '{} {}'.format(self._resp.status, self._resp.reason) def _handle_no_host(self, host, uri, status='404 Not Found'): self._headers = [('Content-Type', 'application/json')] self._status = status self._resp = NoHostResponse(host, uri) @property def headers(self): return self._headers @property def status(self): return self._status def close(self): """The WSGI spec states that if the returned object contains a callable named ``close`` that the Web server must execute it. Failure to close the TCP socket with the back-end service will lead to the API gateway "leaking sockets." Eventually the OS will prevent us from opening any new sockets, and all traffic will grind to a halt. """ if self._conn: # NoHostResponse leaves this as None self._conn.close() def __iter__(self): return self def __next__(self): data = self._resp.readline() if data: return data else: raise StopIteration
def smoke_test_release(release, files, expected_hash, plugins): for release_file in files: if not os.path.isfile(release_file): raise RuntimeError('Smoketest failed missing file %s' % (release_file)) tmp_dir = tempfile.mkdtemp() if release_file.endswith('tar.gz'): run('tar -xzf %s -C %s' % (release_file, tmp_dir)) elif release_file.endswith('zip'): run('unzip %s -d %s' % (release_file, tmp_dir)) else: log('Skip SmokeTest for [%s]' % release_file) continue # nothing to do here es_run_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/elasticsearch') print(' Smoke testing package [%s]' % release_file) es_plugin_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/plugin') plugin_names = {} for name, plugin in plugins: print(' Install plugin [%s] from [%s]' % (name, plugin)) run('%s %s %s' % (es_plugin_path, '-install', plugin)) plugin_names[name] = True if release.startswith("0.90."): background = '' # 0.90.x starts in background automatically else: background = '-d' print(' Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release)) run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.discovery.zen.ping.multicast.enabled=false -Des.node.bench=true -Des.script.disable_dynamic=false %s' % (java_exe(), es_run_path, background)) conn = HTTPConnection('127.0.0.1', 9200, 20) wait_for_node_startup() try: try: conn.request('GET', '') res = conn.getresponse() if res.status == 200: version = json.loads(res.read().decode("utf-8"))['version'] if release != version['number']: raise RuntimeError( 'Expected version [%s] but was [%s]' % (release, version['number'])) if version['build_snapshot']: raise RuntimeError('Expected non snapshot version') if version['build_hash'].strip() != expected_hash: raise RuntimeError( 'HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash'])) print(' Running REST Spec tests against package [%s]' % release_file) run_mvn( 'test -Dtests.cluster=%s -Dtests.class=*.*RestTests' % ("127.0.0.1:9300")) print(' Verify if plugins are listed in _nodes') conn.request('GET', '/_nodes?plugin=true&pretty=true') res = conn.getresponse() if res.status == 200: nodes = json.loads(res.read().decode("utf-8"))['nodes'] for _, node in nodes.items(): node_plugins = node['plugins'] for node_plugin in node_plugins: if not plugin_names.get( node_plugin['name'], False): raise RuntimeError('Unexpeced plugin %s' % node_plugin['name']) del plugin_names[node_plugin['name']] if plugin_names: raise RuntimeError('Plugins not loaded %s' % list(plugin_names.keys())) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) finally: conn.request('POST', '/_cluster/nodes/_local/_shutdown') time.sleep(1) # give the node some time to shut down if conn.getresponse().status != 200: raise RuntimeError( 'Expected HTTP 200 but got %s on node shutdown' % res.status) finally: conn.close() shutil.rmtree(tmp_dir)