def validate_dhcp_resp(request, response): bytes_recv = len(response) if bytes_recv < 0xF6: logger.error("HandleDhcpResponse: Too few bytes received:{0}", bytes_recv) return False logger.verb("BytesReceived:{0}", hex(bytes_recv)) logger.verb("DHCP response:{0}", hex_dump(response, bytes_recv)) # check transactionId, cookie, MAC address cookie should never mismatch # transactionId and MAC address may mismatch if we see a response # meant from another machine if not compare_bytes(request, response, 0xEC, 4): logger.verb("Cookie not match:\nsend={0},\nreceive={1}", hex_dump3(request, 0xEC, 4), hex_dump3(response, 0xEC, 4)) raise DhcpError("Cookie in dhcp respones doesn't match the request") if not compare_bytes(request, response, 4, 4): logger.verb("TransactionID not match:\nsend={0},\nreceive={1}", hex_dump3(request, 4, 4), hex_dump3(response, 4, 4)) raise DhcpError("TransactionID in dhcp respones " "doesn't match the request") if not compare_bytes(request, response, 0x1C, 6): logger.verb("Mac Address not match:\nsend={0},\nreceive={1}", hex_dump3(request, 0x1C, 6), hex_dump3(response, 0x1C, 6)) raise DhcpError("Mac Addr in dhcp respones " "doesn't match the request")
def __init__(self, xml_text): """ Query endpoint server for wire protocol version. Fail if our desired protocol version is not seen. """ logger.verb("Load Version.xml") self.parse(xml_text)
def __init__(self, xml_text): if xml_text is None: raise ValueError("HostingEnvironmentConfig.xml is None") logger.verb("Load HostingEnvironmentConfig.xml") self.vm_name = None self.role_name = None self.deployment_name = None self.parse(xml_text)
def get_ext_handlers(self): logger.verb("Get extension handler config") #Update goal state to get latest extensions config self.client.update_goal_state() goal_state = self.client.get_goal_state() ext_conf = self.client.get_ext_conf() #In wire protocol, incarnation is equivalent to ETag return ext_conf.ext_handlers, goal_state.incarnation
def put_page_blob(self, url, data): logger.verb("Replace old page blob") #Convert string into bytes data=bytearray(data, encoding='utf-8') timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) #Align to 512 bytes page_blob_size = int((len(data) + 511) / 512) * 512 try: resp = self.client.call_storage_service(restutil.http_put, url, "", { "x-ms-date" : timestamp, "x-ms-blob-type" : "PageBlob", "Content-Length": "0", "x-ms-blob-content-length" : ustr(page_blob_size), "x-ms-version" : self.__class__.__storage_version__ }) except HttpError as e: raise ProtocolError((u"Failed to clean up page blob: {0}" u"").format(e)) if resp.status != httpclient.CREATED: raise ProtocolError(("Failed to clean up page blob: {0}" "").format(resp.status)) if url.count("?") < 0: url = "{0}?comp=page".format(url) else: url = "{0}&comp=page".format(url) logger.verb("Upload page blob") page_max = 4 * 1024 * 1024 #Max page size: 4MB start = 0 end = 0 while end < len(data): end = min(len(data), start + page_max) content_size = end - start #Align to 512 bytes page_end = int((end + 511) / 512) * 512 buf_size = page_end - start buf = bytearray(buf_size) buf[0: content_size] = data[start: end] try: resp = self.client.call_storage_service(restutil.http_put, url, bytebuffer(buf), { "x-ms-date" : timestamp, "x-ms-range" : "bytes={0}-{1}".format(start, page_end - 1), "x-ms-page-write" : "update", "x-ms-version" : self.__class__.__storage_version__, "Content-Length": ustr(page_end - start) }) except HttpError as e: raise ProtocolError((u"Failed to upload page blob: {0}" u"").format(e)) if resp is None or resp.status != httpclient.CREATED: raise ProtocolError(("Failed to upload page blob: {0}" "").format(resp.status)) start = end
def collect_event(self, evt_file_name): try: logger.verb("Found event file: {0}", evt_file_name) with open(evt_file_name, "rb") as evt_file: #if fail to open or delete the file, throw exception data_str = evt_file.read().decode("utf-8",'ignore') logger.verb("Processed event file: {0}", evt_file_name) os.remove(evt_file_name) return data_str except IOError as e: msg = "Failed to process {0}, {1}".format(evt_file_name, e) raise EventError(msg)
def __init__(self, xml_text): if xml_text is None: raise ValueError("ovf-env is None") logger.verb("Load ovf-env.xml") self.hostname = None self.username = None self.user_password = None self.customdata = None self.disable_ssh_password_auth = True self.ssh_pubkeys = [] self.ssh_keypairs = [] self.parse(xml_text)
def __init__(self, xml_text): if xml_text is None: raise ValueError("ovf-env is None") logger.verb("Load ovf-env.xml") self.hostname = None self.username = None self.user_password = None self.customdata = None self.disable_ssh_password_auth = True self.ssh_pubkeys = [] self.ssh_keypairs = [] self.parse(xml_text)
def collect_event(self, evt_file_name): try: logger.verb("Found event file: {0}", evt_file_name) with open(evt_file_name, "rb") as evt_file: #if fail to open or delete the file, throw exception data_str = evt_file.read().decode("utf-8", 'ignore') logger.verb("Processed event file: {0}", evt_file_name) os.remove(evt_file_name) return data_str except IOError as e: msg = "Failed to process {0}, {1}".format(evt_file_name, e) raise EventError(msg)
def parse(self, xml_text): xml_doc = parse_doc(xml_text) preferred = find(xml_doc, "Preferred") self.preferred = findtext(preferred, "Version") logger.info("Fabric preferred wire protocol version:{0}", self.preferred) self.supported = [] supported = find(xml_doc, "Supported") supported_version = findall(supported, "Version") for node in supported_version: version = gettext(node) logger.verb("Fabric supported wire protocol version:{0}", version) self.supported.append(version)
def __init__(self, xml_text): if xml_text is None: raise ValueError("GoalState.xml is None") logger.verb("Load GoalState.xml") self.incarnation = None self.expected_state = None self.hosting_env_uri = None self.shared_conf_uri = None self.certs_uri = None self.ext_uri = None self.role_instance_id = None self.container_id = None self.load_balancer_probe_port = None self.parse(xml_text)
def http_request(method, url, data, headers=None, max_retry=3, chk_proxy=False): """ Sending http request to server On error, sleep 10 and retry max_retry times. """ logger.verb("HTTP Req: {0} {1}", method, url) logger.verb(" Data={0}", data) logger.verb(" Header={0}", headers) host, port, secure, rel_uri = _parse_url(url) #Check proxy proxy_host, proxy_port = (None, None) if chk_proxy: proxy_host, proxy_port = get_http_proxy() #If httplib module is not built with ssl support. Fallback to http if secure and not hasattr(httpclient, "HTTPSConnection"): logger.warn("httplib is not built with ssl support") secure = False #If httplib module doesn't support https tunnelling. Fallback to http if secure and \ proxy_host is not None and \ proxy_port is not None and \ not hasattr(httpclient.HTTPSConnection, "set_tunnel"): logger.warn("httplib doesn't support https tunnelling(new in python 2.7)") secure = False for retry in range(0, max_retry): try: resp = _http_request(method, host, rel_uri, port=port, data=data, secure=secure, headers=headers, proxy_host=proxy_host, proxy_port=proxy_port) logger.verb("HTTP Resp: Status={0}", resp.status) logger.verb(" Header={0}", resp.getheaders()) return resp except httpclient.HTTPException as e: logger.warn('HTTPException {0}, args:{1}', e, repr(e.args)) except IOError as e: logger.warn('Socket IOError {0}, args:{1}', e, repr(e.args)) if retry < max_retry - 1: logger.info("Retry={0}, {1} {2}", retry, method, url) time.sleep(RETRY_WAITING_INTERVAL) if url is not None and len(url) > 100: url_log = url[0: 100] #In case the url is too long else: url_log = url raise HttpError("HTTP Err: {0} {1}".format(method, url_log))
def upload(self, url): #TODO upload extension only if content has changed logger.verb("Upload status blob") blob_type = self.get_blob_type(url) data = self.to_json() try: if blob_type == "BlockBlob": self.put_block_blob(url, data) elif blob_type == "PageBlob": self.put_page_blob(url, data) else: raise ProtocolError("Unknown blob type: {0}".format(blob_type)) except HttpError as e: raise ProtocolError("Failed to upload status blob: {0}".format(e))
def send_event(self, provider_id, event_str): uri = TELEMETRY_URI.format(self.endpoint) data_format = ('<?xml version="1.0"?>' '<TelemetryData version="1.0">' '<Provider id="{0}">{1}' '</Provider>' '</TelemetryData>') data = data_format.format(provider_id, event_str) try: header = self.get_header_for_xml_content() resp = self.call_wireserver(restutil.http_post, uri, data, header) except HttpError as e: raise ProtocolError("Failed to send events:{0}".format(e)) if resp.status != httpclient.OK: logger.verb(resp.read()) raise ProtocolError("Failed to send events:{0}".format(resp.status))
def put_block_blob(self, url, data): logger.verb("Upload block blob") timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) try: resp = self.client.call_storage_service(restutil.http_put, url, data, { "x-ms-date" : timestamp, "x-ms-blob-type" : "BlockBlob", "Content-Length": ustr(len(data)), "x-ms-version" : self.__class__.__storage_version__ }) except HttpError as e: raise ProtocolError((u"Failed to upload block blob: {0}" u"").format(e)) if resp.status != httpclient.CREATED: raise ProtocolError(("Failed to upload block blob: {0}" "").format(resp.status))
def fetch_manifest(self, version_uris): for version_uri in version_uris: logger.verb("Fetch ext handler manifest: {0}", version_uri.uri) try: resp = self.call_storage_service(restutil.http_get, version_uri.uri, None, chk_proxy=True) except HttpError as e: raise ProtocolError(ustr(e)) if resp.status == httpclient.OK: return self.decode_config(resp.read()) logger.warn("Failed to fetch ExtensionManifest: {0}, {1}", resp.status, version_uri.uri) logger.info("Will retry later, in {0} seconds", LONG_WAITING_INTERVAL) time.sleep(LONG_WAITING_INTERVAL) raise ProtocolError(("Failed to fetch ExtensionManifest from " "all sources"))
def get_blob_type(self, url): #Check blob type logger.verb("Check blob type.") timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) try: resp = self.client.call_storage_service(restutil.http_head, url, { "x-ms-date" : timestamp, 'x-ms-version' : self.__class__.__storage_version__ }) except HttpError as e: raise ProtocolError((u"Failed to get status blob type: {0}" u"").format(e)) if resp is None or resp.status != httpclient.OK: raise ProtocolError(("Failed to get status blob type: {0}" "").format(resp.status)) blob_type = resp.getheader("x-ms-blob-type") logger.verb("Blob type={0}".format(blob_type)) return blob_type
def run(self): ext_handlers, etag = None, None try: self.protocol = self.distro.protocol_util.get_protocol() ext_handlers, etag = self.protocol.get_ext_handlers() except ProtocolError as e: add_event(name="WALA", is_success=False, message=ustr(e)) return if self.last_etag is not None and self.last_etag == etag: logger.verb("No change to ext handler config:{0}, skip", etag) self.log_report = False else: logger.info("Handle new ext handler config") self.log_report = True #Log status report success on new config self.handle_ext_handlers(ext_handlers) self.last_etag = etag self.report_ext_handlers_status(ext_handlers)
def socket_send(request): sock = None try: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(("0.0.0.0", 68)) sock.sendto(request, ("<broadcast>", 67)) sock.settimeout(10) logger.verb("Send DHCP request: Setting socket.timeout=10, " "entering recv") response = sock.recv(1024) return response except IOError as e: raise DhcpError("{0}".format(e)) finally: if sock is not None: sock.close()
def run(self): ext_handlers, etag = None, None try: self.protocol = self.distro.protocol_util.get_protocol() ext_handlers, etag = self.protocol.get_ext_handlers() except ProtocolError as e: add_event(name="WALA", is_success=False, message=ustr(e)) return if self.last_etag is not None and self.last_etag == etag: logger.verb("No change to ext handler config:{0}, skip", etag) self.log_report = False else: logger.info("Handle new ext handler config") self.log_report = True #Log status report success on new config self.handle_ext_handlers(ext_handlers) self.last_etag = etag self.report_ext_handlers_status(ext_handlers)
def run_get_output(cmd, chk_err=True, log_cmd=True): """ Wrapper for subprocess.check_output. Execute 'cmd'. Returns return code and STDOUT, trapping expected exceptions. Reports exceptions to Error if chk_err parameter is True """ if log_cmd: logger.verb(u"run cmd '{0}'", cmd) try: output=subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) output = ustr(output, encoding='utf-8', errors="backslashreplace") except subprocess.CalledProcessError as e : output = ustr(e.output, encoding='utf-8', errors="backslashreplace") if chk_err: if log_cmd: logger.error(u"run cmd '{0}' failed", e.cmd) logger.error(u"Error Code:{0}", e.returncode) logger.error(u"Result:{0}", output) return e.returncode, output return 0, output
def run_get_output(cmd, chk_err=True, log_cmd=True): """ Wrapper for subprocess.check_output. Execute 'cmd'. Returns return code and STDOUT, trapping expected exceptions. Reports exceptions to Error if chk_err parameter is True """ if log_cmd: logger.verb(u"run cmd '{0}'", cmd) try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) output = ustr(output, encoding='utf-8', errors="backslashreplace") except subprocess.CalledProcessError as e: output = ustr(e.output, encoding='utf-8', errors="backslashreplace") if chk_err: if log_cmd: logger.error(u"run cmd '{0}' failed", e.cmd) logger.error(u"Error Code:{0}", e.returncode) logger.error(u"Result:{0}", output) return e.returncode, output return 0, output
def parse_route(response, option, i, length, bytes_recv): # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx logger.verb("Routes at offset: {0} with length:{1}", hex(i), hex(length)) routes = [] if length < 5: logger.error("Data too small for option:{0}", option) j = i + 2 while j < (i + length + 2): mask_len_bits = str_to_ord(response[j]) mask_len_bytes = (((mask_len_bits + 7) & ~7) >> 3) mask = 0xFFFFFFFF & (0xFFFFFFFF << (32 - mask_len_bits)) j += 1 net = unpack_big_endian(response, j, mask_len_bytes) net <<= (32 - mask_len_bytes * 8) net &= mask j += mask_len_bytes gateway = unpack_big_endian(response, j, 4) j += 4 routes.append((net, mask, gateway)) if j != (i + length + 2): logger.error("Unable to parse routes") return routes
def report_ext_handlers_status(self, ext_handlers): """Go thru handler_state dir, collect and report status""" vm_status = VMStatus() vm_status.vmAgent.version = AGENT_VERSION vm_status.vmAgent.status = "Ready" vm_status.vmAgent.message = "Guest Agent is running" if ext_handlers is not None: for ext_handler in ext_handlers.extHandlers: try: self.report_ext_handler_status(vm_status, ext_handler) except ExtensionError as e: add_event(name="WALA", is_success=False, message=ustr(e)) logger.verb("Report vm agent status") try: self.protocol.report_vm_status(vm_status) except ProtocolError as e: message = "Failed to report vm agent status: {0}".format(e) add_event(name="WALA", is_success=False, message=message) if self.log_report: logger.info("Successfully reported vm agent status")
def report_ext_handlers_status(self, ext_handlers): """Go thru handler_state dir, collect and report status""" vm_status = VMStatus() vm_status.vmAgent.version = AGENT_VERSION vm_status.vmAgent.status = "Ready" vm_status.vmAgent.message = "Guest Agent is running" if ext_handlers is not None: for ext_handler in ext_handlers.extHandlers: try: self.report_ext_handler_status(vm_status, ext_handler) except ExtensionError as e: add_event(name="WALA", is_success=False, message=ustr(e)) logger.verb("Report vm agent status") try: self.protocol.report_vm_status(vm_status) except ProtocolError as e: message = "Failed to report vm agent status: {0}".format(e) add_event(name="WALA", is_success=False, message=message) if self.log_report: logger.info("Successfully reported vm agent status")
def prevent_throttling(self): """ Try to avoid throttling of wire server """ now = time.time() if now - self.last_request < 1: logger.verb("Last request issued less than 1 second ago") logger.verb("Sleep {0} second to avoid throttling.", SHORT_WAITING_INTERVAL) time.sleep(SHORT_WAITING_INTERVAL) self.last_request = now self.req_count += 1 if self.req_count % 3 == 0: logger.verb("Sleep {0} second to avoid throttling.", SHORT_WAITING_INTERVAL) time.sleep(SHORT_WAITING_INTERVAL) self.req_count = 0
def http_request(method, url, data, headers=None, max_retry=3, chk_proxy=False): """ Sending http request to server On error, sleep 10 and retry max_retry times. """ logger.verb("HTTP Req: {0} {1}", method, url) logger.verb(" Data={0}", data) logger.verb(" Header={0}", headers) host, port, secure, rel_uri = _parse_url(url) #Check proxy proxy_host, proxy_port = (None, None) if chk_proxy: proxy_host, proxy_port = get_http_proxy() #If httplib module is not built with ssl support. Fallback to http if secure and not hasattr(httpclient, "HTTPSConnection"): logger.warn("httplib is not built with ssl support") secure = False #If httplib module doesn't support https tunnelling. Fallback to http if secure and \ proxy_host is not None and \ proxy_port is not None and \ not hasattr(httpclient.HTTPSConnection, "set_tunnel"): logger.warn( "httplib doesn't support https tunnelling(new in python 2.7)") secure = False for retry in range(0, max_retry): try: resp = _http_request(method, host, rel_uri, port=port, data=data, secure=secure, headers=headers, proxy_host=proxy_host, proxy_port=proxy_port) logger.verb("HTTP Resp: Status={0}", resp.status) logger.verb(" Header={0}", resp.getheaders()) return resp except httpclient.HTTPException as e: logger.warn('HTTPException {0}, args:{1}', e, repr(e.args)) except IOError as e: logger.warn('Socket IOError {0}, args:{1}', e, repr(e.args)) if retry < max_retry - 1: logger.info("Retry={0}, {1} {2}", retry, method, url) time.sleep(RETRY_WAITING_INTERVAL) if url is not None and len(url) > 100: url_log = url[0:100] #In case the url is too long else: url_log = url raise HttpError("HTTP Err: {0} {1}".format(method, url_log))
def build_dhcp_request(mac_addr): """ Build DHCP request string. """ # # typedef struct _DHCP { # UINT8 Opcode; /* op: BOOTREQUEST or BOOTREPLY */ # UINT8 HardwareAddressType; /* htype: ethernet */ # UINT8 HardwareAddressLength; /* hlen: 6 (48 bit mac address) */ # UINT8 Hops; /* hops: 0 */ # UINT8 TransactionID[4]; /* xid: random */ # UINT8 Seconds[2]; /* secs: 0 */ # UINT8 Flags[2]; /* flags: 0 or 0x8000 for broadcast */ # UINT8 ClientIpAddress[4]; /* ciaddr: 0 */ # UINT8 YourIpAddress[4]; /* yiaddr: 0 */ # UINT8 ServerIpAddress[4]; /* siaddr: 0 */ # UINT8 RelayAgentIpAddress[4]; /* giaddr: 0 */ # UINT8 ClientHardwareAddress[16]; /* chaddr: 6 byte eth MAC address */ # UINT8 ServerName[64]; /* sname: 0 */ # UINT8 BootFileName[128]; /* file: 0 */ # UINT8 MagicCookie[4]; /* 99 130 83 99 */ # /* 0x63 0x82 0x53 0x63 */ # /* options -- hard code ours */ # # UINT8 MessageTypeCode; /* 53 */ # UINT8 MessageTypeLength; /* 1 */ # UINT8 MessageType; /* 1 for DISCOVER */ # UINT8 End; /* 255 */ # } DHCP; # # tuple of 244 zeros # (struct.pack_into would be good here, but requires Python 2.5) request = [0] * 244 trans_id = gen_trans_id() # Opcode = 1 # HardwareAddressType = 1 (ethernet/MAC) # HardwareAddressLength = 6 (ethernet/MAC/48 bits) for a in range(0, 3): request[a] = [1, 1, 6][a] # fill in transaction id (random number to ensure response matches request) for a in range(0, 4): request[4 + a] = str_to_ord(trans_id[a]) logger.verb("BuildDhcpRequest: transactionId:%s,%04X" % ( hex_dump2(trans_id), unpack_big_endian(request, 4, 4))) # fill in ClientHardwareAddress for a in range(0, 6): request[0x1C + a] = str_to_ord(mac_addr[a]) # DHCP Magic Cookie: 99, 130, 83, 99 # MessageTypeCode = 53 DHCP Message Type # MessageTypeLength = 1 # MessageType = DHCPDISCOVER # End = 255 DHCP_END for a in range(0, 8): request[0xEC + a] = [99, 130, 83, 99, 53, 1, 1, 255][a] return array.array("B", request)
def parse_dhcp_resp(response): """ Parse DHCP response: Returns endpoint server or None on error. """ logger.verb("parse Dhcp Response") bytes_recv = len(response) endpoint = None gateway = None routes = None # Walk all the returned options, parsing out what we need, ignoring the # others. We need the custom option 245 to find the the endpoint we talk to, # as well as, to handle some Linux DHCP client incompatibilities, # options 3 for default gateway and 249 for routes. And 255 is end. i = 0xF0 # offset to first option while i < bytes_recv: option = str_to_ord(response[i]) length = 0 if (i + 1) < bytes_recv: length = str_to_ord(response[i + 1]) logger.verb("DHCP option {0} at offset:{1} with length:{2}", hex(option), hex(i), hex(length)) if option == 255: logger.verb("DHCP packet ended at offset:{0}", hex(i)) break elif option == 249: routes = parse_route(response, option, i, length, bytes_recv) elif option == 3: gateway = parse_ip_addr(response, option, i, length, bytes_recv) logger.verb("Default gateway:{0}, at {1}", gateway, hex(i)) elif option == 245: endpoint = parse_ip_addr(response, option, i, length, bytes_recv) logger.verb("Azure wire protocol endpoint:{0}, at {1}", gateway, hex(i)) else: logger.verb("Skipping DHCP option:{0} at {1} with length {2}", hex(option), hex(i), hex(length)) i += length + 2 return endpoint, gateway, routes
def __init__(self, xml_text): logger.verb("Load ExtensionsConfig.xml") self.ext_handlers = ExtHandlerList() self.status_upload_blob = None if xml_text is not None: self.parse(xml_text)
def __init__(self, client, xml_text): logger.verb("Load Certificates.xml") self.client = client self.cert_list = CertList() self.parse(xml_text)
def __init__(self, xml_text): logger.verb("Load SharedConfig.xml") self.parse(xml_text)
def get_ext_handler_pkgs(self, ext_handler): logger.verb("Get extension handler package") goal_state = self.client.get_goal_state() man = self.client.get_ext_manifest(ext_handler, goal_state) return man.pkg_list
def __init__(self, xml_text): if xml_text is None: raise ValueError("ExtensionManifest is None") logger.verb("Load ExtensionManifest.xml") self.pkg_list = ExtHandlerPackageList() self.parse(xml_text)