def test_None(): assert safe_int(None) == 0 assert safe_int(None, 7) == 7 assert safe_float(None) == 0.0 assert safe_float(None, default=7.7) == 7.7 assert safe_dict(None) == {} assert safe_dict(None, {'Jeff': 'Tanner'}) == {'Jeff': 'Tanner'} assert safe_str(None) == '' assert safe_str(None, "stas") == "stas" assert safe_cast(None, str) is None assert safe_cast(None, str, default="TuliTuliTuli") == "TuliTuliTuli"
def write_user_config(userconfig, *args): if (safe_str(args[0], default=False)): write_path = uni.Path(fg.config_path, args[0]) else: write_path = uni.Path(fg.config_path, fg.today) fluid_dump(write_path, userconfig) print("User-config written.")
def format(self, record): fmt_record_items = {} for key, meta in TUNE_STANDARD_FORMAT_SUPPORTED_KEYS.items(): if key in record.__dict__: if 'label' in meta and meta['label']: key_lower = str(key).lower() fmt_record_item = "{}: ".format(key_lower) else: fmt_record_item = "" fmt_record_item.lower() if 'fmt' in meta: meta_fmt = meta['fmt'] else: meta_fmt = '{},' try: if meta['type'] == str: fmt_record_item += \ "\"{}\", ".format(safe_str(record.__dict__[key])) elif meta['type'] == int: fmt_record_item += \ "{:,}, ".format(safe_int(record.__dict__[key])) elif meta['type'] == float: fmt_record_item += \ "{0:.2f}, ".format(safe_float(record.__dict__[key])) elif meta['type'] == dict: fmt_record_item += \ "{}, ".format( json.loads( json.dumps(dict(record.__dict__[key])) ) ) else: fmt_record_item += meta_fmt.format( record.__dict__[key]) except ValueError as va_ex: fmt_record_item += "{}: '{}'".format( type(record.__dict__[key]).__name__, str(record.__dict__[key])) hash = "{0:03d}".format(meta['order']) fmt_record_items[hash] = fmt_record_item str_fmt_record = "" for key in sorted(fmt_record_items.keys()): value = fmt_record_items[key] str_fmt_record += value return str_fmt_record[:-2]
def _error_message(error_message, error_code): error_message_ = None exit_code_description_ = requests_fortified_error_desc(error_code).rstrip('\.') error_message_prefix_ = "{0}: {1}".format(error_code, exit_code_description_) error_message = safe_str(error_message).strip() if error_message: error_message_ = "%s: '%s'" % (error_message_prefix_, six.text_type(error_message)) elif exit_code_description_: error_message_ = error_message_prefix_ return error_message_
def _error_message(error_message, error_code): error_message_ = None exit_code_description_ = tune_request_error_desc(error_code).rstrip( '\.') error_message_prefix_ = f"{error_code}: {exit_code_description_}" error_message = safe_str(error_message).strip() if error_message: error_message_ = "%s: '%s'" % (error_message_prefix_, six.text_type(error_message)) elif exit_code_description_: error_message_ = error_message_prefix_ return error_message_
def _error_message(error_message, error_code): error_message_ = None exit_code_description_ = requests_fortified_error_desc( error_code).rstrip('\.') error_message_prefix_ = "{0}: {1}".format(error_code, exit_code_description_) error_message = safe_str(error_message).strip() if error_message: error_message_ = "%s: '%s'" % (error_message_prefix_, six.text_type(error_message)) elif exit_code_description_: error_message_ = error_message_prefix_ return error_message_
def tune_v2_request_retry_func(self, response): """Request Retry Function Args: response: Returns: Boolean """ try: response_json = response.json() except Exception as ex: # No JSON response available. raise TuneReportingError( error_message='Invalid JSON response: {}'.format(response.text), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_JSON_DECODING ) self.logger.debug("TMC API V2: Check for Retry: Start", extra=response_json) tune_v2_status_code = None tune_v2_errors_messages = "" if 'status_code' in response_json: tune_v2_status_code = response_json['status_code'] if 'errors' in response_json: tune_v2_errors = response_json['errors'] if isinstance(tune_v2_errors, dict) \ and 'message' in tune_v2_errors: tune_v2_errors_messages += tune_v2_errors['message'] elif isinstance(tune_v2_errors, list): for tune_v2_error in tune_v2_errors: if isinstance(tune_v2_error, dict) \ and 'message' in tune_v2_error: tune_v2_errors_messages += tune_v2_error['message'] tune_v2_status_type = get_http_status_type(tune_v2_status_code) response_extra = { 'status_code': tune_v2_status_code, 'status_type': tune_v2_status_type, } if tune_v2_errors_messages: response_extra.update({'error_messages': safe_str(tune_v2_errors_messages)}) self.logger.debug("TMC API Base: Check for Retry: Response", extra=response_extra) if tune_v2_status_code == 200: self.logger.debug("TMC API Base: Check for Retry: Success", extra=response_extra) return False if tune_v2_status_code in [401, 403]: self.logger.error("TMC API: Request: Error", extra={'status_code': tune_v2_status_code}) raise TuneRequestError( error_message=tune_v2_errors_messages, error_code=tune_v2_status_code, ) if tune_v2_status_code in [404, 500]: if "Api key was not found." in tune_v2_errors_messages: self.logger.error("TMC API: Request: Error", extra={'tune_v2_status_code': tune_v2_status_code}) raise TuneRequestError( error_message=tune_v2_errors_messages, error_code=tune_v2_status_code, ) self.logger.warning("TMC API Base: Check for Retry: Retry Candidate", extra=response_extra) return True self.logger.warning("TMC API Base: Check for Retry: No Retry", extra=response_extra) return False
def _request_data( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_headers=None, request_auth=None, request_cert=None, cookie_payload=None, request_label=None, timeout=60, build_request_curl=True, allow_redirects=True, verify=True, stream=False ): """Request Data from requests. Args: request_method: request_method for the new :class:`Request` object. logger: logging instance request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. request_cert: (optional) Cert tuple to enable Client side certificates. timeout: (optional) How long to wait for the server to send data before giving up. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. Returns: requests.Response """ if request_label is None: request_label = 'Request Data' if not request_method: raise RequestsFortifiedValueError(error_message="Parameter 'request_method' not defined") if not request_url: raise RequestsFortifiedValueError(error_message="Parameter 'request_url' not defined") self.built_request_curl = None self.logger.debug( "{0}: Session: Details".format(request_label), extra={ 'cookie_payload': self.requests_session_client.session.cookies.get_dict(), 'request_label': request_label } ) response = None headers = None if request_headers: headers = request_headers request_method = request_method.upper() if request_data and isinstance(request_data, str): if len(request_data) <= 20: request_data_extra = request_data else: request_data_extra = request_data[:20] + ' ...' else: request_data_extra = safe_str(request_data) request_extra = { 'request_method': request_method, 'request_url': request_url, 'timeout': timeout, 'request_params': safe_dict(request_params), 'request_data': request_data_extra, 'request_headers': safe_dict(headers), 'request_label': request_label } self.logger.debug("{0}: Details".format(request_label), extra=request_extra) self.built_request_curl = None kwargs = {} if headers: kwargs.update({'headers': headers}) if request_auth: kwargs.update({'auth': request_auth}) if request_cert: kwargs.update({'cert': request_cert}) if timeout and isinstance(timeout, int): kwargs.update({'timeout': timeout}) if allow_redirects: kwargs.update({'allow_redirects': allow_redirects}) if stream: kwargs.update({'stream': stream}) if cookie_payload: kwargs.update({'cookies': cookie_payload}) kwargs.update({'verify': verify}) try: if build_request_curl: # In case no authentication information has been provided, # use session's cookies information, if exists if not request_auth and self.session.cookies and len(self.session.cookies) > 0: request_auth = self.session.cookies self.built_request_curl = command_line_request_curl( request_method=request_method, request_url=request_url, request_headers=headers, request_params=request_params, request_data=request_data, request_json=request_json, request_auth=request_auth, request_timeout=timeout, request_allow_redirects=allow_redirects ) self.logger.note( "{0}: Curl".format(request_label), extra={ 'request_method': request_method, 'request_label': request_label, 'request_curl': self.built_request_curl } ) if hasattr(response, 'url'): self.logger.debug( "{0}: {1}".format(request_label, request_method), extra={'response_url': response.url} ) if request_params: kwargs.update({'params': request_params}) if request_data: kwargs.update({'data': request_data}) if request_json: kwargs.update({'json': request_json}) if headers: kwargs.update({'headers': headers}) kwargs.update({'request_method': request_method, 'request_url': request_url}) response = self.requests_session_client.request(**kwargs) except Exception as ex: self.logger.error( "{0}: Request Base: Error".format(request_label), extra={ 'request_label': request_label, 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) } ) raise if response is None: self.logger.error( "{0}: Response: Failed".format(request_label), extra={ 'request_curl': self.built_request_curl }, ) raise RequestsFortifiedModuleError( error_message="{0}: Response: Failed".format(request_label), error_code=RequestsFortifiedErrorCodes.REQ_ERR_UNEXPECTED_VALUE, error_request_curl=self.built_request_curl ) http_status_code = response.status_code response_headers = json.loads(json.dumps(dict(response.headers))) http_status_type = \ get_http_status_type(http_status_code) http_status_desc = \ get_http_status_desc(http_status_code) response_extra = { 'http_status_code': http_status_code, 'http_status_type': http_status_type, 'http_status_desc': http_status_desc, 'response_headers': safe_dict(response_headers), } self.logger.debug( "{0}: Response: Details".format(request_label), extra=response_extra, ) http_status_successful = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL ) http_status_redirection = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION ) if http_status_successful or http_status_redirection: if hasattr(response, 'url') and \ response.url and \ len(response.url) > 0: response_extra.update({'response_url': response.url}) self.logger.debug( "{0}: Cookie Payload".format(request_label), extra={ 'cookie_payload': self.requests_session_client.session.cookies.get_dict(), 'request_label': request_label } ) assert response return response else: response_extra.update({'error_request_curl': self.built_request_curl}) self.logger.error("{0}: Response: Failed".format(request_label), extra=response_extra) json_response_error = \ build_response_error_details( response=response, request_label=request_label, request_url=request_url ) extra_error = copy.deepcopy(json_response_error) if self.logger_level == logging.INFO: error_response_details = \ extra_error.get('response_details', None) if error_response_details and \ isinstance(error_response_details, str) and \ len(error_response_details) > 100: extra_error['response_details'] = error_response_details[:100] + ' ...' if self.built_request_curl and \ 'error_request_curl' not in extra_error: extra_error.update({'error_request_curl': self.built_request_curl}) self.logger.error("{0}: Error: Response: Details".format(request_label), extra=extra_error) kwargs = { 'error_status': json_response_error.get("response_status", None), 'error_reason': json_response_error.get("response_reason", None), 'error_details': json_response_error.get("response_details", None), 'error_request_curl': self.built_request_curl } if http_status_code in [ HttpStatusCode.BAD_REQUEST, HttpStatusCode.UNAUTHORIZED, HttpStatusCode.FORBIDDEN, HttpStatusCode.NOT_FOUND, HttpStatusCode.METHOD_NOT_ALLOWED, HttpStatusCode.NOT_ACCEPTABLE, HttpStatusCode.REQUEST_TIMEOUT, HttpStatusCode.CONFLICT, HttpStatusCode.GONE, HttpStatusCode.UNPROCESSABLE_ENTITY, HttpStatusCode.TOO_MANY_REQUESTS, ]: kwargs.update({'error_code': http_status_code}) raise RequestsFortifiedClientError(**kwargs) if http_status_code in [ HttpStatusCode.INTERNAL_SERVER_ERROR, HttpStatusCode.NOT_IMPLEMENTED, HttpStatusCode.BAD_GATEWAY, HttpStatusCode.SERVICE_UNAVAILABLE, HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED, ]: kwargs.update({'error_code': http_status_code}) raise RequestsFortifiedServiceError(**kwargs) kwargs.update({'error_code': json_response_error['response_status_code']}) extra_unhandled = copy.deepcopy(kwargs) extra_unhandled.update({'http_status_code': http_status_code}) self.logger.error("{0}: Error: Unhandled".format(request_label), extra=extra_unhandled) raise RequestsFortifiedModuleError(**kwargs)
def main(tmc_api_key): TIMEZONE_COLLECT = "America/New_York" tune_v2_advertiser_stats_actuals = \ TuneV2AdvertiserStatsActuals( timezone=TIMEZONE_COLLECT, logger_level=logging.INFO, logger_format=TuneLoggingFormat.JSON ) dw_file_path = "data.{}".format(TuneV2AdvertiserStatsFormats.JSON) if os.path.exists(dw_file_path): os.remove(dw_file_path) tz = pytz.timezone(TIMEZONE_COLLECT) yesterday = datetime.now(tz).date() - timedelta(days=1) str_yesterday = str(yesterday) request_params = { 'format': TuneV2AdvertiserStatsFormats.CSV, 'fields': ("ad_clicks," "ad_clicks_unique," "ad_impressions," "ad_impressions_unique," "ad_network_id," "advertiser_id," "country.code," "date_hour," "events," "installs," "is_reengagement," "payouts," "publisher_id," "publisher_sub_ad.ref," "publisher_sub_adgroup.ref," "publisher_sub_campaign.ref," "publisher_sub_publisher.ref," "publisher_sub_site.ref," "site_id"), 'group': ("country_id," "is_reengagement," "publisher_id," "publisher_sub_ad_id," "publisher_sub_adgroup_id," "publisher_sub_campaign_id," "publisher_sub_publisher_id," "publisher_sub_site_id," "site_id"), 'timezone': "America/Los_Angeles" } try: tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key) response = tune_v2_advertiser_stats_actuals.stream( auth_value=tmc_api_key, auth_type=TuneV2AuthenticationTypes.API_KEY, auth_type_use=TuneV2AuthenticationTypes.API_KEY, start_date=str_yesterday, end_date=str_yesterday, request_params=request_params, request_retry={ 'delay': 15, 'timeout': 30, 'tries': 5 }) line_count = 0 csv_keys_list = None json_keys_dict = { "publisher_sub_campaign.ref": "sub_campaign_ref", "publisher_sub_ad.ref": "sub_ad_ref", "publisher_sub_adgroup.ref": "sub_adgroup_ref", "publisher_sub_publisher.ref": "sub_publisher_ref", "publisher_sub_site.ref": "sub_site_ref", "publisher_sub_placement.ref": "sub_placement_ref", "publisher_sub_campaign.name": "sub_campaign_name", "publisher_sub_ad.name": "sub_ad_name", "publisher_sub_adgroup.name": "sub_adgroup_name", "publisher_sub_publisher.name": "sub_publisher_name", "publisher_sub_site.name": "sub_site_name", "publisher_sub_placement.name": "sub_placement_name", "publisher_sub_campaign_id": "sub_campaign_id", "publisher_sub_ad_id": "sub_ad_id", "publisher_sub_adgroup_id": "sub_adgroup_id", "publisher_sub_publisher_id": "sub_publisher_id", "publisher_sub_site_id": "sub_site_id", "publisher_sub_placement_id": "publisher_sub_placement_id", "country.code": "country_code", "ad_impressions": "received_impressions_gross", "ad_impressions_unique": "received_impressions_unique", "ad_clicks": "received_clicks_gross", "ad_clicks_unique": "received_clicks_unique", "events": "received_engagements", "installs": "received_installs", "payouts": "cost" } json_types_dict = { "client_id": int, "partner_id": int, "vendor_id": int, "date": str, "hour": int, "timezone": str, "granularity": str, "site_ref_id": str, "site_ref_type": str, "partner_ref_id": int, "partner_ref_type": str, "partner_vendor_ref_id": int, "partner_vendor_ref_type": str, "sub_campaign_type": str, "sub_campaign_ref": str, "sub_ad_ref": str, "sub_adgroup_ref": str, "sub_publisher_ref": str, "sub_site_ref": str, "sub_placement_ref": str, "sub_campaign_name": str, "sub_ad_name": str, "sub_adgroup_name": str, "sub_publisher_name": str, "sub_site_name": str, "sub_placement_name": str, "sub_campaign_name": str, "sub_ad_name": str, "sub_adgroup_name": str, "sub_publisher_name": str, "sub_site_name": str, "sub_placement_name": str, "sub_campaign_id": int, "sub_ad_id": int, "sub_adgroup_id": int, "sub_publisher_id": int, "sub_site_id": int, "publisher_sub_placement_id": int, "country_code": str, "received_impressions_gross": int, "received_impressions_unique": int, "received_clicks_gross": int, "received_clicks_unique": int, "received_installs": int, "received_engagements": int, "received_conversions": int, "cost": float, "cost_currency": str, "site_id": int, "publisher_id": int, "advertiser_id": int, "ad_network_id": int, "ad_impressions": int, } client_id = 0 partner_id = 0 vendor_id = 0 timezone = "TBD" granularity = "TBD" config_extra = { "client_id": client_id, "partner_id": partner_id, "vendor_id": vendor_id, "timezone": timezone, "granularity": granularity, "cost_currency": "USD", "received_conversions": 0, "site_ref_type": "tmc", "partner_ref_type": "tmc", "partner_vendor_ref_type": "tmc" } with open(file=dw_file_path, mode='w') as dw_file_w: for bytes_line in response.iter_lines(chunk_size=4096): if bytes_line: # filter out keep-alive new chunks line_count += 1 str_line = bytes_line.decode("utf-8") if line_count == 1: csv_keys_list = str_line.split(',') for index, csv_key in enumerate(csv_keys_list): if csv_key in json_keys_dict: csv_keys_list[index] = json_keys_dict[csv_key] continue elif line_count > 2: dw_file_w.write('\n') csv_values_list = str_line.split(',') json__dict = {} is_reengagement = 0 received_installs = 0 received_engagements = 0 for csv_key, csv_value in zip(csv_keys_list, csv_values_list): csv_value_strip = csv_value.strip('"') if csv_key == "date_hour": parts_date_time = csv_value_strip.split(" ") rdate_yyyy_mm_dd = parts_date_time[0] parts_time = parts_date_time[1].split(":") rhour = safe_int(parts_time[0]) json__dict.update({"date": rdate_yyyy_mm_dd}) json__dict.update({"hour": rhour}) elif csv_key == "is_reengagement": is_reengagement = safe_int(csv_value_strip) elif csv_key == "received_installs": received_installs = safe_int(csv_value_strip) json__dict.update( {'received_installs': received_installs}) elif csv_key == "received_engagements": received_engagements = safe_int(csv_value_strip) json__dict.update( {'received_engagements': received_engagements}) else: if csv_key in json_types_dict: if json_types_dict[csv_key] == str: csv_value_typed = safe_str(csv_value_strip) elif json_types_dict[csv_key] == int: csv_value_typed = safe_int(csv_value_strip) elif json_types_dict[csv_key] == float: csv_value_typed = safe_float( csv_value_strip) else: csv_value_typed = safe_str(csv_value_strip) else: csv_value_typed = safe_str(csv_value_strip) json__dict.update({csv_key: csv_value_typed}) if is_reengagement == 1: engagements = received_engagements else: engagements = 0 if engagements > 0 and received_installs > 0: sub_campaign_type = "acquisition_engagement" elif received_installs > 0: sub_campaign_type = "acquisition" elif engagements > 0: sub_campaign_type = "engagement" else: sub_campaign_type = "" json__dict.update({'sub_campaign_type': sub_campaign_type}) json__dict.update(config_extra) json_str = json.dumps(json__dict) dw_file_w.write(json_str) dw_file_w.flush() statinfo = os.stat(dw_file_path) extra = { 'response_status_code': response.status_code, 'response_headers': response.headers, 'dw_file_path': dw_file_path, 'dw_file_size': convert_size(statinfo.st_size), 'line_count': line_count, 'csv_header_list': csv_keys_list } pprint(extra) except TuneRequestBaseError as tmc_req_ex: print_traceback(tmc_req_ex) pprint(tmc_req_ex.to_dict()) print(str(tmc_req_ex)) except TuneReportingError as tmc_rep_ex: pprint(tmc_rep_ex.to_dict()) print(str(tmc_rep_ex)) except Exception as ex: print_traceback(ex) print(get_exception_message(ex))
#!/usr/bin/env python3 # -*- coding: utf-8 -*- # @copyright 2017 TUNE, Inc. (http://www.tune.com) # @namespace smart-cast from pprintpp import pprint from safe_cast import ( safe_int, safe_float, safe_str, ) pprint(safe_int(4)) pprint(safe_float(4)) pprint(safe_str(4)) pprint(safe_int('4')) pprint(safe_float('4')) pprint(safe_str('4')) pprint(safe_int(4.0)) pprint(safe_float(4.0)) pprint(safe_str(4.0)) pprint(safe_int('4.0')) pprint(safe_float('4.0')) pprint(safe_str('4.0')) pprint(safe_int('1.0')) pprint(safe_int('1'))
def validate_response( response, request_curl, request_label=None, ): """Validate response Args: response: request_curl: request_label: Returns: """ response_extra = {} if request_label is None: request_label = 'Validate Response' if not response: error_message = f'{request_label}: Failed: None' log.error(error_message, extra=response_extra) raise TuneRequestModuleError( error_message=error_message, error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE) log.debug(f'{request_label}: Defined', extra=response_extra) response_extra.update({'http_status_code': response.status_code}) # Not using hasattr on purpose # Assuming positive approach will give us speedup since when attribute exists # hasattr takes double the time. # Anyway we use text attribute here to logging purpose only try: response_extra.update({'response_text_length': len(response.text)}) except AttributeError: pass if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_extra.update( {'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_extra.update({ 'Content-Length': bytes_to_human(response_headers_content_length) }) if 'Content-Encoding' in response.headers: response_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_extra.update( {'Content-Encoding': response_content_encoding}) if 'Transfer-Encoding' in response.headers: response_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_extra.update( {'Transfer-Encoding': response_transfer_encoding}) if not is_http_status_successful(http_status_code=response.status_code): error_message = f'{request_label}: Failed' log.error(error_message, extra=response_extra) raise TuneRequestModuleError( error_message=error_message, error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE) log.debug(f'{request_label}: Success', extra=response_extra)
def build_response_error_details(request_label, request_url, response): """Build gather status of Requests' response. Args: request_label: request_url: response: Returns: """ http_status_code = \ response.status_code http_status_type = \ get_http_status_type(http_status_code) http_status_desc = \ get_http_status_desc(http_status_code) response_status = "%s: %s: %s" % (http_status_code, http_status_type, http_status_desc) response_error_details = { 'request_url': request_url, 'request_label': request_label, 'response_status': response_status, 'response_status_code': http_status_code, 'response_status_type': http_status_type, 'response_status_desc': http_status_desc } if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_error_details.update({'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers and \ response.headers['Content-Length']: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_error_details.update({'Content-Length': response_headers_content_length}) if 'Transfer-Encoding' in response.headers and \ response.headers['Transfer-Encoding']: response_headers_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_error_details.update({'Transfer-Encoding': response_headers_transfer_encoding}) if 'Content-Encoding' in response.headers and \ response.headers['Content-Encoding']: response_headers_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_error_details.update({'Content-Encoding': response_headers_content_encoding}) if hasattr(response, "reason") and response.reason: response_error_details.update({'response_reason': response.reason}) response_details = None response_details_source = None try: response_details = response.json() response_details_source = 'json' except Exception: if hasattr(response, 'text') and \ response.text and \ len(response.text) > 0: response_details = response.text response_details_source = 'text' if response_details.startswith('<html'): response_details_source = 'html' soup_html = bs4.BeautifulSoup(response_details, "html.parser") # kill all script and style elements for script in soup_html(["script", "style"]): script.extract() # rip it out text_html = soup_html.get_text() lines_html = [line for line in text_html.split('\n') if line.strip() != ''] lines_html = [line.strip(' ') for line in lines_html] response_details = lines_html elif response_details.startswith('<?xml'): response_details_source = 'xml' response_details = json.dumps(xmltodict.parse(response_details)) response_error_details.update({ 'response_details': response_details, 'response_details_source': response_details_source }) # pprint(response_error_details) return response_error_details
def test_safe_str(): # test type: assert isinstance(safe_str(0), str) assert isinstance(safe_str(0.0), str) assert isinstance(safe_str('0'), str) # test str cast: assert safe_str('') == '' assert safe_str('Hello Jeff') == 'Hello Jeff' assert safe_str('@%#$%^&*') == '@%#$%^&*' assert safe_str(None) == '' # test numeric cast: assert safe_str(0) == '0' assert safe_str(10) == '10' assert safe_str(-1) == '-1' assert safe_str(10.52) == '10.52' assert safe_str(-1.32) == '-1.32'
#!/usr/bin/env python3 # -*- coding: utf-8 -*- # @copyright 2018 TUNE, Inc. (http://www.tune.com) # @namespace smart-cast from safe_cast import (safe_int, safe_float, safe_str, safe_dict, safe_cost, safe_fraction) print("\n") print('==================================') print('safe_str()') print('==================================') print(safe_str(.1)) print(safe_str(.12)) print(safe_str(.123)) print(safe_str(.1234)) print(safe_str(.12345)) print(safe_str(4)) print(safe_str('4')) print(safe_str('4.0')) print(safe_str(4.0)) print("\n") print('==================================') print('safe_int()') print('==================================') print(safe_int(.1)) print(safe_int(.12)) print(safe_int(.123)) print(safe_int(.1234)) print(safe_int(.12345))
def _request_data( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_headers=None, request_auth=None, cookie_payload=None, request_label=None, timeout=60, build_request_curl=True, allow_redirects=True, verify=True, stream=False ): """Request Data from requests. Args: request_method: request_method for the new :class:`Request` object. logger: logging instance request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. timeout: (optional) How long to wait for the server to send data before giving up. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. Returns: requests.Response """ if request_label is None: request_label = 'Request Data' if not request_method: raise TuneRequestValueError(error_message="Parameter 'request_method' not defined") if not request_url: raise TuneRequestValueError(error_message="Parameter 'request_url' not defined") self.built_request_curl = None self.logger.debug( '{}: Session: Details'.format(request_label), extra={'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label} ) response = None headers = None if request_headers: headers = request_headers request_method = request_method.upper() if request_data and isinstance(request_data, str): if len(request_data) <= 20: request_data_extra = request_data else: request_data_extra = request_data[:20] + ' ...' else: request_data_extra = safe_str(request_data) request_extra = { 'request_method': request_method, 'request_url': request_url, 'timeout': timeout, 'request_params': safe_dict(request_params), 'request_data': request_data_extra, 'request_headers': safe_dict(headers), 'request_label': request_label } self.logger.debug('{}: Details'.format(request_label), extra=request_extra) self.built_request_curl = None kwargs = {} if headers: kwargs.update({'headers': headers}) if request_auth: kwargs.update({'auth': request_auth}) if timeout and isinstance(timeout, int): kwargs.update({'timeout': timeout}) if allow_redirects: kwargs.update({'allow_redirects': allow_redirects}) if stream: kwargs.update({'stream': stream}) if cookie_payload: kwargs.update({'cookies': cookie_payload}) kwargs.update({'verify': verify}) try: if build_request_curl: # In case no authentication information has been provided, # use session's cookies information, if exists if not request_auth and self.session.cookies and len(self.session.cookies) > 0: request_auth = self.session.cookies self.built_request_curl = command_line_request_curl( request_method=request_method, request_url=request_url, request_headers=headers, request_params=request_params, request_data=request_data, request_json=request_json, request_auth=request_auth, request_timeout=timeout, request_allow_redirects=allow_redirects ) self.logger.debug( '{}: Curl'.format(request_label), extra={ 'request_method': request_method, 'request_label': request_label, 'request_curl': self.built_request_curl } ) if hasattr(response, 'url'): self.logger.debug('{}: {}'.format(request_label, request_method), extra={'response_url': response.url}) if request_params: kwargs.update({'params': request_params}) if request_data: kwargs.update({'data': request_data}) if request_json: kwargs.update({'json': request_json}) if headers: kwargs.update({'headers': headers}) kwargs.update({'request_method': request_method, 'request_url': request_url}) response = self.tune_request.request(**kwargs) except Exception as ex: self.logger.error( '{}: Request Base: Error'.format(request_label), extra={ 'request_label': request_label, 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) } ) raise if response is None: self.logger.error( '{}: Response: Failed'.format(request_label), extra={'request_curl': self.built_request_curl}, ) raise TuneRequestModuleError( error_message='{}: Response: Failed'.format(request_label), error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE, error_request_curl=self.built_request_curl ) http_status_code = response.status_code response_headers = json.loads(json.dumps(dict(response.headers))) http_status_type = \ http_status_code_to_type(http_status_code) http_status_desc = \ http_status_code_to_desc(http_status_code) response_extra = { 'http_status_code': http_status_code, 'http_status_type': http_status_type, 'http_status_desc': http_status_desc, 'response_headers': safe_dict(response_headers), } self.logger.debug( '{}: Response: Details'.format(request_label), extra=response_extra, ) http_status_successful = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL ) http_status_redirection = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION ) if http_status_successful or http_status_redirection: if hasattr(response, 'url') and \ response.url and \ len(response.url) > 0: response_extra.update({'response_url': response.url}) self.logger.debug( '{}: Cookie Payload'.format(request_label), extra={'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label} ) assert response return response else: response_extra.update({'error_request_curl': self.built_request_curl}) self.logger.error('{}: Response: Failed'.format(request_label), extra=response_extra) json_response_error = \ build_response_error_details( response=response, request_label=request_label, request_url=request_url ) extra_error = copy.deepcopy(json_response_error) if self.logger_level == logging.INFO: error_response_details = \ extra_error.get('response_details', None) if error_response_details and \ isinstance(error_response_details, str) and \ len(error_response_details) > 100: extra_error['response_details'] = error_response_details[:100] + ' ...' if self.built_request_curl and \ 'error_request_curl' not in extra_error: extra_error.update({'error_request_curl': self.built_request_curl}) self.logger.error('{}: Error: Response: Details'.format(request_label), extra=extra_error) kwargs = { 'error_status': json_response_error.get("response_status", None), 'error_reason': json_response_error.get("response_reason", None), 'error_details': json_response_error.get("response_details", None), 'error_request_curl': self.built_request_curl } if http_status_code in [ HttpStatusCode.BAD_REQUEST, HttpStatusCode.UNAUTHORIZED, HttpStatusCode.FORBIDDEN, HttpStatusCode.NOT_FOUND, HttpStatusCode.METHOD_NOT_ALLOWED, HttpStatusCode.NOT_ACCEPTABLE, HttpStatusCode.REQUEST_TIMEOUT, HttpStatusCode.CONFLICT, HttpStatusCode.GONE, HttpStatusCode.UNPROCESSABLE_ENTITY, HttpStatusCode.TOO_MANY_REQUESTS, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestClientError(**kwargs) if http_status_code in [ HttpStatusCode.INTERNAL_SERVER_ERROR, HttpStatusCode.NOT_IMPLEMENTED, HttpStatusCode.BAD_GATEWAY, HttpStatusCode.SERVICE_UNAVAILABLE, HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestServiceError(**kwargs) kwargs.update({'error_code': json_response_error['response_status_code']}) extra_unhandled = copy.deepcopy(kwargs) extra_unhandled.update({'http_status_code': http_status_code}) self.logger.error('{}: Error: Unhandled'.format(request_label), extra=extra_unhandled) raise TuneRequestModuleError(**kwargs)
def validate_response( response, request_curl, request_label=None, ): """Validate response Args: response: request_curl: request_label: Returns: """ response_extra = {} if request_label is None: request_label = 'Validate Response' if not response: error_message = "{0}: Failed: None".format(request_label) log.error(error_message, extra=response_extra) raise RequestsFortifiedModuleError( error_message=error_message, error_request_curl=request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_SOFTWARE ) log.debug("{0}: Defined".format(request_label), extra=response_extra) response_extra.update({'http_status_code': response.status_code}) # Not using hasattr on purpose # Assuming positive approach will give us speedup since when attribute exists # hasattr takes double the time. # Anyway we use text attribute here to logging purpose only try: response_extra.update({'response_text_length': len(response.text)}) except AttributeError: pass if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_extra.update({'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_extra.update({'Content-Length': bytes_to_human(response_headers_content_length)}) if 'Content-Encoding' in response.headers: response_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_extra.update({'Content-Encoding': response_content_encoding}) if 'Transfer-Encoding' in response.headers: response_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_extra.update({'Transfer-Encoding': response_transfer_encoding}) if not is_http_status_successful(http_status_code=response.status_code): error_message = "{0}: Failed".format(request_label) log.error(error_message, extra=response_extra) raise RequestsFortifiedModuleError( error_message=error_message, error_request_curl=request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_SOFTWARE ) log.debug("{0}: Success".format(request_label), extra=response_extra)
def build_response_error_details(request_label, request_url, response): """Build gather status of Requests' response. Args: request_label: request_url: response: Returns: """ http_status_code = \ response.status_code http_status_type = \ get_http_status_type(http_status_code) http_status_desc = \ get_http_status_desc(http_status_code) response_status = f"{http_status_code}: {http_status_type}: {http_status_desc}" response_error_details = { 'request_url': request_url, 'request_label': request_label, 'response_status': response_status, 'response_status_code': http_status_code, 'response_status_type': http_status_type, 'response_status_desc': http_status_desc } if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_error_details.update( {'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers and \ response.headers['Content-Length']: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_error_details.update( {'Content-Length': response_headers_content_length}) if 'Transfer-Encoding' in response.headers and \ response.headers['Transfer-Encoding']: response_headers_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_error_details.update( {'Transfer-Encoding': response_headers_transfer_encoding}) if 'Content-Encoding' in response.headers and \ response.headers['Content-Encoding']: response_headers_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_error_details.update( {'Content-Encoding': response_headers_content_encoding}) if hasattr(response, "reason") and response.reason: response_error_details.update({'response_reason': response.reason}) response_details = None response_details_source = None try: response_details = response.json() response_details_source = 'json' except Exception: if hasattr(response, 'text') and \ response.text and \ len(response.text) > 0: response_details = response.text response_details_source = 'text' if response_details.startswith('<html'): response_details_source = 'html' soup_html = bs4.BeautifulSoup(response_details, "html.parser") # kill all script and style elements for script in soup_html(["script", "style"]): script.extract() # rip it out text_html = soup_html.get_text() lines_html = [ line for line in text_html.split('\n') if line.strip() != '' ] lines_html = [line.strip(' ') for line in lines_html] response_details = lines_html elif response_details.startswith('<?xml'): response_details_source = 'xml' response_details = json.dumps( xmltodict.parse(response_details)) response_error_details.update({ 'response_details': response_details, 'response_details_source': response_details_source }) # pprint(response_error_details) return response_error_details