def transform(stub, request, **kwargs): function = kwargs['function'] stage = kwargs.get('stage') url_args = kwargs['url_args'] trace = kwargs['trace'] hooks = kwargs.pop('hooks') module_system_date = kwargs.get('module_system_date') try: transformer = hooks.make_transformer(stub) if module_system_date and isinstance(module_system_date, basestring): module_system_date = as_date(module_system_date) system_date = kwargs.get('system_date') if isinstance(system_date, basestring): system_date = as_date('system_date') unsafevars = ('request', 'function', 'cache', 'stage', 'module_system_date', 'system_date', 'trace') url_args.update(x for x in stub.args().iteritems() if x[0] not in ('session',)) for var in unsafevars: url_args.pop(var, None) return transformer.transform(request, module_system_date=module_system_date, system_date=system_date, function=function, cache=kwargs['cache'], stage=stage, trace=trace, **url_args) except Exception, e: _, t, v, tbinfo = compact_traceback() msg = u'error={0}, traceback is: ({1}: {2} {3})'.format(e, t, v, tbinfo) log.error(msg) raise TransformError(code=400, title='Error transforming during ' '{0} stage:{1}'.format(function, stage), traceback=msg)
def transform(stub, request, **kwargs): function = kwargs['function'] stage = kwargs.get('stage') url_args = kwargs['url_args'] trace = kwargs['trace'] hooks = kwargs.pop('hooks') module_system_date = kwargs.get('module_system_date') try: transformer = hooks.make_transformer(stub) if module_system_date and isinstance(module_system_date, basestring): module_system_date = as_date(module_system_date) system_date = kwargs.get('system_date') if isinstance(system_date, basestring): system_date = as_date('system_date') unsafevars = ('request', 'function', 'cache', 'stage', 'module_system_date', 'system_date', 'trace') url_args.update(x for x in stub.args().iteritems() if x[0] not in ('session', )) for var in unsafevars: url_args.pop(var, None) return transformer.transform(request, module_system_date=module_system_date, system_date=system_date, function=function, cache=kwargs['cache'], stage=stage, trace=trace, **url_args) except Exception, e: _, t, v, tbinfo = compact_traceback() msg = u'error={0}, traceback is: ({1}: {2} {3})'.format( e, t, v, tbinfo) log.error(msg) raise TransformError(code=400, title='Error transforming during ' '{0} stage:{1}'.format(function, stage), traceback=msg)
def eval_text(self, templ, request, **kwargs): stub = kwargs['stub'] kwargs['recorded'] = stub.recorded() recorded = kwargs.get('recorded') if recorded: kwargs['recorded'] = as_date(recorded) return run_template(templ, request=request, request_text=request.request_body(), # legacy # utility functions roll_date=roll_date, today=today_str, as_date=as_date, parse_xml=parse_xml, **kwargs)
def eval_text(self, templ, request, **kwargs): request_text = request.request_body() stub = kwargs['stub'] kwargs['recorded'] = stub.recorded() recorded = kwargs.get('recorded') if recorded: kwargs['recorded'] = as_date(recorded) return run_template( templ, request_text=request_text, # utility functions roll_date=roll_date, today=today_str, as_date=as_date, parse_xml=parse_xml, **kwargs)
def purge_stubs(): # importing helper handler from testing deps from stubo.testing import DummyRequestHandler parser = ArgumentParser( description="Purge stubs older than given expiry date." ) parser.add_argument('-l', '--list', action='store_const', const=True, dest='list_only', help="Just list the stubs to delete.") parser.add_argument('-e', '--expiry', default=14, dest='expiry', help="expiry is number of days from now (default is 14).") parser.add_argument('--host', default='all', dest='host', help="specify the host uri to use (defaults to all)") parser.add_argument('-c', '--config', dest='config', help='Path to configuration file (defaults to $CWD/etc/dev.ini)', metavar='FILE') args = parser.parse_args() list_only = args.list_only or False expiry_days = args.expiry expiry = datetime.today().date() - timedelta(int(expiry_days)) host = args.host config = args.config or get_default_config() logging.config.fileConfig(config) settings = read_config(config) dbenv = default_env.copy() dbenv.update((k[6:], coerce_mongo_param(k[6:], v)) for k, v in \ settings.iteritems() if k.startswith('mongo.')) log.debug('mongo params: {0}'.format(dbenv)) log.info('purge stubs whereby all sessions in the scenario were last used before {0}'.format(expiry)) db_conn = init_mongo(dbenv).connection slave, master = start_redis(settings) response = list_scenarios(host) if 'error' in response: print response['error'] sys.exit(-1) handler = DummyRequestHandler() session_handler = DummyRequestHandler() for scenario_key in response['data']['scenarios']: log.debug("*** scenario '{0}' ***".format(scenario_key)) hostname, scenario = scenario_key.split(':') if host != 'all' and host != hostname: continue handler.host = hostname handler.request.host = '{0}:8001'.format(hostname) session_handler.host = hostname session_handler.request.host = '{0}:8001'.format(hostname) handler.request.arguments['scenario'] = [scenario] status = get_status(handler) if 'error' in status: log.warn('get_status error: {0}'.format(status['error'])) else: scenario_last_used = [] sessions = status['data']['sessions'] for session in zip(*sessions)[0]: log.debug("*** -> session '{0}' ***".format(session)) session_handler.request.arguments['session'] = [session] session_status = get_status(session_handler) if 'error' in session_status: log.warn('get_status error: {0}'.format(status['error'])) else: last_used = session_status['data']['session'].get('last_used', '-') if last_used != '-': scenario_last_used.append(as_date(last_used[0:10])) if scenario_last_used and (max(scenario_last_used) < expiry): log.info("sessions in scenario '{0}' were last used '{1}' which" " is before expiry date '{2}'".format(scenario_key, max(scenario_last_used), expiry)) if not list_only: response = delete_stubs(handler, scenario_name=scenario, force=True) if 'error' in response: log.error('delete stubs error: {0}'.format(response['error'])) else: log.info('deleted stubs: {0}'.format(response['data']))
def get_response(handler, session_name): request = handler.request stubo_request = StuboRequest(request) request_body = stubo_request.request_body() if not request_body: # TODO: not an error if they match on other attrs raise exception_response(400, title='No text in body') cache = Cache(get_hostname(request)) if cache.blacklisted(): raise exception_response( 400, title="Sorry the host URL '{0}' has been " "blacklisted. Please contact Stub-O-Matic support.".format( cache.host)) scenario_key = cache.find_scenario_key(session_name) scenario_name = scenario_key.partition(':')[-1] handler.track.scenario = scenario_name request_id = compute_hash(request_body) module_system_date = handler.get_argument('system_date', None) url_args = handler.track.request_params if not module_system_date: # BA LEGACY module_system_date = handler.get_argument('stubbedSystemDate', None) trace_matcher = TrackTrace(handler.track, 'matcher') user_cache = handler.settings['ext_cache'] # check cached requests cached_request = cache.get_request(scenario_name, session_name, request_id) if cached_request: response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request else: retry_count = 5 if handler.settings.get('is_cluster', False) else 1 session, retries = cache.get_session_with_delay( scenario_name, session_name, retry_count=retry_count, retry_interval=1) if retries > 0: log.warn("replication was slow for session: {0} {1}, it took {2} "\ "secs!".format(scenario_key, session_name, retries+1)) if session['status'] != 'playback': raise exception_response( 500, title='cache status != playback. session={0}'.format(session)) system_date = session['system_date'] if not system_date: raise exception_response( 500, title="slave session {0} not available for scenario {1}". format(session_name, scenario_key)) session['ext_cache'] = user_cache results = match(stubo_request, session, trace_matcher, as_date(system_date), url_args=url_args, hooks=handler.settings['hooks'], module_system_date=module_system_date) # 0 is best match hits_info, stub = results[0] log.debug(u'match result: (hits_info={0}, stub={1})'.format( hits_info, stub.payload)) hits, stub_number = hits_info if not hits: raise exception_response(400, title='E017:No matching response found') response_ids = stub.response_ids() delay_policy_name = stub.delay_policy_name() recorded = stub.recorded() module_info = stub.module() request_index_key = add_request( session, request_id, stub, system_date, stub_number, handler.settings['request_cache_limit']) if not stub.response_body(): stub.set_response_body( stub.get_response_body_from_cache(request_index_key)) if delay_policy_name: stub.load_delay_from_cache(delay_policy_name) if cached_request: stub = StubCache({}, scenario_key, session_name) stub.load_from_cache(response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key) trace_response = TrackTrace(handler.track, 'response') if module_info: trace_response.info('module used', str(module_info)) response_text = stub.response_body() if not response_text: raise exception_response( 500, title='Unable to find response in ' 'cache using session: {0}:{1}, response_ids: {2}'.format( scenario_key, session_name, response_ids)) # get latest delay policy delay_policy = stub.delay_policy() if delay_policy: delay = calculate_delay(delay_policy) if delay: msg = 'apply delay: {0} => {1}'.format(delay_policy, delay) log.debug(msg) handler.track['delay'] = delay trace_response.info(msg) trace_response.info('found response') module_system_date = as_date(module_system_date) if module_system_date \ else module_system_date stub, _ = transform(stub, stubo_request, module_system_date=module_system_date, system_date=as_date(system_date), function='get/response', cache=user_cache, hooks=handler.settings['hooks'], stage='response', trace=trace_response, url_args=url_args) transfomed_response_text = stub.response_body()[0] # Note transformed_response_text can be encoded in utf8 if response_text[0] != transfomed_response_text: trace_response.diff('response:transformed', dict(response=response_text[0]), dict(response=transfomed_response_text)) return transfomed_response_text
def get_response(handler, session_name): request = handler.request stubo_request = StuboRequest(request) cache = Cache(get_hostname(request)) if cache.blacklisted(): raise exception_response(400, title="Sorry the host URL '{0}' has been " "blacklisted. Please contact Stub-O-Matic support.".format(cache.host)) scenario_key = cache.find_scenario_key(session_name) scenario_name = scenario_key.partition(':')[-1] handler.track.scenario = scenario_name request_id = stubo_request.id() module_system_date = handler.get_argument('system_date', None) url_args = handler.track.request_params if not module_system_date: # LEGACY module_system_date = handler.get_argument('stubbedSystemDate', None) trace_matcher = TrackTrace(handler.track, 'matcher') user_cache = handler.settings['ext_cache'] # check cached requests cached_request = cache.get_request(scenario_name, session_name, request_id) if cached_request: response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request else: retry_count = 5 if handler.settings.get('is_cluster', False) else 1 session, retries = cache.get_session_with_delay(scenario_name, session_name, retry_count=retry_count, retry_interval=1) if retries > 0: log.warn("replication was slow for session: {0} {1}, it took {2} "\ "secs!".format(scenario_key, session_name, retries+1)) if session['status'] != 'playback': raise exception_response(500, title='cache status != playback. session={0}'.format(session)) system_date = session['system_date'] if not system_date: raise exception_response(500, title="slave session {0} not available for scenario {1}".format( session_name, scenario_key)) session['ext_cache'] = user_cache result = match(stubo_request, session, trace_matcher, as_date(system_date), url_args=url_args, hooks=handler.settings['hooks'], module_system_date=module_system_date) if not result[0]: raise exception_response(400, title='E017:No matching response found') _, stub_number, stub = result response_ids = stub.response_ids() delay_policy_name = stub.delay_policy_name() recorded = stub.recorded() module_info = stub.module() request_index_key = add_request(session, request_id, stub, system_date, stub_number, handler.settings['request_cache_limit']) if not stub.response_body(): _response = stub.get_response_from_cache(request_index_key) stub.set_response_body(_response['body']) if delay_policy_name: stub.load_delay_from_cache(delay_policy_name) if cached_request: stub = StubCache({}, scenario_key, session_name) stub.load_from_cache(response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key) trace_response = TrackTrace(handler.track, 'response') if module_info: trace_response.info('module used', str(module_info)) response_text = stub.response_body() if not response_text: raise exception_response(500, title='Unable to find response in ' 'cache using session: {0}:{1}, response_ids: {2}'.format( scenario_key, session_name, response_ids)) # get latest delay policy delay_policy = stub.delay_policy() if delay_policy: delay = Delay.parse_args(delay_policy) if delay: delay = delay.calculate() msg = 'apply delay: {0} => {1}'.format(delay_policy, delay) log.debug(msg) handler.track['delay'] = delay trace_response.info(msg) trace_response.info('found response') module_system_date = as_date(module_system_date) if module_system_date \ else module_system_date stub, _ = transform(stub, stubo_request, module_system_date=module_system_date, system_date=as_date(system_date), function='get/response', cache=user_cache, hooks=handler.settings['hooks'], stage='response', trace=trace_response, url_args=url_args) transfomed_response_text = stub.response_body()[0] # Note transformed_response_text can be encoded in utf8 if response_text[0] != transfomed_response_text: trace_response.diff('response:transformed', dict(response=response_text[0]), dict(response=transfomed_response_text)) if stub.response_status() != 200: handler.set_status(stub.response_status()) if stub.response_headers(): for k, v in stub.response_headers().iteritems(): handler.set_header(k, v) return transfomed_response_text
def get_response(handler, session_name): # getting request value request = handler.request stubo_request = StuboRequest(request) cache = Cache(get_hostname(request)) scenario_key = cache.find_scenario_key(session_name) scenario_name = scenario_key.partition(':')[-1] handler.track.scenario = scenario_name # request_id - computed hash request_id = stubo_request.id() module_system_date = handler.get_argument('system_date', None) url_args = handler.track.request_params if not module_system_date: # LEGACY module_system_date = handler.get_argument('stubbedSystemDate', None) trace_matcher = TrackTrace(handler.track, 'matcher') user_cache = handler.settings['ext_cache'] # check cached requests cached_request = cache.get_request(scenario_name, session_name, request_id) if cached_request: response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request else: retry_count = 5 if handler.settings.get('is_cluster', False) else 1 session, retries = cache.get_session_with_delay( scenario_name, session_name, retry_count=retry_count, retry_interval=1) if retries > 0: log.warn( "replication was slow for session: {0} {1}, it took {2} secs!". format(scenario_key, session_name, retries + 1)) if session['status'] != 'playback': raise exception_response( 500, title='cache status != playback. session={0}'.format(session)) system_date = session['system_date'] if not system_date: raise exception_response( 500, title="slave session {0} not available for scenario {1}". format(session_name, scenario_key)) session['ext_cache'] = user_cache result = match(stubo_request, session, trace_matcher, as_date(system_date), url_args=url_args, hooks=handler.settings['hooks'], module_system_date=module_system_date) if not result[0]: raise exception_response(400, title='E017:No matching response found') _, stub_number, stub = result response_ids = stub.response_ids() delay_policy_name = stub.delay_policy_name() recorded = stub.recorded() module_info = stub.module() request_index_key = add_request( session, request_id, stub, system_date, stub_number, handler.settings['request_cache_limit']) if not stub.response_body(): _response = stub.get_response_from_cache(request_index_key) stub.set_response_body(_response['body']) if delay_policy_name: stub.load_delay_from_cache(delay_policy_name) if cached_request: stub = StubCache({}, scenario_key, session_name) stub.load_from_cache(response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key) trace_response = TrackTrace(handler.track, 'response') if module_info: trace_response.info('module used', str(module_info)) response_text = stub.response_body() if not response_text: raise exception_response( 500, title='Unable to find response in cache using session: {0}:{1}, ' 'response_ids: {2}'.format(scenario_key, session_name, response_ids)) # get latest delay policy delay_policy = stub.delay_policy() if delay_policy: delay = Delay.parse_args(delay_policy) if delay: delay = delay.calculate() msg = 'apply delay: {0} => {1}'.format(delay_policy, delay) log.debug(msg) handler.track['delay'] = delay trace_response.info(msg) trace_response.info('found response') module_system_date = as_date(module_system_date) if module_system_date \ else module_system_date stub, _ = transform(stub, stubo_request, module_system_date=module_system_date, system_date=as_date(system_date), function='get/response', cache=user_cache, hooks=handler.settings['hooks'], stage='response', trace=trace_response, url_args=url_args) transfomed_response_text = stub.response_body()[0] # Note transformed_response_text can be encoded in utf8 if response_text[0] != transfomed_response_text: trace_response.diff('response:transformed', dict(response=response_text[0]), dict(response=transfomed_response_text)) if stub.response_status() != 200: handler.set_status(stub.response_status()) if stub.response_headers(): for k, v in stub.response_headers().iteritems(): handler.set_header(k, v) return transfomed_response_text
def get_response_v2(handler, full_scenario_name, session_name): # main result dictionary that will be returned result_dict = {} request = handler.request stubo_request = StuboRequest(request) cache = Cache(get_hostname(request)) scenario_name = full_scenario_name.partition(':')[-1] handler.track.scenario = scenario_name request_id = stubo_request.id() module_system_date = handler.get_argument('system_date', None) url_args = handler.track.request_params # trace_matcher = TrackTrace(handler.track, 'matcher') user_cache = handler.settings['ext_cache'] # check cached requests cached_request = cache.get_request(scenario_name, session_name, request_id) if cached_request: response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request else: retry_count = 5 if handler.settings.get('is_cluster', False) else 1 session, retries = cache.get_session_with_delay( scenario_name, session_name, retry_count=retry_count, retry_interval=1) if retries > 0: log.warn( "replication was slow for session: {0} {1}, it took {2} secs!". format(full_scenario_name, session_name, retries + 1)) if session['status'] != 'playback': result_dict[ "error"] = 'cache status != playback. session={0}'.format( session) result_dict["statusCode"] = 412 return result_dict system_date = session['system_date'] if not system_date: result_dict[ "error"] = "slave session {0} not available for scenario {1}".format( session_name, full_scenario_name) result_dict["statusCode"] = 412 return result_dict trace_matcher = TrackTrace(handler.track, 'matcher') session['ext_cache'] = user_cache result = match(stubo_request, session, trace_matcher, as_date(system_date), url_args=url_args, hooks=handler.settings['hooks'], module_system_date=module_system_date) # matching request not found if not result[0]: result_dict["error"] = "Not matching request found" result_dict["statusCode"] = 404 return result_dict _, stub_number, stub = result response_ids = stub.response_ids() delay_policy_name = stub.delay_policy_name() recorded = stub.recorded() module_info = stub.module() request_index_key = add_request( session, request_id, stub, system_date, stub_number, handler.settings['request_cache_limit']) if not stub.response_body(): _response = stub.get_response_from_cache(request_index_key) stub.set_response_body(_response['body']) if delay_policy_name: stub.load_delay_from_cache(delay_policy_name) if cached_request: stub = StubCache({}, full_scenario_name, session_name) stub.load_from_cache(response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key) trace_response = TrackTrace(handler.track, 'response') if module_info: trace_response.info('module used', str(module_info)) response_text = stub.response_body() if not response_text: result_dict[ "error"] = 'Unable to find response in cache using session: {0}:{1}, ' 'response_ids: {2}'.format(full_scenario_name, session_name, response_ids) result_dict["statusCode"] = 400 return result_dict # get latest delay policy delay_policy = stub.delay_policy() if delay_policy: delay = Delay.parse_args(delay_policy) if delay: delay = delay.calculate() msg = 'apply delay: {0} => {1}'.format(delay_policy, delay) log.debug(msg) handler.track['delay'] = delay trace_response.info(msg) trace_response.info('found response') module_system_date = as_date(module_system_date) if module_system_date \ else module_system_date stub, _ = transform(stub, stubo_request, module_system_date=module_system_date, system_date=as_date(system_date), function='get/response', cache=user_cache, hooks=handler.settings['hooks'], stage='response', trace=trace_response, url_args=url_args) transfomed_response_text = stub.response_body()[0] # Note transformed_response_text can be encoded in utf8 if response_text[0] != transfomed_response_text: trace_response.diff('response:transformed', dict(response=response_text[0]), dict(response=transfomed_response_text)) result_dict["body"] = transfomed_response_text result_dict["headers"] = stub.response_headers() result_dict["statusCode"] = stub.response_status() return result_dict