def create_kinesis_stream(stream_name, shards=1, env=None, delete=False): env = get_environment(env) # stream stream = KinesisStream(id=stream_name, num_shards=shards) conn = connect_to_service('kinesis', env=env) stream.connect(conn) if delete: run_safe(lambda: stream.destroy(), print_error=False) stream.create() stream.wait_for() return stream
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={}, endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}, region_name=None, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]): env = aws_stack.get_environment(env) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) # decide which credentials provider to use credentialsProvider = None if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and ('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)): # use special credentials provider that can assume IAM roles and handle temporary STS auth tokens credentialsProvider = 'cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider' # pass through env variables to child process for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']: if var_name in os.environ and var_name not in env_vars: env_vars[var_name] = os.environ[var_name] if aws_stack.is_local_env(env): # need to disable CBOR protocol, enforce use of plain JSON, # see https://github.com/mhart/kinesalite/issues/31 env_vars['AWS_CBOR_DISABLE'] = 'true' if kcl_log_level or (len(log_subscribers) > 0): if not log_file: log_file = LOG_FILE_PATTERN.replace('*', short_uid()) TMP_FILES.append(log_file) run('touch %s' % log_file) # start log output reader thread which will read the KCL log # file and print each line to stdout of this process... reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level, 'log_prefix': 'KCL', 'log_subscribers': log_subscribers}) reader_thread.start() # construct stream info stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars) props_file = stream_info['properties_file'] # set kcl config options kwargs = { 'metricsLevel': 'NONE', 'initialPositionInStream': 'LATEST' } # set parameters for local connection if aws_stack.is_local_env(env): kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS) kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB) kwargs['kinesisProtocol'] = get_service_protocol() kwargs['dynamodbProtocol'] = get_service_protocol() kwargs['disableCertChecking'] = 'true' kwargs.update(configs) # create config file kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script, streamName=stream_name, applicationName=stream_info['app_name'], credentialsProvider=credentialsProvider, region_name=region_name, **kwargs) TMP_FILES.append(props_file) # start stream consumer stream = KinesisStream(id=stream_name, params=stream_info) thread_consumer = KinesisProcessorThread.start_consumer(stream) TMP_THREADS.append(thread_consumer) return thread_consumer
def get_kinesis_streams(filter=".*", pool={}, env=None, region=None): if MOCK_OBJ: return [] result = [] try: kinesis_client = _connect("kinesis", region=region) out = kinesis_client.list_streams() for name in out["StreamNames"]: if re.match(filter, name): details = kinesis_client.describe_stream(StreamName=name) arn = details["StreamDescription"]["StreamARN"] stream = KinesisStream(arn) pool[arn] = stream stream.shards = get_kinesis_shards(stream_details=details, env=env, region=region) result.append(stream) except Exception: pass return result
def get_kinesis_streams(filter='.*', pool={}, env=None): if MOCK_OBJ: return [] result = [] try: out = cmd_kinesis('list-streams', env) out = json.loads(out) for name in out['StreamNames']: if re.match(filter, name): details = cmd_kinesis('describe-stream --stream-name %s' % name, env=env) details = json.loads(details) arn = details['StreamDescription']['StreamARN'] stream = KinesisStream(arn) pool[arn] = stream stream.shards = get_kinesis_shards(stream_details=details, env=env) result.append(stream) except socket.error: pass return result
def get_kinesis_streams(filter='.*', pool={}, env=None): if MOCK_OBJ: return [] result = [] try: kinesis_client = aws_stack.connect_to_service('kinesis') out = kinesis_client.list_streams() for name in out['StreamNames']: if re.match(filter, name): details = kinesis_client.describe_stream(StreamArn=name) arn = details['StreamDescription']['StreamARN'] stream = KinesisStream(arn) pool[arn] = stream stream.shards = get_kinesis_shards(stream_details=details, env=env) result.append(stream) except Exception: pass return result
def create_kinesis_stream(stream_name, shards=1, env=None, delete=False): env = get_environment(env) stream = KinesisStream(id=stream_name, num_shards=shards) conn = connect_to_service("kinesis", env=env) stream.connect(conn) if delete: run_safe(lambda: stream.destroy(), print_error=False) stream.create() # Note: Returning the stream without awaiting its creation (via wait_for()) to avoid API call timeouts/retries. return stream
def wait_for_stream_created(table_name): stream_name = get_kinesis_stream_name(table_name) stream = KinesisStream(id=stream_name, num_shards=1) kinesis = aws_stack.connect_to_service('kinesis', env=get_environment(None)) stream.connect(kinesis) stream.wait_for()
def wait_for_stream_created(table_name): stream_name = get_kinesis_stream_name(table_name) stream = KinesisStream(id=stream_name, num_shards=1) kinesis = aws_stack.create_external_boto_client( "kinesis", env=get_environment(None)) stream.connect(kinesis) stream.wait_for()
def start_kcl_client_process( stream_name, listener_script, log_file=None, env=None, configs=None, endpoint_url=None, ddb_lease_table_suffix=None, env_vars=None, region_name=None, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=None, ): if configs is None: configs = {} if env_vars is None: env_vars = {} if log_subscribers is None: log_subscribers = [] env = aws_stack.get_environment(env) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) # decide which credentials provider to use credentialsProvider = None if ("AWS_ASSUME_ROLE_ARN" in os.environ or "AWS_ASSUME_ROLE_ARN" in env_vars) and ("AWS_ASSUME_ROLE_SESSION_NAME" in os.environ or "AWS_ASSUME_ROLE_SESSION_NAME" in env_vars): # use special credentials provider that can assume IAM roles and handle temporary STS auth tokens credentialsProvider = "cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider" # pass through env variables to child process for var_name in [ "AWS_ASSUME_ROLE_ARN", "AWS_ASSUME_ROLE_SESSION_NAME", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", ]: if var_name in os.environ and var_name not in env_vars: env_vars[var_name] = os.environ[var_name] if aws_stack.is_local_env(env): # need to disable CBOR protocol, enforce use of plain JSON, # see https://github.com/mhart/kinesalite/issues/31 env_vars["AWS_CBOR_DISABLE"] = "true" if kcl_log_level or (len(log_subscribers) > 0): if not log_file: log_file = LOG_FILE_PATTERN.replace("*", short_uid()) TMP_FILES.append(log_file) run("touch %s" % log_file) # start log output reader thread which will read the KCL log # file and print each line to stdout of this process... reader_thread = OutputReaderThread({ "file": log_file, "level": kcl_log_level, "log_prefix": "KCL", "log_subscribers": log_subscribers, }) reader_thread.start() # construct stream info stream_info = get_stream_info( stream_name, log_file, env=env, endpoint_url=endpoint_url, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars, ) props_file = stream_info["properties_file"] # set kcl config options kwargs = {"metricsLevel": "NONE", "initialPositionInStream": "LATEST"} # set parameters for local connection if aws_stack.is_local_env(env): kwargs[ "kinesisEndpoint"] = f"{LOCALHOST}:{config.service_port('kinesis')}" kwargs[ "dynamodbEndpoint"] = f"{LOCALHOST}:{config.service_port('dynamodb')}" kwargs["kinesisProtocol"] = config.get_protocol() kwargs["dynamodbProtocol"] = config.get_protocol() kwargs["disableCertChecking"] = "true" kwargs.update(configs) # create config file kclipy_helper.create_config_file( config_file=props_file, executableName=listener_script, streamName=stream_name, applicationName=stream_info["app_name"], credentialsProvider=credentialsProvider, region_name=region_name, **kwargs, ) TMP_FILES.append(props_file) # start stream consumer stream = KinesisStream(id=stream_name, params=stream_info) thread_consumer = KinesisProcessorThread.start_consumer(stream) TMP_THREADS.append(thread_consumer) return thread_consumer