threads = [] start_time = datetime.datetime.now() for shard_id in xrange(len(shards)): worker_name = 'shard_worker:%s' % shard_id print('#-> shardId:', shards[shard_id]['ShardId']) worker = KinesisWorker( stream_name=args.stream_name, shard_id=shards[shard_id]['ShardId'], # iterator_type=iter_type_trim, # uses TRIM_HORIZON iterator_type=iter_type_latest, # uses LATEST worker_time=args.worker_time, sleep_interval=args.sleep_interval, echo=args.echo, name=worker_name) worker.daemon = True threads.append(worker) print('#-> starting: ', worker_name) worker.start() # Wait for all threads to complete for t in threads: t.join() finish_time = datetime.datetime.now() duration = (finish_time - start_time).total_seconds() total_records = poster.sum_posts(threads) print("-=> Exiting Worker Main <=-") print(" Total Records:", total_records) print(" Total Time:", duration) print(" Records / sec:", total_records / duration) print(" Worker sleep interval:", args.sleep_interval)
threads = [] start_time = datetime.datetime.now() for shard_id in xrange(len(shards)): worker_name = 'shard_worker:%s' % shard_id print ('#-> shardId:', shards[shard_id]['ShardId']) worker = KinesisWorker( stream_name=args.stream_name, shard_id=shards[shard_id]['ShardId'], # iterator_type=iter_type_trim, # uses TRIM_HORIZON iterator_type=iter_type_latest, # uses LATEST worker_time=args.worker_time, sleep_interval=args.sleep_interval, name=worker_name ) worker.daemon = True threads.append(worker) print ('#-> starting: ', worker_name) worker.start() # Wait for all threads to complete for t in threads: t.join() finish_time = datetime.datetime.now() duration = (finish_time - start_time).total_seconds() total_records = poster.sum_posts(threads) print ("-=> Exiting Worker Main <=-") print (" Total Records:", total_records) print (" Total Time:", duration) print (" Records / sec:", total_records / duration) print (" Worker sleep interval:", args.sleep_interval)