threads = []
start_time = datetime.datetime.now()
for shard_id in xrange(len(shards)):
worker_name = 'shard_worker:%s' % shard_id
print('#-> shardId:', shards[shard_id]['ShardId'])
worker = KinesisWorker(
stream_name=args.stream_name,
shard_id=shards[shard_id]['ShardId'],
# iterator_type=iter_type_trim, # uses TRIM_HORIZON
iterator_type=iter_type_latest, # uses LATEST
worker_time=args.worker_time,
sleep_interval=args.sleep_interval,
echo=args.echo,
name=worker_name)
worker.daemon = True
threads.append(worker)
print('#-> starting: ', worker_name)
worker.start()
# Wait for all threads to complete
for t in threads:
t.join()
finish_time = datetime.datetime.now()
duration = (finish_time - start_time).total_seconds()
total_records = poster.sum_posts(threads)
print("-=> Exiting Worker Main <=-")
print(" Total Records:", total_records)
print(" Total Time:", duration)
print(" Records / sec:", total_records / duration)
print(" Worker sleep interval:", args.sleep_interval)
threads = []
start_time = datetime.datetime.now()
for shard_id in xrange(len(shards)):
worker_name = 'shard_worker:%s' % shard_id
print ('#-> shardId:', shards[shard_id]['ShardId'])
worker = KinesisWorker(
stream_name=args.stream_name,
shard_id=shards[shard_id]['ShardId'],
# iterator_type=iter_type_trim, # uses TRIM_HORIZON
iterator_type=iter_type_latest, # uses LATEST
worker_time=args.worker_time,
sleep_interval=args.sleep_interval,
name=worker_name
)
worker.daemon = True
threads.append(worker)
print ('#-> starting: ', worker_name)
worker.start()
# Wait for all threads to complete
for t in threads:
t.join()
finish_time = datetime.datetime.now()
duration = (finish_time - start_time).total_seconds()
total_records = poster.sum_posts(threads)
print ("-=> Exiting Worker Main <=-")
print (" Total Records:", total_records)
print (" Total Time:", duration)
print (" Records / sec:", total_records / duration)
print (" Worker sleep interval:", args.sleep_interval)