def docker_worker(submissions):
pool = eventlet.GreenPool(size=submissions)
killer = GracefulKiller()
mongo = DbDriver(config)
if hasattr(config, 'consul'):
logging.info('Using consul for retrieving the docker endpoint')
config.docker['api'] = None
eventlet.spawn_n(read_swarm_manager, config.docker,
config.consul["host"], config.consul["port"],
config.consul["key"])
elif hasattr(config, 'docker'):
logging.info('Using static docker endpoint from config')
else:
raise ValueError(
'No consul or docker configured. Please add one of the two to the config.'
)
while True:
try:
for _ in xrange(min(mongo.no_of_submissions(), submissions)):
pool.spawn(check_submission)
logging.info('Available threads: {}'.format(pool.free()))
logging.info('Running threads: {}'.format(pool.running()))
logging.info('Waiting for {} seconds'.format(2**(submissions -
pool.running())))
eventlet.sleep(2**(submissions - pool.running()))
except (SystemExit, KeyboardInterrupt):
break
if killer.kill_now:
break
def docker_worker(submissions):
pool = eventlet.GreenPool(size=submissions)
killer = GracefulKiller()
mongo = DbDriver(config)
if hasattr(config, 'consul'):
logging.info('Using consul for retrieving the docker endpoint')
config.docker['api'] = None
eventlet.spawn_n(read_swarm_manager, config.docker, config.consul["host"], config.consul["port"],
config.consul["key"])
elif hasattr(config, 'docker'):
logging.info('Using static docker endpoint from config')
else:
raise ValueError('No consul or docker configured. Please add one of the two to the config.')
while True:
try:
for _ in xrange(min(mongo.no_of_submissions(), submissions)):
pool.spawn(check_submission)
logging.info('Available threads: {}'.format(pool.free()))
logging.info('Running threads: {}'.format(pool.running()))
logging.info('Waiting for {} seconds'.format(2**(submissions-pool.running())))
eventlet.sleep(2**(submissions-pool.running()))
except (SystemExit, KeyboardInterrupt):
break
if killer.kill_now:
break
def check_submission():
mongo = DbDriver(config)
docker = DockerController(config.docker['api'], config.docker['network'])
record = mongo.get_one_record()
if not record:
logging.debug("Quiting. No new submission.")
return
# Filter out illegal votes
if record['vote'].title() not in config.misc['allowed_votes']:
mongo.update_record_status(record['_id'],
'discarded',
statusmsg="Invalid Vote record: {}".format(
record['vote']))
return
coordinates = get_coordinates(record['location'])
if coordinates:
mongo.update_record_location(record['_id'], coordinates.latitude,
coordinates.longitude)
else:
mongo.update_record_status(record['_id'],
'failed',
statusmsg='Failed to geo-locate {}'.format(
record['location']))
return
for image in set(record['repo']):
if image == "bogus_bday_image:latest":
logging.info('Submission SUCCESSFUL for {}'.format(record['name']))
mongo.update_record_status(
record['_id'],
'successful',
statusmsg='Magic image passed validation')
return
# Ensure unique submissions
if "/" in image and mongo.sHandle.find_one(
{"dockerHubUser": image.split("/")[0]}, {"_id": 1}):
logging.info(
"Duplicate submission. User already has a successful submission."
)
mongo.update_record_status(
record['_id'],
'duplicated',
statusmsg=
"Duplicate submission. User already has a successful submission."
)
return
logging.info("Downloading image {} for user {}".format(
image, record['name']))
try:
image_result = docker.download_image(image_name=image)
except:
mongo.update_record_status(record['_id'], 'submitted')
traceback.print_exc(file=sys.stdout)
return
if not image_result:
break
logging.info('Starting container {} for user {}'.format(
image, record['name']))
try:
container_id, container_ip = docker.run_container(image)
except ContainerError, e:
mongo.update_record_status(record['_id'],
'failed',
statusmsg=str(e))
return
except DockerError, e:
logging.error('Something went wrong with Docker: {}'.format(
str(e)))
mongo.update_record_status(record['_id'], 'submitted')
return
def check_submission():
mongo = DbDriver(config)
docker = DockerController(config.docker['api'], config.docker['network'])
record = mongo.get_one_record()
if not record:
logging.debug("Quiting. No new submission.")
return
# Filter out illegal votes
if record['vote'].title() not in config.misc['allowed_votes']:
mongo.update_record_status(record['_id'], 'discarded',
statusmsg="Invalid Vote record: {}".format(record['vote']))
return
coordinates = get_coordinates(record['location'])
if coordinates:
mongo.update_record_location(record['_id'], coordinates.latitude, coordinates.longitude)
else:
mongo.update_record_status(record['_id'], 'failed',
statusmsg='Failed to geo-locate {}'.format(record['location']))
return
for image in set(record['repo']):
if image == "bogus_bday_image:latest":
logging.info('Submission SUCCESSFUL for {}'.format(record['name']))
mongo.update_record_status(record['_id'], 'successful', statusmsg='Magic image passed validation')
return
# Ensure unique submissions
if "/" in image and mongo.sHandle.find_one({ "dockerHubUser": image.split("/")[0] }, { "_id": 1}):
logging.info("Duplicate submission. User already has a successful submission.")
mongo.update_record_status(record['_id'], 'duplicated',
statusmsg="Duplicate submission. User already has a successful submission.")
return
logging.info("Downloading image {} for user {}".format(image, record['name']))
try:
image_result = docker.download_image(image_name=image)
except:
mongo.update_record_status(record['_id'], 'submitted')
traceback.print_exc(file=sys.stdout)
return
if not image_result:
break
logging.info('Starting container {} for user {}'.format(image, record['name']))
try:
container_id, container_ip = docker.run_container(image)
except ContainerError, e:
mongo.update_record_status(record['_id'], 'failed', statusmsg=str(e))
return
except DockerError, e:
logging.error('Something went wrong with Docker: {}'.format(str(e)))
mongo.update_record_status(record['_id'], 'submitted')
return
