def add_ids_to_queue(args, and_expression):
engine = create_engine(DB_LOGIN)
connection = engine.connect()
# noinspection PyBroadException
try:
queue = None
if not args.test:
sqs_helper = SqsHelper('us-east-1')
queue = sqs_helper.get_queue(args.queue_name)
select_galaxy = select([GALAXY.c.galaxy_id]).where(and_expression).order_by(func.RAND())
counter = 1
for galaxy in connection.execute(select_galaxy):
if counter % 100 == 0:
LOG.info('Added {0}'.format(counter))
counter += 1
message_text = '{0}'.format(galaxy[GALAXY.c.galaxy_id])
if args.test:
LOG.info(message_text)
else:
message = Message()
message.set_body(message_text)
queue.write(message)
except Exception:
LOG.exception('Error adding ids')
def process_data_sqs(bucket_name, filename, split_training=None):
no_message_counter = 0
while True:
sqs_helper = SqsHelper("us-east-1")
queue = sqs_helper.get_queue(QUEUE_GET_DATA_SET)
# No queue yet
if queue is None:
break
# Read the message
message = queue.read(wait_time_seconds=10)
if message is None:
if no_message_counter >= 5:
break
else:
no_message_counter += 1
else:
# Get the galaxy id and delete the message
json_message = json.loads(message.get_body())
LOG.info("json_message: {0}".format(json_message))
queue.delete_message(message)
no_message_counter = 0
process_data = ProcessData02(
json_message["batch_number"], json_message["galaxy_ids"], bucket_name, filename, split_training
)
process_data.run()
def __call__(self):
# noinspection PyBroadException
try:
sqs_helper = SqsHelper('us-east-1')
queue = sqs_helper.get_queue(self.args.queue_name)
for galaxy_id in range(self.range_id * SEGMENT, (self.range_id * SEGMENT) + SEGMENT):
if self.args.verbosity >= 1 or galaxy_id % 100 == 0:
LOG.info('Adding {0}'.format(galaxy_id))
message = Message()
message.set_body('{0}'.format(galaxy_id))
queue.write(message)
except Exception:
LOG.exception('Task died')
def command_sqs(args):
LOG.info('args: {0}'.format(args))
no_message_counter = 0
while True:
sqs_helper = SqsHelper('us-east-1')
queue = sqs_helper.get_queue(QUEUE_NORMALISE_DATA)
# No queue yet
if queue is None:
break
# Read the message
message = queue.read(wait_time_seconds=10)
if message is None:
if no_message_counter >= 5:
break
else:
no_message_counter += 1
else:
# Get the galaxy id and delete the message
galaxy_id = int(message.get_body())
LOG.info('Galaxy id: {0}'.format(galaxy_id))
queue.delete_message(message)
no_message_counter = 0
galaxies = []
engine = create_engine(DB_LOGIN)
connection = engine.connect()
for galaxy in connection.execute(
select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)):
galaxies.append(galaxy)
if len(galaxies) > 0:
process_galaxy(connection, galaxies)
connection.close()
def command_sqs(args):
"""
:param args:
:return:
"""
LOG.info(
'ip_address: {0}, database_name: {1}'.format(
args.ip_address,
args.database
)
)
if not check_ip_address(args.ip_address, 27017):
LOG.error('The address {0}:27017 is not available'.format(args.ip_address))
return
no_message_counter = 0
while True:
sqs_helper = SqsHelper('us-east-1')
queue = sqs_helper.get_queue(args.queue)
# No queue yet
if queue is None:
break
# Read the message
message = queue.read(wait_time_seconds=10)
if message is None:
if no_message_counter >= 5:
break
else:
no_message_counter += 1
else:
# Get the galaxy id and delete the message
galaxy_id = int(message.get_body())
LOG.info('Galaxy id: {0}'.format(galaxy_id))
queue.delete_message(message)
no_message_counter = 0
galaxies = []
connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address))
engine = create_engine(DB_LOGIN)
connection_mysql = engine.connect()
for galaxy in connection_mysql.execute(
select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)):
galaxies.append(galaxy)
if len(galaxies) > 0:
process_galaxy(
connection_mysql,
connection_mongodb,
args.database,
galaxies,
args.radius
)
connection_mysql.close()
connection_mongodb.close()
def add_ids_to_queue(args):
engine = create_engine(DB_LOGIN)
connection = engine.connect()
# noinspection PyBroadException
try:
queue = None
if not args.test:
sqs_helper = SqsHelper('us-east-1')
queue = sqs_helper.get_queue(args.queue_name)
select_galaxy = select([GALAXY.c.galaxy_id]).where(
and_(
GALAXY.c.run_id == args.run_id[0],
GALAXY.c.dimension_x == args.dimension[0],
GALAXY.c.dimension_y == args.dimension[0],
GALAXY.c.galaxy_type.in_(args.galaxy_types)
)
).order_by(func.RAND())
counter = 1
batch_counter = 1
group_galaxies = []
for galaxy in connection.execute(select_galaxy):
if counter % 100 == 0:
LOG.info('Added {0}'.format(counter))
counter += 1
group_galaxies.append(galaxy[GALAXY.c.galaxy_id])
if len(group_galaxies) == args.groups_of[0]:
json_message = {
'batch_number': batch_counter,
'galaxy_ids': group_galaxies
}
batch_counter += 1
message_text = json.dumps(json_message)
if args.test:
LOG.info(message_text)
else:
message = Message()
message.set_body(message_text)
queue.write(message)
group_galaxies = []
# Mop up any left overs
if len(group_galaxies) > 0:
json_message = {
'batch_number': batch_counter,
'galaxy_ids': group_galaxies
}
message_text = json.dumps(json_message)
if args.test:
LOG.info(message_text)
else:
message = Message()
message.set_body(message_text)
queue.write(message)
except Exception:
LOG.exception('Error adding ids')
