def add_ids_to_queue(args, and_expression): engine = create_engine(DB_LOGIN) connection = engine.connect() # noinspection PyBroadException try: queue = None if not args.test: sqs_helper = SqsHelper('us-east-1') queue = sqs_helper.get_queue(args.queue_name) select_galaxy = select([GALAXY.c.galaxy_id]).where(and_expression).order_by(func.RAND()) counter = 1 for galaxy in connection.execute(select_galaxy): if counter % 100 == 0: LOG.info('Added {0}'.format(counter)) counter += 1 message_text = '{0}'.format(galaxy[GALAXY.c.galaxy_id]) if args.test: LOG.info(message_text) else: message = Message() message.set_body(message_text) queue.write(message) except Exception: LOG.exception('Error adding ids')
def process_data_sqs(bucket_name, filename, split_training=None): no_message_counter = 0 while True: sqs_helper = SqsHelper("us-east-1") queue = sqs_helper.get_queue(QUEUE_GET_DATA_SET) # No queue yet if queue is None: break # Read the message message = queue.read(wait_time_seconds=10) if message is None: if no_message_counter >= 5: break else: no_message_counter += 1 else: # Get the galaxy id and delete the message json_message = json.loads(message.get_body()) LOG.info("json_message: {0}".format(json_message)) queue.delete_message(message) no_message_counter = 0 process_data = ProcessData02( json_message["batch_number"], json_message["galaxy_ids"], bucket_name, filename, split_training ) process_data.run()
def __call__(self): # noinspection PyBroadException try: sqs_helper = SqsHelper('us-east-1') queue = sqs_helper.get_queue(self.args.queue_name) for galaxy_id in range(self.range_id * SEGMENT, (self.range_id * SEGMENT) + SEGMENT): if self.args.verbosity >= 1 or galaxy_id % 100 == 0: LOG.info('Adding {0}'.format(galaxy_id)) message = Message() message.set_body('{0}'.format(galaxy_id)) queue.write(message) except Exception: LOG.exception('Task died')
def command_sqs(args): LOG.info('args: {0}'.format(args)) no_message_counter = 0 while True: sqs_helper = SqsHelper('us-east-1') queue = sqs_helper.get_queue(QUEUE_NORMALISE_DATA) # No queue yet if queue is None: break # Read the message message = queue.read(wait_time_seconds=10) if message is None: if no_message_counter >= 5: break else: no_message_counter += 1 else: # Get the galaxy id and delete the message galaxy_id = int(message.get_body()) LOG.info('Galaxy id: {0}'.format(galaxy_id)) queue.delete_message(message) no_message_counter = 0 galaxies = [] engine = create_engine(DB_LOGIN) connection = engine.connect() for galaxy in connection.execute( select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)): galaxies.append(galaxy) if len(galaxies) > 0: process_galaxy(connection, galaxies) connection.close()
def command_sqs(args): """ :param args: :return: """ LOG.info( 'ip_address: {0}, database_name: {1}'.format( args.ip_address, args.database ) ) if not check_ip_address(args.ip_address, 27017): LOG.error('The address {0}:27017 is not available'.format(args.ip_address)) return no_message_counter = 0 while True: sqs_helper = SqsHelper('us-east-1') queue = sqs_helper.get_queue(args.queue) # No queue yet if queue is None: break # Read the message message = queue.read(wait_time_seconds=10) if message is None: if no_message_counter >= 5: break else: no_message_counter += 1 else: # Get the galaxy id and delete the message galaxy_id = int(message.get_body()) LOG.info('Galaxy id: {0}'.format(galaxy_id)) queue.delete_message(message) no_message_counter = 0 galaxies = [] connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address)) engine = create_engine(DB_LOGIN) connection_mysql = engine.connect() for galaxy in connection_mysql.execute( select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)): galaxies.append(galaxy) if len(galaxies) > 0: process_galaxy( connection_mysql, connection_mongodb, args.database, galaxies, args.radius ) connection_mysql.close() connection_mongodb.close()
def add_ids_to_queue(args): engine = create_engine(DB_LOGIN) connection = engine.connect() # noinspection PyBroadException try: queue = None if not args.test: sqs_helper = SqsHelper('us-east-1') queue = sqs_helper.get_queue(args.queue_name) select_galaxy = select([GALAXY.c.galaxy_id]).where( and_( GALAXY.c.run_id == args.run_id[0], GALAXY.c.dimension_x == args.dimension[0], GALAXY.c.dimension_y == args.dimension[0], GALAXY.c.galaxy_type.in_(args.galaxy_types) ) ).order_by(func.RAND()) counter = 1 batch_counter = 1 group_galaxies = [] for galaxy in connection.execute(select_galaxy): if counter % 100 == 0: LOG.info('Added {0}'.format(counter)) counter += 1 group_galaxies.append(galaxy[GALAXY.c.galaxy_id]) if len(group_galaxies) == args.groups_of[0]: json_message = { 'batch_number': batch_counter, 'galaxy_ids': group_galaxies } batch_counter += 1 message_text = json.dumps(json_message) if args.test: LOG.info(message_text) else: message = Message() message.set_body(message_text) queue.write(message) group_galaxies = [] # Mop up any left overs if len(group_galaxies) > 0: json_message = { 'batch_number': batch_counter, 'galaxy_ids': group_galaxies } message_text = json.dumps(json_message) if args.test: LOG.info(message_text) else: message = Message() message.set_body(message_text) queue.write(message) except Exception: LOG.exception('Error adding ids')