def command_create(args): """ create XX.XX.XX.XX db-test :param args: :return: """ if not check_ip_address(args.ip_address, 27017): LOG.error('The address {0}:27017 is not available'.format(args.ip_address)) return connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address)) # Drop the old database connection_mongodb.drop_database(args.database) # Recreate the database database = Database(connection_mongodb, args.database) # Create indexes on collections galaxies = Collection(database, COLLECTION_GALAXIES, True) galaxies.create_index('galaxy_id') galaxy_data = Collection(database, COLLECTION_GALAXY_DATA_SED, True) galaxy_data.create_index('galaxy_id') galaxy_data = Collection(database, COLLECTION_GALAXY_DATA_ORIGINAL, True) galaxy_data.create_index('galaxy_id')
def command_galaxy(args): """ galaxy XX.XX.XX.XX db-test 30 100-120 :param args: :return: """ LOG.info( 'ip_address: {0}, database_name: {1}'.format( args.ip_address, args.database ) ) if not check_ip_address(args.ip_address, 27017): LOG.error('The address {0}:27017 is not available'.format(args.ip_address)) return galaxy_ids = [] for galaxy_id in args.galaxy_ids: if galaxy_id.find('-') > 1: list_range = galaxy_id.split('-') galaxy_ids.extend(range(int(list_range[0]), int(list_range[1]) + 1)) else: galaxy_ids.append(int(galaxy_id)) galaxies = [] connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address)) engine = create_engine(DB_LOGIN) connection_mysql = engine.connect() for galaxy_id in galaxy_ids: galaxy = connection_mysql.execute( select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id) ).first() if galaxy is not None: galaxies.append(galaxy) if len(galaxies) > 0: process_galaxy( connection_mysql, connection_mongodb, args.database, galaxies, args.radius ) connection_mysql.close() connection_mongodb.close()
def command_sqs(args): """ :param args: :return: """ LOG.info( 'ip_address: {0}, database_name: {1}'.format( args.ip_address, args.database ) ) if not check_ip_address(args.ip_address, 27017): LOG.error('The address {0}:27017 is not available'.format(args.ip_address)) return no_message_counter = 0 while True: sqs_helper = SqsHelper('us-east-1') queue = sqs_helper.get_queue(args.queue) # No queue yet if queue is None: break # Read the message message = queue.read(wait_time_seconds=10) if message is None: if no_message_counter >= 5: break else: no_message_counter += 1 else: # Get the galaxy id and delete the message galaxy_id = int(message.get_body()) LOG.info('Galaxy id: {0}'.format(galaxy_id)) queue.delete_message(message) no_message_counter = 0 galaxies = [] connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address)) engine = create_engine(DB_LOGIN) connection_mysql = engine.connect() for galaxy in connection_mysql.execute( select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)): galaxies.append(galaxy) if len(galaxies) > 0: process_galaxy( connection_mysql, connection_mongodb, args.database, galaxies, args.radius ) connection_mysql.close() connection_mongodb.close()