Ejemplo n.º 1
0
def command_create(args):
    """
    create XX.XX.XX.XX db-test

    :param args:
    :return:
    """
    if not check_ip_address(args.ip_address, 27017):
        LOG.error('The address {0}:27017 is not available'.format(args.ip_address))
        return

    connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address))

    # Drop the old database
    connection_mongodb.drop_database(args.database)

    # Recreate the database
    database = Database(connection_mongodb, args.database)

    # Create indexes on collections
    galaxies = Collection(database, COLLECTION_GALAXIES, True)
    galaxies.create_index('galaxy_id')

    galaxy_data = Collection(database, COLLECTION_GALAXY_DATA_SED, True)
    galaxy_data.create_index('galaxy_id')

    galaxy_data = Collection(database, COLLECTION_GALAXY_DATA_ORIGINAL, True)
    galaxy_data.create_index('galaxy_id')
Ejemplo n.º 2
0
def command_galaxy(args):
    """
    galaxy XX.XX.XX.XX db-test 30 100-120
    :param args:
    :return:
    """
    LOG.info(
        'ip_address: {0}, database_name: {1}'.format(
            args.ip_address,
            args.database
        )
    )

    if not check_ip_address(args.ip_address, 27017):
        LOG.error('The address {0}:27017 is not available'.format(args.ip_address))
        return

    galaxy_ids = []
    for galaxy_id in args.galaxy_ids:
        if galaxy_id.find('-') > 1:
            list_range = galaxy_id.split('-')
            galaxy_ids.extend(range(int(list_range[0]), int(list_range[1]) + 1))
        else:
            galaxy_ids.append(int(galaxy_id))

    galaxies = []

    connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address))

    engine = create_engine(DB_LOGIN)
    connection_mysql = engine.connect()

    for galaxy_id in galaxy_ids:
        galaxy = connection_mysql.execute(
            select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)
        ).first()
        if galaxy is not None:
            galaxies.append(galaxy)

    if len(galaxies) > 0:
        process_galaxy(
            connection_mysql,
            connection_mongodb,
            args.database,
            galaxies,
            args.radius
        )

    connection_mysql.close()
    connection_mongodb.close()
Ejemplo n.º 3
0
def command_sqs(args):
    """

    :param args:
    :return:
    """
    LOG.info(
        'ip_address: {0}, database_name: {1}'.format(
            args.ip_address,
            args.database
        )
    )

    if not check_ip_address(args.ip_address, 27017):
        LOG.error('The address {0}:27017 is not available'.format(args.ip_address))
        return

    no_message_counter = 0
    while True:
        sqs_helper = SqsHelper('us-east-1')
        queue = sqs_helper.get_queue(args.queue)

        # No queue yet
        if queue is None:
            break

        # Read the message
        message = queue.read(wait_time_seconds=10)

        if message is None:
            if no_message_counter >= 5:
                break
            else:
                no_message_counter += 1
        else:
            # Get the galaxy id and delete the message
            galaxy_id = int(message.get_body())
            LOG.info('Galaxy id: {0}'.format(galaxy_id))

            queue.delete_message(message)

            no_message_counter = 0
            galaxies = []

            connection_mongodb = MongoClient('mongodb://{0}:27017/'.format(args.ip_address))

            engine = create_engine(DB_LOGIN)
            connection_mysql = engine.connect()

            for galaxy in connection_mysql.execute(
                    select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)):
                galaxies.append(galaxy)

            if len(galaxies) > 0:
                process_galaxy(
                    connection_mysql,
                    connection_mongodb,
                    args.database,
                    galaxies,
                    args.radius
                )

            connection_mysql.close()
            connection_mongodb.close()