def process_galaxy(connection, galaxies, verbosity):
    """
    Process the galaxies
    """
    # Build the tables
    build_dynamic_tables(connection)

    for galaxy in galaxies:
        galaxy_id = galaxy[GALAXY.c.galaxy_id]
        steps_done = get_step_done_ids(connection, galaxy_id, STEP_DONE_ID_MEAN_STANDARD_DEVIATION, True)
        if verbosity:
            LOG.info(
                'galaxy_id: {0}, galaxy_name: {1}'.format(
                    galaxy_id,
                    galaxy[GALAXY.c.name]
                )
            )
        tables_required = get_tables_required(steps_done)

        if len(tables_required) > 0:
            for table in tables_required:
                with connection.begin():
                    create_mean_standard_deviation = CreateMeanStandardDeviation(connection, table, galaxy_id)
                    create_mean_standard_deviation.calculate_details()
                    create_mean_standard_deviation.update_details()
                    add_step_done_id(
                        connection,
                        galaxy_id,
                        STEP_DONE_ID_MEAN_STANDARD_DEVIATION,
                        table.name
                    )
Ejemplo n.º 2
0
def process_galaxy(connection, galaxies, verbosity):
    """
    Process the galaxies
    """
    # Build the tables
    build_dynamic_tables(connection)

    for galaxy in galaxies:
        galaxy_id = galaxy[GALAXY.c.galaxy_id]
        steps_done = get_step_done_ids(connection, galaxy_id, STEP_DONE_ID_BUILD_MASK)
        if verbosity:
            LOG.info(
                'galaxy_id: {0}, galaxy_name: {1}'.format(
                    galaxy_id,
                    galaxy[GALAXY.c.name]
                )
            )
        if STEP_DONE_ID_BUILD_MASK not in steps_done:
            with connection.begin():
                build_mask = ProcessGalaxyMasks(connection, galaxy_id)
                build_mask.run()
                add_step_done_id(
                    connection,
                    galaxy_id,
                    STEP_DONE_ID_BUILD_MASK
                )
Ejemplo n.º 3
0
def process_galaxy(connection, galaxies, verbosity):
    """
    Process the galaxies
    """
    # Build the tables
    build_dynamic_tables(connection)

    for galaxy in galaxies:
        galaxy_id = galaxy[GALAXY.c.galaxy_id]
        steps_done = get_step_done_ids(
            connection,
            galaxy_id,
            [STEP_DONE_ID_BUILD_MASK, STEP_DONE_ID_CENTROID_CENTRE_OF_MASS])

        # have we built the mask
        if STEP_DONE_ID_BUILD_MASK in steps_done:
            if STEP_DONE_ID_CENTROID_CENTRE_OF_MASS not in steps_done:
                with connection.begin():
                    centroid_center_of_mass = CentroidCentreOfMass(connection, galaxy, verbosity)
                    centroid_center_of_mass.calculate_results()

                    if len(centroid_center_of_mass.layers) > 0:
                        connection.execute(
                            IMAGE_DETAIL.insert(),
                            galaxy_id=galaxy_id,
                            x_image=centroid_center_of_mass.dimension_x / 2.0,
                            y_image=centroid_center_of_mass.dimension_y / 2.0,
                            x_centroid=numpy.asscalar(centroid_center_of_mass.centroid[0]),
                            y_centroid=numpy.asscalar(centroid_center_of_mass.centroid[1]),
                            x_centre_of_mass=numpy.asscalar(centroid_center_of_mass.centre_of_mass[0]),
                            y_centre_of_mass=numpy.asscalar(centroid_center_of_mass.centre_of_mass[1])
                        )
                    else:
                        connection.execute(
                            IMAGE_DETAIL.insert(),
                            galaxy_id=galaxy_id,
                            x_image=centroid_center_of_mass.dimension_x / 2.0,
                            y_image=centroid_center_of_mass.dimension_y / 2.0,
                            x_centroid=-1.0,
                            y_centroid=-1.0,
                            x_centre_of_mass=-1.0,
                            y_centre_of_mass=-1.0
                        )
                    add_step_done_id(connection, galaxy_id, STEP_DONE_ID_CENTROID_CENTRE_OF_MASS)

            elif verbosity >= 1:
                LOG.info('Nothing to do for the galaxy {0}'.format(galaxy_id))

        elif verbosity >= 1:
            LOG.info('The mask has not been built for the galaxy {0}'.format(galaxy_id))
Ejemplo n.º 4
0
def process_galaxy(connection, galaxies, verbosity):
    """
    Process the galaxies
    """
    # Build the tables
    build_dynamic_tables(connection)

    for galaxy in galaxies:
        time_start = time.time()

        galaxy_id = galaxy[GALAXY.c.galaxy_id]
        steps_done = get_step_done_ids(connection, galaxy_id, STEP_DONE_ID_BUILD_GALAXY_DETAILS, True)
        if verbosity:
            LOG.info(
                'galaxy_id: {0}, galaxy_name: {1}'.format(
                    galaxy_id,
                    galaxy[GALAXY.c.name]
                )
            )
        tables_required = get_tables_required(steps_done)

        if len(tables_required) > 0:
            for table in tables_required:
                with connection.begin():
                    time_start_update = time.time()
                    create_galaxy_detail = CreateGalaxyDetail(connection, table, galaxy_id)
                    create_galaxy_detail.calculate_details()
                    create_galaxy_detail.insert_details()
                    add_step_done_id(
                        connection,
                        galaxy_id,
                        STEP_DONE_ID_BUILD_GALAXY_DETAILS,
                        table.name
                    )
                    time_end_update = time.time()
                    LOG.info(
                        'galaxy_id: {0}, table: {1}, time: {2:.3f}s'.format(
                            galaxy_id,
                            table.name,
                            time_end_update - time_start_update
                        )
                    )
        time_end = time.time()
        LOG.info(
            'galaxy_id: {0}, time: {1:.3f}s'.format(
                galaxy_id,
                time_end - time_start
            )
        )
Ejemplo n.º 5
0
def process_galaxy(connection, galaxies):
    """
    Process the galaxies
    """
    # Build the tables
    build_dynamic_tables(connection)

    for galaxy in galaxies:
        galaxy_id = galaxy[GALAXY.c.galaxy_id]
        steps_done = get_step_done_ids(
            connection,
            galaxy_id,
            [STEP_DONE_ID_BUILD_GALAXY_DETAILS, STEP_DONE_ID_NORMALISE_Z_MIN_MAX],
            True
        )

        tables_required = get_tables_required(steps_done)

        if len(tables_required) > 0:
            time_start = time.time()
            for table in tables_required:
                with connection.begin():
                    time_start_update = time.time()
                    mean_standard = connection.execute(
                        select([GALAXY_DETAIL]).where(
                            and_(
                                GALAXY_DETAIL.c.galaxy_id == galaxy_id,
                                GALAXY_DETAIL.c.table_name == table.name
                            )
                        )
                    ).first()
                    insert_data = []
                    if mean_standard is not None:
                        mean = mean_standard[GALAXY_DETAIL.c.mean]
                        standard_deviation = mean_standard[GALAXY_DETAIL.c.standard_deviation]
                        min_value = mean_standard[GALAXY_DETAIL.c.min_value]
                        max_value = mean_standard[GALAXY_DETAIL.c.max_value]
                        max_minus_min = max_value - min_value
                        if max_minus_min is not None and max_minus_min != 0.0 \
                                and standard_deviation is not None and standard_deviation != 0.0:
                            if table.name.startswith('original_value__'):
                                select_statement = select(
                                    [table]
                                ).where(
                                    and_(
                                        table.c.galaxy_id == galaxy_id,
                                        table.c.galaxy_id == MASK_POINT.c.galaxy_id,
                                        table.c.x == MASK_POINT.c.x,
                                        table.c.y == MASK_POINT.c.y,
                                        table.c.value > 0.0
                                    )
                                )
                            else:
                                select_statement = select(
                                    [table]
                                ).where(
                                    and_(
                                        table.c.galaxy_id == galaxy_id,
                                        table.c.galaxy_id == MASK_POINT.c.galaxy_id,
                                        table.c.x == MASK_POINT.c.x,
                                        table.c.y == MASK_POINT.c.y
                                    )
                                )

                            for table_data in connection.execute(select_statement):
                                value = table_data[table.c.value]
                                z_score = (value - mean) / standard_deviation
                                insert_data.append(
                                    {
                                        'original_id': table_data[table.c.id],
                                        'value_zscore': z_score,
                                        'value_min_max': (value - min_value) / max_minus_min,
                                        'value_ln': math.log(value - min_value + 1),
                                        'value_softmax': 1.0 / (1.0 + math.exp(-z_score))
                                    }
                                )

                    if len(insert_data) > 0:
                        table_extended = DATA_TABLES_EXTENDED[table.name + '__extended']
                        connection.execute(table_extended.insert(), insert_data)

                    add_step_done_id(
                        connection,
                        galaxy_id,
                        STEP_DONE_ID_NORMALISE_Z_MIN_MAX,
                        table.name
                    )
                    time_end_update = time.time()
                    LOG.info(
                        'galaxy_id: {0}, table: {1}__extended, time: {2:.3f}s, inserts: {3}'.format(
                            galaxy_id,
                            table.name,
                            time_end_update - time_start_update,
                            len(insert_data)
                        )
                    )

            time_end = time.time()
            LOG.info('galaxy_id: {0}, time: {1:.3f}s'.format(galaxy_id, time_end - time_start))
Ejemplo n.º 6
0
def process_galaxy(connection, galaxies, verbosity, profile_name='aws-pogs'):
    """
    Process the galaxies
    """
    # Build the tables
    build_dynamic_tables(connection)

    for galaxy in galaxies:
        galaxy_id = galaxy[GALAXY.c.galaxy_id]
        steps_done = get_step_done_ids(
            connection,
            galaxy_id,
            [STEP_DONE_ID_NO_HDF5_FILE, STEP_DONE_ID_ORIGINAL_VALUES, STEP_DONE_ID_SED_DATA],
            True,
            True
        )

        # Work out what SED data is required
        data_required_sed = get_data_required_sed(connection, steps_done)
        data_required_original = get_data_required_original(steps_done)

        if len(data_required_sed) > 0 or len(data_required_original) > 0:
            # Copy the file from S3
            s3_helper = S3Helper(profile_name=profile_name)
            galaxy_id = int(galaxy[GALAXY.c.galaxy_id])
            galaxy_name = get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy_id)
            s3_name = os.path.join('{0:04d}000'.format(galaxy_id / 1000), galaxy_name) + '.hdf5'
            hdf5_file_name = os.path.join(FAST_DISK, galaxy_name) + '.hdf5'

            copy_ok = s3_helper.copy_file_to_disk(BUCKET_NAME, s3_name, hdf5_file_name)

            if copy_ok:
                h5_file = h5py.File(hdf5_file_name, 'r')
                LOG.info('Processing SED for name: {0}, run_id: {1}, galaxy_id: {2}'.format(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy_id))
                # Do we have anything to do?
                if len(data_required_sed) > 0:
                    # noinspection PyBroadException
                    try:
                        # Store the SED fit values
                        add_sed_data(connection, galaxy_id, h5_file, data_required_sed)
                    except Exception:
                        LOG.exception('An exception occurred in process_galaxy processing the SED values')
                else:
                    if verbosity >= 1:
                        LOG.info('Nothing to add - SED')

                if len(data_required_original) > 0:
                    # noinspection PyBroadException
                    try:
                        add_original_data(connection, galaxy_id, h5_file, data_required_original)
                    except Exception:
                        LOG.exception('An exception occurred in process_galaxy processing the original values')
                else:
                    if verbosity >= 1:
                        LOG.info('Nothing to add - Original Data')

                # Clean up after ourselves
                h5_file.close()
                os.remove(hdf5_file_name)

            else:
                LOG.error('The file for name: {0}, run_id: {1}, galaxy_id: {2} does not exist'.format(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy_id))