def process_galaxy(connection, galaxies, verbosity):
"""
Process the galaxies
"""
# Build the tables
build_dynamic_tables(connection)
for galaxy in galaxies:
galaxy_id = galaxy[GALAXY.c.galaxy_id]
steps_done = get_step_done_ids(connection, galaxy_id, STEP_DONE_ID_MEAN_STANDARD_DEVIATION, True)
if verbosity:
LOG.info(
'galaxy_id: {0}, galaxy_name: {1}'.format(
galaxy_id,
galaxy[GALAXY.c.name]
)
)
tables_required = get_tables_required(steps_done)
if len(tables_required) > 0:
for table in tables_required:
with connection.begin():
create_mean_standard_deviation = CreateMeanStandardDeviation(connection, table, galaxy_id)
create_mean_standard_deviation.calculate_details()
create_mean_standard_deviation.update_details()
add_step_done_id(
connection,
galaxy_id,
STEP_DONE_ID_MEAN_STANDARD_DEVIATION,
table.name
)
def process_galaxy(connection, galaxies, verbosity):
"""
Process the galaxies
"""
# Build the tables
build_dynamic_tables(connection)
for galaxy in galaxies:
galaxy_id = galaxy[GALAXY.c.galaxy_id]
steps_done = get_step_done_ids(connection, galaxy_id, STEP_DONE_ID_BUILD_MASK)
if verbosity:
LOG.info(
'galaxy_id: {0}, galaxy_name: {1}'.format(
galaxy_id,
galaxy[GALAXY.c.name]
)
)
if STEP_DONE_ID_BUILD_MASK not in steps_done:
with connection.begin():
build_mask = ProcessGalaxyMasks(connection, galaxy_id)
build_mask.run()
add_step_done_id(
connection,
galaxy_id,
STEP_DONE_ID_BUILD_MASK
)
def process_galaxy(connection, galaxies, verbosity):
"""
Process the galaxies
"""
# Build the tables
build_dynamic_tables(connection)
for galaxy in galaxies:
galaxy_id = galaxy[GALAXY.c.galaxy_id]
steps_done = get_step_done_ids(
connection,
galaxy_id,
[STEP_DONE_ID_BUILD_MASK, STEP_DONE_ID_CENTROID_CENTRE_OF_MASS])
# have we built the mask
if STEP_DONE_ID_BUILD_MASK in steps_done:
if STEP_DONE_ID_CENTROID_CENTRE_OF_MASS not in steps_done:
with connection.begin():
centroid_center_of_mass = CentroidCentreOfMass(connection, galaxy, verbosity)
centroid_center_of_mass.calculate_results()
if len(centroid_center_of_mass.layers) > 0:
connection.execute(
IMAGE_DETAIL.insert(),
galaxy_id=galaxy_id,
x_image=centroid_center_of_mass.dimension_x / 2.0,
y_image=centroid_center_of_mass.dimension_y / 2.0,
x_centroid=numpy.asscalar(centroid_center_of_mass.centroid[0]),
y_centroid=numpy.asscalar(centroid_center_of_mass.centroid[1]),
x_centre_of_mass=numpy.asscalar(centroid_center_of_mass.centre_of_mass[0]),
y_centre_of_mass=numpy.asscalar(centroid_center_of_mass.centre_of_mass[1])
)
else:
connection.execute(
IMAGE_DETAIL.insert(),
galaxy_id=galaxy_id,
x_image=centroid_center_of_mass.dimension_x / 2.0,
y_image=centroid_center_of_mass.dimension_y / 2.0,
x_centroid=-1.0,
y_centroid=-1.0,
x_centre_of_mass=-1.0,
y_centre_of_mass=-1.0
)
add_step_done_id(connection, galaxy_id, STEP_DONE_ID_CENTROID_CENTRE_OF_MASS)
elif verbosity >= 1:
LOG.info('Nothing to do for the galaxy {0}'.format(galaxy_id))
elif verbosity >= 1:
LOG.info('The mask has not been built for the galaxy {0}'.format(galaxy_id))
def process_galaxy(connection, galaxies, verbosity):
"""
Process the galaxies
"""
# Build the tables
build_dynamic_tables(connection)
for galaxy in galaxies:
time_start = time.time()
galaxy_id = galaxy[GALAXY.c.galaxy_id]
steps_done = get_step_done_ids(connection, galaxy_id, STEP_DONE_ID_BUILD_GALAXY_DETAILS, True)
if verbosity:
LOG.info(
'galaxy_id: {0}, galaxy_name: {1}'.format(
galaxy_id,
galaxy[GALAXY.c.name]
)
)
tables_required = get_tables_required(steps_done)
if len(tables_required) > 0:
for table in tables_required:
with connection.begin():
time_start_update = time.time()
create_galaxy_detail = CreateGalaxyDetail(connection, table, galaxy_id)
create_galaxy_detail.calculate_details()
create_galaxy_detail.insert_details()
add_step_done_id(
connection,
galaxy_id,
STEP_DONE_ID_BUILD_GALAXY_DETAILS,
table.name
)
time_end_update = time.time()
LOG.info(
'galaxy_id: {0}, table: {1}, time: {2:.3f}s'.format(
galaxy_id,
table.name,
time_end_update - time_start_update
)
)
time_end = time.time()
LOG.info(
'galaxy_id: {0}, time: {1:.3f}s'.format(
galaxy_id,
time_end - time_start
)
)
def process_galaxy(connection, galaxies):
"""
Process the galaxies
"""
# Build the tables
build_dynamic_tables(connection)
for galaxy in galaxies:
galaxy_id = galaxy[GALAXY.c.galaxy_id]
steps_done = get_step_done_ids(
connection,
galaxy_id,
[STEP_DONE_ID_BUILD_GALAXY_DETAILS, STEP_DONE_ID_NORMALISE_Z_MIN_MAX],
True
)
tables_required = get_tables_required(steps_done)
if len(tables_required) > 0:
time_start = time.time()
for table in tables_required:
with connection.begin():
time_start_update = time.time()
mean_standard = connection.execute(
select([GALAXY_DETAIL]).where(
and_(
GALAXY_DETAIL.c.galaxy_id == galaxy_id,
GALAXY_DETAIL.c.table_name == table.name
)
)
).first()
insert_data = []
if mean_standard is not None:
mean = mean_standard[GALAXY_DETAIL.c.mean]
standard_deviation = mean_standard[GALAXY_DETAIL.c.standard_deviation]
min_value = mean_standard[GALAXY_DETAIL.c.min_value]
max_value = mean_standard[GALAXY_DETAIL.c.max_value]
max_minus_min = max_value - min_value
if max_minus_min is not None and max_minus_min != 0.0 \
and standard_deviation is not None and standard_deviation != 0.0:
if table.name.startswith('original_value__'):
select_statement = select(
[table]
).where(
and_(
table.c.galaxy_id == galaxy_id,
table.c.galaxy_id == MASK_POINT.c.galaxy_id,
table.c.x == MASK_POINT.c.x,
table.c.y == MASK_POINT.c.y,
table.c.value > 0.0
)
)
else:
select_statement = select(
[table]
).where(
and_(
table.c.galaxy_id == galaxy_id,
table.c.galaxy_id == MASK_POINT.c.galaxy_id,
table.c.x == MASK_POINT.c.x,
table.c.y == MASK_POINT.c.y
)
)
for table_data in connection.execute(select_statement):
value = table_data[table.c.value]
z_score = (value - mean) / standard_deviation
insert_data.append(
{
'original_id': table_data[table.c.id],
'value_zscore': z_score,
'value_min_max': (value - min_value) / max_minus_min,
'value_ln': math.log(value - min_value + 1),
'value_softmax': 1.0 / (1.0 + math.exp(-z_score))
}
)
if len(insert_data) > 0:
table_extended = DATA_TABLES_EXTENDED[table.name + '__extended']
connection.execute(table_extended.insert(), insert_data)
add_step_done_id(
connection,
galaxy_id,
STEP_DONE_ID_NORMALISE_Z_MIN_MAX,
table.name
)
time_end_update = time.time()
LOG.info(
'galaxy_id: {0}, table: {1}__extended, time: {2:.3f}s, inserts: {3}'.format(
galaxy_id,
table.name,
time_end_update - time_start_update,
len(insert_data)
)
)
time_end = time.time()
LOG.info('galaxy_id: {0}, time: {1:.3f}s'.format(galaxy_id, time_end - time_start))
def process_galaxy(connection, galaxies, verbosity, profile_name='aws-pogs'):
"""
Process the galaxies
"""
# Build the tables
build_dynamic_tables(connection)
for galaxy in galaxies:
galaxy_id = galaxy[GALAXY.c.galaxy_id]
steps_done = get_step_done_ids(
connection,
galaxy_id,
[STEP_DONE_ID_NO_HDF5_FILE, STEP_DONE_ID_ORIGINAL_VALUES, STEP_DONE_ID_SED_DATA],
True,
True
)
# Work out what SED data is required
data_required_sed = get_data_required_sed(connection, steps_done)
data_required_original = get_data_required_original(steps_done)
if len(data_required_sed) > 0 or len(data_required_original) > 0:
# Copy the file from S3
s3_helper = S3Helper(profile_name=profile_name)
galaxy_id = int(galaxy[GALAXY.c.galaxy_id])
galaxy_name = get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy_id)
s3_name = os.path.join('{0:04d}000'.format(galaxy_id / 1000), galaxy_name) + '.hdf5'
hdf5_file_name = os.path.join(FAST_DISK, galaxy_name) + '.hdf5'
copy_ok = s3_helper.copy_file_to_disk(BUCKET_NAME, s3_name, hdf5_file_name)
if copy_ok:
h5_file = h5py.File(hdf5_file_name, 'r')
LOG.info('Processing SED for name: {0}, run_id: {1}, galaxy_id: {2}'.format(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy_id))
# Do we have anything to do?
if len(data_required_sed) > 0:
# noinspection PyBroadException
try:
# Store the SED fit values
add_sed_data(connection, galaxy_id, h5_file, data_required_sed)
except Exception:
LOG.exception('An exception occurred in process_galaxy processing the SED values')
else:
if verbosity >= 1:
LOG.info('Nothing to add - SED')
if len(data_required_original) > 0:
# noinspection PyBroadException
try:
add_original_data(connection, galaxy_id, h5_file, data_required_original)
except Exception:
LOG.exception('An exception occurred in process_galaxy processing the original values')
else:
if verbosity >= 1:
LOG.info('Nothing to add - Original Data')
# Clean up after ourselves
h5_file.close()
os.remove(hdf5_file_name)
else:
LOG.error('The file for name: {0}, run_id: {1}, galaxy_id: {2} does not exist'.format(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy_id))