def _save_results(self, connection, map_pixel_results, list_insert_filters,
list_pixel_parameters, map_pixel_histograms):
"""
Add the pixel to the database
"""
if self._pxresult_id is not None and not self.noinsert:
# Update the filters
connection.execute(PIXEL_RESULT.update().where(
PIXEL_RESULT.c.pxresult_id == self._pxresult_id).values(
map_pixel_results))
connection.execute(PIXEL_FILTER.insert(), list_insert_filters)
# Because I need to get the PK back we have to do each one separately
for pixel_parameter in list_pixel_parameters:
result = connection.execute(PIXEL_PARAMETER.insert(),
pixel_parameter)
id = result.inserted_primary_key
# Add the ID to the list
list_pixel_histograms = map_pixel_histograms[
pixel_parameter['parameter_name_id']]
for map_values in list_pixel_histograms:
map_values['pxparameter_id'] = id[0]
connection.execute(PIXEL_HISTOGRAM.insert(),
list_pixel_histograms)
def _save_results(self, connection, map_pixel_results, list_insert_filters, list_pixel_parameters, map_pixel_histograms):
"""
Add the pixel to the database
"""
if self._pxresult_id is not None and not self.noinsert:
# Update the filters
connection.execute(PIXEL_RESULT.update().where(PIXEL_RESULT.c.pxresult_id == self._pxresult_id).values(map_pixel_results))
connection.execute(PIXEL_FILTER.insert(), list_insert_filters)
# Because I need to get the PK back we have to do each one separately
for pixel_parameter in list_pixel_parameters:
result = connection.execute(PIXEL_PARAMETER.insert(), pixel_parameter)
id = result.inserted_primary_key
# Add the ID to the list
list_pixel_histograms = map_pixel_histograms[pixel_parameter['parameter_name_id']]
for map_values in list_pixel_histograms:
map_values['pxparameter_id'] = id[0]
connection.execute(PIXEL_HISTOGRAM.insert(), list_pixel_histograms)
if pixel_parameter[PIXEL_PARAMETER.c.high_prob_bin] is None:
pixel_histogram = connection.execute(select([PIXEL_HISTOGRAM]).where(
and_(PIXEL_HISTOGRAM.c.pxresult_id == row[PIXEL_RESULT.c.pxresult_id],
PIXEL_HISTOGRAM.c.pxparameter_id == pixel_parameter[PIXEL_PARAMETER.c.pxparameter_id],
PIXEL_HISTOGRAM.c.hist_value ==
select([func.max(PIXEL_HISTOGRAM.c.hist_value)]).
where(and_(PIXEL_HISTOGRAM.c.pxresult_id == row[PIXEL_RESULT.c.pxresult_id],
PIXEL_HISTOGRAM.c.pxparameter_id == pixel_parameter[PIXEL_PARAMETER.c.pxparameter_id]))))).first()
if pixel_histogram is not None:
array_highest_prob_bin_v[row__y, row__x, index] = pixel_histogram[PIXEL_HISTOGRAM.c.x_axis]
# Update the database
connection.execute(PIXEL_PARAMETER.update().
where(PIXEL_PARAMETER.c.pxparameter_id == pixel_parameter[PIXEL_PARAMETER.c.pxparameter_id]).
values(high_prob_bin = pixel_histogram[PIXEL_HISTOGRAM.c.x_axis]))
else:
array_highest_prob_bin_v[row__y, row__x, index] = pixel_parameter[PIXEL_PARAMETER.c.high_prob_bin]
# Commit any changes
transaction.commit()
name_count = 0
utc_now = datetime.utcnow().strftime('%Y-%m-%dT%H:%m:%S')
for name in IMAGE_NAMES:
hdu = pyfits.PrimaryHDU(array_best_fit[:,:,name_count])
hdu_list = pyfits.HDUList([hdu])
# Write the header
hdu_list[0].header.update('MAGPHYST', name, 'MAGPHYS Parameter')
PIXEL_HISTOGRAM.c.pxresult_id == row[
PIXEL_RESULT.c.pxresult_id],
PIXEL_HISTOGRAM.c.pxparameter_id ==
pixel_parameter[
PIXEL_PARAMETER.c.pxparameter_id])
)))).first()
if pixel_histogram is not None:
array_highest_prob_bin_v[row__y, row__x,
index] = pixel_histogram[
PIXEL_HISTOGRAM.c.x_axis]
# Update the database
connection.execute(PIXEL_PARAMETER.update().where(
PIXEL_PARAMETER.c.pxparameter_id ==
pixel_parameter[
PIXEL_PARAMETER.c.pxparameter_id]).values(
high_prob_bin=pixel_histogram[
PIXEL_HISTOGRAM.c.x_axis]))
else:
array_highest_prob_bin_v[
row__y, row__x, index] = pixel_parameter[
PIXEL_PARAMETER.c.high_prob_bin]
# Commit any changes
transaction.commit()
name_count = 0
utc_now = datetime.utcnow().strftime('%Y-%m-%dT%H:%m:%S')
for name in IMAGE_NAMES:
hdu = pyfits.PrimaryHDU(array_best_fit[:, :, name_count])
if False:
for area_id1 in connection_aws.execute(
select([
AREA.c.area_id
]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(
AREA.c.area_id)):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(
select([PIXEL_RESULT.c.pxresult_id
]).where(PIXEL_RESULT.c.area_id ==
area_id1[0]).order_by(
PIXEL_RESULT.c.pxresult_id)):
deleted_pixel_count += 1
connection_aws.execute(PIXEL_FILTER.delete().where(
PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_PARAMETER.delete().where(
PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_HISTOGRAM.delete().where(
PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_RESULT.delete().where(
PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
for image_filters_used in connection_aws.execute(select([IMAGE_FILTERS_USED]).where(IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id_aws)):
insert_only(IMAGE_FILTERS_USED, image_filters_used, connection_pleiades)
transaction_pleiades.commit()
copy_end_time = time.time()
# Now we can delete the bits we don't need
deleted_area_count = 0
deleted_pixel_count = 0
if False:
for area_id1 in connection_aws.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(AREA.c.area_id)):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.area_id == area_id1[0]).order_by(PIXEL_RESULT.c.pxresult_id)):
deleted_pixel_count += 1
connection_aws.execute(PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
LOG.info('Copied %d areas %d pixels.', area_count, pixel_count)
LOG.info('Deleted %d areas %d pixels.', deleted_area_count, deleted_pixel_count)
if False:
for area_id1 in connection_aws.execute(
select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(AREA.c.area_id)
):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(
select([PIXEL_RESULT.c.pxresult_id])
.where(PIXEL_RESULT.c.area_id == area_id1[0])
.order_by(PIXEL_RESULT.c.pxresult_id)
):
deleted_pixel_count += 1
connection_aws.execute(
PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0])
)
connection_aws.execute(
PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0])
)
connection_aws.execute(
PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0])
)
connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
