def regenerated_original_images(galaxy_name, run_id, galaxy_id, s3Helper, connection):
"""
We need to regenerate the image
:param galaxy_name:
:param run_id:
:param galaxy_id:
:return: if we succeed
"""
all_ok = False
# Get the fits file
bucket = s3Helper.get_bucket(get_files_bucket())
galaxy_file_name = get_galaxy_file_name(galaxy_name, run_id, galaxy_id)
key_name = '{0}/{0}.fits'.format(galaxy_name)
key = bucket.get_key(key_name)
if key is None:
LOG.error('The fits file does not seem to exists')
return all_ok
path_name = get_temp_file('fits')
key.get_contents_to_filename(path_name)
# Now regenerate
try:
image = FitsImage(connection)
image.build_image(path_name, galaxy_file_name, galaxy_id, get_galaxy_image_bucket())
all_ok = True
except Exception:
LOG.exception('Major error')
all_ok = False
finally:
os.remove(path_name)
return all_ok
def regenerated_original_images(galaxy_name, run_id, galaxy_id, s3_helper,
connection):
"""
We need to regenerate the image
:param galaxy_name:
:param run_id:
:param galaxy_id:
:return: if we succeed
"""
all_ok = False
# Get the fits file
bucket = s3_helper.get_bucket(get_saved_files_bucket())
galaxy_file_name = get_galaxy_file_name(galaxy_name, run_id, galaxy_id)
key_name = '{0}/{0}.fits'.format(galaxy_name)
key = bucket.get_key(key_name)
if key is None:
LOG.error('The fits file does not seem to exists')
return all_ok
path_name = get_temp_file('fits')
key.get_contents_to_filename(path_name)
# Now regenerate
try:
image = FitsImage(connection)
image.build_image(path_name, galaxy_file_name, galaxy_id,
get_galaxy_image_bucket())
all_ok = True
except Exception:
LOG.exception('Major error')
all_ok = False
finally:
os.remove(path_name)
return all_ok
def process_file(self, registration):
"""
Process a registration.
:param registration:
"""
self._filename = registration[REGISTER.c.filename]
self._galaxy_name = registration[REGISTER.c.galaxy_name]
self._galaxy_type = registration[REGISTER.c.galaxy_type]
self._priority = registration[REGISTER.c.priority]
self._redshift = registration[REGISTER.c.redshift]
self._run_id = registration[REGISTER.c.run_id]
self._sigma = registration[REGISTER.c.sigma]
self._sigma_filename = registration[REGISTER.c.sigma_filename]
# Have we files that we can use for this?
self._rounded_redshift = self._get_rounded_redshift()
if self._rounded_redshift is None:
LOG.error('No models matching the redshift of %.4f', self._redshift)
return 0
self._hdu_list = pyfits.open(self._filename, memmap=True)
self._layer_count = len(self._hdu_list)
# Do we need to open and sort the S/N Ratio file
if self._sigma_filename is not None:
self._sigma = 0.0
self._signal_noise_hdu = pyfits.open(self._sigma_filename, memmap=True)
if self._layer_count != len(self._signal_noise_hdu):
LOG.error('The layer counts do not match %d vs %d', self._layer_count, len(self._signal_noise_hdu))
return 0, 0
else:
self._sigma = float(self._sigma)
self._end_y = self._hdu_list[0].data.shape[0]
self._end_x = self._hdu_list[0].data.shape[1]
LOG.info("Image dimensions: %(x)d x %(y)d x %(z)d => %(pix).2f Mpixels" % {'x': self._end_x, 'y': self._end_y, 'z': self._layer_count, 'pix': self._end_x * self._end_y / 1000000.0})
# Get the flops estimate amd cobblestone factor
run = self._connection.execute(select([RUN]).where(RUN.c.run_id == self._run_id)).first()
self._fpops_est_per_pixel = run[RUN.c.fpops_est]
self._cobblestone_scaling_factor = run[RUN.c.cobblestone_factor]
# Create and save the object
datetime_now = datetime.now()
result = self._connection.execute(GALAXY.insert().values(name=self._galaxy_name,
dimension_x=self._end_x,
dimension_y=self._end_y,
dimension_z=self._layer_count,
redshift=self._redshift,
sigma=self._sigma,
create_time=datetime_now,
image_time=datetime_now,
galaxy_type=self._galaxy_type,
ra_cent=0,
dec_cent=0,
pixel_count=0,
pixels_processed=0,
run_id=self._run_id))
self._galaxy_id = result.inserted_primary_key[0]
LOG.info("Writing %s to database", self._galaxy_name)
# Store the fits header
self._store_fits_header()
# Get the filters we're using for this run and sort the layers
self._get_filters_sort_layers()
# Build the template file we need if necessary
self._build_template_file()
# Copy the filter and model files we need
self._copy_important_files()
# Now break up the galaxy into chunks
self._break_up_galaxy()
self._connection.execute(GALAXY.update().where(GALAXY.c.galaxy_id == self._galaxy_id).values(pixel_count=self._pixel_count))
LOG.info('Building the images')
galaxy_file_name = get_galaxy_file_name(self._galaxy_name, self._run_id, self._galaxy_id)
s3helper = S3Helper()
image = FitsImage(self._connection)
image.build_image(self._filename, galaxy_file_name, self._galaxy_id, get_galaxy_image_bucket())
# Copy the fits file to S3 - renamed to make it unique
bucket_name = get_files_bucket()
s3helper.add_file_to_bucket(bucket_name, get_key_fits(self._galaxy_name, self._run_id, self._galaxy_id), self._filename)
if self._sigma_filename is not None:
s3helper.add_file_to_bucket(bucket_name, get_key_sigma_fits(self._galaxy_name, self._run_id, self._galaxy_id), self._sigma_filename)
return self._work_units_added, self._pixel_count