def migrate_image_files(connection, image_bucket_name, file_bucket_name, s3helper):
for file_name in glob.glob('/home/ec2-user/galaxyImages/*/*'):
(name, version, extension) = get_name_version(file_name)
# Only migrate the images in the original galaxy is still in the database
galaxy = connection.execute(select([GALAXY]).where(and_(GALAXY.c.name == name, GALAXY.c.version_number == version))).first()
if galaxy is not None:
if extension == '.fits':
add_file_to_bucket1(file_bucket_name, get_key_fits(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id]), file_name, s3helper)
else:
galaxy_key = get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id])
if file_name.endswith('_tn_colour_1.png'):
add_file_to_bucket1(image_bucket_name, get_thumbnail_colour_image_key(galaxy_key, 1), file_name, s3helper)
elif file_name.endswith('_colour_1.png'):
add_file_to_bucket1(image_bucket_name, get_colour_image_key(galaxy_key, 1), file_name, s3helper)
elif file_name.endswith('_colour_2.png'):
add_file_to_bucket1(image_bucket_name, get_colour_image_key(galaxy_key, 2), file_name, s3helper)
elif file_name.endswith('_colour_3.png'):
add_file_to_bucket1(image_bucket_name, get_colour_image_key(galaxy_key, 3), file_name, s3helper)
elif file_name.endswith('_colour_4.png'):
add_file_to_bucket1(image_bucket_name, get_colour_image_key(galaxy_key, 4), file_name, s3helper)
elif file_name.endswith('_mu.png'):
add_file_to_bucket1(image_bucket_name, get_build_png_name(galaxy_key, 'mu'), file_name, s3helper)
elif file_name.endswith('_m.png'):
add_file_to_bucket1(image_bucket_name, get_build_png_name(galaxy_key, 'm'), file_name, s3helper)
elif file_name.endswith('_ldust.png'):
add_file_to_bucket1(image_bucket_name, get_build_png_name(galaxy_key, 'ldust'), file_name, s3helper)
elif file_name.endswith('_sfr.png'):
add_file_to_bucket1(image_bucket_name, get_build_png_name(galaxy_key, 'sfr'), file_name, s3helper)
def process_file(self, registration):
"""
Process a registration.
:param registration:
"""
self._filename = registration[REGISTER.c.filename]
self._galaxy_name = registration[REGISTER.c.galaxy_name]
self._galaxy_type = registration[REGISTER.c.galaxy_type]
self._priority = registration[REGISTER.c.priority]
self._redshift = registration[REGISTER.c.redshift]
self._run_id = registration[REGISTER.c.run_id]
self._sigma = registration[REGISTER.c.sigma]
self._sigma_filename = registration[REGISTER.c.sigma_filename]
# Have we files that we can use for this?
self._rounded_redshift = self._get_rounded_redshift()
if self._rounded_redshift is None:
LOG.error('No models matching the redshift of %.4f', self._redshift)
return 0
self._hdu_list = pyfits.open(self._filename, memmap=True)
self._layer_count = len(self._hdu_list)
# Do we need to open and sort the S/N Ratio file
if self._sigma_filename is not None:
self._sigma = 0.0
self._signal_noise_hdu = pyfits.open(self._sigma_filename, memmap=True)
if self._layer_count != len(self._signal_noise_hdu):
LOG.error('The layer counts do not match %d vs %d', self._layer_count, len(self._signal_noise_hdu))
return 0, 0
else:
self._sigma = float(self._sigma)
self._end_y = self._hdu_list[0].data.shape[0]
self._end_x = self._hdu_list[0].data.shape[1]
LOG.info("Image dimensions: %(x)d x %(y)d x %(z)d => %(pix).2f Mpixels" % {'x': self._end_x, 'y': self._end_y, 'z': self._layer_count, 'pix': self._end_x * self._end_y / 1000000.0})
# Get the flops estimate amd cobblestone factor
run = self._connection.execute(select([RUN]).where(RUN.c.run_id == self._run_id)).first()
self._fpops_est_per_pixel = run[RUN.c.fpops_est]
self._cobblestone_scaling_factor = run[RUN.c.cobblestone_factor]
# Create and save the object
datetime_now = datetime.now()
result = self._connection.execute(GALAXY.insert().values(name=self._galaxy_name,
dimension_x=self._end_x,
dimension_y=self._end_y,
dimension_z=self._layer_count,
redshift=self._redshift,
sigma=self._sigma,
create_time=datetime_now,
image_time=datetime_now,
galaxy_type=self._galaxy_type,
ra_cent=0,
dec_cent=0,
pixel_count=0,
pixels_processed=0,
run_id=self._run_id))
self._galaxy_id = result.inserted_primary_key[0]
LOG.info("Writing %s to database", self._galaxy_name)
# Store the fits header
self._store_fits_header()
# Get the filters we're using for this run and sort the layers
self._get_filters_sort_layers()
# Build the template file we need if necessary
self._build_template_file()
# Copy the filter and model files we need
self._copy_important_files()
# Now break up the galaxy into chunks
self._break_up_galaxy()
self._connection.execute(GALAXY.update().where(GALAXY.c.galaxy_id == self._galaxy_id).values(pixel_count=self._pixel_count))
LOG.info('Building the images')
galaxy_file_name = get_galaxy_file_name(self._galaxy_name, self._run_id, self._galaxy_id)
s3helper = S3Helper()
image = FitsImage(self._connection)
image.build_image(self._filename, galaxy_file_name, self._galaxy_id, get_galaxy_image_bucket())
# Copy the fits file to S3 - renamed to make it unique
bucket_name = get_files_bucket()
s3helper.add_file_to_bucket(bucket_name, get_key_fits(self._galaxy_name, self._run_id, self._galaxy_id), self._filename)
if self._sigma_filename is not None:
s3helper.add_file_to_bucket(bucket_name, get_key_sigma_fits(self._galaxy_name, self._run_id, self._galaxy_id), self._sigma_filename)
return self._work_units_added, self._pixel_count