# This relies on a ~/.boto file holding the '<aws access key>', '<aws secret key>' s3_connection = boto.connect_s3() bucket = s3_connection.get_bucket('boinc.theskynet.org-galaxy_images') def copy_file(file_name): key = Key(bucket) key.key = file_name key.set_contents_from_filename(file_name) for galaxy in connection.execute(select([GALAXY])): LOG.info('Processing %d - %s', galaxy[GALAXY.c.galaxy_id], galaxy[GALAXY.c.name]) file_prefix_name = galaxy[GALAXY.c.name] + "_" + str( galaxy[GALAXY.c.version_number]) for i in [1, 2, 3, 4]: file_name = directory_mod.get_colour_image_path( WG_IMAGE_DIRECTORY, file_prefix_name, i, False) copy_file(file_name) file_name = directory_mod.get_thumbnail_colour_image_path( WG_IMAGE_DIRECTORY, file_prefix_name, i, False) copy_file(file_name) fits_file_name = directory_mod.get_file_path(WG_IMAGE_DIRECTORY, file_prefix_name + '.fits', False) copy_file(fits_file_name)
else: LOG.info('Building FITS files for all the galaxies') for galaxy in connection.execute(query): galaxy__name = galaxy[GALAXY.c.name] galaxy__version_number = galaxy[GALAXY.c.version_number] galaxy__dimension_x = galaxy[GALAXY.c.dimension_x] galaxy__dimension_y = galaxy[GALAXY.c.dimension_y] LOG.info('Working on galaxy %s (%d) %d x %d', galaxy__name, galaxy__version_number, galaxy__dimension_x, galaxy__dimension_y) # Do we have an old version need_to_run = True # Create the directory to hold the fits files if galaxy__version_number == 1: directory = directory_mod.get_file_path(OUTPUT_DIRECTORY, galaxy__name, True) else: directory = directory_mod.get_file_path(OUTPUT_DIRECTORY, '{0}_V{1}'.format(galaxy__name, galaxy__version_number), True) if not os.path.exists(directory): os.makedirs(directory) else: need_to_run = check_need_to_run(directory, galaxy) # If we don't need to run - don't if not need_to_run: continue # A vagary of PyFits/NumPy is the order of the x & y indexes is reversed # See page 13 of the PyFITS User Guide array_best_fit = numpy.empty((galaxy__dimension_y, galaxy__dimension_x, len(IMAGE_NAMES)), dtype=numpy.float)
from image import directory_mod LOG = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO, format='%(asctime)-15s:' + logging.BASIC_FORMAT) ENGINE = create_engine(DB_LOGIN) connection = ENGINE.connect() # This relies on a ~/.boto file holding the '<aws access key>', '<aws secret key>' s3_connection = boto.connect_s3() bucket = s3_connection.get_bucket('boinc.theskynet.org-galaxy_images') def copy_file(file_name): key = Key(bucket) key.key = file_name key.set_contents_from_filename(file_name) for galaxy in connection.execute(select([GALAXY])): LOG.info('Processing %d - %s', galaxy[GALAXY.c.galaxy_id], galaxy[GALAXY.c.name]) file_prefix_name = galaxy[GALAXY.c.name] + "_" + str(galaxy[GALAXY.c.version_number]) for i in [1, 2, 3, 4]: file_name = directory_mod.get_colour_image_path(WG_IMAGE_DIRECTORY, file_prefix_name, i, False) copy_file(file_name) file_name = directory_mod.get_thumbnail_colour_image_path(WG_IMAGE_DIRECTORY, file_prefix_name, i, False) copy_file(file_name) fits_file_name = directory_mod.get_file_path(WG_IMAGE_DIRECTORY, file_prefix_name + '.fits', False) copy_file(fits_file_name)
LOG.info('Building FITS files for all the galaxies') for galaxy in connection.execute(query): galaxy__name = galaxy[GALAXY.c.name] galaxy__version_number = galaxy[GALAXY.c.version_number] galaxy__dimension_x = galaxy[GALAXY.c.dimension_x] galaxy__dimension_y = galaxy[GALAXY.c.dimension_y] LOG.info('Working on galaxy %s (%d) %d x %d', galaxy__name, galaxy__version_number, galaxy__dimension_x, galaxy__dimension_y) # Do we have an old version need_to_run = True # Create the directory to hold the fits files if galaxy__version_number == 1: directory = directory_mod.get_file_path(OUTPUT_DIRECTORY, galaxy__name, True) else: directory = directory_mod.get_file_path( OUTPUT_DIRECTORY, '{0}_V{1}'.format(galaxy__name, galaxy__version_number), True) if not os.path.exists(directory): os.makedirs(directory) else: need_to_run = check_need_to_run(directory, galaxy) # If we don't need to run - don't if not need_to_run: continue # A vagary of PyFits/NumPy is the order of the x & y indexes is reversed