# This relies on a ~/.boto file holding the '<aws access key>', '<aws secret key>'
s3_connection = boto.connect_s3()
bucket = s3_connection.get_bucket('boinc.theskynet.org-galaxy_images')
def copy_file(file_name):
key = Key(bucket)
key.key = file_name
key.set_contents_from_filename(file_name)
for galaxy in connection.execute(select([GALAXY])):
LOG.info('Processing %d - %s', galaxy[GALAXY.c.galaxy_id],
galaxy[GALAXY.c.name])
file_prefix_name = galaxy[GALAXY.c.name] + "_" + str(
galaxy[GALAXY.c.version_number])
for i in [1, 2, 3, 4]:
file_name = directory_mod.get_colour_image_path(
WG_IMAGE_DIRECTORY, file_prefix_name, i, False)
copy_file(file_name)
file_name = directory_mod.get_thumbnail_colour_image_path(
WG_IMAGE_DIRECTORY, file_prefix_name, i, False)
copy_file(file_name)
fits_file_name = directory_mod.get_file_path(WG_IMAGE_DIRECTORY,
file_prefix_name + '.fits',
False)
copy_file(fits_file_name)
else:
LOG.info('Building FITS files for all the galaxies')
for galaxy in connection.execute(query):
galaxy__name = galaxy[GALAXY.c.name]
galaxy__version_number = galaxy[GALAXY.c.version_number]
galaxy__dimension_x = galaxy[GALAXY.c.dimension_x]
galaxy__dimension_y = galaxy[GALAXY.c.dimension_y]
LOG.info('Working on galaxy %s (%d) %d x %d', galaxy__name, galaxy__version_number, galaxy__dimension_x, galaxy__dimension_y)
# Do we have an old version
need_to_run = True
# Create the directory to hold the fits files
if galaxy__version_number == 1:
directory = directory_mod.get_file_path(OUTPUT_DIRECTORY, galaxy__name, True)
else:
directory = directory_mod.get_file_path(OUTPUT_DIRECTORY, '{0}_V{1}'.format(galaxy__name, galaxy__version_number), True)
if not os.path.exists(directory):
os.makedirs(directory)
else:
need_to_run = check_need_to_run(directory, galaxy)
# If we don't need to run - don't
if not need_to_run:
continue
# A vagary of PyFits/NumPy is the order of the x & y indexes is reversed
# See page 13 of the PyFITS User Guide
array_best_fit = numpy.empty((galaxy__dimension_y, galaxy__dimension_x, len(IMAGE_NAMES)), dtype=numpy.float)
from image import directory_mod
LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO, format='%(asctime)-15s:' + logging.BASIC_FORMAT)
ENGINE = create_engine(DB_LOGIN)
connection = ENGINE.connect()
# This relies on a ~/.boto file holding the '<aws access key>', '<aws secret key>'
s3_connection = boto.connect_s3()
bucket = s3_connection.get_bucket('boinc.theskynet.org-galaxy_images')
def copy_file(file_name):
key = Key(bucket)
key.key = file_name
key.set_contents_from_filename(file_name)
for galaxy in connection.execute(select([GALAXY])):
LOG.info('Processing %d - %s', galaxy[GALAXY.c.galaxy_id], galaxy[GALAXY.c.name])
file_prefix_name = galaxy[GALAXY.c.name] + "_" + str(galaxy[GALAXY.c.version_number])
for i in [1, 2, 3, 4]:
file_name = directory_mod.get_colour_image_path(WG_IMAGE_DIRECTORY, file_prefix_name, i, False)
copy_file(file_name)
file_name = directory_mod.get_thumbnail_colour_image_path(WG_IMAGE_DIRECTORY, file_prefix_name, i, False)
copy_file(file_name)
fits_file_name = directory_mod.get_file_path(WG_IMAGE_DIRECTORY, file_prefix_name + '.fits', False)
copy_file(fits_file_name)
LOG.info('Building FITS files for all the galaxies')
for galaxy in connection.execute(query):
galaxy__name = galaxy[GALAXY.c.name]
galaxy__version_number = galaxy[GALAXY.c.version_number]
galaxy__dimension_x = galaxy[GALAXY.c.dimension_x]
galaxy__dimension_y = galaxy[GALAXY.c.dimension_y]
LOG.info('Working on galaxy %s (%d) %d x %d', galaxy__name,
galaxy__version_number, galaxy__dimension_x, galaxy__dimension_y)
# Do we have an old version
need_to_run = True
# Create the directory to hold the fits files
if galaxy__version_number == 1:
directory = directory_mod.get_file_path(OUTPUT_DIRECTORY, galaxy__name,
True)
else:
directory = directory_mod.get_file_path(
OUTPUT_DIRECTORY, '{0}_V{1}'.format(galaxy__name,
galaxy__version_number), True)
if not os.path.exists(directory):
os.makedirs(directory)
else:
need_to_run = check_need_to_run(directory, galaxy)
# If we don't need to run - don't
if not need_to_run:
continue
# A vagary of PyFits/NumPy is the order of the x & y indexes is reversed