def process_ami(modulus, remainder):
"""
We're running on the AMI instance - so actually do the work
Find the files and move them to S3
:return:
"""
# Connect to the database - the login string is set in the database package
# Start the shutdown signal poller to check when this instance must close
start_poll()
engine = create_engine(DB_LOGIN)
connection = engine.connect()
try:
# Check the processed data
try:
LOG.info('Updating state information')
processed_data(connection, modulus, remainder)
except Exception:
LOG.exception('processed_data(): an exception occurred')
# Store files
try:
LOG.info('Storing files')
store_files(connection, modulus, remainder)
except Exception:
LOG.exception('store_files(): an exception occurred')
# Delete galaxy data - commits happen inside
try:
LOG.info('Deleting galaxy data')
delete_galaxy_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_galaxy_data(): an exception occurred')
# Delete register data - commits happen inside
try:
LOG.info('Deleting register data')
delete_register_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_register_data(): an exception occurred')
# Archive to HDF5
try:
LOG.info('Archiving to HDF5')
archive_to_hdf5(connection, modulus, remainder)
except Exception:
LOG.exception('archive_to_hdf5(): an exception occurred')
except SystemExit:
LOG.info(
'Spot Instance Terminate Notice received, archive_task is shutting down'
)
finally:
connection.close()
def process_ami(modulus, remainder):
"""
We're running on the AMI instance - so actually do the work
Find the files and move them to S3
:return:
"""
# Connect to the database - the login string is set in the database package
# Start the shutdown signal poller to check when this instance must close
start_poll()
engine = create_engine(DB_LOGIN)
connection = engine.connect()
try:
# Check the processed data
try:
LOG.info('Updating state information')
processed_data(connection, modulus, remainder)
except Exception:
LOG.exception('processed_data(): an exception occurred')
# Store files
try:
LOG.info('Storing files')
store_files(connection, modulus, remainder)
except Exception:
LOG.exception('store_files(): an exception occurred')
# Delete galaxy data - commits happen inside
try:
LOG.info('Deleting galaxy data')
delete_galaxy_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_galaxy_data(): an exception occurred')
# Delete register data - commits happen inside
try:
LOG.info('Deleting register data')
delete_register_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_register_data(): an exception occurred')
# Archive to HDF5
try:
LOG.info('Archiving to HDF5')
archive_to_hdf5(connection, modulus, remainder)
except Exception:
LOG.exception('archive_to_hdf5(): an exception occurred')
except SystemExit:
LOG.info('Spot Instance Terminate Notice received, archive_task is shutting down')
finally:
connection.close()
def build_png_image_ami():
"""
Build the images
:return:
"""
# First check the galaxy exists in the database
engine = create_engine(DB_LOGIN)
connection = engine.connect()
try:
query = select([GALAXY]).distinct().where(and_(AREA.c.galaxy_id == GALAXY.c.galaxy_id, AREA.c.update_time >= GALAXY.c.image_time))
galaxy_count = 0
s3helper = S3Helper()
bucket_name = get_galaxy_image_bucket()
# Start the shutdown signal poller to check when this instance must close
start_poll()
galaxy_list = []
for galaxy in connection.execute(query):
galaxy_list.append(galaxy)
total_galaxies = len(galaxy_list)
processed_galaxies = 0
processed_print_point = 50
for galaxy in galaxy_list:
if processed_galaxies == processed_print_point:
LOG.info('{0} out of {1} galaxies processed'.format(processed_galaxies, total_galaxies))
processed_print_point += 50
processed_galaxies += 1
LOG.info('Working on galaxy %s', galaxy[GALAXY.c.name])
array = numpy.empty((galaxy[GALAXY.c.dimension_y], galaxy[GALAXY.c.dimension_x], len(PNG_IMAGE_NAMES)), dtype=numpy.float)
array.fill(numpy.NaN)
# Return the rows
pixel_count = 0
pixels_processed = 0
for row in connection.execute(select([PIXEL_RESULT]).where((PIXEL_RESULT.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]) and PIXEL_RESULT.c.x > -1)):
row__x = row[PIXEL_RESULT.c.x]
row__y = row[PIXEL_RESULT.c.y]
pixel_count += 1
if row[PIXEL_RESULT.c.workunit_id] is not None:
pixels_processed += 1
# Defend against bad values
if row[PIXEL_RESULT.c.mu] is not None:
array[row__y, row__x, 0] = row[PIXEL_RESULT.c.mu]
if row[PIXEL_RESULT.c.m] is not None:
array[row__y, row__x, 1] = row[PIXEL_RESULT.c.m]
if row[PIXEL_RESULT.c.ldust] is not None:
array[row__y, row__x, 2] = row[PIXEL_RESULT.c.ldust]
if row[PIXEL_RESULT.c.sfr] is not None:
# the SFR is a log
array[row__y, row__x, 3] = math.pow(10, row[PIXEL_RESULT.c.sfr])
connection.execute(GALAXY.update()
.where(GALAXY.c.galaxy_id == galaxy[GALAXY.c.galaxy_id])
.values(image_time=datetime.datetime.now(), pixel_count=pixel_count, pixels_processed=pixels_processed))
galaxy_count += 1
# Now write the files
black_rgb = (0, 0, 0)
for name in PNG_IMAGE_NAMES:
value = 0
height = galaxy[GALAXY.c.dimension_y]
width = galaxy[GALAXY.c.dimension_x]
idx = 0
if name == 'mu':
idx = 0
elif name == 'm':
idx = 1
elif name == 'ldust':
idx = 2
elif name == 'sfr':
idx = 3
values = []
for x in range(0, width - 1):
for y in range(0, height - 1):
value = array[y, x, idx]
if not math.isnan(value) and value > 0:
values.append(value)
values.sort()
if len(values) > 1000:
top_count = int(len(values) * 0.005)
top_value = values[len(values) - top_count]
elif len(values) > 0:
top_value = values[len(values) - 1]
else:
top_value = 1
if len(values) > 1:
median_value = values[int(len(values) / 2)]
elif len(values) > 0:
median_value = values[0]
else:
median_value = 1
sigma = 1 / median_value
multiplier = 255.0 / math.asinh(top_value * sigma)
image = Image.new("RGB", (width, height), black_rgb)
for x in range(0, width - 1):
for y in range(0, height - 1):
value = array[y, x, idx]
if not math.isnan(value) and value > 0:
value = int(math.asinh(value * sigma) * multiplier)
if value > 255:
value = 255
red = FIRE_R[value]
green = FIRE_G[value]
blue = FIRE_B[value]
image.putpixel((x, height - y - 1), (red, green, blue))
file_name = '{0}/image.png'.format(POGS_TMP)
image.save(file_name)
s3helper.add_file_to_bucket(bucket_name,
get_build_png_name(get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id]),
name),
file_name)
if shutdown() is True:
LOG.info('Spot Instance Terminate Notice received, build_png_image is shutting down')
break
except:
LOG.exception('An exception occurred.')
finally:
connection.close()
LOG.info('Built images for %d galaxies', galaxy_count)
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
from utils.shutdown_detection import start_poll, shutdown
import time
def main():
i = 0
try:
while True:
print "I am doing things! {0}".format(i)
i += 1
time.sleep(1)
if shutdown() is True:
raise SystemExit
except SystemExit:
print 'Exiting...'
if __name__ == "__main__":
start_poll()
main()
