def process_ami(modulus, remainder): """ We're running on the AMI instance - so actually do the work Find the files and move them to S3 :return: """ # Connect to the database - the login string is set in the database package # Start the shutdown signal poller to check when this instance must close start_poll() engine = create_engine(DB_LOGIN) connection = engine.connect() try: # Check the processed data try: LOG.info('Updating state information') processed_data(connection, modulus, remainder) except Exception: LOG.exception('processed_data(): an exception occurred') # Store files try: LOG.info('Storing files') store_files(connection, modulus, remainder) except Exception: LOG.exception('store_files(): an exception occurred') # Delete galaxy data - commits happen inside try: LOG.info('Deleting galaxy data') delete_galaxy_data(connection, modulus, remainder) except Exception: LOG.exception('delete_galaxy_data(): an exception occurred') # Delete register data - commits happen inside try: LOG.info('Deleting register data') delete_register_data(connection, modulus, remainder) except Exception: LOG.exception('delete_register_data(): an exception occurred') # Archive to HDF5 try: LOG.info('Archiving to HDF5') archive_to_hdf5(connection, modulus, remainder) except Exception: LOG.exception('archive_to_hdf5(): an exception occurred') except SystemExit: LOG.info( 'Spot Instance Terminate Notice received, archive_task is shutting down' ) finally: connection.close()
def process_ami(modulus, remainder): """ We're running on the AMI instance - so actually do the work Find the files and move them to S3 :return: """ # Connect to the database - the login string is set in the database package # Start the shutdown signal poller to check when this instance must close start_poll() engine = create_engine(DB_LOGIN) connection = engine.connect() try: # Check the processed data try: LOG.info('Updating state information') processed_data(connection, modulus, remainder) except Exception: LOG.exception('processed_data(): an exception occurred') # Store files try: LOG.info('Storing files') store_files(connection, modulus, remainder) except Exception: LOG.exception('store_files(): an exception occurred') # Delete galaxy data - commits happen inside try: LOG.info('Deleting galaxy data') delete_galaxy_data(connection, modulus, remainder) except Exception: LOG.exception('delete_galaxy_data(): an exception occurred') # Delete register data - commits happen inside try: LOG.info('Deleting register data') delete_register_data(connection, modulus, remainder) except Exception: LOG.exception('delete_register_data(): an exception occurred') # Archive to HDF5 try: LOG.info('Archiving to HDF5') archive_to_hdf5(connection, modulus, remainder) except Exception: LOG.exception('archive_to_hdf5(): an exception occurred') except SystemExit: LOG.info('Spot Instance Terminate Notice received, archive_task is shutting down') finally: connection.close()
def build_png_image_ami(): """ Build the images :return: """ # First check the galaxy exists in the database engine = create_engine(DB_LOGIN) connection = engine.connect() try: query = select([GALAXY]).distinct().where(and_(AREA.c.galaxy_id == GALAXY.c.galaxy_id, AREA.c.update_time >= GALAXY.c.image_time)) galaxy_count = 0 s3helper = S3Helper() bucket_name = get_galaxy_image_bucket() # Start the shutdown signal poller to check when this instance must close start_poll() galaxy_list = [] for galaxy in connection.execute(query): galaxy_list.append(galaxy) total_galaxies = len(galaxy_list) processed_galaxies = 0 processed_print_point = 50 for galaxy in galaxy_list: if processed_galaxies == processed_print_point: LOG.info('{0} out of {1} galaxies processed'.format(processed_galaxies, total_galaxies)) processed_print_point += 50 processed_galaxies += 1 LOG.info('Working on galaxy %s', galaxy[GALAXY.c.name]) array = numpy.empty((galaxy[GALAXY.c.dimension_y], galaxy[GALAXY.c.dimension_x], len(PNG_IMAGE_NAMES)), dtype=numpy.float) array.fill(numpy.NaN) # Return the rows pixel_count = 0 pixels_processed = 0 for row in connection.execute(select([PIXEL_RESULT]).where((PIXEL_RESULT.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]) and PIXEL_RESULT.c.x > -1)): row__x = row[PIXEL_RESULT.c.x] row__y = row[PIXEL_RESULT.c.y] pixel_count += 1 if row[PIXEL_RESULT.c.workunit_id] is not None: pixels_processed += 1 # Defend against bad values if row[PIXEL_RESULT.c.mu] is not None: array[row__y, row__x, 0] = row[PIXEL_RESULT.c.mu] if row[PIXEL_RESULT.c.m] is not None: array[row__y, row__x, 1] = row[PIXEL_RESULT.c.m] if row[PIXEL_RESULT.c.ldust] is not None: array[row__y, row__x, 2] = row[PIXEL_RESULT.c.ldust] if row[PIXEL_RESULT.c.sfr] is not None: # the SFR is a log array[row__y, row__x, 3] = math.pow(10, row[PIXEL_RESULT.c.sfr]) connection.execute(GALAXY.update() .where(GALAXY.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]) .values(image_time=datetime.datetime.now(), pixel_count=pixel_count, pixels_processed=pixels_processed)) galaxy_count += 1 # Now write the files black_rgb = (0, 0, 0) for name in PNG_IMAGE_NAMES: value = 0 height = galaxy[GALAXY.c.dimension_y] width = galaxy[GALAXY.c.dimension_x] idx = 0 if name == 'mu': idx = 0 elif name == 'm': idx = 1 elif name == 'ldust': idx = 2 elif name == 'sfr': idx = 3 values = [] for x in range(0, width - 1): for y in range(0, height - 1): value = array[y, x, idx] if not math.isnan(value) and value > 0: values.append(value) values.sort() if len(values) > 1000: top_count = int(len(values) * 0.005) top_value = values[len(values) - top_count] elif len(values) > 0: top_value = values[len(values) - 1] else: top_value = 1 if len(values) > 1: median_value = values[int(len(values) / 2)] elif len(values) > 0: median_value = values[0] else: median_value = 1 sigma = 1 / median_value multiplier = 255.0 / math.asinh(top_value * sigma) image = Image.new("RGB", (width, height), black_rgb) for x in range(0, width - 1): for y in range(0, height - 1): value = array[y, x, idx] if not math.isnan(value) and value > 0: value = int(math.asinh(value * sigma) * multiplier) if value > 255: value = 255 red = FIRE_R[value] green = FIRE_G[value] blue = FIRE_B[value] image.putpixel((x, height - y - 1), (red, green, blue)) file_name = '{0}/image.png'.format(POGS_TMP) image.save(file_name) s3helper.add_file_to_bucket(bucket_name, get_build_png_name(get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id]), name), file_name) if shutdown() is True: LOG.info('Spot Instance Terminate Notice received, build_png_image is shutting down') break except: LOG.exception('An exception occurred.') finally: connection.close() LOG.info('Built images for %d galaxies', galaxy_count)
# Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, # MA 02111-1307 USA # from utils.shutdown_detection import start_poll, shutdown import time def main(): i = 0 try: while True: print "I am doing things! {0}".format(i) i += 1 time.sleep(1) if shutdown() is True: raise SystemExit except SystemExit: print 'Exiting...' if __name__ == "__main__": start_poll() main()