LOG.info('Range from %s to %s', list[0], list[1]) galaxy_ids = range(int(list[0]), int(list[1]) + 1) else: galaxy_ids = args['galaxy_id'] for galaxy_id_str in galaxy_ids: galaxy_id1 = int(galaxy_id_str) galaxy = connection.execute(select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id1)).first() if galaxy is None: LOG.info('Error: Galaxy with galaxy_id of %d was not found', galaxy_id1) else: # Have we got work units out there for this galaxy? LOG.info('Working on galaxy %s (%d)', galaxy[GALAXY.c.name], galaxy[GALAXY.c.version_number]) deleted_galaxy = 0 transaction = connection.begin() count = 0 for pxresult_id in connection.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]).order_by(PIXEL_RESULT.c.pxresult_id)): if not (count % 1000): LOG.info('Deleting low pixel_histogram values from galaxy {0} pixel {1} : Deleted total {2} galaxy {3}'.format(galaxy[GALAXY.c.galaxy_id], pxresult_id[0], deleted_total, deleted_galaxy)) result_proxy = connection.execute(PIXEL_HISTOGRAM.delete().where(and_(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id[0], PIXEL_HISTOGRAM.c.hist_value < MIN_HIST_VALUE))) deleted_total += result_proxy.rowcount deleted_galaxy += result_proxy.rowcount count += 1 transaction.commit() LOG.info('Removed %d really small histogram values from %s (%d)', deleted_galaxy, galaxy.name, galaxy.version_number) connection.close() LOG.info('Done - removed %d really small histogram values.', deleted_total)
select([ AREA.c.area_id ]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by( AREA.c.area_id)): deleted_area_count += 1 for pxresult_id1 in connection_aws.execute( select([PIXEL_RESULT.c.pxresult_id ]).where(PIXEL_RESULT.c.area_id == area_id1[0]).order_by( PIXEL_RESULT.c.pxresult_id)): deleted_pixel_count += 1 connection_aws.execute(PIXEL_FILTER.delete().where( PIXEL_FILTER.c.pxresult_id == pxresult_id1[0])) connection_aws.execute(PIXEL_PARAMETER.delete().where( PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0])) connection_aws.execute(PIXEL_HISTOGRAM.delete().where( PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0])) connection_aws.execute(PIXEL_RESULT.delete().where( PIXEL_RESULT.c.area_id == area_id1[0])) transaction_aws.commit() transaction_aws = connection_aws.begin() # Give the rest of the world a chance to access the database time.sleep(1) transaction_aws.commit() end_time = time.time() LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1) LOG.info('Copied %d areas %d pixels.', area_count, pixel_count) LOG.info('Deleted %d areas %d pixels.', deleted_area_count,
): deleted_area_count += 1 for pxresult_id1 in connection_aws.execute( select([PIXEL_RESULT.c.pxresult_id]) .where(PIXEL_RESULT.c.area_id == area_id1[0]) .order_by(PIXEL_RESULT.c.pxresult_id) ): deleted_pixel_count += 1 connection_aws.execute( PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]) ) connection_aws.execute( PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]) ) connection_aws.execute( PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]) ) connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0])) transaction_aws.commit() transaction_aws = connection_aws.begin() # Give the rest of the world a chance to access the database time.sleep(1) transaction_aws.commit() end_time = time.time() LOG.info("Galaxy with galaxy_id of %d was archived.", galaxy_id1) LOG.info("Copied %d areas %d pixels.", area_count, pixel_count) LOG.info("Deleted %d areas %d pixels.", deleted_area_count, deleted_pixel_count)
insert_only(IMAGE_FILTERS_USED, image_filters_used, connection_pleiades) transaction_pleiades.commit() copy_end_time = time.time() # Now we can delete the bits we don't need deleted_area_count = 0 deleted_pixel_count = 0 if False: for area_id1 in connection_aws.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(AREA.c.area_id)): deleted_area_count += 1 for pxresult_id1 in connection_aws.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.area_id == area_id1[0]).order_by(PIXEL_RESULT.c.pxresult_id)): deleted_pixel_count += 1 connection_aws.execute(PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0])) connection_aws.execute(PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0])) connection_aws.execute(PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0])) connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0])) transaction_aws.commit() transaction_aws = connection_aws.begin() # Give the rest of the world a chance to access the database time.sleep(1) transaction_aws.commit() end_time = time.time() LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1) LOG.info('Copied %d areas %d pixels.', area_count, pixel_count) LOG.info('Deleted %d areas %d pixels.', deleted_area_count, deleted_pixel_count) total_time = end_time - start_time
# Have we got work units out there for this galaxy? LOG.info('Working on galaxy %s (%d)', galaxy[GALAXY.c.name], galaxy[GALAXY.c.version_number]) deleted_galaxy = 0 transaction = connection.begin() count = 0 for pxresult_id in connection.execute( select([ PIXEL_RESULT.c.pxresult_id ]).where(PIXEL_RESULT.c.galaxy_id == galaxy[ GALAXY.c.galaxy_id]).order_by(PIXEL_RESULT.c.pxresult_id)): if not (count % 1000): LOG.info( 'Deleting low pixel_histogram values from galaxy {0} pixel {1} : Deleted total {2} galaxy {3}' .format(galaxy[GALAXY.c.galaxy_id], pxresult_id[0], deleted_total, deleted_galaxy)) result_proxy = connection.execute(PIXEL_HISTOGRAM.delete().where( and_(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id[0], PIXEL_HISTOGRAM.c.hist_value < MIN_HIST_VALUE))) deleted_total += result_proxy.rowcount deleted_galaxy += result_proxy.rowcount count += 1 transaction.commit() LOG.info('Removed %d really small histogram values from %s (%d)', deleted_galaxy, galaxy.name, galaxy.version_number) connection.close() LOG.info('Done - removed %d really small histogram values.', deleted_total)