LOG.info('Range from %s to %s', list[0], list[1])
galaxy_ids = range(int(list[0]), int(list[1]) + 1)
else:
galaxy_ids = args['galaxy_id']
for galaxy_id_str in galaxy_ids:
galaxy_id1 = int(galaxy_id_str)
galaxy = connection.execute(select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id1)).first()
if galaxy is None:
LOG.info('Error: Galaxy with galaxy_id of %d was not found', galaxy_id1)
else:
# Have we got work units out there for this galaxy?
LOG.info('Working on galaxy %s (%d)', galaxy[GALAXY.c.name], galaxy[GALAXY.c.version_number])
deleted_galaxy = 0
transaction = connection.begin()
count = 0
for pxresult_id in connection.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]).order_by(PIXEL_RESULT.c.pxresult_id)):
if not (count % 1000):
LOG.info('Deleting low pixel_histogram values from galaxy {0} pixel {1} : Deleted total {2} galaxy {3}'.format(galaxy[GALAXY.c.galaxy_id], pxresult_id[0], deleted_total, deleted_galaxy))
result_proxy = connection.execute(PIXEL_HISTOGRAM.delete().where(and_(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id[0], PIXEL_HISTOGRAM.c.hist_value < MIN_HIST_VALUE)))
deleted_total += result_proxy.rowcount
deleted_galaxy += result_proxy.rowcount
count += 1
transaction.commit()
LOG.info('Removed %d really small histogram values from %s (%d)', deleted_galaxy, galaxy.name, galaxy.version_number)
connection.close()
LOG.info('Done - removed %d really small histogram values.', deleted_total)
select([
AREA.c.area_id
]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(
AREA.c.area_id)):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(
select([PIXEL_RESULT.c.pxresult_id
]).where(PIXEL_RESULT.c.area_id ==
area_id1[0]).order_by(
PIXEL_RESULT.c.pxresult_id)):
deleted_pixel_count += 1
connection_aws.execute(PIXEL_FILTER.delete().where(
PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_PARAMETER.delete().where(
PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_HISTOGRAM.delete().where(
PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_RESULT.delete().where(
PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
LOG.info('Copied %d areas %d pixels.', area_count, pixel_count)
LOG.info('Deleted %d areas %d pixels.', deleted_area_count,
):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(
select([PIXEL_RESULT.c.pxresult_id])
.where(PIXEL_RESULT.c.area_id == area_id1[0])
.order_by(PIXEL_RESULT.c.pxresult_id)
):
deleted_pixel_count += 1
connection_aws.execute(
PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0])
)
connection_aws.execute(
PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0])
)
connection_aws.execute(
PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0])
)
connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
LOG.info("Galaxy with galaxy_id of %d was archived.", galaxy_id1)
LOG.info("Copied %d areas %d pixels.", area_count, pixel_count)
LOG.info("Deleted %d areas %d pixels.", deleted_area_count, deleted_pixel_count)
insert_only(IMAGE_FILTERS_USED, image_filters_used, connection_pleiades)
transaction_pleiades.commit()
copy_end_time = time.time()
# Now we can delete the bits we don't need
deleted_area_count = 0
deleted_pixel_count = 0
if False:
for area_id1 in connection_aws.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(AREA.c.area_id)):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.area_id == area_id1[0]).order_by(PIXEL_RESULT.c.pxresult_id)):
deleted_pixel_count += 1
connection_aws.execute(PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
LOG.info('Copied %d areas %d pixels.', area_count, pixel_count)
LOG.info('Deleted %d areas %d pixels.', deleted_area_count, deleted_pixel_count)
total_time = end_time - start_time
# Have we got work units out there for this galaxy?
LOG.info('Working on galaxy %s (%d)', galaxy[GALAXY.c.name],
galaxy[GALAXY.c.version_number])
deleted_galaxy = 0
transaction = connection.begin()
count = 0
for pxresult_id in connection.execute(
select([
PIXEL_RESULT.c.pxresult_id
]).where(PIXEL_RESULT.c.galaxy_id == galaxy[
GALAXY.c.galaxy_id]).order_by(PIXEL_RESULT.c.pxresult_id)):
if not (count % 1000):
LOG.info(
'Deleting low pixel_histogram values from galaxy {0} pixel {1} : Deleted total {2} galaxy {3}'
.format(galaxy[GALAXY.c.galaxy_id], pxresult_id[0],
deleted_total, deleted_galaxy))
result_proxy = connection.execute(PIXEL_HISTOGRAM.delete().where(
and_(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id[0],
PIXEL_HISTOGRAM.c.hist_value < MIN_HIST_VALUE)))
deleted_total += result_proxy.rowcount
deleted_galaxy += result_proxy.rowcount
count += 1
transaction.commit()
LOG.info('Removed %d really small histogram values from %s (%d)',
deleted_galaxy, galaxy.name, galaxy.version_number)
connection.close()
LOG.info('Done - removed %d really small histogram values.', deleted_total)