LOG.info('Range from %s to %s', list[0], list[1])
    galaxy_ids = range(int(list[0]), int(list[1]) + 1)
else:
    galaxy_ids = args['galaxy_id']

for galaxy_id_str in galaxy_ids:
    galaxy_id1 = int(galaxy_id_str)
    galaxy = connection.execute(select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id1)).first()
    if galaxy is None:
        LOG.info('Error: Galaxy with galaxy_id of %d was not found', galaxy_id1)
    else:
        # Have we got work units out there for this galaxy?
        LOG.info('Working on galaxy %s (%d)', galaxy[GALAXY.c.name], galaxy[GALAXY.c.version_number])

        deleted_galaxy = 0
        transaction = connection.begin()
        count = 0
        for pxresult_id in connection.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]).order_by(PIXEL_RESULT.c.pxresult_id)):
            if not (count % 1000):
                LOG.info('Deleting low pixel_histogram values from galaxy {0} pixel {1} : Deleted total {2} galaxy {3}'.format(galaxy[GALAXY.c.galaxy_id], pxresult_id[0], deleted_total, deleted_galaxy))
            result_proxy = connection.execute(PIXEL_HISTOGRAM.delete().where(and_(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id[0], PIXEL_HISTOGRAM.c.hist_value < MIN_HIST_VALUE)))
            deleted_total += result_proxy.rowcount
            deleted_galaxy += result_proxy.rowcount
            count += 1

        transaction.commit()
        LOG.info('Removed %d really small histogram values from %s (%d)', deleted_galaxy, galaxy.name, galaxy.version_number)

connection.close()
LOG.info('Done - removed %d really small histogram values.', deleted_total)
Beispiel #2
0
                        select([
                            AREA.c.area_id
                        ]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(
                            AREA.c.area_id)):
                    deleted_area_count += 1
                    for pxresult_id1 in connection_aws.execute(
                            select([PIXEL_RESULT.c.pxresult_id
                                    ]).where(PIXEL_RESULT.c.area_id ==
                                             area_id1[0]).order_by(
                                                 PIXEL_RESULT.c.pxresult_id)):
                        deleted_pixel_count += 1
                        connection_aws.execute(PIXEL_FILTER.delete().where(
                            PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
                        connection_aws.execute(PIXEL_PARAMETER.delete().where(
                            PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
                        connection_aws.execute(PIXEL_HISTOGRAM.delete().where(
                            PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))

                    connection_aws.execute(PIXEL_RESULT.delete().where(
                        PIXEL_RESULT.c.area_id == area_id1[0]))

                    transaction_aws.commit()
                    transaction_aws = connection_aws.begin()

                    # Give the rest of the world a chance to access the database
                    time.sleep(1)

            transaction_aws.commit()
            end_time = time.time()
            LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
            LOG.info('Copied %d areas %d pixels.', area_count, pixel_count)
            LOG.info('Deleted %d areas %d pixels.', deleted_area_count,
Beispiel #3
0
                ):
                    deleted_area_count += 1
                    for pxresult_id1 in connection_aws.execute(
                        select([PIXEL_RESULT.c.pxresult_id])
                        .where(PIXEL_RESULT.c.area_id == area_id1[0])
                        .order_by(PIXEL_RESULT.c.pxresult_id)
                    ):
                        deleted_pixel_count += 1
                        connection_aws.execute(
                            PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0])
                        )
                        connection_aws.execute(
                            PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0])
                        )
                        connection_aws.execute(
                            PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0])
                        )

                    connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))

                    transaction_aws.commit()
                    transaction_aws = connection_aws.begin()

                    # Give the rest of the world a chance to access the database
                    time.sleep(1)

            transaction_aws.commit()
            end_time = time.time()
            LOG.info("Galaxy with galaxy_id of %d was archived.", galaxy_id1)
            LOG.info("Copied %d areas %d pixels.", area_count, pixel_count)
            LOG.info("Deleted %d areas %d pixels.", deleted_area_count, deleted_pixel_count)
Beispiel #4
0
                insert_only(IMAGE_FILTERS_USED, image_filters_used, connection_pleiades)

            transaction_pleiades.commit()
            copy_end_time = time.time()

            # Now we can delete the bits we don't need
            deleted_area_count = 0
            deleted_pixel_count = 0
            if False:
                for area_id1 in connection_aws.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(AREA.c.area_id)):
                    deleted_area_count += 1
                    for pxresult_id1 in connection_aws.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.area_id == area_id1[0]).order_by(PIXEL_RESULT.c.pxresult_id)):
                        deleted_pixel_count += 1
                        connection_aws.execute(PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
                        connection_aws.execute(PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
                        connection_aws.execute(PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))

                    connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))

                    transaction_aws.commit()
                    transaction_aws = connection_aws.begin()

                    # Give the rest of the world a chance to access the database
                    time.sleep(1)

            transaction_aws.commit()
            end_time = time.time()
            LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
            LOG.info('Copied %d areas %d pixels.', area_count, pixel_count)
            LOG.info('Deleted %d areas %d pixels.', deleted_area_count, deleted_pixel_count)
            total_time = end_time - start_time
Beispiel #5
0
        # Have we got work units out there for this galaxy?
        LOG.info('Working on galaxy %s (%d)', galaxy[GALAXY.c.name],
                 galaxy[GALAXY.c.version_number])

        deleted_galaxy = 0
        transaction = connection.begin()
        count = 0
        for pxresult_id in connection.execute(
                select([
                    PIXEL_RESULT.c.pxresult_id
                ]).where(PIXEL_RESULT.c.galaxy_id == galaxy[
                    GALAXY.c.galaxy_id]).order_by(PIXEL_RESULT.c.pxresult_id)):
            if not (count % 1000):
                LOG.info(
                    'Deleting low pixel_histogram values from galaxy {0} pixel {1} : Deleted total {2} galaxy {3}'
                    .format(galaxy[GALAXY.c.galaxy_id], pxresult_id[0],
                            deleted_total, deleted_galaxy))
            result_proxy = connection.execute(PIXEL_HISTOGRAM.delete().where(
                and_(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id[0],
                     PIXEL_HISTOGRAM.c.hist_value < MIN_HIST_VALUE)))
            deleted_total += result_proxy.rowcount
            deleted_galaxy += result_proxy.rowcount
            count += 1

        transaction.commit()
        LOG.info('Removed %d really small histogram values from %s (%d)',
                 deleted_galaxy, galaxy.name, galaxy.version_number)

connection.close()
LOG.info('Done - removed %d really small histogram values.', deleted_total)