def _save_results(self, connection, map_pixel_results, list_insert_filters,
list_pixel_parameters, map_pixel_histograms):
"""
Add the pixel to the database
"""
if self._pxresult_id is not None and not self.noinsert:
# Update the filters
connection.execute(PIXEL_RESULT.update().where(
PIXEL_RESULT.c.pxresult_id == self._pxresult_id).values(
map_pixel_results))
connection.execute(PIXEL_FILTER.insert(), list_insert_filters)
# Because I need to get the PK back we have to do each one separately
for pixel_parameter in list_pixel_parameters:
result = connection.execute(PIXEL_PARAMETER.insert(),
pixel_parameter)
id = result.inserted_primary_key
# Add the ID to the list
list_pixel_histograms = map_pixel_histograms[
pixel_parameter['parameter_name_id']]
for map_values in list_pixel_histograms:
map_values['pxparameter_id'] = id[0]
connection.execute(PIXEL_HISTOGRAM.insert(),
list_pixel_histograms)
def _save_results(self, connection, map_pixel_results):
"""
Add the pixel to the database
"""
if self._pxresult_id is not None and not self.noinsert:
# Update the filters
connection.execute(PIXEL_RESULT.update().where(PIXEL_RESULT.c.pxresult_id == self._pxresult_id).values(map_pixel_results))
def _save_results(self, map_pixel_results):
"""
Add the pixel to the database
"""
if self._pxresult_id is not None and not self.noinsert:
# Update the filters
self._database_queue.append(PIXEL_RESULT.update().where(PIXEL_RESULT.c.pxresult_id == self._pxresult_id).values(map_pixel_results))
def _save_results(self, connection, map_pixel_results):
"""
Add the pixel to the database
"""
if self._pxresult_id is not None and not self.noinsert:
# Update the filters
connection.execute(PIXEL_RESULT.update().where(PIXEL_RESULT.c.pxresult_id == self._pxresult_id).values(map_pixel_results))
def delete_galaxy(connection, galaxy_ids):
try:
for galaxy_id_str in galaxy_ids:
transaction = connection.begin()
galaxy_id1 = int(galaxy_id_str)
galaxy = connection.execute(
select([GALAXY
]).where(GALAXY.c.galaxy_id == galaxy_id1)).first()
if galaxy is None:
LOG.info('Error: Galaxy with galaxy_id of %d was not found',
galaxy_id1)
else:
LOG.info('Deleting Galaxy with galaxy_id of %d - %s',
galaxy_id1, galaxy[GALAXY.c.name])
area_count = connection.execute(
select([func.count(AREA.c.area_id)
]).where(AREA.c.galaxy_id == galaxy[
GALAXY.c.galaxy_id])).first()[0]
counter = 1
for area_id1 in connection.execute(
select(
[AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy[
GALAXY.c.galaxy_id]).order_by(AREA.c.area_id)):
LOG.info("Deleting galaxy {0} area {1}. {2} of {3}".format(
galaxy_id_str, area_id1[0], counter, area_count))
connection.execute(PIXEL_RESULT.delete().where(
PIXEL_RESULT.c.area_id == area_id1[0]))
# Give the rest of the world a chance to access the database
time.sleep(0.1)
counter += 1
# Now empty the bucket
s3helper = S3Helper()
bucket = s3helper.get_bucket(get_files_bucket())
galaxy_file_name = get_galaxy_file_name(
galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id],
galaxy[GALAXY.c.galaxy_id])
for key in bucket.list(
prefix='{0}/sed/'.format(galaxy_file_name)):
# Ignore the key
if key.key.endswith('/'):
continue
bucket.delete_key(key)
# Now the folder
key = Key(bucket)
key.key = '{0}/sed/'.format(galaxy_file_name)
bucket.delete_key(key)
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id1)
connection.execute(
GALAXY.update().where(GALAXY.c.galaxy_id == galaxy_id1).values(
status_id=DELETED, status_time=datetime.datetime.now()))
transaction.commit()
except Exception:
LOG.exception('Major error')
def _create_areas(self, pix_y):
"""
Create a area - we try to make them squares, but they aren't as the images have dead zones
"""
area_insert = AREA.insert()
pixel_result_insert = PIXEL_RESULT.insert()
pix_x = 0
while pix_x < self._end_x:
max_x, pixels = self._get_pixels(pix_x, pix_y)
if len(pixels) > 0:
area = Area(pix_x, pix_y, max_x, min(pix_y + WG_ROW_HEIGHT, self._end_y))
result1 = self._connection.execute(area_insert.values(galaxy_id=self._galaxy_id,
top_x=area.top_x,
top_y=area.top_y,
bottom_x=area.bottom_x,
bottom_y=area.bottom_y))
area.area_id = result1.inserted_primary_key[0]
for pixel in pixels:
result2 = self._connection.execute(pixel_result_insert.values(galaxy_id=self._galaxy_id,
area_id=area.area_id,
y=pixel.y,
x=pixel.x))
pixel.pixel_id = result2.inserted_primary_key[0]
self._pixel_count += 1
# Write the pixels
self._create_output_file(area, pixels)
self._work_units_added += 1
pix_x = max_x + 1
def remove_database_entries(connection, galaxy_id):
connection.execute(
PIXEL_RESULT.delete().where(PIXEL_RESULT.c.galaxy_id == galaxy_id))
connection.execute(IMAGE_FILTERS_USED.delete().where(
IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
connection.execute(
FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))
def delete_galaxy(connection, galaxy_ids):
for galaxy_id in galaxy_ids:
transaction = connection.begin()
galaxy = connection.execute(select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)).first()
if galaxy is None:
LOG.info('Error: Galaxy with galaxy_id of %d was not found', galaxy_id)
else:
LOG.info('Deleting Galaxy with galaxy_id of %d - %s', galaxy_id, galaxy[GALAXY.c.name])
area_count = connection.execute(select([func.count(AREA.c.area_id)]).where(AREA.c.galaxy_id == galaxy[GALAXY.c.galaxy_id])).first()[0]
counter = 1
for area_id1 in connection.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]).order_by(AREA.c.area_id)):
LOG.info("Deleting galaxy {0} area {1}. {2} of {3}".format(galaxy_id, area_id1[0], counter, area_count))
connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
# Give the rest of the world a chance to access the database
time.sleep(0.1)
counter += 1
if shutdown() is True:
transaction.rollback()
raise SystemExit
LOG.info("Deleting FITS headers for galaxy {0}".format(galaxy_id))
connection.execute(FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]))
# Now empty the bucket of the sed files
s3helper = S3Helper()
bucket = s3helper.get_bucket(get_sed_files_bucket())
galaxy_file_name = get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id])
for key in bucket.list(prefix='{0}/'.format(galaxy_file_name)):
# Ignore the key
if key.key.endswith('/'):
continue
bucket.delete_key(key)
if shutdown() is True:
transaction.rollback()
raise SystemExit
# Now the folder
key = Key(bucket)
key.key = '{0}/'.format(galaxy_file_name)
bucket.delete_key(key)
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
connection.execute(GALAXY.update().where(GALAXY.c.galaxy_id == galaxy_id).values(status_id=DELETED, status_time=datetime.datetime.now()))
if shutdown() is True:
transaction.rollback()
raise SystemExit
transaction.commit()
def lock(time_p):
connection = ENGINE.connect()
transaction = connection.begin()
i = 999
while True:
wu_id = random.randrange(5, 60, 1)
connection.execute(AREA.update().where(AREA.c.area_id == wu_id).values(
workunit_id=wu_id, update_time=datetime.datetime.now()))
connection.execute(PIXEL_RESULT.update().where(
PIXEL_RESULT.c.pxresult_id == wu_id).values(y=i, x=i))
i += 1
if i > 100000:
break
transaction.rollback()
def lock(time_p):
connection = ENGINE.connect()
transaction = connection.begin()
i = 999
while True:
wu_id = random.randrange(5, 60, 1)
connection.execute(
AREA.update()
.where(AREA.c.area_id == wu_id)
.values(workunit_id=wu_id, update_time=datetime.datetime.now()))
connection.execute(PIXEL_RESULT.update().where(PIXEL_RESULT.c.pxresult_id == wu_id).values(y=i,
x=i))
i += 1
if i > 100000:
break
transaction.rollback()
def delete_galaxy(connection, galaxy_id):
if DRY_RUN:
LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id))
else:
transaction = connection.begin()
for area_id1 in connection.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA.c.area_id)):
connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
connection.execute(AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0]))
connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
connection.execute(FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
connection.execute(IMAGE_FILTERS_USED.delete().where(IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
transaction.commit()
def delete_galaxy(connection, galaxy_ids):
try:
for galaxy_id_str in galaxy_ids:
transaction = connection.begin()
galaxy_id1 = int(galaxy_id_str)
galaxy = connection.execute(select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id1)).first()
if galaxy is None:
LOG.info('Error: Galaxy with galaxy_id of %d was not found', galaxy_id1)
else:
LOG.info('Deleting Galaxy with galaxy_id of %d - %s', galaxy_id1, galaxy[GALAXY.c.name])
area_count = connection.execute(select([func.count(AREA.c.area_id)]).where(AREA.c.galaxy_id == galaxy[GALAXY.c.galaxy_id])).first()[0]
counter = 1
for area_id1 in connection.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]).order_by(AREA.c.area_id)):
LOG.info("Deleting galaxy {0} area {1}. {2} of {3}".format(galaxy_id_str, area_id1[0], counter, area_count))
connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
# Give the rest of the world a chance to access the database
time.sleep(0.1)
counter += 1
# Now empty the bucket
s3helper = S3Helper()
bucket = s3helper.get_bucket(get_files_bucket())
galaxy_file_name = get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id])
for key in bucket.list(prefix='{0}/sed/'.format(galaxy_file_name)):
# Ignore the key
if key.key.endswith('/'):
continue
bucket.delete_key(key)
# Now the folder
key = Key(bucket)
key.key = '{0}/sed/'.format(galaxy_file_name)
bucket.delete_key(key)
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id1)
connection.execute(GALAXY.update().where(GALAXY.c.galaxy_id == galaxy_id1).values(status_id=DELETED, status_time=datetime.datetime.now()))
transaction.commit()
except Exception:
LOG.exception('Major error')
def _save_results(self, connection, map_pixel_results, list_insert_filters, list_pixel_parameters, map_pixel_histograms):
"""
Add the pixel to the database
"""
if self._pxresult_id is not None and not self.noinsert:
# Update the filters
connection.execute(PIXEL_RESULT.update().where(PIXEL_RESULT.c.pxresult_id == self._pxresult_id).values(map_pixel_results))
connection.execute(PIXEL_FILTER.insert(), list_insert_filters)
# Because I need to get the PK back we have to do each one separately
for pixel_parameter in list_pixel_parameters:
result = connection.execute(PIXEL_PARAMETER.insert(), pixel_parameter)
id = result.inserted_primary_key
# Add the ID to the list
list_pixel_histograms = map_pixel_histograms[pixel_parameter['parameter_name_id']]
for map_values in list_pixel_histograms:
map_values['pxparameter_id'] = id[0]
connection.execute(PIXEL_HISTOGRAM.insert(), list_pixel_histograms)
def delete_galaxy(connection, galaxy_id):
if DRY_RUN:
LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id))
else:
transaction = connection.begin()
for area_id1 in connection.execute(
select([AREA.c.area_id
]).where(AREA.c.galaxy_id == galaxy_id).order_by(
AREA.c.area_id)):
connection.execute(PIXEL_RESULT.delete().where(
PIXEL_RESULT.c.area_id == area_id1[0]))
connection.execute(
AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0]))
connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
connection.execute(
FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
connection.execute(IMAGE_FILTERS_USED.delete().where(
IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
connection.execute(
GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
transaction.commit()
def remove_database_entries(connection, galaxy_id):
connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.galaxy_id == galaxy_id))
connection.execute(IMAGE_FILTERS_USED.delete().where(IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
connection.execute(FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))
transaction_pleiades.commit()
copy_end_time = time.time()
# Now we can delete the bits we don't need
deleted_area_count = 0
deleted_pixel_count = 0
if False:
for area_id1 in connection_aws.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id_aws).order_by(AREA.c.area_id)):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(select([PIXEL_RESULT.c.pxresult_id]).where(PIXEL_RESULT.c.area_id == area_id1[0]).order_by(PIXEL_RESULT.c.pxresult_id)):
deleted_pixel_count += 1
connection_aws.execute(PIXEL_FILTER.delete().where(PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_PARAMETER.delete().where(PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_HISTOGRAM.delete().where(PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
LOG.info('Copied %d areas %d pixels.', area_count, pixel_count)
LOG.info('Deleted %d areas %d pixels.', deleted_area_count, deleted_pixel_count)
total_time = end_time - start_time
LOG.info('Total time %d mins %.1f secs', int(total_time / 60), total_time % 60)
copy_time = copy_end_time - start_time
def delete_galaxy(connection, galaxy_ids):
for galaxy_id in galaxy_ids:
transaction = connection.begin()
galaxy = connection.execute(
select([GALAXY]).where(GALAXY.c.galaxy_id == galaxy_id)).first()
if galaxy is None:
LOG.info('Error: Galaxy with galaxy_id of %d was not found',
galaxy_id)
else:
LOG.info('Deleting Galaxy with galaxy_id of %d - %s', galaxy_id,
galaxy[GALAXY.c.name])
area_count = connection.execute(
select([func.count(AREA.c.area_id)]).where(
AREA.c.galaxy_id == galaxy[GALAXY.c.galaxy_id])).first()[0]
counter = 1
for area_id1 in connection.execute(
select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy[
GALAXY.c.galaxy_id]).order_by(AREA.c.area_id)):
LOG.info("Deleting galaxy {0} area {1}. {2} of {3}".format(
galaxy_id, area_id1[0], counter, area_count))
connection.execute(PIXEL_RESULT.delete().where(
PIXEL_RESULT.c.area_id == area_id1[0]))
# Give the rest of the world a chance to access the database
time.sleep(0.1)
counter += 1
if shutdown() is True:
transaction.rollback()
raise SystemExit
LOG.info("Deleting FITS headers for galaxy {0}".format(galaxy_id))
connection.execute(FITS_HEADER.delete().where(
FITS_HEADER.c.galaxy_id == galaxy[GALAXY.c.galaxy_id]))
# Now empty the bucket of the sed files
s3helper = S3Helper()
bucket = s3helper.get_bucket(get_sed_files_bucket())
galaxy_file_name = get_galaxy_file_name(galaxy[GALAXY.c.name],
galaxy[GALAXY.c.run_id],
galaxy[GALAXY.c.galaxy_id])
for key in bucket.list(prefix='{0}/'.format(galaxy_file_name)):
# Ignore the key
if key.key.endswith('/'):
continue
bucket.delete_key(key)
if shutdown() is True:
transaction.rollback()
raise SystemExit
# Now the folder
key = Key(bucket)
key.key = '{0}/'.format(galaxy_file_name)
bucket.delete_key(key)
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
connection.execute(
GALAXY.update().where(GALAXY.c.galaxy_id == galaxy_id).values(
status_id=DELETED, status_time=datetime.datetime.now()))
if shutdown() is True:
transaction.rollback()
raise SystemExit
transaction.commit()
AREA.c.area_id)):
deleted_area_count += 1
for pxresult_id1 in connection_aws.execute(
select([PIXEL_RESULT.c.pxresult_id
]).where(PIXEL_RESULT.c.area_id ==
area_id1[0]).order_by(
PIXEL_RESULT.c.pxresult_id)):
deleted_pixel_count += 1
connection_aws.execute(PIXEL_FILTER.delete().where(
PIXEL_FILTER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_PARAMETER.delete().where(
PIXEL_PARAMETER.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_HISTOGRAM.delete().where(
PIXEL_HISTOGRAM.c.pxresult_id == pxresult_id1[0]))
connection_aws.execute(PIXEL_RESULT.delete().where(
PIXEL_RESULT.c.area_id == area_id1[0]))
transaction_aws.commit()
transaction_aws = connection_aws.begin()
# Give the rest of the world a chance to access the database
time.sleep(1)
transaction_aws.commit()
end_time = time.time()
LOG.info('Galaxy with galaxy_id of %d was archived.', galaxy_id1)
LOG.info('Copied %d areas %d pixels.', area_count, pixel_count)
LOG.info('Deleted %d areas %d pixels.', deleted_area_count,
deleted_pixel_count)
total_time = end_time - start_time
LOG.info('Total time %d mins %.1f secs', int(total_time / 60),