def delete_register_entries(connection, register_ids):
for register_id in register_ids:
transaction = connection.begin()
register = connection.execute(select([REGISTER]).where(REGISTER.c.register_id == register_id)).first()
if register is None:
LOG.info('Error: Register entry with register_id of %d was not found', register_id)
else:
LOG.info("Deleting tags for register entry {0}".format(register_id))
connection.execute(TAG_REGISTER.delete().where(TAG_REGISTER.c.register_id == register_id))
LOG.info('Deleting register entry with register_id of %d - %s', register_id, register[REGISTER.c.galaxy_name])
connection.execute(REGISTER.delete().where(REGISTER.c.register_id == register_id))
transaction.commit()
def delete_register_entries(connection, register_ids):
for register_id in register_ids:
transaction = connection.begin()
register = connection.execute(
select([REGISTER
]).where(REGISTER.c.register_id == register_id)).first()
if register is None:
LOG.info(
'Error: Register entry with register_id of %d was not found',
register_id)
else:
LOG.info(
"Deleting tags for register entry {0}".format(register_id))
connection.execute(TAG_REGISTER.delete().where(
TAG_REGISTER.c.register_id == register_id))
LOG.info('Deleting register entry with register_id of %d - %s',
register_id, register[REGISTER.c.galaxy_name])
connection.execute(
REGISTER.delete().where(REGISTER.c.register_id == register_id))
transaction.commit()
fit2wu = Fit2Wu(connection, download_dir, fanout)
(work_units_added,
pixel_count) = fit2wu.process_file(registration)
# One WU = MIN_QUORUM Results
total_work_units_added += (work_units_added * MIN_QUORUM)
if os.path.exists(registration[REGISTER.c.filename]):
os.remove(registration[REGISTER.c.filename])
if registration.sigma_filename is not None and os.path.exists(
registration.sigma_filename):
os.remove(registration[REGISTER.c.sigma_filename])
connection.execute(REGISTER.update().where(
REGISTER.c.register_id == registration[
REGISTER.c.register_id]).values(
create_time=datetime.now()))
connection.execute(TAG_REGISTER.delete().where(
TAG_REGISTER.c.register_id == registration[
REGISTER.c.register_id]))
LOG.info('Done - added %d Results', total_work_units_added)
# Closing BOINC DB
if return_value == 0:
LOG.info('Closing BOINC DB')
return_value = py_boinc.boinc_db_close()
# Log how many are left in the queue
count = connection.execute(
select([func.count(REGISTER.c.register_id)
]).where(REGISTER.c.create_time == None)).first()[0]
LOG.info('Galaxies in queue = %d', count)
def add_to_database(connection, galaxy):
"""
Adds the specified galaxies to the database
:param connection:
:param galaxy:
:return:
"""
galaxy_name = galaxy['name']
redshift = galaxy['redshift']
galaxy_type = galaxy['type']
input_file = galaxy['input_file']
priority = galaxy['priority']
run_id = galaxy['run_id']
sigma_in = galaxy['sigma']
tags = galaxy['tags']
integrated = galaxy['int']
integrated_snr = galaxy['int_snr']
rad = galaxy['rad']
rad_snr = galaxy['rad_snr']
transaction = connection.begin()
try:
try:
sigma = float(sigma_in)
sigma_filename = None
except ValueError:
sigma = 0.0
sigma_filename = sigma_in
result = connection.execute(
REGISTER.insert(),
galaxy_name=galaxy_name,
redshift=redshift,
galaxy_type=galaxy_type,
filename=input_file,
priority=priority,
register_time=datetime.now(),
run_id=run_id,
sigma=sigma,
sigma_filename=sigma_filename,
int_filename=integrated,
int_sigma_filename=integrated_snr,
rad_filename=rad,
rad_sigma_filename=rad_snr
)
register_id = result.inserted_primary_key[0]
# Get the tag ids
tag_ids = set()
for tag_text in tags:
tag_text = tag_text.strip()
if len(tag_text) > 0:
tag = connection.execute(select([TAG]).where(TAG.c.tag_text == tag_text)).first()
if tag is None:
result = connection.execute(
TAG.insert(),
tag_text=tag_text
)
tag_id = result.inserted_primary_key[0]
else:
tag_id = tag[TAG.c.tag_id]
tag_ids.add(tag_id)
# Add the tag ids
for tag_id in tag_ids:
connection.execute(
TAG_REGISTER.insert(),
tag_id=tag_id,
register_id=register_id
)
transaction.commit()
LOG.info('Registered %s %s %f %s %d %d', galaxy_name, galaxy_type, redshift, input_file, priority, run_id)
for tag_text in tags:
LOG.info('Tag: {0}'.format(tag_text))
except Exception:
transaction.rollback()
raise
sigma_filename=sigma_filename)
register_id = result.inserted_primary_key[0]
# Get the tag ids
tag_ids = set()
for tag_text in TAGS:
tag_text = tag_text.strip()
if len(tag_text) > 0:
tag = connection.execute(select([TAG]).where(TAG.c.tag_text == tag_text)).first()
if tag is None:
result = connection.execute(TAG.insert(),
tag_text=tag_text)
tag_id = result.inserted_primary_key[0]
else:
tag_id = tag[TAG.c.tag_id]
tag_ids.add(tag_id)
# Add the tag ids
for tag_id in tag_ids:
connection.execute(TAG_REGISTER.insert(),
tag_id=tag_id,
register_id=register_id)
transaction.commit()
LOG.info('Registered %s %s %f %s %d %d', GALAXY_NAME, GALAXY_TYPE, REDSHIFT, INPUT_FILE, PRIORITY, RUN_ID)
for tag_text in TAGS:
LOG.info('Tag: {0}'.format(tag_text))
if os.path.exists(registration[REGISTER.c.filename]):
os.remove(registration[REGISTER.c.filename])
if registration[REGISTER.c.sigma_filename] is not None and os.path.exists(registration[REGISTER.c.sigma_filename]):
os.remove(registration[REGISTER.c.sigma_filename])
if registration[REGISTER.c.rad_filename] is not None and os.path.exists(registration[REGISTER.c.rad_filename]):
os.remove(registration[REGISTER.c.rad_filename])
if registration[REGISTER.c.rad_sigma_filename] is not None and os.path.exists(registration[REGISTER.c.rad_sigma_filename]):
os.remove(registration[REGISTER.c.rad_sigma_filename])
if registration[REGISTER.c.int_filename] is not None and os.path.exists(registration[REGISTER.c.int_filename]):
os.remove(registration[REGISTER.c.int_filename])
if registration[REGISTER.c.int_sigma_filename] is not None and os.path.exists(registration[REGISTER.c.int_sigma_filename]):
os.remove(registration[REGISTER.c.int_sigma_filename])
connection.execute(REGISTER.update().where(REGISTER.c.register_id == registration[REGISTER.c.register_id]).values(create_time=datetime.now()))
connection.execute(TAG_REGISTER.delete().where(TAG_REGISTER.c.register_id == registration[REGISTER.c.register_id]))
LOG.info('Total areas added {0}, pixels added {1}'.format(total_areas, total_pixels))
if len(areaave) > 0:
LOG.info('Total db time: {0}'.format(total_db_time))
asum = 0
for rtime in areaave:
asum += rtime
LOG.info('Average time per transaction: {0}'.format(asum / len(areaave)))
LOG.info('Total BOINC db time: {0}'.format(total_boinc_db_time))
asum_boinc = 0
for rtime in areaave_boinc:
asum_boinc += rtime
LOG.info('Average BOINC db time per transaction: {0}'.format(asum_boinc / len(areaave_boinc)))
