def delete_register_entries(connection, register_ids):
for register_id in register_ids:
transaction = connection.begin()
register = connection.execute(select([REGISTER]).where(REGISTER.c.register_id == register_id)).first()
if register is None:
LOG.info('Error: Register entry with register_id of %d was not found', register_id)
else:
LOG.info("Deleting tags for register entry {0}".format(register_id))
connection.execute(TAG_REGISTER.delete().where(TAG_REGISTER.c.register_id == register_id))
LOG.info('Deleting register entry with register_id of %d - %s', register_id, register[REGISTER.c.galaxy_name])
connection.execute(REGISTER.delete().where(REGISTER.c.register_id == register_id))
transaction.commit()
def delete_register_entries(connection, register_ids):
for register_id in register_ids:
transaction = connection.begin()
register = connection.execute(
select([REGISTER
]).where(REGISTER.c.register_id == register_id)).first()
if register is None:
LOG.info(
'Error: Register entry with register_id of %d was not found',
register_id)
else:
LOG.info(
"Deleting tags for register entry {0}".format(register_id))
connection.execute(TAG_REGISTER.delete().where(
TAG_REGISTER.c.register_id == register_id))
LOG.info('Deleting register entry with register_id of %d - %s',
register_id, register[REGISTER.c.galaxy_name])
connection.execute(
REGISTER.delete().where(REGISTER.c.register_id == register_id))
transaction.commit()
registration = connection.execute(
select([REGISTER
]).where(REGISTER.c.create_time == None).order_by(
REGISTER.c.priority.desc(),
REGISTER.c.register_time)).first()
if registration is None:
LOG.info('No registrations waiting')
break
else:
# As the load work unit component adds data to the data base we need autocommit on to ensure each pixel matches
#transaction = connection.begin()
if not os.path.isfile(registration[REGISTER.c.filename]):
LOG.error('The file %s does not exist',
registration[REGISTER.c.filename])
connection.execute(REGISTER.update().where(
REGISTER.c.register_id == registration[
REGISTER.c.register_id]).values(
create_time=datetime.now()))
elif registration[
REGISTER.c.
sigma_filename] is not None and not os.path.isfile(
registration[REGISTER.c.sigma_filename]):
LOG.error('The file %s does not exist',
registration[REGISTER.c.sigma_filename])
connection.execute(REGISTER.update().where(
REGISTER.c.register_id == registration[
REGISTER.c.register_id]).values(
create_time=datetime.now()))
else:
LOG.info('Processing %s %d',
registration[REGISTER.c.galaxy_name],
registration[REGISTER.c.priority])
def add_to_database(connection, galaxy):
"""
Adds the specified galaxies to the database
:param connection:
:param galaxy:
:return:
"""
galaxy_name = galaxy['name']
redshift = galaxy['redshift']
galaxy_type = galaxy['type']
input_file = galaxy['input_file']
priority = galaxy['priority']
run_id = galaxy['run_id']
sigma_in = galaxy['sigma']
tags = galaxy['tags']
integrated = galaxy['int']
integrated_snr = galaxy['int_snr']
rad = galaxy['rad']
rad_snr = galaxy['rad_snr']
transaction = connection.begin()
try:
try:
sigma = float(sigma_in)
sigma_filename = None
except ValueError:
sigma = 0.0
sigma_filename = sigma_in
result = connection.execute(
REGISTER.insert(),
galaxy_name=galaxy_name,
redshift=redshift,
galaxy_type=galaxy_type,
filename=input_file,
priority=priority,
register_time=datetime.now(),
run_id=run_id,
sigma=sigma,
sigma_filename=sigma_filename,
int_filename=integrated,
int_sigma_filename=integrated_snr,
rad_filename=rad,
rad_sigma_filename=rad_snr
)
register_id = result.inserted_primary_key[0]
# Get the tag ids
tag_ids = set()
for tag_text in tags:
tag_text = tag_text.strip()
if len(tag_text) > 0:
tag = connection.execute(select([TAG]).where(TAG.c.tag_text == tag_text)).first()
if tag is None:
result = connection.execute(
TAG.insert(),
tag_text=tag_text
)
tag_id = result.inserted_primary_key[0]
else:
tag_id = tag[TAG.c.tag_id]
tag_ids.add(tag_id)
# Add the tag ids
for tag_id in tag_ids:
connection.execute(
TAG_REGISTER.insert(),
tag_id=tag_id,
register_id=register_id
)
transaction.commit()
LOG.info('Registered %s %s %f %s %d %d', galaxy_name, galaxy_type, redshift, input_file, priority, run_id)
for tag_text in tags:
LOG.info('Tag: {0}'.format(tag_text))
except Exception:
transaction.rollback()
raise
registration = connection.execute(
select([REGISTER])
.where(REGISTER.c.create_time == None)
.order_by(REGISTER.c.priority.desc(), REGISTER.c.register_time)
).first()
if registration is None:
LOG.info("No registrations waiting")
break
else:
# As the load work unit component adds data to the data base we need autocommit on to ensure each pixel matches
# transaction = connection.begin()
if not os.path.isfile(registration[REGISTER.c.filename]):
LOG.error("The file %s does not exist", registration[REGISTER.c.filename])
connection.execute(
REGISTER.update()
.where(REGISTER.c.register_id == registration[REGISTER.c.register_id])
.values(create_time=datetime.now())
)
elif registration[REGISTER.c.sigma_filename] is not None and not os.path.isfile(
registration[REGISTER.c.sigma_filename]
):
LOG.error("The file %s does not exist", registration[REGISTER.c.sigma_filename])
connection.execute(
REGISTER.update()
.where(REGISTER.c.register_id == registration[REGISTER.c.register_id])
.values(create_time=datetime.now())
)
else:
LOG.info(
"Processing %s %d", registration[REGISTER.c.galaxy_name], registration[REGISTER.c.priority]
)
exit(1)
# Connect to the database - the login string is set in the database package
ENGINE = create_engine(DB_LOGIN)
connection = ENGINE.connect()
transaction = connection.begin()
# If it is a float store it as the sigma otherwise assume it is a string pointing to a file containing the sigmas
try:
sigma = float(SIGMA)
sigma_filename = None
except ValueError:
sigma = 0.0
sigma_filename = SIGMA
result = connection.execute(REGISTER.insert(),
galaxy_name=GALAXY_NAME,
redshift=REDSHIFT,
galaxy_type=GALAXY_TYPE,
filename=INPUT_FILE,
priority=PRIORITY,
register_time=datetime.now(),
run_id=RUN_ID,
sigma=sigma,
sigma_filename=sigma_filename)
register_id = result.inserted_primary_key[0]
# Get the tag ids
tag_ids = set()
for tag_text in TAGS:
if not os.path.isfile(INPUT_FILE):
LOG.error('The file %s does not exist', INPUT_FILE)
exit(1)
# Connect to the database - the login string is set in the database package
ENGINE = create_engine(DB_LOGIN)
connection = ENGINE.connect()
transaction = connection.begin()
# If it is a float store it as the sigma otherwise assume it is a string pointing to a file containing the sigmas
try:
sigma = float(SIGMA)
sigma_filename = None
except ValueError:
sigma = 0.0
sigma_filename = SIGMA
connection.execute(REGISTER.insert(),
galaxy_name=GALAXY_NAME,
redshift=REDSHIFT,
galaxy_type=GALAXY_TYPE,
filename=INPUT_FILE,
priority=PRIORITY,
register_time=datetime.now(),
run_id=RUN_ID,
sigma=sigma,
sigma_filename=sigma_filename)
transaction.commit()
LOG.info('Registered %s %s %f %s %d %d', GALAXY_NAME, GALAXY_TYPE, REDSHIFT, INPUT_FILE, PRIORITY, RUN_ID)
exit(1)
# Connect to the database - the login string is set in the database package
ENGINE = create_engine(DB_LOGIN)
connection = ENGINE.connect()
transaction = connection.begin()
# If it is a float store it as the sigma otherwise assume it is a string pointing to a file containing the sigmas
try:
sigma = float(SIGMA)
sigma_filename = None
except ValueError:
sigma = 0.0
sigma_filename = SIGMA
connection.execute(
REGISTER.insert(),
galaxy_name=GALAXY_NAME,
redshift=REDSHIFT,
galaxy_type=GALAXY_TYPE,
filename=INPUT_FILE,
priority=PRIORITY,
register_time=datetime.now(),
run_id=RUN_ID,
sigma=sigma,
sigma_filename=sigma_filename,
)
transaction.commit()
LOG.info("Registered %s %s %f %s %d %d", GALAXY_NAME, GALAXY_TYPE, REDSHIFT, INPUT_FILE, PRIORITY, RUN_ID)