def preview_file_already_processed(path, db_address=_DEFAULT_DB):
db_session = get_session(db_address=db_address)
query = db_session.query(PreviewImage)
criteria = PreviewImage.filename == os.path.basename(path)
criteria &= PreviewImage.checksum == file_utils.get_md5(path)
query = query.filter(criteria)
already_processed = len(query.all()) > 0
db_session.close()
return already_processed
def preview_file_already_processed(path, db_address=_DEFAULT_DB):
db_session = get_session(db_address=db_address)
query = db_session.query(PreviewImage)
criteria = PreviewImage.filename == os.path.basename(path)
criteria &= PreviewImage.checksum == file_utils.get_md5(path)
query = query.filter(criteria)
already_processed = len(query.all()) > 0
db_session.close()
return already_processed
def set_preview_file_as_processed(path, db_address=_DEFAULT_DB):
filename = os.path.basename(path)
checksum = file_utils.get_md5(path)
db_session = get_session(db_address=db_address)
query = db_session.query(PreviewImage)
criteria = PreviewImage.filename == filename
criteria &= PreviewImage.checksum == checksum
query = query.filter(criteria)
if len(query.all()) == 0:
preview_image = PreviewImage(filename=filename, checksum=checksum)
db_session.add(preview_image)
db_session.commit()
db_session.close()
def set_preview_file_as_processed(path, db_address=_DEFAULT_DB):
filename = os.path.basename(path)
checksum = file_utils.get_md5(path)
db_session = get_session(db_address=db_address)
query = db_session.query(PreviewImage)
criteria = PreviewImage.filename == filename
criteria &= PreviewImage.checksum == checksum
query = query.filter(criteria)
if len(query.all()) == 0:
preview_image = PreviewImage(filename=filename, checksum=checksum)
db_session.add(preview_image)
db_session.commit()
db_session.close()
def need_to_make_preview(path, db_address=_DEFAULT_DB, max_tries=5):
# Get the preview image in db. If it doesn't exist add it.
preview_image = get_preview_image(path, db_address=db_address)
# If there was an issue with the database return none and move on and return false
if preview_image is None:
need_to_process = False
else:
try:
# As long as the preview file exists, check the md5.
checksum = file_utils.get_md5(path)
if preview_image.checksum == checksum and (preview_image.tries >= max_tries or
preview_image.success):
need_to_process = False
else:
need_to_process = True
preview_image.checksum = checksum
commit_preview_image(preview_image, db_address)
except IOError as e:
logger.error('{0}. {1}'.format(e, path), extra={'tags': {'filename': os.path.basename(path)}})
need_to_process = False
return need_to_process
def need_to_make_preview(path, db_address=_DEFAULT_DB, max_tries=5):
try:
telescope = get_telescope_for_file(path, db_address=db_address)
if not telescope.schedulable:
return False
except TelescopeMissingException:
logger.error('Telescope/Camera not in database for {f}'.format(path))
return False
# Get the preview image in db. If it doesn't exist add it.
preview_image = get_preview_image(path, db_address=db_address)
# If there was an issue with the database return none and move on and return false
need_to_process = False
# Check the md5.
checksum = file_utils.get_md5(path)
if preview_image.checksum != checksum and (preview_image.tries <= max_tries and not preview_image.success):
need_to_process = True
preview_image.checksum = checksum
commit_preview_image(preview_image, db_address)
return need_to_process
def need_to_process_image(path, ignore_schedulability=False, db_address=dbs._DEFAULT_DB, max_tries=5):
"""
Figure out if we need to try to make a process a given file.
Parameters
----------
path: str
Full path to the image possibly needing to be processed
ignore_schedulability: bool
Process non-schedulable instruments
db_address: str
SQLAlchemy style URL to the database with the status of previous reductions
max_tries: int
Maximum number of retries to reduce an image
Returns
-------
need_to_process: bool
True if we should try to process the image
Notes
-----
If the file has changed on disk, we reset the success flags and the number of tries to zero.
We only attempt to make images if the instrument is in the database and passes the given criteria.
"""
logger.info("Checking if file needs to be processed", extra_tags={"filename": path})
if not (path.endswith('.fits') or path.endswith('.fits.fz')):
logger.warning("Filename does not have a .fits extension, stopping reduction", extra_tags={"filename": path})
return False
header = fits_utils.get_primary_header(path)
if not image_utils.image_can_be_processed(header, db_address):
return False
try:
instrument = dbs.get_instrument(header, db_address=db_address)
except ValueError:
return False
if not ignore_schedulability and not instrument.schedulable:
logger.info('Image will not be processed because instrument is not schedulable', extra_tags={"filename": path})
return False
# Get the image in db. If it doesn't exist add it.
image = dbs.get_processed_image(path, db_address=db_address)
need_to_process = False
# Check the md5.
checksum = file_utils.get_md5(path)
# Reset the number of tries if the file has changed on disk
if image.checksum != checksum:
need_to_process = True
image.checksum = checksum
image.tries = 0
image.success = False
dbs.commit_processed_image(image, db_address)
# Check if we need to try again
elif image.tries < max_tries and not image.success:
need_to_process = True
dbs.commit_processed_image(image, db_address)
return need_to_process
def need_to_process_image(path, context):
"""
Figure out if we need to try to make a process a given file.
Parameters
----------
path: str
Full path to the image possibly needing to be processed
context: banzai.context.Context
Context object with runtime environment info
Returns
-------
need_to_process: bool
True if we should try to process the image
Notes
-----
If the file has changed on disk, we reset the success flags and the number of tries to zero.
We only attempt to make images if the instrument is in the database and passes the given criteria.
"""
logger.info("Checking if file needs to be processed",
extra_tags={"filename": path})
if not (path.endswith('.fits') or path.endswith('.fits.fz')):
logger.warning(
"Filename does not have a .fits extension, stopping reduction",
extra_tags={"filename": path})
return False
header = fits_utils.get_primary_header(path)
if not image_utils.image_can_be_processed(header, context):
return False
try:
instrument = dbs.get_instrument(header, db_address=context.db_address)
except ValueError:
return False
if not context.ignore_schedulability and not instrument.schedulable:
logger.info(
'Image will not be processed because instrument is not schedulable',
extra_tags={"filename": path})
return False
# Get the image in db. If it doesn't exist add it.
image = dbs.get_processed_image(path, db_address=context.db_address)
need_to_process = False
# Check the md5.
checksum = file_utils.get_md5(path)
# Reset the number of tries if the file has changed on disk
if image.checksum != checksum:
need_to_process = True
image.checksum = checksum
image.tries = 0
image.success = False
dbs.commit_processed_image(image, context.db_address)
# Check if we need to try again
elif image.tries < context.max_tries and not image.success:
need_to_process = True
dbs.commit_processed_image(image, context.db_address)
return need_to_process