def ingest(self, urls):
"""
Add images to helioviewer images db.
(1) Make sure the file exists
(2) Make sure the file is 'good', and quarantine if it is not.
(3) Apply the ESA JPIP encoding.
(4) Ingest
(5) Update database to say that the file has been successfully
'ingested'.
"""
# Get filepaths
filepaths = []
images = []
corrupt = []
for url in urls:
path = os.path.join(
self.incoming,
os.path.basename(url)) # @TODO: Better path computation
if os.path.isfile(path):
filepaths.append(path)
# Add to hvpull/Helioviewer.org databases
for filepath in filepaths:
filename = os.path.basename(filepath)
# Parse header and validate metadata
try:
try:
image_params = sunpy.read_header(filepath)
except:
raise BadImage("HEADER")
logging.warn('BadImage("HEADER") error raised')
self._validate(image_params)
except BadImage, e:
logging.warn("Quarantining invalid image: %s", filename)
logging.warn("BadImage found; error message= %s",
e.get_message())
shutil.move(filepath, os.path.join(self.quarantine, filename))
mark_as_corrupt(self._db, filename, e.get_message())
corrupt.append(filename)
continue
# If everything looks good, move to archive and add to database
print image_params['date']
date_str = image_params['date'].strftime('%Y/%m/%d')
# Transcode
try:
if image_params['instrument'] == "AIA":
self._transcode(filepath, cprecincts=[128, 128])
else:
self._transcode(filepath)
except KduTranscodeError, e:
logging.warning("kdu_transcode: " + e.get_message())
continue
def ingest(self, urls):
"""
Add images to helioviewer images db.
(1) Make sure the file exists
(2) Make sure the file is 'good', and quarantine if it is not.
(3) Apply the ESA JPIP encoding.
(4) Ingest
(5) Update database to say that the file has been successfully
'ingested'.
"""
# Get filepaths
filepaths = []
images = []
corrupt = []
for url in urls:
path = os.path.join(self.incoming, os.path.basename(url)) # @TODO: Better path computation
if os.path.isfile(path):
filepaths.append(path)
# Add to hvpull/Helioviewer.org databases
for filepath in filepaths:
filename = os.path.basename(filepath)
# Parse header and validate metadata
try:
try:
image_params = sunpy.read_header(filepath)
except:
raise BadImage("HEADER")
logging.warn('BadImage("HEADER") error raised')
self._validate(image_params)
except BadImage, e:
logging.warn("Quarantining invalid image: %s", filename)
logging.warn("BadImage found; error message= %s", e.get_message())
shutil.move(filepath, os.path.join(self.quarantine, filename))
mark_as_corrupt(self._db, filename, e.get_message())
corrupt.append(filename)
continue
# If everything looks good, move to archive and add to database
print image_params['date']
date_str = image_params['date'].strftime('%Y/%m/%d')
# Transcode
try:
if image_params['instrument'] == "AIA":
self._transcode(filepath, cprecincts=[128, 128])
else:
self._transcode(filepath)
except KduTranscodeError, e:
logging.warning("kdu_transcode: " + e.get_message())
continue
def __init__(self):
self.print_greeting()
path = self.get_filepath()
# Locate jp2 images in specified filepath
filepaths = find_images(path)
# Check to make sure some images were found
if len(filepaths) is 0:
print("No JPEG 2000 images found. Exiting installation.")
sys.exit(2)
# Setup database schema if needed
cursor, mysql = self.get_db_cursor()
print("Processing Images...")
# Extract image parameters, 10,000 at a time
while len(filepaths) > 0:
subset = filepaths[:10000]
filepaths = filepaths[10000:]
images = []
for filepath in subset:
try:
image = sunpy.read_header(filepath)
image['filepath'] = filepath
images.append(image)
except:
#raise BadImage("HEADER")
print("Skipping corrupt image: %s" %
os.path.basename(filepath))
continue
# Insert image information into database
if len(images) > 0:
process_jp2_images(images, path, cursor, mysql)
# clean up afterwards
images = []
gc.collect()
# close db connection
cursor.close()
print("Finished!")
def __init__(self):
self.print_greeting()
path = self.get_filepath()
# Locate jp2 images in specified filepath
filepaths = find_images(path)
# Check to make sure some images were found
if len(filepaths) is 0:
print("No JPEG 2000 images found. Exiting installation.")
sys.exit(2)
# Setup database schema if needed
cursor, mysql = self.get_db_cursor()
print("Processing Images...")
# Extract image parameters, 10,000 at a time
while len(filepaths) > 0:
subset = filepaths[:10000]
filepaths = filepaths[10000:]
images = []
for filepath in subset:
try:
image = sunpy.read_header(filepath)
image['filepath'] = filepath
images.append(image)
except:
#raise BadImage("HEADER")
print("Skipping corrupt image: %s" %
os.path.basename(filepath))
continue
# Insert image information into database
if len(images) > 0:
process_jp2_images(images, path, cursor, mysql)
# clean up afterwards
images = []
gc.collect()
# close db connection
cursor.close()
print("Finished!")
def process_images(self):
''' Process JPEG 2000 archive and enter information into the database '''
admin, adminpass, hvdb, hvuser, hvpass, jp2dir, mysql = self.get_form_fields(
)
self.ui.startProcessingBtn.setEnabled(False)
self.ui.statusMsg.setText("Creating database schema")
cursor = setup_database_schema(admin, adminpass, hvdb, hvuser, hvpass,
mysql)
# Extract image parameters, 10,000 at a time
while len(self.filepaths) > 0:
subset = self.filepaths[:10000]
self.filepaths = self.filepaths[10000:]
images = []
for filepath in subset:
try:
image = sunpy.read_header(filepath)
image['filepath'] = filepath
images.append(image)
except:
#raise BadImage("HEADER")
print("Skipping corrupt image: %s" %
os.path.basename(filepath))
continue
# Insert image information into database
if len(images) > 0:
process_jp2_images(images, jp2dir, cursor, mysql,
self.update_progress)
# clean up afterwards
images = []
gc.collect()
cursor.close()
#self.ui.installProgress.setValue(len(images))
self.ui.statusMsg.setText("Finished!")
self.install_finished = True
def process_images(self):
''' Process JPEG 2000 archive and enter information into the database '''
admin, adminpass, hvdb, hvuser, hvpass, jp2dir, mysql = self.get_form_fields()
self.ui.startProcessingBtn.setEnabled(False)
self.ui.statusMsg.setText("Creating database schema")
cursor = setup_database_schema(admin, adminpass, hvdb, hvuser, hvpass, mysql)
# Extract image parameters, 10,000 at a time
while len(self.filepaths) > 0:
subset = self.filepaths[:10000]
self.filepaths = self.filepaths[10000:]
images = []
for filepath in subset:
try:
image = sunpy.read_header(filepath)
image['filepath'] = filepath
images.append(image)
except:
#raise BadImage("HEADER")
print("Skipping corrupt image: %s" %
os.path.basename(filepath))
continue
# Insert image information into database
if len(images) > 0:
process_jp2_images(images, jp2dir, cursor, mysql, self.update_progress)
# clean up afterwards
images = []
gc.collect()
cursor.close()
#self.ui.installProgress.setValue(len(images))
self.ui.statusMsg.setText("Finished!")
self.install_finished = True
def main(argv):
'''Main application access point'''
options = get_options()
init_logger('update.log')
print('Processing Images...')
# Get a list of images to process
filepaths = find_images(options.source)
if len(filepaths) is 0:
return
images = []
# Move images to main archive
for filepath in filepaths:
dest = os.path.join(options.destination,
os.path.relpath(filepath, options.source))
# Parse image header
image_params = sunpy.read_header(filepath)
image_params['filepath'] = dest
images.append(image_params)
directory = os.path.dirname(dest)
if not os.path.isdir(directory):
os.makedirs(directory)
shutil.move(filepath, dest)
# Add images to the database
cursor = get_db_cursor(options.dbname, options.dbuser, options.dbpass)
process_jp2_images(images, options.destination, cursor, True)
cursor.close()
print('Finished!')
