def ingest(self, urls):
"""
Add images to helioviewer images db.
(1) Make sure the file exists
(2) Make sure the file is 'good', and quarantine if it is not.
(3) Apply the ESA JPIP encoding.
(4) Ingest
(5) Update database to say that the file has been successfully
'ingested'.
"""
# Get filepaths
filepaths = []
images = []
corrupt = []
for url in urls:
path = os.path.join(
self.incoming,
os.path.basename(url)) # @TODO: Better path computation
if os.path.isfile(path):
filepaths.append(path)
# Add to hvpull/Helioviewer.org databases
for filepath in filepaths:
filename = os.path.basename(filepath)
# Parse header and validate metadata
try:
try:
image_params = sunpy.read_header(filepath)
except:
raise BadImage("HEADER")
logging.warn('BadImage("HEADER") error raised')
self._validate(image_params)
except BadImage, e:
logging.warn("Quarantining invalid image: %s", filename)
logging.warn("BadImage found; error message= %s",
e.get_message())
shutil.move(filepath, os.path.join(self.quarantine, filename))
mark_as_corrupt(self._db, filename, e.get_message())
corrupt.append(filename)
continue
# If everything looks good, move to archive and add to database
print image_params['date']
date_str = image_params['date'].strftime('%Y/%m/%d')
# Transcode
try:
if image_params['instrument'] == "AIA":
self._transcode(filepath, cprecincts=[128, 128])
else:
self._transcode(filepath)
except KduTranscodeError, e:
logging.warning("kdu_transcode: " + e.get_message())
continue
def ingest(self, urls):
"""
Add images to helioviewer images db.
(1) Make sure the file exists
(2) Make sure the file is 'good', and quarantine if it is not.
(3) Apply the ESA JPIP encoding.
(4) Ingest
(5) Update database to say that the file has been successfully
'ingested'.
"""
# Get filepaths
filepaths = []
images = []
corrupt = []
for url in urls:
path = os.path.join(self.incoming, os.path.basename(url)) # @TODO: Better path computation
if os.path.isfile(path):
filepaths.append(path)
# Add to hvpull/Helioviewer.org databases
for filepath in filepaths:
filename = os.path.basename(filepath)
# Parse header and validate metadata
try:
try:
image_params = sunpy.read_header(filepath)
except:
raise BadImage("HEADER")
logging.warn('BadImage("HEADER") error raised')
self._validate(image_params)
except BadImage, e:
logging.warn("Quarantining invalid image: %s", filename)
logging.warn("BadImage found; error message= %s", e.get_message())
shutil.move(filepath, os.path.join(self.quarantine, filename))
mark_as_corrupt(self._db, filename, e.get_message())
corrupt.append(filename)
continue
# If everything looks good, move to archive and add to database
print image_params['date']
date_str = image_params['date'].strftime('%Y/%m/%d')
# Transcode
try:
if image_params['instrument'] == "AIA":
self._transcode(filepath, cprecincts=[128, 128])
else:
self._transcode(filepath)
except KduTranscodeError, e:
logging.warning("kdu_transcode: " + e.get_message())
continue
def ingest(self, urls):
"""
Add images to helioviewer data db.
(1) Make sure the file exists
(2) Make sure the file is 'good', and quarantine if it is not.
(3) Apply the ESA JPIP encoding.
(4) Ingest
(5) Update database to say that the file has been successfully
'ingested'.
"""
# Get filepaths
filepaths = []
images = []
corrupt = []
for url in urls:
path = os.path.join(self.incoming, os.path.basename(url)) # @TODO: Better path computation
if os.path.isfile(path):
filepaths.append(path)
# Add to hvpull/Helioviewer.org databases
for filepath in filepaths:
filename = os.path.basename(filepath)
# Parse header and validate metadata
try:
try:
image_params = create_image_data(filepath)
except:
raise BadImage("HEADER")
logging.warn('BadImage("HEADER") error raised')
self._validate(image_params)
except BadImage as e:
logging.warn("Quarantining invalid image: %s", filename)
logging.warn("BadImage found; error message= %s", e.get_message())
shutil.move(filepath, os.path.join(self.quarantine, filename))
mark_as_corrupt(self._cursor, filename, e.get_message())
corrupt.append(filename)
continue
# If everything looks good, move to archive and add to database
# print image_params['date']
date_str = image_params['date'].strftime('%Y/%m/%d')
# The files must be transcoded in order to work with JHelioviewer.
# Therefore, any problem with the transcoding process must raise
# an error.
try:
if image_params['instrument'] == "AIA":
self._transcode(filepath, cprecincts=[128, 128])
else:
self._transcode(filepath)
except KduTranscodeError, e:
logging.error("kdu_transcode: " + e.get_message())
# Move to archive
if image_params['observatory'] == "Hinode":
directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['filter1']), str(image_params['filter2']))
else:
directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['measurement']))
dest = os.path.join(directory, filename)
image_params['filepath'] = dest
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError:
logging.error("Unable to create the directory '" +
directory + "'. Please ensure that you "
"have the proper permissions and try again.")
self.shutdown_requested = True
try:
shutil.move(filepath, dest)
except IOError:
logging.error("Unable to move files to destination. Is there "
"enough free space?")
self.shutdown_requested = True
# Add to list to send to main database
images.append(image_params)
def ingest(self, urls):
"""
Add images to helioviewer data db.
(1) Make sure the file exists
(2) Make sure the file is 'good', and quarantine if it is not.
(3) Apply the ESA JPIP encoding.
(4) Ingest
(5) Update database to say that the file has been successfully
'ingested'.
"""
# Get filepaths
filepaths = []
images = []
corrupt = []
for url in urls:
path = os.path.join(
self.incoming,
os.path.basename(url)) # @TODO: Better path computation
if os.path.isfile(path):
filepaths.append(path)
# Add to hvpull/Helioviewer.org databases
for filepath in filepaths:
filename = os.path.basename(filepath)
# Parse header and validate metadata
try:
try:
image_params = create_image_data(filepath)
except:
raise BadImage("HEADER")
logging.warn('BadImage("HEADER") error raised')
self._validate(image_params)
except BadImage as e:
logging.warn("Quarantining invalid image: %s", filename)
logging.warn("BadImage found; error message= %s",
e.get_message())
shutil.move(filepath, os.path.join(self.quarantine, filename))
mark_as_corrupt(self._cursor, filename, e.get_message())
corrupt.append(filename)
continue
# If everything looks good, move to archive and add to database
# print image_params['date']
date_str = image_params['date'].strftime('%Y/%m/%d')
# The files must be transcoded in order to work with JHelioviewer.
# Therefore, any problem with the transcoding process must raise
# an error.
try:
if image_params['instrument'] == "AIA":
self._transcode(filepath, cprecincts=[128, 128])
else:
self._transcode(filepath)
except KduTranscodeError, e:
logging.error("kdu_transcode: " + e.get_message())
# Move to archive
if image_params['observatory'] == "Hinode":
directory = os.path.join(self.image_archive,
image_params['nickname'], date_str,
str(image_params['filter1']),
str(image_params['filter2']))
else:
directory = os.path.join(self.image_archive,
image_params['nickname'], date_str,
str(image_params['measurement']))
dest = os.path.join(directory, filename)
image_params['filepath'] = dest
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError:
logging.error("Unable to create the directory '" +
directory + "'. Please ensure that you "
"have the proper permissions and try again.")
self.shutdown_requested = True
try:
shutil.move(filepath, dest)
except IOError:
logging.error("Unable to move files to destination. Is there "
"enough free space?")
self.shutdown_requested = True
# Add to list to send to main database
images.append(image_params)