def ingest(self, urls): """ Add images to helioviewer images db. (1) Make sure the file exists (2) Make sure the file is 'good', and quarantine if it is not. (3) Apply the ESA JPIP encoding. (4) Ingest (5) Update database to say that the file has been successfully 'ingested'. """ # Get filepaths filepaths = [] images = [] corrupt = [] for url in urls: path = os.path.join( self.incoming, os.path.basename(url)) # @TODO: Better path computation if os.path.isfile(path): filepaths.append(path) # Add to hvpull/Helioviewer.org databases for filepath in filepaths: filename = os.path.basename(filepath) # Parse header and validate metadata try: try: image_params = sunpy.read_header(filepath) except: raise BadImage("HEADER") logging.warn('BadImage("HEADER") error raised') self._validate(image_params) except BadImage, e: logging.warn("Quarantining invalid image: %s", filename) logging.warn("BadImage found; error message= %s", e.get_message()) shutil.move(filepath, os.path.join(self.quarantine, filename)) mark_as_corrupt(self._db, filename, e.get_message()) corrupt.append(filename) continue # If everything looks good, move to archive and add to database print image_params['date'] date_str = image_params['date'].strftime('%Y/%m/%d') # Transcode try: if image_params['instrument'] == "AIA": self._transcode(filepath, cprecincts=[128, 128]) else: self._transcode(filepath) except KduTranscodeError, e: logging.warning("kdu_transcode: " + e.get_message()) continue
def ingest(self, urls): """ Add images to helioviewer images db. (1) Make sure the file exists (2) Make sure the file is 'good', and quarantine if it is not. (3) Apply the ESA JPIP encoding. (4) Ingest (5) Update database to say that the file has been successfully 'ingested'. """ # Get filepaths filepaths = [] images = [] corrupt = [] for url in urls: path = os.path.join(self.incoming, os.path.basename(url)) # @TODO: Better path computation if os.path.isfile(path): filepaths.append(path) # Add to hvpull/Helioviewer.org databases for filepath in filepaths: filename = os.path.basename(filepath) # Parse header and validate metadata try: try: image_params = sunpy.read_header(filepath) except: raise BadImage("HEADER") logging.warn('BadImage("HEADER") error raised') self._validate(image_params) except BadImage, e: logging.warn("Quarantining invalid image: %s", filename) logging.warn("BadImage found; error message= %s", e.get_message()) shutil.move(filepath, os.path.join(self.quarantine, filename)) mark_as_corrupt(self._db, filename, e.get_message()) corrupt.append(filename) continue # If everything looks good, move to archive and add to database print image_params['date'] date_str = image_params['date'].strftime('%Y/%m/%d') # Transcode try: if image_params['instrument'] == "AIA": self._transcode(filepath, cprecincts=[128, 128]) else: self._transcode(filepath) except KduTranscodeError, e: logging.warning("kdu_transcode: " + e.get_message()) continue
def ingest(self, urls): """ Add images to helioviewer data db. (1) Make sure the file exists (2) Make sure the file is 'good', and quarantine if it is not. (3) Apply the ESA JPIP encoding. (4) Ingest (5) Update database to say that the file has been successfully 'ingested'. """ # Get filepaths filepaths = [] images = [] corrupt = [] for url in urls: path = os.path.join(self.incoming, os.path.basename(url)) # @TODO: Better path computation if os.path.isfile(path): filepaths.append(path) # Add to hvpull/Helioviewer.org databases for filepath in filepaths: filename = os.path.basename(filepath) # Parse header and validate metadata try: try: image_params = create_image_data(filepath) except: raise BadImage("HEADER") logging.warn('BadImage("HEADER") error raised') self._validate(image_params) except BadImage as e: logging.warn("Quarantining invalid image: %s", filename) logging.warn("BadImage found; error message= %s", e.get_message()) shutil.move(filepath, os.path.join(self.quarantine, filename)) mark_as_corrupt(self._cursor, filename, e.get_message()) corrupt.append(filename) continue # If everything looks good, move to archive and add to database # print image_params['date'] date_str = image_params['date'].strftime('%Y/%m/%d') # The files must be transcoded in order to work with JHelioviewer. # Therefore, any problem with the transcoding process must raise # an error. try: if image_params['instrument'] == "AIA": self._transcode(filepath, cprecincts=[128, 128]) else: self._transcode(filepath) except KduTranscodeError, e: logging.error("kdu_transcode: " + e.get_message()) # Move to archive if image_params['observatory'] == "Hinode": directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['filter1']), str(image_params['filter2'])) else: directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['measurement'])) dest = os.path.join(directory, filename) image_params['filepath'] = dest if not os.path.exists(directory): try: os.makedirs(directory) except OSError: logging.error("Unable to create the directory '" + directory + "'. Please ensure that you " "have the proper permissions and try again.") self.shutdown_requested = True try: shutil.move(filepath, dest) except IOError: logging.error("Unable to move files to destination. Is there " "enough free space?") self.shutdown_requested = True # Add to list to send to main database images.append(image_params)
def ingest(self, urls): """ Add images to helioviewer data db. (1) Make sure the file exists (2) Make sure the file is 'good', and quarantine if it is not. (3) Apply the ESA JPIP encoding. (4) Ingest (5) Update database to say that the file has been successfully 'ingested'. """ # Get filepaths filepaths = [] images = [] corrupt = [] for url in urls: path = os.path.join( self.incoming, os.path.basename(url)) # @TODO: Better path computation if os.path.isfile(path): filepaths.append(path) # Add to hvpull/Helioviewer.org databases for filepath in filepaths: filename = os.path.basename(filepath) # Parse header and validate metadata try: try: image_params = create_image_data(filepath) except: raise BadImage("HEADER") logging.warn('BadImage("HEADER") error raised') self._validate(image_params) except BadImage as e: logging.warn("Quarantining invalid image: %s", filename) logging.warn("BadImage found; error message= %s", e.get_message()) shutil.move(filepath, os.path.join(self.quarantine, filename)) mark_as_corrupt(self._cursor, filename, e.get_message()) corrupt.append(filename) continue # If everything looks good, move to archive and add to database # print image_params['date'] date_str = image_params['date'].strftime('%Y/%m/%d') # The files must be transcoded in order to work with JHelioviewer. # Therefore, any problem with the transcoding process must raise # an error. try: if image_params['instrument'] == "AIA": self._transcode(filepath, cprecincts=[128, 128]) else: self._transcode(filepath) except KduTranscodeError, e: logging.error("kdu_transcode: " + e.get_message()) # Move to archive if image_params['observatory'] == "Hinode": directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['filter1']), str(image_params['filter2'])) else: directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['measurement'])) dest = os.path.join(directory, filename) image_params['filepath'] = dest if not os.path.exists(directory): try: os.makedirs(directory) except OSError: logging.error("Unable to create the directory '" + directory + "'. Please ensure that you " "have the proper permissions and try again.") self.shutdown_requested = True try: shutil.move(filepath, dest) except IOError: logging.error("Unable to move files to destination. Is there " "enough free space?") self.shutdown_requested = True # Add to list to send to main database images.append(image_params)