Ejemplo n.º 1
0
def main(argv):
    '''Main application access point'''
    options = get_options()

    init_logger('update.log')

    print('Processing Images...')

    # Get a list of images to process
    filepaths = find_images(options.source)

    if len(filepaths) is 0:
        return

    images = []

    # Move images to main archive
    for filepath in filepaths:
        dest = os.path.join(options.destination,
                            os.path.relpath(filepath, options.source))

        # Parse image header
        image_params = create_image_data(filepath)
        image_params['filepath'] = dest

        images.append(image_params)

        directory = os.path.dirname(dest)

        if not os.path.isdir(directory):
            os.makedirs(directory)

        shutil.move(filepath, dest)

    # Add images to the database
    db, cursor = get_db_cursor(options.dbhost, options.dbname, options.dbuser,
                               options.dbpass)
    process_jp2_images(images, options.destination, cursor, True)
    cursor.close()

    print('Finished!')
Ejemplo n.º 2
0
def main(argv):
    '''Main application access point'''
    options = get_options()
    
    init_logger('update.log')
    
    print('Processing Images...')
    
    # Get a list of images to process
    filepaths = find_images(options.source)
    
    if len(filepaths) is 0:
        return

    images = []

    # Move images to main archive
    for filepath in filepaths:
        dest = os.path.join(options.destination, 
                            os.path.relpath(filepath, options.source))
        
        # Parse image header
        image_params = create_image_data(filepath)
        image_params['filepath'] = dest
        
        images.append(image_params)

        directory = os.path.dirname(dest)
        
        if not os.path.isdir(directory):
            os.makedirs(directory)

        shutil.move(filepath, dest)
    
    # Add images to the database
    db, cursor = get_db_cursor(options.dbhost, options.dbname, options.dbuser, options.dbpass)
    process_jp2_images(images, options.destination, cursor, True)    
    cursor.close()
    
    print('Finished!')
Ejemplo n.º 3
0
    def ingest(self, urls):
        """
        Add images to helioviewer data db.
          (1) Make sure the file exists
          (2) Make sure the file is 'good', and quarantine if it is not.
          (3) Apply the ESA JPIP encoding.
          (4) Ingest
          (5) Update database to say that the file has been successfully
              'ingested'.
        """
        # Get filepaths
        filepaths = []
        images = []
        corrupt = []

        for url in urls:
            path = os.path.join(self.incoming, os.path.basename(url)) # @TODO: Better path computation
            if os.path.isfile(path):
                filepaths.append(path)

        # Add to hvpull/Helioviewer.org databases
        for filepath in filepaths:
            filename = os.path.basename(filepath)

            # Parse header and validate metadata
            try:
                try:
                    image_params = create_image_data(filepath)
                except:
                    raise BadImage("HEADER")
                    logging.warn('BadImage("HEADER") error raised')
                self._validate(image_params)
            except BadImage as e:
                logging.warn("Quarantining invalid image: %s", filename)
                logging.warn("BadImage found; error message= %s", e.get_message())
                shutil.move(filepath, os.path.join(self.quarantine, filename))
                mark_as_corrupt(self._cursor, filename, e.get_message())
                corrupt.append(filename)
                continue

            # If everything looks good, move to archive and add to database
            # print image_params['date']
            date_str = image_params['date'].strftime('%Y/%m/%d')

            # The files must be transcoded in order to work with JHelioviewer.
            # Therefore, any problem with the transcoding process must raise
            # an error. 
            try:
                if image_params['instrument'] == "AIA":
                    self._transcode(filepath, cprecincts=[128, 128])
                else:
                    self._transcode(filepath)
            except KduTranscodeError, e:
                logging.error("kdu_transcode: " + e.get_message())

            # Move to archive
            if image_params['observatory'] == "Hinode":
                directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['filter1']), str(image_params['filter2']))
            else:
                directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['measurement']))
            
            dest = os.path.join(directory, filename)

            image_params['filepath'] = dest

            if not os.path.exists(directory):
                try:
                    os.makedirs(directory)
                except OSError:
                    logging.error("Unable to create the directory '" +
                                  directory + "'. Please ensure that you "
                                  "have the proper permissions and try again.")
                    self.shutdown_requested = True

            try:
                shutil.move(filepath, dest)
            except IOError:
                logging.error("Unable to move files to destination. Is there "
                              "enough free space?")
                self.shutdown_requested = True

            # Add to list to send to main database
            images.append(image_params)
Ejemplo n.º 4
0
    def ingest(self, urls):
        """
        Add images to helioviewer data db.
          (1) Make sure the file exists
          (2) Make sure the file is 'good', and quarantine if it is not.
          (3) Apply the ESA JPIP encoding.
          (4) Ingest
          (5) Update database to say that the file has been successfully
              'ingested'.
        """
        # Get filepaths
        filepaths = []
        images = []
        corrupt = []

        for url in urls:
            path = os.path.join(
                self.incoming,
                os.path.basename(url))  # @TODO: Better path computation
            if os.path.isfile(path):
                filepaths.append(path)

        # Add to hvpull/Helioviewer.org databases
        for filepath in filepaths:
            filename = os.path.basename(filepath)

            # Parse header and validate metadata
            try:
                try:
                    image_params = create_image_data(filepath)
                except:
                    raise BadImage("HEADER")
                    logging.warn('BadImage("HEADER") error raised')
                self._validate(image_params)
            except BadImage as e:
                logging.warn("Quarantining invalid image: %s", filename)
                logging.warn("BadImage found; error message= %s",
                             e.get_message())
                shutil.move(filepath, os.path.join(self.quarantine, filename))
                mark_as_corrupt(self._cursor, filename, e.get_message())
                corrupt.append(filename)
                continue

            # If everything looks good, move to archive and add to database
            # print image_params['date']
            date_str = image_params['date'].strftime('%Y/%m/%d')

            # The files must be transcoded in order to work with JHelioviewer.
            # Therefore, any problem with the transcoding process must raise
            # an error.
            try:
                if image_params['instrument'] == "AIA":
                    self._transcode(filepath, cprecincts=[128, 128])
                else:
                    self._transcode(filepath)
            except KduTranscodeError, e:
                logging.error("kdu_transcode: " + e.get_message())

            # Move to archive
            if image_params['observatory'] == "Hinode":
                directory = os.path.join(self.image_archive,
                                         image_params['nickname'], date_str,
                                         str(image_params['filter1']),
                                         str(image_params['filter2']))
            else:
                directory = os.path.join(self.image_archive,
                                         image_params['nickname'], date_str,
                                         str(image_params['measurement']))

            dest = os.path.join(directory, filename)

            image_params['filepath'] = dest

            if not os.path.exists(directory):
                try:
                    os.makedirs(directory)
                except OSError:
                    logging.error("Unable to create the directory '" +
                                  directory + "'. Please ensure that you "
                                  "have the proper permissions and try again.")
                    self.shutdown_requested = True

            try:
                shutil.move(filepath, dest)
            except IOError:
                logging.error("Unable to move files to destination. Is there "
                              "enough free space?")
                self.shutdown_requested = True

            # Add to list to send to main database
            images.append(image_params)