def process_grib_for_variable(self, variable, verbose=False): '''Extract the specified variable from the grib file and archive it. Precondition: A grib file, with the name get_external_filename(), exists in current working directory. wgrib must be installed on the system Postcondition: A grib and header for variable will exist in current working directory with the name given by get_internal_filename() ''' logger = logging.getLogger(__name__) grib_file = self.get_external_filename() hdr_name = self.get_internal_filename(variable, 'hdr') grb_name = self.get_internal_filename(variable, 'grb') if os.path.isfile(grb_name) and os.path.isfile(hdr_name): logger.info( '{0} and {1} already exist. Skipping extraction.'.format( hdr_name, grb_name)) return logger.info("Processing [{0}]".format(grib_file)) # Create inventory/header file to extract the variable data cmd_create_temp_header = [ 'wgrib', grib_file, '|', 'grep', variable, '>', hdr_name ] # Create grib files for each variable cmd_create_final_grib = [ 'cat', hdr_name, '|', 'wgrib', grib_file, '-i', '-grib', '-o', grb_name ] # Create new inventory/header file for the variable cmd_create_final_header = [ 'wgrib', grb_name, '|', 'grep', variable, '>', hdr_name ] cmds = [ cmd_create_temp_header, cmd_create_final_grib, cmd_create_final_header ] for cmd_list in cmds: cmd = ' '.join(cmd_list) output = '' logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if verbose: if len(output) > 0: logger.info(output)
def archive_aux_data(self): ''' Description: Defines the main processing method for the class. ''' # Figure out the names of the files to retrieve names = list(self.get_name_list()) # Establish a logged in session session = Web.Session( block_size=Config.get('http_transfer_block_size')) # Log in session.login(Config.get('ucar.login_credentials.login_url'), Config.get('ucar.login_credentials.login_data')) for name in names: filename = '{0}.tar'.format(name) self.logger.info('Retrieving {0}'.format(filename)) year = name[7:11] url = Config.get('ucar.url_format').format(year, filename) session.http_transfer_file(url, filename) # Extract the tar'd data cmd = ['tar', '-xvf', filename] cmd = ' '.join(cmd) grib_files = System.execute_cmd(cmd) if grib_files is not None and len(grib_files) > 0: self.logger.info(grib_files) # For each parameter we need for variable in Config.get('narr_variables'): self.logger.info('Processing Variable [{0}]'.format(variable)) for grib_file in grib_files.split(): self.process_grib_for_variable(variable, grib_file) # Cleanup - Extracted grib files for grib_file in grib_files.split(): if os.path.exists(grib_file): os.unlink(grib_file) # Cleanup - The Tar ball if os.path.exists(filename): os.unlink(filename)
def process_grib_for_variable(self, variable, verbose=False): '''Extract the specified variable from the grib file and archive it. Precondition: A grib file, with the name get_external_filename(), exists in current working directory. wgrib must be installed on the system Postcondition: A grib and header for variable will exist in current working directory with the name given by get_internal_filename() ''' logger = logging.getLogger(__name__) grib_file = self.get_external_filename() hdr_name = self.get_internal_filename(variable, 'hdr') grb_name = self.get_internal_filename(variable, 'grb') if os.path.isfile(grb_name) and os.path.isfile(hdr_name): logger.info('{0} and {1} already exist. Skipping extraction.' .format(hdr_name, grb_name)) return logger.info("Processing [{0}]".format(grib_file)) # Create inventory/header file to extract the variable data cmd_create_temp_header = ['wgrib', grib_file, '|', 'grep', variable, '>', hdr_name] # Create grib files for each variable cmd_create_final_grib = ['cat', hdr_name, '|', 'wgrib', grib_file, '-i', '-grib', '-o', grb_name] # Create new inventory/header file for the variable cmd_create_final_header = ['wgrib', grb_name, '|', 'grep', variable, '>', hdr_name] cmds = [cmd_create_temp_header, cmd_create_final_grib, cmd_create_final_header] for cmd_list in cmds: cmd = ' '.join(cmd_list) output = '' logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if verbose: if len(output) > 0: logger.info(output)
def process_grib_for_variable(self, variable, grib_file): ''' Description: Extract the specified variable from the grib file and archive it. ''' self.logger.info("Processing [{0}]".format(grib_file)) # Get the date information from the grib file parts = grib_file.split('.') year = int(parts[1][:4]) month = int(parts[1][4:6]) day = int(parts[1][6:8]) hour = int(parts[1][8:]) # Figure out the filenames to create hdr_name = (Config.get('archive_name_format') .format(variable, year, month, day, hour*100, 'hdr')) grb_name = (Config.get('archive_name_format') .format(variable, year, month, day, hour*100, 'grb')) # Create inventory/header file to extract the variable data cmd = ['wgrib', grib_file, '|', 'grep', variable, '>', hdr_name] cmd = ' '.join(cmd) self.logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if output is not None and len(output) > 0: self.logger.info(output) # Create grib files for each variable cmd = ['cat', hdr_name, '|', 'wgrib', grib_file, '-i', '-grib', '-o', grb_name] cmd = ' '.join(cmd) output = '' self.logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if output is not None and len(output) > 0: self.logger.info(output) # Create new inventory/header file for the variable cmd = ['wgrib', grb_name, '|', 'grep', variable, '>', hdr_name] cmd = ' '.join(cmd) self.logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if output is not None and len(output) > 0: self.logger.info(output) # Determine the directory to place the data and create it if it does # not exist dest_path = (Config.get('archive_directory_format') .format(self.base_aux_dir, year, month, day)) System.create_directory(dest_path) # Archive the files self.logger.info('Archiving into [{0}]'.format(dest_path)) # GRIB dest_file = os.path.join(dest_path, grb_name) shutil.copyfile(grb_name, dest_file) # HEADER dest_file = os.path.join(dest_path, hdr_name) shutil.copyfile(hdr_name, dest_file) # Cleanup the working directory if os.path.exists(grb_name): os.unlink(grb_name) if os.path.exists(hdr_name): os.unlink(hdr_name)
def process_grib_for_variable(self, variable, grib_file): ''' Description: Extract the specified variable from the grib file and archive it. ''' self.logger.info("Processing [{0}]".format(grib_file)) # Get the date information from the grib file parts = grib_file.split('.') year = int(parts[1][:4]) month = int(parts[1][4:6]) day = int(parts[1][6:8]) hour = int(parts[1][8:]) # Figure out the filenames to create hdr_name = (Config.get('archive_name_format').format( variable, year, month, day, hour * 100, 'hdr')) grb_name = (Config.get('archive_name_format').format( variable, year, month, day, hour * 100, 'grb')) # Create inventory/header file to extract the variable data cmd = ['wgrib', grib_file, '|', 'grep', variable, '>', hdr_name] cmd = ' '.join(cmd) self.logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if output is not None and len(output) > 0: self.logger.info(output) # Create grib files for each variable cmd = [ 'cat', hdr_name, '|', 'wgrib', grib_file, '-i', '-grib', '-o', grb_name ] cmd = ' '.join(cmd) output = '' self.logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if output is not None and len(output) > 0: self.logger.info(output) # Create new inventory/header file for the variable cmd = ['wgrib', grb_name, '|', 'grep', variable, '>', hdr_name] cmd = ' '.join(cmd) self.logger.info('Executing [{0}]'.format(cmd)) output = System.execute_cmd(cmd) if output is not None and len(output) > 0: self.logger.info(output) # Determine the directory to place the data and create it if it does # not exist dest_path = (Config.get('archive_directory_format').format( self.base_aux_dir, year, month, day)) System.create_directory(dest_path) # Archive the files self.logger.info('Archiving into [{0}]'.format(dest_path)) # GRIB dest_file = os.path.join(dest_path, grb_name) shutil.copyfile(grb_name, dest_file) # HEADER dest_file = os.path.join(dest_path, hdr_name) shutil.copyfile(hdr_name, dest_file) # Cleanup the working directory if os.path.exists(grb_name): os.unlink(grb_name) if os.path.exists(hdr_name): os.unlink(hdr_name)