def move_to_archive(self, variable):
'''Moves grb and hdr files to archive location.
Precondition:
Header and Grib files for variable exist in current directory
Postcondition:
Header and Grib files for variable exist in archive directory
Header and Grib files for variable don't exist in current directory
'''
logger = logging.getLogger(__name__)
dest_path = self.get_internal_drectory() # Determine the directory
hdr_name = self.get_internal_filename(variable, 'hdr')
grb_name = self.get_internal_filename(variable, 'grb')
System.create_directory(dest_path) # create it if it does not exist
# Archive the files
logger.info('Archiving into [{0}]'.format(dest_path))
# GRIB
dest_file = os.path.join(dest_path, grb_name)
shutil.copyfile(grb_name, dest_file)
# HEADER
dest_file = os.path.join(dest_path, hdr_name)
shutil.copyfile(hdr_name, dest_file)
# Cleanup the working directory
if os.path.exists(grb_name):
os.unlink(grb_name)
if os.path.exists(hdr_name):
os.unlink(hdr_name)
def move_to_archive(self, variable):
'''Moves grb and hdr files to archive location.
Precondition:
Header and Grib files for variable exist in current directory
Postcondition:
Header and Grib files for variable exist in archive directory
Header and Grib files for variable don't exist in current directory
'''
logger = logging.getLogger(__name__)
dest_path = self.get_internal_drectory() # Determine the directory
hdr_name = self.get_internal_filename(variable, 'hdr')
grb_name = self.get_internal_filename(variable, 'grb')
System.create_directory(dest_path) # create it if it does not exist
# Archive the files
logger.info('Archiving into [{0}]'.format(dest_path))
# GRIB
dest_file = os.path.join(dest_path, grb_name)
shutil.copyfile(grb_name, dest_file)
# HEADER
dest_file = os.path.join(dest_path, hdr_name)
shutil.copyfile(hdr_name, dest_file)
# Cleanup the working directory
if os.path.exists(grb_name):
os.unlink(grb_name)
if os.path.exists(hdr_name):
os.unlink(hdr_name)
def process_grib_for_variable(self, variable, verbose=False):
'''Extract the specified variable from the grib file and archive it.
Precondition:
A grib file, with the name get_external_filename(), exists in
current working directory.
wgrib must be installed on the system
Postcondition:
A grib and header for variable will exist in current working
directory with the name given by get_internal_filename()
'''
logger = logging.getLogger(__name__)
grib_file = self.get_external_filename()
hdr_name = self.get_internal_filename(variable, 'hdr')
grb_name = self.get_internal_filename(variable, 'grb')
if os.path.isfile(grb_name) and os.path.isfile(hdr_name):
logger.info(
'{0} and {1} already exist. Skipping extraction.'.format(
hdr_name, grb_name))
return
logger.info("Processing [{0}]".format(grib_file))
# Create inventory/header file to extract the variable data
cmd_create_temp_header = [
'wgrib', grib_file, '|', 'grep', variable, '>', hdr_name
]
# Create grib files for each variable
cmd_create_final_grib = [
'cat', hdr_name, '|', 'wgrib', grib_file, '-i', '-grib', '-o',
grb_name
]
# Create new inventory/header file for the variable
cmd_create_final_header = [
'wgrib', grb_name, '|', 'grep', variable, '>', hdr_name
]
cmds = [
cmd_create_temp_header, cmd_create_final_grib,
cmd_create_final_header
]
for cmd_list in cmds:
cmd = ' '.join(cmd_list)
output = ''
logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if verbose:
if len(output) > 0:
logger.info(output)
def archive_aux_data(self):
'''
Description:
Defines the main processing method for the class.
'''
# Figure out the names of the files to retrieve
names = list(self.get_name_list())
# Establish a logged in session
session = Web.Session(
block_size=Config.get('http_transfer_block_size'))
# Log in
session.login(Config.get('ucar.login_credentials.login_url'),
Config.get('ucar.login_credentials.login_data'))
for name in names:
filename = '{0}.tar'.format(name)
self.logger.info('Retrieving {0}'.format(filename))
year = name[7:11]
url = Config.get('ucar.url_format').format(year, filename)
session.http_transfer_file(url, filename)
# Extract the tar'd data
cmd = ['tar', '-xvf', filename]
cmd = ' '.join(cmd)
grib_files = System.execute_cmd(cmd)
if grib_files is not None and len(grib_files) > 0:
self.logger.info(grib_files)
# For each parameter we need
for variable in Config.get('narr_variables'):
self.logger.info('Processing Variable [{0}]'.format(variable))
for grib_file in grib_files.split():
self.process_grib_for_variable(variable, grib_file)
# Cleanup - Extracted grib files
for grib_file in grib_files.split():
if os.path.exists(grib_file):
os.unlink(grib_file)
# Cleanup - The Tar ball
if os.path.exists(filename):
os.unlink(filename)
def archive_aux_data(self):
'''
Description:
Defines the main processing method for the class.
'''
# Figure out the names of the files to retrieve
names = list(self.get_name_list())
# Establish a logged in session
session = Web.Session(
block_size=Config.get('http_transfer_block_size'))
# Log in
session.login(Config.get('ucar.login_credentials.login_url'),
Config.get('ucar.login_credentials.login_data'))
for name in names:
filename = '{0}.tar'.format(name)
self.logger.info('Retrieving {0}'.format(filename))
year = name[7:11]
url = Config.get('ucar.url_format').format(year, filename)
session.http_transfer_file(url, filename)
# Extract the tar'd data
cmd = ['tar', '-xvf', filename]
cmd = ' '.join(cmd)
grib_files = System.execute_cmd(cmd)
if grib_files is not None and len(grib_files) > 0:
self.logger.info(grib_files)
# For each parameter we need
for variable in Config.get('narr_variables'):
self.logger.info('Processing Variable [{0}]'.format(variable))
for grib_file in grib_files.split():
self.process_grib_for_variable(variable, grib_file)
# Cleanup - Extracted grib files
for grib_file in grib_files.split():
if os.path.exists(grib_file):
os.unlink(grib_file)
# Cleanup - The Tar ball
if os.path.exists(filename):
os.unlink(filename)
def process_grib_for_variable(self, variable, verbose=False):
'''Extract the specified variable from the grib file and archive it.
Precondition:
A grib file, with the name get_external_filename(), exists in
current working directory.
wgrib must be installed on the system
Postcondition:
A grib and header for variable will exist in current working
directory with the name given by get_internal_filename()
'''
logger = logging.getLogger(__name__)
grib_file = self.get_external_filename()
hdr_name = self.get_internal_filename(variable, 'hdr')
grb_name = self.get_internal_filename(variable, 'grb')
if os.path.isfile(grb_name) and os.path.isfile(hdr_name):
logger.info('{0} and {1} already exist. Skipping extraction.'
.format(hdr_name, grb_name))
return
logger.info("Processing [{0}]".format(grib_file))
# Create inventory/header file to extract the variable data
cmd_create_temp_header = ['wgrib', grib_file, '|', 'grep', variable,
'>', hdr_name]
# Create grib files for each variable
cmd_create_final_grib = ['cat', hdr_name, '|',
'wgrib', grib_file, '-i', '-grib',
'-o', grb_name]
# Create new inventory/header file for the variable
cmd_create_final_header = ['wgrib', grb_name, '|', 'grep', variable,
'>', hdr_name]
cmds = [cmd_create_temp_header, cmd_create_final_grib,
cmd_create_final_header]
for cmd_list in cmds:
cmd = ' '.join(cmd_list)
output = ''
logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if verbose:
if len(output) > 0:
logger.info(output)
def process_grib_for_variable(self, variable, grib_file):
'''
Description:
Extract the specified variable from the grib file and archive it.
'''
self.logger.info("Processing [{0}]".format(grib_file))
# Get the date information from the grib file
parts = grib_file.split('.')
year = int(parts[1][:4])
month = int(parts[1][4:6])
day = int(parts[1][6:8])
hour = int(parts[1][8:])
# Figure out the filenames to create
hdr_name = (Config.get('archive_name_format')
.format(variable, year, month, day, hour*100, 'hdr'))
grb_name = (Config.get('archive_name_format')
.format(variable, year, month, day, hour*100, 'grb'))
# Create inventory/header file to extract the variable data
cmd = ['wgrib', grib_file, '|', 'grep', variable, '>', hdr_name]
cmd = ' '.join(cmd)
self.logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if output is not None and len(output) > 0:
self.logger.info(output)
# Create grib files for each variable
cmd = ['cat', hdr_name, '|',
'wgrib', grib_file, '-i', '-grib', '-o', grb_name]
cmd = ' '.join(cmd)
output = ''
self.logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if output is not None and len(output) > 0:
self.logger.info(output)
# Create new inventory/header file for the variable
cmd = ['wgrib', grb_name, '|', 'grep', variable, '>', hdr_name]
cmd = ' '.join(cmd)
self.logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if output is not None and len(output) > 0:
self.logger.info(output)
# Determine the directory to place the data and create it if it does
# not exist
dest_path = (Config.get('archive_directory_format')
.format(self.base_aux_dir, year, month, day))
System.create_directory(dest_path)
# Archive the files
self.logger.info('Archiving into [{0}]'.format(dest_path))
# GRIB
dest_file = os.path.join(dest_path, grb_name)
shutil.copyfile(grb_name, dest_file)
# HEADER
dest_file = os.path.join(dest_path, hdr_name)
shutil.copyfile(hdr_name, dest_file)
# Cleanup the working directory
if os.path.exists(grb_name):
os.unlink(grb_name)
if os.path.exists(hdr_name):
os.unlink(hdr_name)
def process_grib_for_variable(self, variable, grib_file):
'''
Description:
Extract the specified variable from the grib file and archive it.
'''
self.logger.info("Processing [{0}]".format(grib_file))
# Get the date information from the grib file
parts = grib_file.split('.')
year = int(parts[1][:4])
month = int(parts[1][4:6])
day = int(parts[1][6:8])
hour = int(parts[1][8:])
# Figure out the filenames to create
hdr_name = (Config.get('archive_name_format').format(
variable, year, month, day, hour * 100, 'hdr'))
grb_name = (Config.get('archive_name_format').format(
variable, year, month, day, hour * 100, 'grb'))
# Create inventory/header file to extract the variable data
cmd = ['wgrib', grib_file, '|', 'grep', variable, '>', hdr_name]
cmd = ' '.join(cmd)
self.logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if output is not None and len(output) > 0:
self.logger.info(output)
# Create grib files for each variable
cmd = [
'cat', hdr_name, '|', 'wgrib', grib_file, '-i', '-grib', '-o',
grb_name
]
cmd = ' '.join(cmd)
output = ''
self.logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if output is not None and len(output) > 0:
self.logger.info(output)
# Create new inventory/header file for the variable
cmd = ['wgrib', grb_name, '|', 'grep', variable, '>', hdr_name]
cmd = ' '.join(cmd)
self.logger.info('Executing [{0}]'.format(cmd))
output = System.execute_cmd(cmd)
if output is not None and len(output) > 0:
self.logger.info(output)
# Determine the directory to place the data and create it if it does
# not exist
dest_path = (Config.get('archive_directory_format').format(
self.base_aux_dir, year, month, day))
System.create_directory(dest_path)
# Archive the files
self.logger.info('Archiving into [{0}]'.format(dest_path))
# GRIB
dest_file = os.path.join(dest_path, grb_name)
shutil.copyfile(grb_name, dest_file)
# HEADER
dest_file = os.path.join(dest_path, hdr_name)
shutil.copyfile(hdr_name, dest_file)
# Cleanup the working directory
if os.path.exists(grb_name):
os.unlink(grb_name)
if os.path.exists(hdr_name):
os.unlink(hdr_name)
