def handle(self):
# Parse date and path
profile_date = date.fromisoformat(self.argument('profile-date'))
csv_path = Path(self.argument('csv-path'))
csv_path.mkdir(parents=True, exist_ok=True)
location = self.argument('location')
config = Config()
# Read the WRF profile file
try:
df = wrf.read_wrf_daily_profile(config,
self.argument('location'),
profile_date)
except FileNotFoundError as ex:
self.line_error('<error>WRF Profile file not found!</error>')
self.line_error(f'(Path: {ex.filename})')
sys.exit(1)
# Split into days and write each into a file
for _, group in df.groupby('timestamp'):
group_timestamp = group.iloc[0, 0].strftime('%Y%m%d_%H%M')
filename = csv_path / f'{location}_{group_timestamp}.csv'
self.line(f'<info>Writing</info> {filename}')
group.iloc[:, 1:].to_csv(filename, index=False)
def handle(self):
ids = self.argument("id")
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Login to SCC
successes = []
failures = []
with scc_session(credentials) as scc:
with Progress(console=console) as progress:
task = progress.add_task("Asking for re-runs...", total=len(ids))
for id in ids:
try:
scc.rerun_processing(id)
console.print(f"[info]Re-running[/info] {id}")
successes.append(id)
except Exception as ex:
console.print(
f"-> [error]Could not make request:[/error] {id}", style="bold")
console.print(f'[error]{type(ex).__name__}:[/error] {str(ex)}')
failures.append(id)
progress.advance(task)
# Print a summary
summary = "---\n"
if len(successes) > 0:
summary += "**Re-running:**\n"
for id in successes:
summary += f"* {id}\n"
if len(failures) > 0:
summary += "\n**Failed to rerun:**\n"
for id in failures:
summary += f"* {id}\n"
console.print(Markdown(summary))
def handle(self):
# Check output directory
output_directory = Path(self.argument('output-directory'))
output_directory.mkdir(parents=True, exist_ok=True)
# Check if list or IDs are defined
id_frame = None
id_list_file = self.argument('list')
if id_list_file is None:
ids = self.option('id')
if ids is None or len(ids) == 0:
self.line_error('Either a list file or some measurement IDs must be provided!')
return 1
else:
id_frame = pd.read_csv(id_list_file, index_col='id')
ids = id_frame.index
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Download files for each ID
with scc_session(credentials) as scc:
for id in track(ids, description='Downloading products', console=console):
# Check if processing is done
measurement = scc.get_measurement(id)
if measurement.is_processing:
console.print(f'[warn]File[/warn] {id} [warn]is still processing.[/warn]')
continue
for file in scc.download_products(id, output_directory):
console.print(f'[info]Downloaded[/info] {file}')
if id_frame is not None:
id_frame.loc[id, 'Products_Downloaded'] = True
def handle(self):
# Create output directory
output_path = Path(self.argument('output-path'))
output_path.mkdir(parents=True, exist_ok=True)
# Parse arguments
input_path = Path(self.argument('input'))
should_round = self.option('round')
interval = self.option('interval')
if interval is None:
interval = 60 # Default duration is 1 hour/60 minutes
interval = timedelta(minutes=interval)
# Try to get location
location_name = self.argument('location')
location = locations.get_location_by_name(location_name)
if location is None:
locations.unknown_location_error(location_name)
return 1
# Check for radiosonde files
config = Config()
profiles = None
if not self.option('no-radiosonde'):
day = pollyxt.get_measurement_period(input_path)[0].date()
try:
profiles = radiosondes.wrf.read_wrf_daily_profile(
config, location, day)
except FileNotFoundError as ex:
self.line_error(
f'<error>No radiosonde file found for </error>{location.name}<error> at </error>{day.isoformat()}'
)
self.line_error(
'<error>Use the --no-radiosonde option to skip this.')
return 1
# Convert files
skip_calibration = self.option('no-calibration')
converter = scc_netcdf.convert_pollyxt_file(
input_path,
output_path,
location,
interval,
should_round,
calibration=(not skip_calibration))
for id, path, timestamp in converter:
self.line('<info>Created file with measurement ID </info>' + id +
'<info> at </info>' + str(path))
# Attempt to write radiosonde for this profile
if profiles is not None:
p = profiles[profiles['timestamp'] == timestamp.replace(
minute=0, second=0)]
if len(p) > 0:
path = output_path / f'rs_{id[:-2]}.nc'
radiosondes.create_radiosonde_netcdf(p, location, path)
self.line(
f'<info>Created radiosonde file at</info> {path}')
else:
self.line_error(
f'<error>No radiosonde profile found for </error>{id}')
def handle(self):
# Check output directory
output_path = Path(self.argument('output-path'))
output_path.mkdir(parents=True, exist_ok=True)
# Parse other arguments
should_round = self.option('round')
interval = self.option('interval')
if interval is None:
interval = 60 # Default duration is 1 hour/60 minutes
interval = timedelta(minutes=interval)
# Try to get location
location_name = self.argument('location')
location = locations.get_location_by_name(location_name)
if location is None:
locations.unknown_location_error(location_name)
return 1
# Get list of input files
input_path = Path(self.argument('input'))
if self.option('recursive'):
pattern = '**/*.nc'
else:
pattern = '*.nc'
file_list = list(input_path.glob(pattern))
if len(file_list) == 0:
self.line_error(
f'<error>No netCDF files found in </error>{input_path}')
return 1
progress = self.progress_bar(len(file_list))
# Iterate over list and convert files
skip_calibration = self.option('no-calibration')
for file in file_list:
progress.clear()
self.line(
f'\r-> <comment>Converting</comment> {file} <comment>...</comment>'
)
progress.display()
# Try to find profiles
# Check for radiosonde files
config = Config()
profiles = None
if not self.option('no-radiosonde'):
day = pollyxt.get_measurement_period(file)[0].date()
try:
profiles = radiosondes.wrf.read_wrf_daily_profile(
config, location, day)
except FileNotFoundError as ex:
self.line_error(
f'<error>No radiosonde file found for </error>{location.name}<error> at </error>{day.isoformat()}'
)
self.line_error(
'<error>Use the --no-radiosonde option to skip this.')
return 1
converter = scc_netcdf.convert_pollyxt_file(
file,
output_path,
location,
interval,
should_round,
calibration=(not skip_calibration))
for id, path, timestamp in converter:
progress.clear()
self.line('\r<info>Created file with measurement ID </info>' +
id + '<info> at </info>' + str(path))
# Attempt to write radiosonde for this profile
if profiles is not None:
p = profiles[profiles['timestamp'] == timestamp.replace(
minute=0, second=0)]
if len(p) > 0:
path = output_path / f'rs_{id[:-2]}.nc'
radiosondes.create_radiosonde_netcdf(p, location, path)
self.line(
f'<info>Created radiosonde file at</info> {path}')
else:
self.line_error(
f'<error>No radiosonde profile found for </error>{id}'
)
progress.display()
progress.advance()
progress.finish()
self.line('\n<comment>Done!</comment>')
def handle(self):
# Parse arguments
location_name = self.option('location')
location = None
if location_name is not None:
location = locations.get_location_by_name(location_name)
if location is None:
locations.unknown_location_error(location_name)
return 1
try:
date_start = self.argument('date-start')
date_start = datetime.date.fromisoformat(date_start)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
try:
date_end = self.argument('date-end')
date_end = datetime.date.fromisoformat(date_end)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
hirelpp = option_to_bool(self.option('no-hirelpp'), True)
cloudmask = option_to_bool(self.option('no-cloudmask'), True)
elpp = option_to_bool(self.option('no-elpp'), True)
optical = option_to_bool(self.option('no-optical'), True)
elic = option_to_bool(self.option('no-elic'), True)
download_path = Path(self.argument('download-path'))
download_path.mkdir(exist_ok=True, parents=True)
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Login to SCC
with scc_session(credentials) as scc:
# Look up products
with Progress(console=console) as progress:
task = progress.add_task('Fetching results...', start=False, total=1)
# Query SCC for measurements
pages, measurements = scc.query_measurements(
date_start, date_end, location, credentials)
if len(measurements) == 0:
progress.stop()
console.print('[warn]No measurements found![/warn]')
return 0
if pages > 1:
progress.start_task(task)
progress.update(task, total=pages, completed=1, start=True)
current_page = 2
while current_page <= pages:
_, more_measurements = scc.query_measurements(
date_start, date_end, location, page=current_page)
measurements += more_measurements
current_page += 1
progress.advance(task)
console.log(f'[info]Found[/info] {len(measurements)} [info]measurements.[/info]')
# Download files
measurement_count = len(measurements)
file_count = 0
i = 0
with Progress(console=console) as progress:
task = progress.add_task(
f'Downloading products (1/{measurement_count})...', total=measurement_count)
for m in measurements:
progress.update(
task, description=f'Downloading products ({i}/{measurement_count})...')
for file in scc.download_products(m.id, download_path,
hirelpp and m.has_hirelpp,
cloudmask and m.has_cloudmask,
elpp and m.has_elpp,
optical and (m.has_elda or m.has_eldec),
elic and m.has_elic):
file_count += 1
console.log(f'[info]Downloaded[/info] {file}')
progress.advance(task)
i += 1
console.log(f'[info]Downloaded[/info] {file_count} [info]files![/info]')
def handle(self):
# Parse arguments
location_name = self.option('location')
location = None
if location_name is not None:
location = locations.get_location_by_name(location_name)
if location is None:
locations.unknown_location_error(location_name)
return 1
try:
date_start = self.argument('date-start')
date_start = datetime.date.fromisoformat(date_start)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
try:
date_end = self.argument('date-end')
date_end = datetime.date.fromisoformat(date_end)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Login to SCC to make queries
with scc_session(credentials) as scc:
with Progress(console=console) as progress:
task = progress.add_task('Fetching results...', start=False, total=1)
# Query SCC for measurements
pages, measurements = scc.query_measurements(
date_start, date_end, location, credentials)
if len(measurements) == 0:
progress.stop()
console.print('[warn]No measurements found![/warn]')
return 0
progress.start_task(task)
if pages > 1:
progress.update(task, total=pages, completed=1, start=True)
current_page = 2
while current_page <= pages:
_, more_measurements = scc.query_measurements(
date_start, date_end, location, page=current_page)
measurements += more_measurements
current_page += 1
progress.advance(task)
else:
progress.advance(task)
# Render table
table = Table(show_header=True, header_style="bold")
for col in ['ID', 'Location', 'Start', 'End',
'HiRELPP', 'CloudMask', 'ELPP', 'ELDA', 'ELDEC', 'ELIC', 'ELQUICK', 'Is Processing']:
table.add_column(col)
for m in measurements:
table.add_row(
m.id,
m.location.name,
m.date_start.strftime('%Y-%m-%d %H:%M'),
m.date_end.strftime('%Y-%m-%d %H:%M'),
bool_to_emoji(m.has_hirelpp),
bool_to_emoji(m.has_cloudmask),
bool_to_emoji(m.has_elpp),
bool_to_emoji(m.has_elda),
bool_to_emoji(m.has_eldec),
bool_to_emoji(m.has_elic),
bool_to_emoji(m.has_elquick),
bool_to_emoji(m.is_processing),
)
console.print(table)
# Write to CSV
csv_path = self.option('to-csv')
if csv_path is not None:
csv_path = Path(csv_path)
with open(csv_path, 'w') as f:
f.write(
'id,station_id,location,date_start,date_end,date_creation,date_updated,hirelpp,cloudmask,elpp,elda,eldec,elic,elquick,is_processing\n')
for m in measurements:
f.write(m.to_csv() + '\n')
console.print(f'[info]Wrote .csv file[/info] {csv_path}')
def handle(self):
# Parse arguments
path = Path(self.argument('path'))
if path.is_dir:
files = path.glob('*.nc')
files = filter(lambda x: not x.name.startswith('rs_'), files)
files = filter(lambda x: not x.name.startswith('calibration_'),
files) # TODO Handle calibration files
else:
files = [path]
files = list(files)
if len(files) == 0:
console.print('[error]No files found in given directory[/error]')
return 1
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Upload files
successful_files = []
successful_ids = []
with scc_session(credentials) as scc:
for file in track(files, description='Uploading files...', console=console):
# Read file to find radiosondes
nc = Dataset(file, 'r')
radiosonde_path = file.parent / nc.Sounding_File_Name
dataset_id = nc.Measurement_ID
configuration_id = nc.NOAReACT_Configuration_ID
nc.close()
if not radiosonde_path.exists():
console.print(
f'[error]Cannot find radiosonde file[/error] {radiosonde_path} [error] for measurement [/error] {file} [error]. Skipping file.[/error]')
continue
# Upload file to SCC
try:
scc.upload_file(file, configuration_id, rs_filename=radiosonde_path)
successful_files.append(file)
successful_ids.append(dataset_id)
except exceptions.SCCError as ex:
console.print(
f'[error]Error while uploading[/error] {file}[error]:[/error] {str(ex)}')
except Exception:
console.print(f'[error]Unknown error while uploading[/error] {file}')
console.print_exception()
successful_count = len(successful_ids)
if successful_count == 0:
console.print('[warn]No files were uploaded successfully![/warn]')
return 0
else:
console.print(
f'[info]Successfully uploaded[/info] {successful_count} [info]files.[/info]')
# Write list file if requested
list_file = self.argument('list')
if list_file is not None:
list_file = Path(list_file)
df = pd.DataFrame()
df['Filename'] = successful_files
df['id'] = successful_ids
df['Products_Downloaded'] = False
df.to_csv(list_file, index=False)
self.line(f'<comment>Wrote IDs to </comment>{list_file}')