def handle(self):
# Parse arguments
try:
group, name = self.argument('name').split('.')
except ValueError:
console.print(
'[error]Variable names should be in the[/error] GROUP.NAME [error]format. For example,[/error] `auth.username`'
)
return 1
value = self.argument('value')
config = Config()
if value is None:
# Read variable
try:
value = config[group][name]
console.print(value)
except KeyError:
console.print(
f'[error]No config value with name[/error] {group}.{name}')
console.print('Did you forget to define it?')
return 1
else:
config[group][name] = value
config.write()
return 0
def handle(self):
# Check output directory
output_directory = Path(self.argument('output-directory'))
output_directory.mkdir(parents=True, exist_ok=True)
# Check if list or IDs are defined
id_frame = None
id_list_file = self.argument('list')
if id_list_file is None:
ids = self.option('id')
if ids is None or len(ids) == 0:
self.line_error('Either a list file or some measurement IDs must be provided!')
return 1
else:
id_frame = pd.read_csv(id_list_file, index_col='id')
ids = id_frame.index
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Download files for each ID
with scc_session(credentials) as scc:
for id in track(ids, description='Downloading products', console=console):
# Check if processing is done
measurement = scc.get_measurement(id)
if measurement.is_processing:
console.print(f'[warn]File[/warn] {id} [warn]is still processing.[/warn]')
continue
for file in scc.download_products(id, output_directory):
console.print(f'[info]Downloaded[/info] {file}')
if id_frame is not None:
id_frame.loc[id, 'Products_Downloaded'] = True
def handle(self):
ids = self.argument("id")
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Login to SCC
successes = []
failures = []
with scc_session(credentials) as scc:
with Progress(console=console) as progress:
task = progress.add_task("Asking for re-runs...", total=len(ids))
for id in ids:
try:
scc.rerun_processing(id)
console.print(f"[info]Re-running[/info] {id}")
successes.append(id)
except Exception as ex:
console.print(
f"-> [error]Could not make request:[/error] {id}", style="bold")
console.print(f'[error]{type(ex).__name__}:[/error] {str(ex)}')
failures.append(id)
progress.advance(task)
# Print a summary
summary = "---\n"
if len(successes) > 0:
summary += "**Re-running:**\n"
for id in successes:
summary += f"* {id}\n"
if len(failures) > 0:
summary += "\n**Failed to rerun:**\n"
for id in failures:
summary += f"* {id}\n"
console.print(Markdown(summary))
def unknown_location_error(name: str):
'''
Prints an error message that the given location is not found, along with a
list of known locations
'''
console.print(f'[error]Could not find location [/error]{name}[error].')
console.print('Known locations:')
for l in LOCATIONS:
console.print(f'- {l.name}')
def handle(self):
# Parse arguments
location_name = self.option('location')
location = None
if location_name is not None:
location = locations.get_location_by_name(location_name)
if location is None:
locations.unknown_location_error(location_name)
return 1
try:
date_start = self.argument('date-start')
date_start = datetime.date.fromisoformat(date_start)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
try:
date_end = self.argument('date-end')
date_end = datetime.date.fromisoformat(date_end)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
hirelpp = option_to_bool(self.option('no-hirelpp'), True)
cloudmask = option_to_bool(self.option('no-cloudmask'), True)
elpp = option_to_bool(self.option('no-elpp'), True)
optical = option_to_bool(self.option('no-optical'), True)
elic = option_to_bool(self.option('no-elic'), True)
download_path = Path(self.argument('download-path'))
download_path.mkdir(exist_ok=True, parents=True)
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Login to SCC
with scc_session(credentials) as scc:
# Look up products
with Progress(console=console) as progress:
task = progress.add_task('Fetching results...', start=False, total=1)
# Query SCC for measurements
pages, measurements = scc.query_measurements(
date_start, date_end, location, credentials)
if len(measurements) == 0:
progress.stop()
console.print('[warn]No measurements found![/warn]')
return 0
if pages > 1:
progress.start_task(task)
progress.update(task, total=pages, completed=1, start=True)
current_page = 2
while current_page <= pages:
_, more_measurements = scc.query_measurements(
date_start, date_end, location, page=current_page)
measurements += more_measurements
current_page += 1
progress.advance(task)
console.log(f'[info]Found[/info] {len(measurements)} [info]measurements.[/info]')
# Download files
measurement_count = len(measurements)
file_count = 0
i = 0
with Progress(console=console) as progress:
task = progress.add_task(
f'Downloading products (1/{measurement_count})...', total=measurement_count)
for m in measurements:
progress.update(
task, description=f'Downloading products ({i}/{measurement_count})...')
for file in scc.download_products(m.id, download_path,
hirelpp and m.has_hirelpp,
cloudmask and m.has_cloudmask,
elpp and m.has_elpp,
optical and (m.has_elda or m.has_eldec),
elic and m.has_elic):
file_count += 1
console.log(f'[info]Downloaded[/info] {file}')
progress.advance(task)
i += 1
console.log(f'[info]Downloaded[/info] {file_count} [info]files![/info]')
def handle(self):
# Parse arguments
location_name = self.option('location')
location = None
if location_name is not None:
location = locations.get_location_by_name(location_name)
if location is None:
locations.unknown_location_error(location_name)
return 1
try:
date_start = self.argument('date-start')
date_start = datetime.date.fromisoformat(date_start)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
try:
date_end = self.argument('date-end')
date_end = datetime.date.fromisoformat(date_end)
except ValueError:
logging.error('Could not parse date-start! Please use the ISO format (YYYY-MM-DD)')
return 1
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Login to SCC to make queries
with scc_session(credentials) as scc:
with Progress(console=console) as progress:
task = progress.add_task('Fetching results...', start=False, total=1)
# Query SCC for measurements
pages, measurements = scc.query_measurements(
date_start, date_end, location, credentials)
if len(measurements) == 0:
progress.stop()
console.print('[warn]No measurements found![/warn]')
return 0
progress.start_task(task)
if pages > 1:
progress.update(task, total=pages, completed=1, start=True)
current_page = 2
while current_page <= pages:
_, more_measurements = scc.query_measurements(
date_start, date_end, location, page=current_page)
measurements += more_measurements
current_page += 1
progress.advance(task)
else:
progress.advance(task)
# Render table
table = Table(show_header=True, header_style="bold")
for col in ['ID', 'Location', 'Start', 'End',
'HiRELPP', 'CloudMask', 'ELPP', 'ELDA', 'ELDEC', 'ELIC', 'ELQUICK', 'Is Processing']:
table.add_column(col)
for m in measurements:
table.add_row(
m.id,
m.location.name,
m.date_start.strftime('%Y-%m-%d %H:%M'),
m.date_end.strftime('%Y-%m-%d %H:%M'),
bool_to_emoji(m.has_hirelpp),
bool_to_emoji(m.has_cloudmask),
bool_to_emoji(m.has_elpp),
bool_to_emoji(m.has_elda),
bool_to_emoji(m.has_eldec),
bool_to_emoji(m.has_elic),
bool_to_emoji(m.has_elquick),
bool_to_emoji(m.is_processing),
)
console.print(table)
# Write to CSV
csv_path = self.option('to-csv')
if csv_path is not None:
csv_path = Path(csv_path)
with open(csv_path, 'w') as f:
f.write(
'id,station_id,location,date_start,date_end,date_creation,date_updated,hirelpp,cloudmask,elpp,elda,eldec,elic,elquick,is_processing\n')
for m in measurements:
f.write(m.to_csv() + '\n')
console.print(f'[info]Wrote .csv file[/info] {csv_path}')
def handle(self):
# Parse arguments
path = Path(self.argument('path'))
if path.is_dir:
files = path.glob('*.nc')
files = filter(lambda x: not x.name.startswith('rs_'), files)
files = filter(lambda x: not x.name.startswith('calibration_'),
files) # TODO Handle calibration files
else:
files = [path]
files = list(files)
if len(files) == 0:
console.print('[error]No files found in given directory[/error]')
return 1
# Read application config
config = Config()
try:
credentials = SCC_Credentials(config)
except KeyError:
self.line('<error>Credentials not found in config</error>')
self.line('Use `pollyxt_pipelines config` to set the following variables:')
self.line('- http.username')
self.line('- http.password')
self.line('- auth.username')
self.line('- auth.password')
self.line('For example, `pollyxt_pipelines config http.username scc_user')
return 1
# Upload files
successful_files = []
successful_ids = []
with scc_session(credentials) as scc:
for file in track(files, description='Uploading files...', console=console):
# Read file to find radiosondes
nc = Dataset(file, 'r')
radiosonde_path = file.parent / nc.Sounding_File_Name
dataset_id = nc.Measurement_ID
configuration_id = nc.NOAReACT_Configuration_ID
nc.close()
if not radiosonde_path.exists():
console.print(
f'[error]Cannot find radiosonde file[/error] {radiosonde_path} [error] for measurement [/error] {file} [error]. Skipping file.[/error]')
continue
# Upload file to SCC
try:
scc.upload_file(file, configuration_id, rs_filename=radiosonde_path)
successful_files.append(file)
successful_ids.append(dataset_id)
except exceptions.SCCError as ex:
console.print(
f'[error]Error while uploading[/error] {file}[error]:[/error] {str(ex)}')
except Exception:
console.print(f'[error]Unknown error while uploading[/error] {file}')
console.print_exception()
successful_count = len(successful_ids)
if successful_count == 0:
console.print('[warn]No files were uploaded successfully![/warn]')
return 0
else:
console.print(
f'[info]Successfully uploaded[/info] {successful_count} [info]files.[/info]')
# Write list file if requested
list_file = self.argument('list')
if list_file is not None:
list_file = Path(list_file)
df = pd.DataFrame()
df['Filename'] = successful_files
df['id'] = successful_ids
df['Products_Downloaded'] = False
df.to_csv(list_file, index=False)
self.line(f'<comment>Wrote IDs to </comment>{list_file}')
def upload_file(self,
filename: Path,
system_id: str,
rs_filename: Union[Path, None] = None,
ov_filename: Union[Path, None] = None,
lr_filename: Union[Path, None] = None):
'''
Uploads a file to SCC, together with the auxilary files. There is no return value, but it will
throw for potential errors.
Parameters:
filename: Path to the SCC netCDF file
system_id: SCC Lidar System ID for the system that made the measurement
rs_filename: Path to the radiosonde netCDF file
ov_filename: Path to the overlap netCDF file
lr_filename: Path to the lidar ratio netCDF file
'''
# Check if the given anchillary files already exist before adding them to the request body
files = {}
if rs_filename is not None:
info = self.get_anchillary(rs_filename.name, 'sounding')
if info is not None and info.exists:
console.print(
f'[warn]Radiosonde file[/warn] {rs_filename.name} [warn]already exists on SCC.[/warn]'
)
else:
files['sounding_file'] = open(rs_filename, 'rb')
if ov_filename is not None:
info = self.get_anchillary(ov_filename.name, 'overlap')
if info is not None and info.exists:
console.print(
f'[warn]Overlap file[/warn] {ov_filename.name} [warn]already exists on SCC.[/warn]'
)
else:
files['overlap_file'] = open(ov_filename, 'rb')
if lr_filename is not None:
info = self.get_anchillary(lr_filename.name, 'lidarratio')
if info is not None and info.exists:
console.print(
f'[warn]Lidar ratio file[/warn] {lr_filename.name} [warn]already exists on SCC.[/warn]'
)
else:
files['lidar_ratio_file'] = open(lr_filename, 'rb')
files['data'] = open(filename, 'rb')
# Get the form and submit it
upload_page = self.session.get(constants.upload_url)
body = {'system': system_id}
headers = {
'X-CSRFToken': upload_page.cookies['csrftoken'],
'referer': constants.upload_url
}
upload_submit = self.session.post(constants.upload_url,
data=body,
files=files,
headers=headers)
# Check response
response_body = BeautifulSoup(upload_submit.text, 'html.parser')
alerts = response_body.find_all('div', class_='alert-box')
if len(alerts) > 0:
errors = ', '.join([alert.p.text.strip() for alert in alerts])
raise exceptions.SCCError(errors)
# console.print(upload_submit.text)
if upload_submit.status_code != 200 or upload_submit.url == constants.upload_url:
raise exceptions.UnexpectedResponse('Upload to SCC failed')
def download_products(self,
measurement_id: str,
download_path: Path,
hirelpp=True,
cloudmask=True,
elpp=True,
optical=True,
elic=True):
'''
Downloads products for a given measurement (ID) to the given path.
This function is a generator, yielding the filename of each downloaded file.
Parameters:
measurement_id: Which measurement to download products for
download_path: Where to store the downloaded products
hirelpp: Whether to download HiRELPP files
cloudmask: Whether to download Cloudmask files
elpp: Whether to download ELPP files
optical: Whether to download optical (ELDA or ELDEC) files
elic: Whether to download ELIC files
'''
# Determine URLs to download
to_download = []
if hirelpp:
to_download.append({
'url':
constants.download_hirelpp_pattern.format(measurement_id),
'path':
download_path / f'hirelpp_{measurement_id}.zip'
})
if cloudmask:
to_download.append({
'url':
constants.download_cloudmask_pattern.format(measurement_id),
'path':
download_path / f'cloudmask_{measurement_id}.zip'
})
if elpp:
to_download.append({
'url':
constants.download_preprocessed_pattern.format(measurement_id),
'path':
download_path / f'preprocessed_{measurement_id}.zip'
})
if optical:
to_download.append({
'url':
constants.download_optical_pattern.format(measurement_id),
'path':
download_path / f'optical_{measurement_id}.zip'
})
if elic:
to_download.append({
'url':
constants.download_elic_pattern.format(measurement_id),
'path':
download_path / f'elic_{measurement_id}.zip'
})
if len(to_download) == 0:
raise ValueError('At least one product must be downloaded!')
# Download each file
for download in to_download:
try:
self.download_file(**download)
yield download['path']
except Exception as ex:
console.print(
'[error]Error while downloading file from SCC[/error]')
console.print(f'[error]URL:[/error] {download["url"]}')
console.print(f'[error]Path:[/error] {download["path"]}')
console.print('[error]Exception:[/error]')
console.print_exception()
continue