def main(radial_file, save_path, qc_values):
"""
Main function to parse and qc radial files
:param radial_file: Path to radial file
:param save_path: Path to save quality controlled radial file
:param qc_values: Dictionary containing thresholds for each QC test
"""
try:
r = Radial(radial_file)
except Exception as err:
logging.error('{} - {}'.format(radial_file, err))
return
if r.is_valid():
# run high frequency radar qartod tests on open radial file
r.initialize_qc()
r.qc_qartod_syntax()
r.qc_qartod_maximum_velocity(**qc_values['qc_qartod_maximum_velocity'])
r.qc_qartod_valid_location()
r.qc_qartod_radial_count(**qc_values['qc_qartod_radial_count'])
r.qc_qartod_spatial_median(**qc_values['qc_qartod_spatial_median'])
# r.qc_qartod_avg_radial_bearing(qc_values['average_bearing_threshold'])
# Export radial file to either a radial or netcdf
try:
r.export(os.path.join(save_path, r.file_name), 'radial')
except ValueError as err:
logging.error('{} - {}'.format(radial_file, err))
pass
def main(file_list):
client = MongoClient()
db = client.codar
db.radials.create_indexes([index1, index2])
bulk_info = []
for radial in file_list: # TODO Add multiprocessing here.
# print(radial)
r = Radial(radial)
# print(r)
if r.is_valid():
r.metadata['Site'] = r.metadata['Site']
try:
r.metadata['PatternType'] = r.metadata['PatternType'].lower()
except KeyError:
pass
# print(r.file_name)
r.clean_header(split_origin=True)
r.metadata['filename'] = r.file_name
# assign a system type so we can sort on this
r.metadata['SystemType'] = frequency_check(
r.metadata['TransmitCenterFreqMHz'])
r.metadata['RadialSolutions'] = r.data.__len__()
# Try statements in case a radial file doesn't contain a diagnostic table.
try:
r.metadata[
'diagnostics_hardware'] = r.diagnostics_hardware.to_dict(
orient='r')
except AttributeError as ae:
logging.error(ae)
try:
r.metadata[
'diagnostics_radial'] = r.diagnostics_radial.to_dict(
orient='r')
except AttributeError as ae:
logging.error(ae)
bulk_info.append(InsertOne(r.metadata))
try:
db.radials.bulk_write(bulk_info, ordered=False)
logging.info(
'Bulk radial insert successful. {} radials inserted.'.format(
len(bulk_info)))
except BulkWriteError as bwe:
logging.error(bwe.details)
def main(radial_file, save_path):
"""
Main function to parse and qc radial files
:param radial_file: Path to radial file
:param save_path: Path to save quality controlled radial file
"""
try:
r = Radial(radial_file)
except Exception:
return
if r.is_valid():
try:
r.export(
os.path.join(save_path, r.file_name.replace('.ruv', '.nc')),
'netcdf')
except ValueError:
pass
def parse_radial_file(radial_file):
"""
Parse CODAR radial files utilizing the Radial class and upload to MySQL database.
:param radial_file: Path to CODAR Radial File
"""
basename = os.path.basename(radial_file).split('.')[0]
logging.debug(
'{} - Checking if file is uploaded to MySQL database.'.format(
basename))
uploaded = db.check_file_upload(session, basename, RadialMetadata)
if not uploaded: # Check if the file has been uploaded already. If it hasn't, upload it completely.
logging.debug('{} - Loading'.format(radial_file))
try:
r = Radial(radial_file)
if not r.is_valid():
return
r.clean_header(
) # Clean up header information for entry into mysql database
r.metadata['filename'] = os.path.splitext(
os.path.basename(radial_file))[0]
r.metadata['fileModTime'] = dt.datetime.fromtimestamp(
os.stat(radial_file).st_mtime)
# Fill certain table columns with relational ids
# Check to see if the site has been uploaded to the HfrSites table of the MySQL database
try:
site_info = sites[sites.site == r.metadata['Site']]
site_id = int(site_info.id.iloc[0])
except IndexError:
logging.info(
'{} not found. Uploading site to hfrSites table'.format(
r.metadata['Site']))
site_info = db.update_site_table(
session, r.metadata['Site'],
r.metadata['TransmitCenterFreqMHz'], r.metadata['Origin'])
site_id = int(site_info)
r.metadata['Site'] = site_id
try:
patt_type = pattern_types[pattern_types.type ==
r.metadata['PatternType']]
pattern_id = int(patt_type.id.iloc[0])
except IndexError:
logging.error('{} not found. Pattern type invalid'.format(
r.metadata['PatternType']))
return
r.metadata['PatternType'] = pattern_id
# Add extra information to header
r.metadata['TableType'] = r._tables['1']['TableType']
r.metadata['TableColumns'] = r._tables['1']['TableColumns']
r.metadata['TableColumnTypes'] = r._tables['1']['TableColumnTypes']
r.metadata['TableRows'] = r._tables['1']['TableRows']
# Upload radial header information and update latest radials table
r.metadata = dbr.upload_radial_header(session, r.metadata)
dbr.update_latest_radials(session, r.metadata)
try:
# Upload radial diagnostic data
r.diagnostics_radial = r.diagnostics_radial.drop(
['TIME', 'TYRS', 'TMON', 'TDAY', 'THRS', 'TMIN', 'TSEC'],
axis=1)
r.diagnostics_radial['id_site'] = r.metadata['Site']
r.diagnostics_radial['id_radial'] = r.metadata['radial_id']
dbr.upload_diagnostics(session, RadialDiagnostics,
r.diagnostics_radial,
r.metadata['Site'])
logging.debug(
'{} - Table `{}` - Diagnostic data uploaded '.format(
r.metadata['filename'], 'hfrRadialDiagnostics'))
except:
pass
try:
# Upload hardware diagnostic data
r.diagnostics_hardware = r.diagnostics_hardware.drop(
['TIME', 'TYRS', 'TMON', 'TDAY', 'THRS', 'TMIN', 'TSEC'],
axis=1)
r.diagnostics_hardware['id_site'] = r.metadata['Site']
r.diagnostics_hardware['id_radial'] = r.metadata['radial_id']
dbr.upload_diagnostics(session, HardwareDiagnostics,
r.diagnostics_hardware,
r.metadata['Site'])
logging.debug(
'{} - Table `{}` - Diagnostic data uploaded '.format(
r.metadata['filename'], 'hfrHardwareDiagnostics'))
except:
pass
logging.info('{} - File uploaded successfully'.format(radial_file))
except:
logging.error('{} - File failed to upload'.format(radial_file))
