def main(file):
r = Radial(file)
r.clean_header()
print(r.file_name)
r.metadata['filename'] = r.file_name
# Upload site information to database
try:
site = Site(name=r.metadata['Site'],
center_frequency=r.metadata['TransmitCenterFreqMHz'])
site.save()
except NotUniqueError: # except if its already uploaded
site = Site.objects(name=r.metadata['Site'])[0]
r.metadata['site_code'] = site.id
hardware_diagnostics = HardwareDiagnostics(
**r.diagnostics_hardware.to_dict(orient='list'))
radial_diagnostics = RadialDiagnostics(**r.diagnostics_radial.to_dict(
orient='list'))
radial_metadata = RadialMetadata(**r.metadata)
object_info = {}
object_info['filename'] = r.file_name
object_info['site_code'] = site.id
object_info['radial_metadata'] = radial_metadata
object_info['hardware_diagnostics'] = hardware_diagnostics
object_info['radial_diagnostics'] = radial_diagnostics
# Upload the radial file header information
RadialFile(**object_info).save()
def main(file_list):
client = MongoClient()
db = client.codar
db.radials.create_indexes([index1, index2])
bulk_info = []
for radial in file_list: # TODO Add multiprocessing here.
# print(radial)
r = Radial(radial)
# print(r)
if r.is_valid():
r.metadata['Site'] = r.metadata['Site']
try:
r.metadata['PatternType'] = r.metadata['PatternType'].lower()
except KeyError:
pass
# print(r.file_name)
r.clean_header(split_origin=True)
r.metadata['filename'] = r.file_name
# assign a system type so we can sort on this
r.metadata['SystemType'] = frequency_check(
r.metadata['TransmitCenterFreqMHz'])
r.metadata['RadialSolutions'] = r.data.__len__()
# Try statements in case a radial file doesn't contain a diagnostic table.
try:
r.metadata[
'diagnostics_hardware'] = r.diagnostics_hardware.to_dict(
orient='r')
except AttributeError as ae:
logging.error(ae)
try:
r.metadata[
'diagnostics_radial'] = r.diagnostics_radial.to_dict(
orient='r')
except AttributeError as ae:
logging.error(ae)
bulk_info.append(InsertOne(r.metadata))
try:
db.radials.bulk_write(bulk_info, ordered=False)
logging.info(
'Bulk radial insert successful. {} radials inserted.'.format(
len(bulk_info)))
except BulkWriteError as bwe:
logging.error(bwe.details)
def parse_radial_file(radial_file):
"""
Parse CODAR radial files utilizing the Radial class and upload to MySQL database.
:param radial_file: Path to CODAR Radial File
"""
basename = os.path.basename(radial_file).split('.')[0]
logging.debug(
'{} - Checking if file is uploaded to MySQL database.'.format(
basename))
uploaded = db.check_file_upload(session, basename, RadialMetadata)
if not uploaded: # Check if the file has been uploaded already. If it hasn't, upload it completely.
logging.debug('{} - Loading'.format(radial_file))
try:
r = Radial(radial_file)
if not r.is_valid():
return
r.clean_header(
) # Clean up header information for entry into mysql database
r.metadata['filename'] = os.path.splitext(
os.path.basename(radial_file))[0]
r.metadata['fileModTime'] = dt.datetime.fromtimestamp(
os.stat(radial_file).st_mtime)
# Fill certain table columns with relational ids
# Check to see if the site has been uploaded to the HfrSites table of the MySQL database
try:
site_info = sites[sites.site == r.metadata['Site']]
site_id = int(site_info.id.iloc[0])
except IndexError:
logging.info(
'{} not found. Uploading site to hfrSites table'.format(
r.metadata['Site']))
site_info = db.update_site_table(
session, r.metadata['Site'],
r.metadata['TransmitCenterFreqMHz'], r.metadata['Origin'])
site_id = int(site_info)
r.metadata['Site'] = site_id
try:
patt_type = pattern_types[pattern_types.type ==
r.metadata['PatternType']]
pattern_id = int(patt_type.id.iloc[0])
except IndexError:
logging.error('{} not found. Pattern type invalid'.format(
r.metadata['PatternType']))
return
r.metadata['PatternType'] = pattern_id
# Add extra information to header
r.metadata['TableType'] = r._tables['1']['TableType']
r.metadata['TableColumns'] = r._tables['1']['TableColumns']
r.metadata['TableColumnTypes'] = r._tables['1']['TableColumnTypes']
r.metadata['TableRows'] = r._tables['1']['TableRows']
# Upload radial header information and update latest radials table
r.metadata = dbr.upload_radial_header(session, r.metadata)
dbr.update_latest_radials(session, r.metadata)
try:
# Upload radial diagnostic data
r.diagnostics_radial = r.diagnostics_radial.drop(
['TIME', 'TYRS', 'TMON', 'TDAY', 'THRS', 'TMIN', 'TSEC'],
axis=1)
r.diagnostics_radial['id_site'] = r.metadata['Site']
r.diagnostics_radial['id_radial'] = r.metadata['radial_id']
dbr.upload_diagnostics(session, RadialDiagnostics,
r.diagnostics_radial,
r.metadata['Site'])
logging.debug(
'{} - Table `{}` - Diagnostic data uploaded '.format(
r.metadata['filename'], 'hfrRadialDiagnostics'))
except:
pass
try:
# Upload hardware diagnostic data
r.diagnostics_hardware = r.diagnostics_hardware.drop(
['TIME', 'TYRS', 'TMON', 'TDAY', 'THRS', 'TMIN', 'TSEC'],
axis=1)
r.diagnostics_hardware['id_site'] = r.metadata['Site']
r.diagnostics_hardware['id_radial'] = r.metadata['radial_id']
dbr.upload_diagnostics(session, HardwareDiagnostics,
r.diagnostics_hardware,
r.metadata['Site'])
logging.debug(
'{} - Table `{}` - Diagnostic data uploaded '.format(
r.metadata['filename'], 'hfrHardwareDiagnostics'))
except:
pass
logging.info('{} - File uploaded successfully'.format(radial_file))
except:
logging.error('{} - File failed to upload'.format(radial_file))