def main(file): r = Radial(file) r.clean_header() print(r.file_name) r.metadata['filename'] = r.file_name # Upload site information to database try: site = Site(name=r.metadata['Site'], center_frequency=r.metadata['TransmitCenterFreqMHz']) site.save() except NotUniqueError: # except if its already uploaded site = Site.objects(name=r.metadata['Site'])[0] r.metadata['site_code'] = site.id hardware_diagnostics = HardwareDiagnostics( **r.diagnostics_hardware.to_dict(orient='list')) radial_diagnostics = RadialDiagnostics(**r.diagnostics_radial.to_dict( orient='list')) radial_metadata = RadialMetadata(**r.metadata) object_info = {} object_info['filename'] = r.file_name object_info['site_code'] = site.id object_info['radial_metadata'] = radial_metadata object_info['hardware_diagnostics'] = hardware_diagnostics object_info['radial_diagnostics'] = radial_diagnostics # Upload the radial file header information RadialFile(**object_info).save()
def main(file_list): client = MongoClient() db = client.codar db.radials.create_indexes([index1, index2]) bulk_info = [] for radial in file_list: # TODO Add multiprocessing here. # print(radial) r = Radial(radial) # print(r) if r.is_valid(): r.metadata['Site'] = r.metadata['Site'] try: r.metadata['PatternType'] = r.metadata['PatternType'].lower() except KeyError: pass # print(r.file_name) r.clean_header(split_origin=True) r.metadata['filename'] = r.file_name # assign a system type so we can sort on this r.metadata['SystemType'] = frequency_check( r.metadata['TransmitCenterFreqMHz']) r.metadata['RadialSolutions'] = r.data.__len__() # Try statements in case a radial file doesn't contain a diagnostic table. try: r.metadata[ 'diagnostics_hardware'] = r.diagnostics_hardware.to_dict( orient='r') except AttributeError as ae: logging.error(ae) try: r.metadata[ 'diagnostics_radial'] = r.diagnostics_radial.to_dict( orient='r') except AttributeError as ae: logging.error(ae) bulk_info.append(InsertOne(r.metadata)) try: db.radials.bulk_write(bulk_info, ordered=False) logging.info( 'Bulk radial insert successful. {} radials inserted.'.format( len(bulk_info))) except BulkWriteError as bwe: logging.error(bwe.details)
def parse_radial_file(radial_file): """ Parse CODAR radial files utilizing the Radial class and upload to MySQL database. :param radial_file: Path to CODAR Radial File """ basename = os.path.basename(radial_file).split('.')[0] logging.debug( '{} - Checking if file is uploaded to MySQL database.'.format( basename)) uploaded = db.check_file_upload(session, basename, RadialMetadata) if not uploaded: # Check if the file has been uploaded already. If it hasn't, upload it completely. logging.debug('{} - Loading'.format(radial_file)) try: r = Radial(radial_file) if not r.is_valid(): return r.clean_header( ) # Clean up header information for entry into mysql database r.metadata['filename'] = os.path.splitext( os.path.basename(radial_file))[0] r.metadata['fileModTime'] = dt.datetime.fromtimestamp( os.stat(radial_file).st_mtime) # Fill certain table columns with relational ids # Check to see if the site has been uploaded to the HfrSites table of the MySQL database try: site_info = sites[sites.site == r.metadata['Site']] site_id = int(site_info.id.iloc[0]) except IndexError: logging.info( '{} not found. Uploading site to hfrSites table'.format( r.metadata['Site'])) site_info = db.update_site_table( session, r.metadata['Site'], r.metadata['TransmitCenterFreqMHz'], r.metadata['Origin']) site_id = int(site_info) r.metadata['Site'] = site_id try: patt_type = pattern_types[pattern_types.type == r.metadata['PatternType']] pattern_id = int(patt_type.id.iloc[0]) except IndexError: logging.error('{} not found. Pattern type invalid'.format( r.metadata['PatternType'])) return r.metadata['PatternType'] = pattern_id # Add extra information to header r.metadata['TableType'] = r._tables['1']['TableType'] r.metadata['TableColumns'] = r._tables['1']['TableColumns'] r.metadata['TableColumnTypes'] = r._tables['1']['TableColumnTypes'] r.metadata['TableRows'] = r._tables['1']['TableRows'] # Upload radial header information and update latest radials table r.metadata = dbr.upload_radial_header(session, r.metadata) dbr.update_latest_radials(session, r.metadata) try: # Upload radial diagnostic data r.diagnostics_radial = r.diagnostics_radial.drop( ['TIME', 'TYRS', 'TMON', 'TDAY', 'THRS', 'TMIN', 'TSEC'], axis=1) r.diagnostics_radial['id_site'] = r.metadata['Site'] r.diagnostics_radial['id_radial'] = r.metadata['radial_id'] dbr.upload_diagnostics(session, RadialDiagnostics, r.diagnostics_radial, r.metadata['Site']) logging.debug( '{} - Table `{}` - Diagnostic data uploaded '.format( r.metadata['filename'], 'hfrRadialDiagnostics')) except: pass try: # Upload hardware diagnostic data r.diagnostics_hardware = r.diagnostics_hardware.drop( ['TIME', 'TYRS', 'TMON', 'TDAY', 'THRS', 'TMIN', 'TSEC'], axis=1) r.diagnostics_hardware['id_site'] = r.metadata['Site'] r.diagnostics_hardware['id_radial'] = r.metadata['radial_id'] dbr.upload_diagnostics(session, HardwareDiagnostics, r.diagnostics_hardware, r.metadata['Site']) logging.debug( '{} - Table `{}` - Diagnostic data uploaded '.format( r.metadata['filename'], 'hfrHardwareDiagnostics')) except: pass logging.info('{} - File uploaded successfully'.format(radial_file)) except: logging.error('{} - File failed to upload'.format(radial_file))