def dump_a2pats_file(model_, folder): # type: (model, PathLike) -> bool ''' Dump single A2PATS object Parameters: * `model_`: (model class object) Model object in A2PATS * `folder`: (string or path like string) Location to dump files **Returns**: (boolean) True on success ''' logger.debug('Now exporting A2PATS model {}'.format(model_.name)) filename = '{}.{}'.format(model_.name, MODEL_FILES[model_.type].lower()) # NOTE: The following path joiner works best in Python 3.5+ # due to the implemenation of os.path.join(path, *paths). If # you are the maintainer of this project please be assured # that os.path.join(path, *paths) does not accept all types # of PathLike objects and you can replace this with a call # to os.path.normpath or os.path.realpath should there # be issues on legacy versions of Python. filepath = join(folder, filename) header = to_str_section(model_) sections = [header, ''] + model_.converted_data try: with open(filepath, 'w') as fp: fp.write('\n'.join(sections)) return True except: return False
def split_header_auto(table, default=DEF_HDRRW_CNT): # type: (List[List[str]], int) -> Tuple[List[List[str]], int] ''' Automatically detect header height Parameters: * `table`: (list of list of strings) 2D matrix used to create the table * `default`: (int) default number of header rows **Returns**: (tuple of list of list of strings and integer) Formatted table and number of header rows ''' # TODO logger.debug( 'Unable to determine header height, using default {}'.format(default)) if len(table) < 1: logger.warn( 'Table was empty, nothing to do in app.util.tables.split_header_auto' ) return table, 0 headers = [s.rsplit(' ', 1) for s in table[0]] headers_n = [[h[0] for h in headers], [h[1] if len(h) > 1 else str() for h in headers]] if len(table) < 2: logger.warn('Table data was empty, but headers were rearranged') return headers_n, len(headers_n) return headers_n + table[1:], len(headers_n)
def traverse_xml_tree(parent, stack_size=0): # type: (ElementTree.Element, int) -> dict ''' Traverse the XML tree and return it as a dictionary Parameters: * `parent`: Element tree to traverse * `stack_size`: (integer, optional) Stack size **Returns**: (dictionary) XML tree as dictionary ''' if stack_size == 0: logger.debug('Now traversing XML tree beginning at initial stack') data = dict() for child in parent: if child.text: text = child.text.strip() if not text: text = traverse_xml_tree(child, stack_size + 1) if child.tag in data: if type(data[child.tag]) is not list: data[child.tag] = [data[child.tag]] data[child.tag].append(text) else: data[child.tag] = text return data
def convert(input_file, output_file): # type: (str, str) -> a2pats ''' Load CEESIM file and convert to A²PATS. Output files that do not exist are created automatically. Parameters: * `input_file`: (string) Location of file to import * `output_file`: (string) Location of file to export (does not have to exist) **Returns**: (a2pats class object) A²PATS object (if successful) ''' logger.debug( 'Main app converter called, using provided input and output files') input_data = import_(input_file, ceesim) # -> CEESIM object emitter_modes = split_emitter_modes(input_data.imported_data) for i, emitter_mode in enumerate(emitter_modes): mode_num = "0" + str(i + 1) if len(str(i + 1)) == 1 else str(i + 1) logger.debug("!!Working on Emitter Mode {}".format(mode_num)) scan_type = determine_scan_type(emitter_mode) logger.info("Scan Type: {}".format(scan_type)) path_to_tables = "data/c_tables/" lookup_table = prepare_lookup_table(path_to_tables + scan_file_name[scan_type] + "base.csv", \ path_to_tables + scan_file_name[scan_type] + "base.json") output_data = convert_to_a2pats(emitter_mode, lookup_table) if scan_type == "HELICAL": # stacked circular scans mode_num += " (Needs Edit)" success = dump_a2pats(output_data, output_file + "/{}".format(mode_num)) '''
def flatten_table(ceesim_data, stack_size=1): # type: (dict, int) -> dict ''' Flattens a table Parameters: * `ceesim_data`: (dictionary) Import ceesim_data from JSON * `stack_size`: (int) Number of function calls in recursive mode **Returns**: (dictionary) Flattened 1 level table ''' logger.debug( 'Flattening input table with call stack size at {}'.format(stack_size)) data_ = dict() for key in ceesim_data: if key not in data_: if type(ceesim_data[key]) not in {dict, list}: logger.debug( '{} was not in data, and was added to the flat table'.format(key)) data_[key] = ceesim_data[key] else: logger.debug( 'Now processing lookup table for subkeys of {}'.format(key)) frame = ceesim_data[key] if type(frame) is list: if len(frame) < 1 or frame[0] is not dict: continue else: frame = frame[0] subdict = flatten_table(frame, stack_size + 1) subdict.update(data_) data_ = subdict logger.debug('Now returning flat table with size of {}'.format(len(data_))) return data_
def import_ceesim(fp): # type: (TextIO) -> ceesim ''' Import a CEESIM file for conversion Parameters: * `fp`: File pointer **Returns**: Imported CEESIM object ''' logger.debug( 'CEESIM importer called, begining CEESIM import with XML file') itr = ElementTree.iterparse(fp) strip_xml_namespaces(itr) data = traverse_xml_tree(itr.root) store = ceesim(data) return store
def strip_xml_namespaces(itr): # type: (Iterator) -> None ''' Strip XML namespaces from an iterator provided by XML parser Parameters: * `itr`: Iterator from an ElementTree **Returns**: None, strip happens in place ''' logger.debug('Stripping all namespaces from imported CEESIM file') for _, element in itr: if '}' in element.tag: element.tag = element.tag.split('}', 1)[1] for attribute in list(element.attrib.keys()): if '}' in attribute: new_attribute = attribute.split('}', 1)[1] element.attrib[new_attribute] = element.attrib[attribute] del element.attrib[attribute]
def convert_to_a2pats(ceesim_data, lookup_table): # type: (ceesim, dict) -> a2pats ''' Convert CEESIM data to A²PATS data Parameters: * `ceesim_data`: (ceesim class object) CEESIM emitter mode data to import * `lookup_table`: (dictionary) Looktable table JSON **Returns**: (a2pats class object) A2PATS data ''' logger.info('Beginning CEESIM to A2PATS conversion') flattened_data = flatten_table(ceesim_data) store = a2pats(imported_type='A2PATS') generic_models = generate_models( ceesim_data, flattened_data, lookup_table) logger.debug('Added {} models from generate_models'.format( len(generic_models))) store.models += generic_models return store
def dump_a2pats(obj, folder): # type: (a2pats, PathLike) -> bool ''' Dump A2PATS object Parameters: * `obj`: (a2pats class object) A2PATS object * `folder`: (str or Path like object) Folder location to export **Returns**: (boolean) True on success ''' if isfile(folder): logger.error('{} is a filepath, a folder cannot be placed here. Aborting'.format(folder)) return False if not isdir(folder): logger.debug('{} does not exist, creating folder'.format(folder)) mkdir(folder) logger.info('Now exporting A2PATS to folder {}'.format(folder)) for model_ in obj.models: dump_a2pats_file(model_, folder) return True
def dump(obj, folder, export_type=a2pats): # type: (datastore, PathLike, type) -> bool '''Dump data dynmically Parameters: * `obj`: (datastore class object) Datastore object * `folder`: (str or path like object) Folder location of export * `export_type`: (class) Export type **Returns**: (boolean) True on success ''' logger.debug('Generic dump function called, auto-detcting type...') assert issubclass( type(obj), datastore), 'Your data must be a datastore-like object!' if type(obj) is a2pats: logger.debug('Detected A2PATS, exporting as A2PATS') return dump_a2pats(obj, folder) elif type(obj) is ceesim: logger.debug('Detected CEESIM, exporting as CEESIM') return None assert type( obj) is not export_type, 'You cannot auto-dump a datastore object!'
def create_converted(model, opt): # type: (model, list) -> None ''' Converts in-place model Parameters: * `model`: (model class object) Model object * `opt`: (list) Options from lookup table **Returns**: None, table is filled in place ''' if type(opt["TABLE"]) is int: table_string = build_table_str(ceesim_data, lookup_table, opt[FILE_HDR], opt["SECTION"], opt[PRI_HDR], convert_one_key, obtain_relevant_tags, opt[FILE_HDR] == "ANT") fill_table(model, opt[PRI_HDR], table_string) return None # No other conversion should occur after building table values = [] for tag in opt[TAG_HDR].split('&'): tags = obtain_relevant_tags(ceesim_data, ceesim_flattened, tag) # removed taking the zero-index as it's done below if tags: value = tags[0] logger.debug("Found tag {} for {} in {} with value: {}".format(opt[TAG_HDR], opt["LABEL"], opt[FILE_HDR], value)) else: if opt[TAG_HDR]: logger.debug('Could not find tag {}, using default value for {} in {}: {}'.format(opt[TAG_HDR], opt["LABEL"], opt[FILE_HDR], opt[DEFAULT_HDR])) else: logger.debug('Using default value for tagless {} in {}: {}'.format( opt["LABEL"], opt[FILE_HDR], opt[DEFAULT_HDR])) value = opt[DEFAULT_HDR] values.append(value) converted = convert_one_key(opt, values) fill_table(model, opt[PRI_HDR], converted) # adds to model.converted_data
def import_(fp, classtype=datastore, downgrade_peaceful=True): # type: (Union[str, TextIO], type, bool) -> datastore '''Import data dynamically :param fp: File pointer or string to import file :type fp: str or file-like pointer :param classtype: Class type to import as (must be of instance datastore) :type classtype: datastore or datastore-like :param downgrade_peaceful: Whether or not to ignore errors :type downgrade_peaceful: bool :returns: Database object for typing :rtype: datastore or datastore-like ''' logger.debug( 'Generic importer called, automatically detecting import type') if downgrade_peaceful: if not issubclass(classtype, datastore): # Assume CEESIM import logger.debug('Unable to determine type, assuming CEESIM') classtype = ceesim if type(fp) is str: logger.debug('Now opening file {} for import'.format(fp)) try: fp = open(fp, encoding='utf-8') except: logger.debug('Failed to open, returning empty object') return datastore() elif not isinstance(fp, IOBase): logger.debug('Unrecognizable file pointer, returning empty object') return datastore() else: assert issubclass( classtype, datastore ), 'Your import_ call must be of datastore or datastore-like type!' if type(fp) is str: logger.debug('Now opening file {} for import'.format(fp)) fp = open(fp, encoding='utf-8') assert isinstance( fp, IOBase ), 'Your import_ call must provide a valid files-like pointer or file path!' if classtype is a2pats: logger.debug('A2PATS file provided, importing as A2PATS') return import_a2pats(fp) elif classtype is ceesim: logger.debug('CEESIM file provided, importing as CEESIM') return import_ceesim(fp) else: raise ValueError('This type of datastore isn\'t supported yet!')
def generate_models(ceesim_data, ceesim_flattened, lookup_table): # type: (dict, dict, dict) -> list ''' Generate all non INP/PUL models Parameters: * `ceesim_data`: (dictionary) Imported CEESIM JSON data * `ceesim_flattened`: (dictionary, optional) Flattened CEESIM JSON data * `lookup_table`: (dictionary) Lookup table JSON imported **Returns**: (list) List of models ''' logger.debug('Now generating all non-signal models') def fill_table(model, pos, value): # type: (model, int, str) -> None ''' Fills out table from model with values Parameters: * `model`: (model class object) Model object * `pos`: (int) Position index * `value`: (str) Value to fill **Returns**: None, table is filled in-place ''' if len(model.converted_data) <= pos: model.converted_data += [''] * \ (pos - len(model.converted_data) + 1) model.converted_data[pos] = value def create_converted(model, opt): # type: (model, list) -> None ''' Converts in-place model Parameters: * `model`: (model class object) Model object * `opt`: (list) Options from lookup table **Returns**: None, table is filled in place ''' if type(opt["TABLE"]) is int: table_string = build_table_str(ceesim_data, lookup_table, opt[FILE_HDR], opt["SECTION"], opt[PRI_HDR], convert_one_key, obtain_relevant_tags, opt[FILE_HDR] == "ANT") fill_table(model, opt[PRI_HDR], table_string) return None # No other conversion should occur after building table values = [] for tag in opt[TAG_HDR].split('&'): tags = obtain_relevant_tags(ceesim_data, ceesim_flattened, tag) # removed taking the zero-index as it's done below if tags: value = tags[0] logger.debug("Found tag {} for {} in {} with value: {}".format(opt[TAG_HDR], opt["LABEL"], opt[FILE_HDR], value)) else: if opt[TAG_HDR]: logger.debug('Could not find tag {}, using default value for {} in {}: {}'.format(opt[TAG_HDR], opt["LABEL"], opt[FILE_HDR], opt[DEFAULT_HDR])) else: logger.debug('Using default value for tagless {} in {}: {}'.format( opt["LABEL"], opt[FILE_HDR], opt[DEFAULT_HDR])) value = opt[DEFAULT_HDR] values.append(value) converted = convert_one_key(opt, values) fill_table(model, opt[PRI_HDR], converted) # adds to model.converted_data def add_headers(mfile, model): # type: (str, model) -> None ''' Adds headers to converted model Parameters: * `mfile`: (string) Name of file * `model`: (model) Model class object **Returns**: None, data is converted in place ''' with open("data/headers.csv") as head: headers = reader(head) for header in headers: if header[0] == mfile: left = (int(header[2]) - len(header[1]) - 3) / 2 lleft = int(left) right = lleft if lleft == left: left = lleft right = lleft + 1 htext = "//" + ' '.join( ['*' * lleft, header[1], '*' * right]) h_priority = int(header[3]) if determine_scan_type(ceesim_data) == "LORO" and mfile == "SIG" and (header[1] == "ANTENNA MODEL" or header[1] == "MULTIPLE SIMULTANEOUS SIGNALS"): # Resolves a downstream feature displacement that occurs from the inclusion of all scan data within the .sig file's SCAN MODEL h_priority += 3 fill_table(model, h_priority, htext) models = list() name = form_model_name(ceesim_data, ceesim_flattened) timestamp = obtain_relevant_tags(ceesim_data, ceesim_flattened, "LastUpdateDate")[0] logger.debug('Generic model generator using name: {}'.format(name)) for mtype in AUTO_MODELS: next_model = model(mtype, name, timestamp) if mtype not in MODEL_FILES: logger.warn( 'Could not find mtype {} in model files, skipping'.format(mtype)) continue table_key = MODEL_FILES[mtype] add_headers(table_key, next_model) if table_key not in lookup_table: logger.warn( 'Could not find key {} in lookup table, skipping'.format(table_key)) continue logger.debug( 'Now processing table key {} with mtype {}'.format(table_key, mtype)) for cdict_key in lookup_table[table_key]: key_data = lookup_table[table_key][cdict_key] if MULTI_HDR in key_data and TABLE_DATA in key_data: data_opts = key_data[TABLE_DATA] for opt in data_opts: if opt["SECTION"] == "Main": continue create_converted(next_model, opt) else: if key_data["SECTION"] == "Main": continue create_converted(next_model, key_data) models.append(next_model) # Handles Intrapule here inp_deviations = obtain_relevant_tags(ceesim_data, ceesim_flattened, "LinearFreqDeviation") inp_durations = obtain_relevant_tags(ceesim_data, ceesim_flattened, "LinearFreqDuration") inp_info = zip(inp_deviations, inp_durations) for i, info in enumerate(inp_info): newName = "{}-{}".format(name, i + 1) next_model = model('INTRAPULSE', newName, timestamp) table_key = MODEL_FILES['INTRAPULSE'] add_headers(table_key, next_model) if table_key not in lookup_table: logger.warn( 'Could not find key {} in lookup table, skipping'.format(table_key)) continue logger.debug( 'Now processing table key {} with mtype {}'.format(table_key, 'INTRAPULSE')) for cdict_key in lookup_table[table_key]: key_data = lookup_table[table_key][cdict_key] if MULTI_HDR in key_data and TABLE_DATA in key_data: data_opts = key_data[TABLE_DATA] for opt in data_opts: if opt["SECTION"] == "Main": continue create_converted(next_model, opt) else: if key_data["SECTION"] == "Main": continue if not key_data["TAG"]: create_converted(next_model, key_data) else: if key_data["TAG"] == "LinearFreqDeviation": fill_table(next_model, key_data[PRI_HDR], convert_one_key(key_data, [info[0]])) if key_data["TAG"] == "LinearFreqDeviation&LinearFreqDuration": fill_table(next_model, key_data[PRI_HDR], convert_one_key(key_data, info)) models.append(next_model) return models
async def shutdown_events(): logger.debug("Shutdown!")
async def startup_event(): logger.debug("App startup!") init_db(app)