def run(self): """ Run the main loop. Log to application log if debug level is DEBUG. Then write to database, log to file, check humidity treshold, update leds. """ self.debug() try: log_batch = [] status = [0] * led._LED_COUNT for i in i2c._SENSORS_ADDR: # Acquire sensor data data = i2c.read_i2c_addr(i) log_batch.append(data) if data[3] > led._LEDS_HUMIDITY_THRESHOLD: # Update the treshold status status[i % (led._LED_COUNT)] += 1 if self._LEDS_ENABLED: # Update the leds according to treshold status logging.getLogger().info("Updating LEDs", exc_info=False) indicator = led.Argospi2cWS2811x() indicator.set_leds(status) if self._INFLUX_ENABLED: # Write to database logging.getLogger().info("Contacting influx", exc_info=False) data = influx.prepare_data(log_batch) influx.post_to_database(data) if self._LOG_ENABLED: # Write to local logfile logging.getLogger().info("Log file enabled", exc_info=False) helpers.write_log(self._LOG_FILE, log_batch) except IndexError: logging.getLogger().error("Index out of range.", exc_info=True) except IOError: logging.getLogger().error("Failed to connect to i2c.", exc_info=True)
def _set_feature_type(self, type: str): """ Set on init. Define the feature representation - MEL with 20 or 40 filters - OR MFCC with coefficients 1-13 - OR STFT, either logarithmically scaled or normal transforms - Also set self.feature_shape depending on the feature type :param type: The feature type """ # STFT in mel basis, using 20 filters if type == self.FEATURE_TYPE_MEL_20: self.num_mel_filters = 20 # 501 * 20 shape = self.num_mel_filters # STFT in mel basis, using 20 filters elif type == self.FEATURE_TYPE_MEL_40: self.num_mel_filters = 40 # 501 * 40 shape = self.num_mel_filters # Mel frequency cepstral coefficients 1-13 elif type == self.FEATURE_TYPE_MFCC: # 501 * 12 shape = self.num_coefficients # Short time fourier transform: windows in frequency domains, 501 * (1 + self.frame_length / 2) = 501 * 201 elif type == self.FEATURE_TYPE_STFT or type == self.FEATURE_TYPE_LOG_STFT: # 501 * 201 shape = 1 + self.frame_length // 2 else: write_log(f'Feature type {type} is invalid', True, True) write_log(f'Using feature type {type}') self.feature_type = type self.feature_shape = (int(self.sample_rate * self.seconds_per_record / self.n_overlap) + 1, shape)
def train(self, files: np.ndarray, min_speakers: int, max_speakers: int, feature_type: str): """ Train the network, eg - Create data generators - Train with Adam, Poisson loss, MeanAbsoluteError metric. - Visualize using Tensorboard :param files: All files :param min_speakers The min number of speakers to generate files for :param max_speakers The max number of speakers to generate files for :param feature_type: Feature type to use """ train_generator, (val_x, val_y) = self.__get_train_data( files, min_speakers, max_speakers, feature_type) net = self.compile_net(train_generator.feature_shape) write_log('Training model') history = net.fit( train_generator, validation_data=(val_x, val_y), epochs=self.num_epochs, callbacks=self.callbacks, verbose=1, ) write_log('Model trained') return net, history
def add_economy_to_plane(plane, economy, url): if len(economy) != 0: plane.research = economy[0] plane.purchase = economy[1] if economy[0] == 0 and len(economy) == 12: plane.is_premium = True plane.repair_min_sb = economy[2] plane.repair_min_rb = economy[3] plane.repair_min_ab = economy[4] plane.crew_training = economy[5] plane.experts = economy[6] plane.aces = economy[7] plane.reward_rp = economy[8] plane.reward_sl_sb = economy[9] plane.reward_sl_rb = economy[10] plane.reward_sl_ab = economy[11] else: plane.is_premium = False plane.repair_min_sb = economy[2] plane.repair_max_sb = economy[3] plane.repair_min_rb = economy[4] plane.repair_max_rb = economy[5] plane.repair_min_ab = economy[6] plane.repair_max_ab = economy[7] plane.crew_training = economy[8] plane.experts = economy[9] plane.aces = economy[10] plane.reward_rp = economy[11] plane.reward_sl_sb = economy[12] plane.reward_sl_rb = economy[13] plane.reward_sl_ab = economy[14] else: write_log('economy_insert', 'db_adding_planes.log', url)
def process_general_characteristics(lst, url): if any(word in WORD_TO_REMOVE_PROCESSING for word in lst) or len(lst) == 0 or len(lst) not in [9, 10]: write_log(0, 'processing_planes.log', url) lst = [] return lst lst[0] = lst[0].replace('\xa0', ' ') lst[2] = lst[2].replace(' Rank', '') lst[2] = roman_to_int(lst[2]) lst[3] = float(lst[3]) lst[4] = float(lst[4]) lst[5] = float(lst[5]) if 'Fighter' in lst[6] or 'fighter' in lst[6]: lst[6] = 'fighter' elif 'Bomber' in lst[6] or 'bomber' in lst[6]: lst[6] = 'bomber' elif 'Attacker' in lst[6] or 'attacker' in lst[6]: lst[6] = 'attacker' else: lst[6] = None lst[7] = int(lst[7].replace(' people', '').replace(' person', '')) lst[8] = float(lst[8].replace(' t', '')) if len(lst) == 10: lst[9] = float(lst[9].replace(' kg/s', '')) return lst
def add_general_characteristics(plane, lst, url): if len(lst) != 0: plane.rank = lst[2] plane.battle_rating_sb = lst[3] plane.battle_rating_rb = lst[4] plane.battle_rating_ab = lst[5] if lst[6] == 'fighter': plane.plane_class_id = 1 elif lst[6] == 'attacker': plane.plane_class_id = 2 elif lst[6] == 'bomber': plane.plane_class_id = 3 else: plane.plane_class_id = None plane.crew=lst[7] plane.take_off_weight=lst[8] if len(lst) == 10: plane.burst_mass = lst[9] country_id = get_plane_country(lst[1], url) plane.country_id = country_id else: write_log('general_characteristics_adding', 'db_adding_planes.log', url)
def kvs(key): response = 'Unsupported HTTP method', 501 # Validate the key if key is None or key == '': response = 'You need to send a key', 500 print(request.form) # validate the value if request.method in ['POST', 'PUT']: if request.form is None or request.form['value'] is None: response = 'You need to send a value1', 500 if (not config['allow_empty_values'] and request.form['value'] == ''): response = 'You need to send a value2', 500 # Function dispatcher. Choose function from the functions class based on request type if response[1] != 500: if request.method == 'GET': response = functions.get_key(key) if request.method == 'POST': response = functions.set_key(key, request.form['value']) if request.method == 'PUT': response = functions.mod_key(key, request.form['value'], config['strict_modify']) if request.method == 'DELETE': response = functions.del_key(key, config['strict_delete']) # write the log write_log(key, request, response) # return the response return response
def process_file(file): filename = secure_filename(file.filename) if not filename: filename = hashlib.md5(time.time()).hexdigest() path = os.path.join(config.UPLOAD_FOLDER, hashlib.sha1(filename + str(session['last'])).hexdigest()[:10] + os.path.splitext(filename)[1]) file.save(path) # Strip meta-data if we can. subprocess.call(config.script.EXIF_TOOL.format(**locals()), shell=True) # Scan for virus try: subprocess.check_call(config.script.CLAMDSCAN.format(**locals()), shell=True) except subprocess.CalledProcessError: return jsonify({ 'mode': 'message', 'message': config.messages.VIRUS, 'color': 'red' }) rel_path = path.replace(config.UPLOAD_FOLDER, '') subprocess.call(config.script.UPLOAD_AWS.format(**locals()), shell=True) write_log('Uploaded ' + path) session['filename'] = rel_path return None
def _load_validation(validation_dir, out_dir, log): v_xs, v_ys, v_ids = load_train_data(validation_dir) print(LINE) s = f'Loaded {len(v_xs)} sets of validation data' print(s) if log: write_log(LINE, out_dir) write_log(s, out_dir) return v_xs, v_ys, v_ids
def add_flaps_sods_to_plane(plane, sods, url): if len(sods) != 0: if None in sods: write_log('flaps_sods_adding', 'db_adding_planes.log', url) plane.sod_combat_flaps = sods[0] plane.sod_takeoff_flaps = sods[1] plane.sod_landing_flaps = sods[2] else: write_log('flaps_sods_adding', 'db_adding_planes.log', url)
def process_features_table(lst, url): if len(lst) == 0: write_log(7, 'processing_planes.log', url) return lst else: new_list = [False if word == 'X' else True for word in lst] if len(new_list) == 5: new_list.append(False) return new_list
def process_modules(lst, url): if len(lst) == 0: write_log(11, 'processing_planes.log', url) return [] else: new_list = flatten_list([_list[1::] for _list in lst]) new_list = list(filter(None, new_list)) new_list = [word.replace('.', '') for word in new_list] return new_list
def add_optimal_velocities_to_plane(plane, optimal_vel, url): if len(optimal_vel) != 0: if None in optimal_vel: write_log('optimal_velocities_adding', 'db_adding_planes.log', url) plane.ailerons = optimal_vel[0] plane.rudder = optimal_vel[1] plane.elevators = optimal_vel[2] plane.radiator = optimal_vel[3] else: write_log('optimal_velocities_adding', 'db_adding_planes.log', url)
def get_plane_country(country, url): ''' Checks which country should plane be appended to. ''' country_id = db.session.query(Country.country_id).filter_by(name=country).scalar() if country is not None: return country_id else: write_log('country_adding', 'db_adding_planes.log', url) return None
def process_limits_table(lst, url): if len(lst) == 0: write_log(8, 'processing_planes.log', url) return [] else: lst = lst[2:5] for i in range(len(lst)): if '?' in lst[i] or 'N/A' in lst[i] or '_' in lst[ i] or not re.match('\d', lst[i]): lst[i] = None else: lst[i] = int(lst[i].replace(' ', '').replace(',', '')) return lst
def add_engine_and_sod_to_plane(plane, lst, detailed_info, url): if len(lst) != 0: if lst[0] is not None: plane.max_alt = lst[0] plane.no_engines = lst[1] plane.sod_structural = lst[5] plane.sod_gear = lst[6] engine_id = get_id_or_create_plane_engine(lst, detailed_info, url) else: engine_id = None write_log('engine/ceiling/sod_addding', 'db_adding_planes.log', url) plane.engine_id = engine_id
def add_features_to_plane(plane, features, url): if len(features) != 0: if None in features: write_log('features_adding', 'db_adding_planes.log', url) plane.combat_flaps = features[0] plane.take_off_flaps = features[1] plane.landing_flaps = features[2] plane.air_brakes = features[3] plane.arrestor_gear = features[4] plane.drogue_chute = features[5] plane.radar_warning_receiver = False plane.ballistic_computer = False else: write_log('features_adding', 'db_adding_planes.log', url)
def _print_train_info(loss_function, bce_weights, epochs, lr, weights_are, device_name, out_dir, log): s = LINE + '\n' + f'Loss function: {loss_function} \n' if bce_weights is not None: s = s + f'Loss function channel weights: {bce_weights} \n' s = s + 'Optimiser: Adam \n' + f'Learning rate: {lr} \n' s = s + LINE + '\n' s = s + f'Training {weights_are} U-net for {epochs} ' s = s + 'epochs with batch size 1 \n' s = s + f'Device: {device_name} \n' + LINE print(s) if log: write_log(LINE, out_dir) write_log(s, out_dir)
def add_weapons_to_plane(plane, is_defensive, weapons, url): ''' Adds defensive and offensive weapons to plane, based on is_defensive variable. ''' if len(weapons) % 4 != 0: write_log('weapons_adding', 'db_adding_planes.log', url) else: lst = (list_to_chunks(weapons, 4)) for weapon_list in lst: weapon_id = get_id_or_create_weapon(weapon_list) if is_defensive: plane_weapon = PlaneDefensiveWeapon(quantity=weapon_list[0], rounds=weapon_list[2], weapon_id=weapon_id, plane_id=plane.plane_id) else: plane_weapon = PlaneOffensiveWeapon(quantity=weapon_list[0], rounds=weapon_list[2], weapon_id=weapon_id, plane_id=plane.plane_id) db.session.add(plane_weapon)
def process_defensive_armament(lst, url): if any(word in WORD_TO_REMOVE_PROCESSING for word in lst): write_log(2, 'processing_planes.log', url) lst = [] return lst else: lst = [ word.replace(' rounds', '').replace(' shots/min', '') for word in lst ] lst = [ int(word.replace(' ', '')) if re.match('[0-9 ]+$', word) else word for word in lst ] return separate_quantity_weapon(lst)
def validate_latest_batch(spe): completed, incomplete = species_progress(spe) for runid in incomplete: if kallisto_processed(runid): sweep(runid) completed.append(runid) incomplete.remove(runid) log_path = helpers.initiate_logfile('progress', ['runid', 'status'], spe=f"{spe}-") for runid in completed: to_write = f"{runid}\tcompleted\n" helpers.write_log(to_write, log_path) for runid in incomplete: to_write = f"{runid}\tincomplete\n" helpers.write_log(to_write, log_path) return incomplete
def process_characteristics_table(lst, url): if any(word in WORD_TO_REMOVE_PROCESSING for word in lst) or len(lst) == 0 or len(lst[0]) != 8 or len( lst[1]) not in [6, 8]: write_log(6, 'processing_planes.log', url) lst = [] return lst elif len(lst[1]) == 6: lst[1].insert(2, lst[0][2]) lst[1].insert(7, lst[0][7]) for i in range(0, 2): lst[i] = [n.replace(' ', '').replace(',', '') for n in lst[i]] lst[i] = [filter_characteristics(n) for n in lst[i]] return lst
def test(self, X: np.ndarray, Y: np.ndarray, feature_type: str, plot_result=False): """ Test the network: - Compute the MAE for each count in Y - Compute MAE where y in [1, 10] - Compute MAE where y in [1, 20] - Compute the MAE over all labels :param X: The test data set (list of files) :param Y: The labels :param feature_type: Feature type to use :return MAE """ if self.__net is None: write_log( 'Cannot test the network, as it is not initialized. Please train your model, or load it from filesystem', True, True) write_log('Testing network') generator = TestSetGenerator(X, Y, self.batch_size, feature_type) Y_hat = self.__net.predict(generator) # Convert predictions to int: take median of poisson distribution predictions = np.array( [int(poisson(y_hat[0]).median()) for y_hat in Y_hat]) errors = {} for speaker_count in range(min(Y), max(Y) + 1): indices_with_count = np.argwhere(Y == speaker_count) y_current = Y[indices_with_count] predictions_current = predictions[indices_with_count] error = mean_absolute_error(y_current, predictions_current) errors[speaker_count] = error for max_count in [10, 20]: indices = np.argwhere(np.logical_and(Y >= 1, Y <= max_count)) errors[f'1_to_{max_count}'] = mean_absolute_error( Y[indices], predictions[indices]) errors['mean'] = mean_absolute_error(Y, predictions) if plot_result: self.__plot_test_results(errors) return errors
def process_suspended_armament(lst, url): if any(word in WORD_TO_REMOVE_PROCESSING for word in lst): write_log(4, 'processing_planes.log', url) lst = [] return lst else: new_list = [] for word in lst: setup = list(filter(None, (re.split('(\d{1,3} x )', word)))) for i in range(len(setup)): if i % 2 == 0: setup[i] = int(setup[i].replace(' x ', '')) setup = [x for x in setup if not isinstance(x, int)] new_list.append(setup) new_list = flatten_list(new_list) res = [] [res.append(x) for x in new_list if x not in res] return res
def process_flight_characteristics(lst, url): if any(word in WORD_TO_REMOVE_PROCESSING for word in lst) or len(lst) != 6: write_log(1, 'processing_planes.log', url) lst = [] return lst else: engines = lst[1].split(' х ') lst[0] = int(lst[0].replace(' m', '').replace(' ', '')) lst[4] = int(lst[4].replace(' km/h', '').replace(' ', '').split('.')[0]) lst[5] = int(lst[5].replace(' km/h', '').replace(' ', '').split('.')[0]) if len(engines) == 1: lst[1] = 1 lst.insert(2, engines[0]) else: lst[1] = int(engines[0]) lst.insert(2, engines[1]) return lst
def process_optimal_velocities_table(lst, url): if len(lst) == 0: write_log(9, 'processing_planes.log', url) return lst else: new_list = [] for word in lst: if '_' in word: new_list.append('0') else: new_list.append( word.replace('<', '').replace('N/A', '0').replace('>', '').replace(' ', '')) new_list = [word for word in new_list if word not in WORDS_TO_REMOVE] new_list = [None if word == '0' else int(word) for word in new_list] return new_list
def add_characteristics_to_plane(plane, characteristics, url): if len(characteristics) != 0: plane.max_speed_stock_ab = characteristics[0][0] plane.max_speed_upgraded_ab = characteristics[1][0] plane.max_speed_stock_rb = characteristics[0][1] plane.max_speed_upgraded_rb = characteristics[1][1] if characteristics[0][2] == characteristics[1][2] and characteristics[0][2] is not None: if plane.max_alt is None: plane.max_alt = characteristics[0][2] else: write_log('max_alt_adding', 'db_adding_planes.log', url) plane.turn_stock_ab = characteristics[0][3] plane.turn_upgraded_ab = characteristics[1][3] plane.turn_stock_rb = characteristics[0][4] plane.turn_upgraded_rb = characteristics[1][4] plane.roc_stock_ab = characteristics[0][5] plane.roc_upgraded_ab = characteristics[1][5] plane.roc_stock_rb = characteristics[0][6] plane.roc_upgraded_rb = characteristics[1][6] if characteristics[0][7] == characteristics[1][7] and characteristics[0][7] is not None: plane.take_off_run = characteristics[0][7] else: plane.take_off_run = None write_log('take_off_run_adding', 'db_adding_planes.log', url) else: write_log('characteristics_adding', 'db_adding_planes.log', url)
def _validate(v_xs, v_ys, v_ids, device, unet, v_loss, progress, log, out_dir, validation_dict, e, batch_no): validation_loss = 0.0 with torch.no_grad(): v_y_hats = [] for i in range(len(v_xs)): v_x, v_y = _prep_x_y(v_xs[i], v_ys[i], device) v_y_hat = unet(v_x.float()) v_y_hats.append(v_y_hat) vl = v_loss(v_y_hat, v_y) validation_loss += vl.item() validation_dict['epoch'].append(e) validation_dict['validation_loss'].append(vl.item()) validation_dict['data_id'].append(v_ids[i]) validation_dict['batch_id'].append(batch_no) progress.update(1) score = validation_loss / len(v_xs) s = f'Epoch {e} - validation loss: {score}' print(s) if log: write_log(s, out_dir) return v_y_hats
def _train_loop(no_iter, epochs, xs, ys, ids, device, unet, out_dir, optimiser, loss, loss_dict, validate, v_xs, v_ys, v_ids, validation_dict, v_loss, update_every, log, suffix, channels): v_y_hats = None # loop over training data unet = unet.to(device=device, dtype=torch.float32) with tqdm(total=no_iter, desc='unet training') as progress: for e in range(epochs): _set_epoch_if_epoch_weighted(loss, e) if validate and e == 0: _set_epoch_if_epoch_weighted(v_loss, e, verbose=False) if e == 0: # first validation at the start of the first epoch v_y_hats = _validate(v_xs, v_ys, v_ids, device, unet, v_loss, progress, log, out_dir, validation_dict, e, 0) running_loss = 0.0 y_hats = [] for i in range(len(xs)): l = _train_step(i, xs, ys, ids, device, unet, optimiser, y_hats, loss, loss_dict, e, channels) optimiser.step() running_loss += l.item() progress.update(1) if i % update_every == (update_every - 1): s = f'Epoch {e} - running loss: ' s = s + f'{running_loss / update_every}' print(s) if log: write_log(s, out_dir) running_loss = 0.0 if validate: # validation at the end of the epoch batch_no = ((e + 1) * len(xs)) v_y_hats = _validate(v_xs, v_ys, v_ids, device, unet, v_loss, progress, log, out_dir, validation_dict, e, batch_no) _save_checkpoint(unet.state_dict(), out_dir, f'{suffix}_epoch-{e}') return y_hats, v_y_hats
def process_detailed_engine_info_table(lst, url): new_list = [] is_info_found_flag = False for _list in lst: if _list[0] == 'Stationary': new_list = [ word.replace(',', '').replace('kgf', '').replace(' ', '') for word in _list[1:3] ] is_info_found_flag = True if 'Compressor' in _list[0]: temp = lst[lst.index(_list) + 2] new_list = [ word.replace(',', '').replace('hp', '').replace(' ', '') for word in temp[1::] ] is_info_found_flag = True if is_info_found_flag is not True: write_log(10, 'processing_planes.log', url) return new_list
def process_economy(lst, url): if any(word in WORD_TO_REMOVE_PROCESSING for word in lst) or len(lst) == 0 or len(lst) != 12: write_log(5, 'processing_planes.log', url) lst = [] return lst else: if lst[0] in ['Premium', 'premium']: new_list = [ word.replace(' %', '').replace(' ', '').replace('×2', '') for word in lst ] new_list[0] = 0 if 'or' in new_list[1]: new_list[1] = 0 else: new_list = [ word.replace(' %', '').replace(' ', '') for word in lst ] if 'free' not in lst: repair_sb = new_list[2].partition('/') repair_rb = new_list[3].partition('/') repair_ab = new_list[4].partition('/') new_list[2] = repair_sb[0] new_list[3] = repair_rb[0] new_list[4] = repair_ab[0] new_list.insert(3, repair_sb[2]) new_list.insert(5, repair_rb[2]) new_list.insert(7, repair_ab[2]) else: new_list = [word.replace('free', '0') for word in new_list] for i in range(3, 9): new_list.insert(i, '0') new_list = [int(word) for word in new_list] return new_list
def __parse_all__(content_lines=("",), debug_mode=False, show_error=True): # init variable prev_type_index = 0 # this is the type index of the previous line(see helpers.TYPE_MAP for more) cur_city = "" # this is the current city at the point we are parsing cur_hospital = "" # this is the current hospital that we are parsing cur_info = "" # this is the current info that we are parsing (used in info_subitem) info_dict = {} # the return variable that contain all the indentation_list = [] # this is the list that tracks the parent item indentation # calculation for content_line in content_lines: if not content_line.isspace(): # if this line is not space content_line = content_line.replace('\t', ' ') # replace tab with four spaces if debug_mode: helpers.write_log() helpers.write_log('this is the current line we are processing:') helpers.write_log(content_line) # it is a title, which means this is a city if content_line.startswith('#'): parsed_line = __parse_city__(content_line) if debug_mode: helpers.write_log('this is a city line') if parsed_line == '': if show_error: print('we encounter a warning while parsing readme: ') print('this line will be parsed as empty:') print(content_line[:-1]) print('we will not add this line to the result') if debug_mode: helpers.write_log('WARNING: this line contains illegal string, that will parse as empty') else: print('you can run program in debug mode for further information') else: info_dict.update({parsed_line: {}}) cur_city = parsed_line indentation_list = [] else: # determine the type of information of this line cur_indentation = __get_indent__(content_line) # see if the current indentation in the indent list # (smaller or equal to at least on number in the indentation_list) in_indent_list = False for type_index, indent in enumerate(indentation_list): if cur_indentation < indent: cur_type_index = type_index + 1 indentation_list = indentation_list[:type_index - 1] in_indent_list = True break elif cur_indentation == indent: cur_type_index = type_index indentation_list = indentation_list[:type_index] in_indent_list = True break # larger than all the type index in the list if not in_indent_list: cur_type_index = len(indentation_list) # parse the current line into the info_dict # this line is a hospital if cur_type_index == 0: parsed_line = __parse_hospital__(content_line) if parsed_line == '': if show_error: print('we encounter a warning while parsing readme: ') print('this line will be parsed as empty:') print('"' + content_line[:-1] + '"') print('we will not add this line to the result') if debug_mode: helpers.write_log('WARNING: this line contains illegal string, that will parse as empty') else: print('you can run program in debug mode for further information') else: info_dict[cur_city].update({parsed_line: {}}) cur_hospital = parsed_line if debug_mode: helpers.write_log('the city is:', cur_city) helpers.write_log('hospital is:', cur_hospital) # this is a info line elif cur_type_index == 1: # value is a list, to indicate whether there is a value key, value = __parse_hospital_info__(content_line) try: if value in info_dict[cur_city][cur_hospital][key]: # key in info dict and value duplicated if show_error: print('we encounter a warning will parsing readme:') print('this line is a duplicate info of previous info') print('"' + content_line[:-1] + '"') print('the hospital is', cur_hospital) print('the city is', cur_city) if debug_mode: helpers.write_log( 'WARNING: this line is a duplicate of info') else: print('you can run program in debug mode for further information') else: # key in info dict and value not duplicated info_dict[cur_city][cur_hospital][key].append(value[0]) except KeyError: # key not in info dict info_dict[cur_city][cur_hospital][key] = value cur_info = key if debug_mode: helpers.write_log('the current city is:', cur_city) helpers.write_log('the current hospital is:', cur_hospital) helpers.write_log('the current info name is:', cur_info) helpers.write_log('the current info info is:', info_dict[cur_city][cur_hospital][cur_info]) # this is a info suitem line elif cur_type_index == 2: parsed_line = __parse_info_subitem__(content_line) if parsed_line == '': if show_error: print('we encounter a warning while parsing readme: ') print('this line will be parsed as empty:') print('"' + content_line[:-1] + '"') print('we will not add this line to the result') if debug_mode: helpers.write_log('WARNING: this line contains illegal string, that will parse as empty') else: print('you can run program in debug mode for further information') else: info_dict[cur_city][cur_hospital][cur_info].append(parsed_line) if debug_mode: helpers.write_log('the current city is:', cur_city) helpers.write_log('the current hospital is:', cur_hospital) helpers.write_log('the current info name is:', cur_info) helpers.write_log('the current info info is:', info_dict[cur_city][cur_hospital][cur_info]) else: if show_error: print('we cannot recognize the type of the line') print('here is some information you can send to the issue:') print(' this is the line') print(' ', "'" + content_line[:-1] + "'") print(' previous line has type index:', prev_type_index) print(' here is the indentation list:', indentation_list) print(' current line has', cur_indentation, 'leading white spaces') input('press <enter> to continue') # just pass the previous info to this line to disregard this line cur_type_index = prev_type_index indentation_list.append(cur_indentation) if debug_mode: helpers.write_log('we cannot recognize the type of the line') helpers.write_log('here is some information you can send to the issue:') helpers.write_log(' this is the line') helpers.write_log(' ', "'" + content_line[:-1] + "'") helpers.write_log(' here is the indentation list:', indentation_list) helpers.write_log(' current line has', cur_indentation, 'leading white spaces') # update all the previous variable for the next round prev_type_index = cur_type_index indentation_list.append(cur_indentation) if debug_mode: helpers.write_log('the current line has', cur_indentation, 'leading white space') helpers.write_log('the current line has type index of', cur_type_index) helpers.write_log('the current indentation list is:', indentation_list) print('parse all finished') if debug_mode: helpers.write_log() helpers.write_log('here is a json of the output: ') helpers.write_log(json.dumps(info_dict, ensure_ascii=False, sort_keys=True, indent=4)) return info_dict
def log_city_hospital(): if debug_mode: helpers.write_log('the current city is:', cur_city) helpers.write_log('the current hospital is:', cur_hospital)
def __parse_all__(content_lines=("",), debug_mode=False): # init variable prev_leading_space_num = 0 # this is the number of leading white space in the previous line prev_type_index = 0 # this is the type index of the previous line(see helpers.TYPE_MAP for more) cur_city = "" # this is the current city at the point we are parsing cur_hospital = "" # this is the current hospital that we are parsing info_dict = {} # the return variable that contain all the # calculation for content_line in content_lines: if not content_line.isspace(): # if this line is not space content_line = content_line.replace('\t', ' ') # replace tab with four spaces if debug_mode: helpers.write_log('') helpers.write_log('this is the current line we are processing:') helpers.write_log(content_line) # it is a title, which means this is a city if content_line.startswith('#'): parsed_line = __parse_city__(content_line) if debug_mode: helpers.write_log('this is a city line') if parsed_line == '': print('we encounter a warning while parsing readme: ') print('this line will be parsed as empty:') print(content_line[:-1]) print('we will not add this line to the result') if debug_mode: helpers.write_log('WARNING: this line contains illegal string, that will parse as empty') else: print('you can run program in debug mode for further information') else: info_dict.update({parsed_line: {}}) cur_city = parsed_line else: # determine the type of information of this line cur_leading_space_num = __get_leading_space_num__(content_line) if cur_leading_space_num == prev_leading_space_num: cur_type_index = prev_type_index elif cur_leading_space_num < prev_leading_space_num: cur_type_index = prev_type_index - 1 else: cur_type_index = prev_type_index + 1 # parse the current line into the info_dict # this line is a hospital if cur_type_index == 0: parsed_line = __parse_hospital__(content_line) if parsed_line == '': print('we encounter a warning while parsing readme: ') print('this line will be parsed as empty:') print('"' + content_line[:-1] + '"') print('we will not add this line to the result') if debug_mode: helpers.write_log('WARNING: this line contains illegal string, that will parse as empty') else: print('you can run program in debug mode for further information') else: info_dict[cur_city].update({parsed_line: {}}) cur_hospital = parsed_line # this is a info line elif cur_type_index == 1: key, value = __parse_hospital_info__(content_line) try: if value in info_dict[cur_city][cur_hospital][key]: # key in info dict and value duplicated print('we encounter a warning will parsing readme:') print('this line is a duplicate info of previous info') print('"' + content_line[:-1] + '"') print('the hospital is', cur_hospital) print('the city is', cur_city) if debug_mode: helpers.write_log( 'WARNING: this line is a duplicate of info') else: print('you can run program in debug mode for further information') else: # key in info dict and value not duplicated info_dict[cur_city][cur_hospital][key].append(value) except KeyError: # key not in info dict info_dict[cur_city][cur_hospital][key] = [value, ] else: print('we cannot recognize the type of the line') print('here is some information you can send to the issue:') print(' this is the line') print(' ', "'" + content_line[:-1] + "'") print(' previous line has', prev_leading_space_num, 'leading white spaces') print(' current line has', cur_leading_space_num, 'leading white spaces') input('press <enter> to continue') # just pass the previous info to this line to disregard this line cur_type_index = prev_type_index cur_leading_space_num = prev_leading_space_num if debug_mode: helpers.write_log('we cannot recognize the type of the line') helpers.write_log('here is some information you can send to the issue:') helpers.write_log(' this is the line') helpers.write_log(' ', "'" + content_line[:-1] + "'") helpers.write_log(' previous line has', prev_leading_space_num, 'leading white spaces') helpers.write_log(' current line has', cur_leading_space_num, 'leading white spaces') # update all the previous variable for the next round prev_type_index = cur_type_index prev_leading_space_num = cur_leading_space_num if debug_mode: helpers.write_log('the current line has', cur_leading_space_num, 'leading white space') helpers.write_log('the current line has type index of', cur_type_index) print('parse all finished') if debug_mode: helpers.write_log() helpers.write_log('here is a json of the output: ') helpers.write_log(json.dumps(info_dict, ensure_ascii=False, sort_keys=True, indent=4)) return info_dict
def log_city_hospital_info(): if debug_mode: log_city_hospital() helpers.write_log('the current info name is:', cur_info) helpers.write_log('the current info info is:', info_dict[cur_city][cur_hospital][cur_info])