def run(self):
"""
Run the main loop. Log to application log if debug level is DEBUG.
Then write to database, log to file, check humidity treshold, update leds.
"""
self.debug()
try:
log_batch = []
status = [0] * led._LED_COUNT
for i in i2c._SENSORS_ADDR: # Acquire sensor data
data = i2c.read_i2c_addr(i)
log_batch.append(data)
if data[3] > led._LEDS_HUMIDITY_THRESHOLD: # Update the treshold status
status[i % (led._LED_COUNT)] += 1
if self._LEDS_ENABLED: # Update the leds according to treshold status
logging.getLogger().info("Updating LEDs", exc_info=False)
indicator = led.Argospi2cWS2811x()
indicator.set_leds(status)
if self._INFLUX_ENABLED: # Write to database
logging.getLogger().info("Contacting influx", exc_info=False)
data = influx.prepare_data(log_batch)
influx.post_to_database(data)
if self._LOG_ENABLED: # Write to local logfile
logging.getLogger().info("Log file enabled", exc_info=False)
helpers.write_log(self._LOG_FILE, log_batch)
except IndexError:
logging.getLogger().error("Index out of range.", exc_info=True)
except IOError:
logging.getLogger().error("Failed to connect to i2c.", exc_info=True)
def _set_feature_type(self, type: str):
"""
Set on init.
Define the feature representation
- MEL with 20 or 40 filters
- OR MFCC with coefficients 1-13
- OR STFT, either logarithmically scaled or normal transforms
- Also set self.feature_shape depending on the feature type
:param type: The feature type
"""
# STFT in mel basis, using 20 filters
if type == self.FEATURE_TYPE_MEL_20:
self.num_mel_filters = 20
# 501 * 20
shape = self.num_mel_filters
# STFT in mel basis, using 20 filters
elif type == self.FEATURE_TYPE_MEL_40:
self.num_mel_filters = 40
# 501 * 40
shape = self.num_mel_filters
# Mel frequency cepstral coefficients 1-13
elif type == self.FEATURE_TYPE_MFCC:
# 501 * 12
shape = self.num_coefficients
# Short time fourier transform: windows in frequency domains, 501 * (1 + self.frame_length / 2) = 501 * 201
elif type == self.FEATURE_TYPE_STFT or type == self.FEATURE_TYPE_LOG_STFT:
# 501 * 201
shape = 1 + self.frame_length // 2
else:
write_log(f'Feature type {type} is invalid', True, True)
write_log(f'Using feature type {type}')
self.feature_type = type
self.feature_shape = (int(self.sample_rate * self.seconds_per_record / self.n_overlap) + 1, shape)
def train(self, files: np.ndarray, min_speakers: int, max_speakers: int,
feature_type: str):
"""
Train the network, eg
- Create data generators
- Train with Adam, Poisson loss, MeanAbsoluteError metric.
- Visualize using Tensorboard
:param files: All files
:param min_speakers The min number of speakers to generate files for
:param max_speakers The max number of speakers to generate files for
:param feature_type: Feature type to use
"""
train_generator, (val_x, val_y) = self.__get_train_data(
files, min_speakers, max_speakers, feature_type)
net = self.compile_net(train_generator.feature_shape)
write_log('Training model')
history = net.fit(
train_generator,
validation_data=(val_x, val_y),
epochs=self.num_epochs,
callbacks=self.callbacks,
verbose=1,
)
write_log('Model trained')
return net, history
def add_economy_to_plane(plane, economy, url):
if len(economy) != 0:
plane.research = economy[0]
plane.purchase = economy[1]
if economy[0] == 0 and len(economy) == 12:
plane.is_premium = True
plane.repair_min_sb = economy[2]
plane.repair_min_rb = economy[3]
plane.repair_min_ab = economy[4]
plane.crew_training = economy[5]
plane.experts = economy[6]
plane.aces = economy[7]
plane.reward_rp = economy[8]
plane.reward_sl_sb = economy[9]
plane.reward_sl_rb = economy[10]
plane.reward_sl_ab = economy[11]
else:
plane.is_premium = False
plane.repair_min_sb = economy[2]
plane.repair_max_sb = economy[3]
plane.repair_min_rb = economy[4]
plane.repair_max_rb = economy[5]
plane.repair_min_ab = economy[6]
plane.repair_max_ab = economy[7]
plane.crew_training = economy[8]
plane.experts = economy[9]
plane.aces = economy[10]
plane.reward_rp = economy[11]
plane.reward_sl_sb = economy[12]
plane.reward_sl_rb = economy[13]
plane.reward_sl_ab = economy[14]
else:
write_log('economy_insert', 'db_adding_planes.log', url)
def process_general_characteristics(lst, url):
if any(word in WORD_TO_REMOVE_PROCESSING
for word in lst) or len(lst) == 0 or len(lst) not in [9, 10]:
write_log(0, 'processing_planes.log', url)
lst = []
return lst
lst[0] = lst[0].replace('\xa0', ' ')
lst[2] = lst[2].replace(' Rank', '')
lst[2] = roman_to_int(lst[2])
lst[3] = float(lst[3])
lst[4] = float(lst[4])
lst[5] = float(lst[5])
if 'Fighter' in lst[6] or 'fighter' in lst[6]:
lst[6] = 'fighter'
elif 'Bomber' in lst[6] or 'bomber' in lst[6]:
lst[6] = 'bomber'
elif 'Attacker' in lst[6] or 'attacker' in lst[6]:
lst[6] = 'attacker'
else:
lst[6] = None
lst[7] = int(lst[7].replace(' people', '').replace(' person', ''))
lst[8] = float(lst[8].replace(' t', ''))
if len(lst) == 10:
lst[9] = float(lst[9].replace(' kg/s', ''))
return lst
def add_general_characteristics(plane, lst, url):
if len(lst) != 0:
plane.rank = lst[2]
plane.battle_rating_sb = lst[3]
plane.battle_rating_rb = lst[4]
plane.battle_rating_ab = lst[5]
if lst[6] == 'fighter':
plane.plane_class_id = 1
elif lst[6] == 'attacker':
plane.plane_class_id = 2
elif lst[6] == 'bomber':
plane.plane_class_id = 3
else:
plane.plane_class_id = None
plane.crew=lst[7]
plane.take_off_weight=lst[8]
if len(lst) == 10:
plane.burst_mass = lst[9]
country_id = get_plane_country(lst[1], url)
plane.country_id = country_id
else:
write_log('general_characteristics_adding', 'db_adding_planes.log', url)
def kvs(key):
response = 'Unsupported HTTP method', 501
# Validate the key
if key is None or key == '':
response = 'You need to send a key', 500
print(request.form)
# validate the value
if request.method in ['POST', 'PUT']:
if request.form is None or request.form['value'] is None:
response = 'You need to send a value1', 500
if (not config['allow_empty_values'] and request.form['value'] == ''):
response = 'You need to send a value2', 500
# Function dispatcher. Choose function from the functions class based on request type
if response[1] != 500:
if request.method == 'GET':
response = functions.get_key(key)
if request.method == 'POST':
response = functions.set_key(key, request.form['value'])
if request.method == 'PUT':
response = functions.mod_key(key, request.form['value'], config['strict_modify'])
if request.method == 'DELETE':
response = functions.del_key(key, config['strict_delete'])
# write the log
write_log(key, request, response)
# return the response
return response
def process_file(file):
filename = secure_filename(file.filename)
if not filename:
filename = hashlib.md5(time.time()).hexdigest()
path = os.path.join(config.UPLOAD_FOLDER, hashlib.sha1(filename + str(session['last'])).hexdigest()[:10] + os.path.splitext(filename)[1])
file.save(path)
# Strip meta-data if we can.
subprocess.call(config.script.EXIF_TOOL.format(**locals()), shell=True)
# Scan for virus
try:
subprocess.check_call(config.script.CLAMDSCAN.format(**locals()), shell=True)
except subprocess.CalledProcessError:
return jsonify({
'mode': 'message',
'message': config.messages.VIRUS,
'color': 'red'
})
rel_path = path.replace(config.UPLOAD_FOLDER, '')
subprocess.call(config.script.UPLOAD_AWS.format(**locals()), shell=True)
write_log('Uploaded ' + path)
session['filename'] = rel_path
return None
def _load_validation(validation_dir, out_dir, log):
v_xs, v_ys, v_ids = load_train_data(validation_dir)
print(LINE)
s = f'Loaded {len(v_xs)} sets of validation data'
print(s)
if log:
write_log(LINE, out_dir)
write_log(s, out_dir)
return v_xs, v_ys, v_ids
def add_flaps_sods_to_plane(plane, sods, url):
if len(sods) != 0:
if None in sods:
write_log('flaps_sods_adding', 'db_adding_planes.log', url)
plane.sod_combat_flaps = sods[0]
plane.sod_takeoff_flaps = sods[1]
plane.sod_landing_flaps = sods[2]
else:
write_log('flaps_sods_adding', 'db_adding_planes.log', url)
def process_features_table(lst, url):
if len(lst) == 0:
write_log(7, 'processing_planes.log', url)
return lst
else:
new_list = [False if word == 'X' else True for word in lst]
if len(new_list) == 5:
new_list.append(False)
return new_list
def process_modules(lst, url):
if len(lst) == 0:
write_log(11, 'processing_planes.log', url)
return []
else:
new_list = flatten_list([_list[1::] for _list in lst])
new_list = list(filter(None, new_list))
new_list = [word.replace('.', '') for word in new_list]
return new_list
def add_optimal_velocities_to_plane(plane, optimal_vel, url):
if len(optimal_vel) != 0:
if None in optimal_vel:
write_log('optimal_velocities_adding', 'db_adding_planes.log', url)
plane.ailerons = optimal_vel[0]
plane.rudder = optimal_vel[1]
plane.elevators = optimal_vel[2]
plane.radiator = optimal_vel[3]
else:
write_log('optimal_velocities_adding', 'db_adding_planes.log', url)
def get_plane_country(country, url):
'''
Checks which country should plane be appended to.
'''
country_id = db.session.query(Country.country_id).filter_by(name=country).scalar()
if country is not None:
return country_id
else:
write_log('country_adding', 'db_adding_planes.log', url)
return None
def process_limits_table(lst, url):
if len(lst) == 0:
write_log(8, 'processing_planes.log', url)
return []
else:
lst = lst[2:5]
for i in range(len(lst)):
if '?' in lst[i] or 'N/A' in lst[i] or '_' in lst[
i] or not re.match('\d', lst[i]):
lst[i] = None
else:
lst[i] = int(lst[i].replace(' ', '').replace(',', ''))
return lst
def add_engine_and_sod_to_plane(plane, lst, detailed_info, url):
if len(lst) != 0:
if lst[0] is not None:
plane.max_alt = lst[0]
plane.no_engines = lst[1]
plane.sod_structural = lst[5]
plane.sod_gear = lst[6]
engine_id = get_id_or_create_plane_engine(lst, detailed_info, url)
else:
engine_id = None
write_log('engine/ceiling/sod_addding', 'db_adding_planes.log', url)
plane.engine_id = engine_id
def add_features_to_plane(plane, features, url):
if len(features) != 0:
if None in features:
write_log('features_adding', 'db_adding_planes.log', url)
plane.combat_flaps = features[0]
plane.take_off_flaps = features[1]
plane.landing_flaps = features[2]
plane.air_brakes = features[3]
plane.arrestor_gear = features[4]
plane.drogue_chute = features[5]
plane.radar_warning_receiver = False
plane.ballistic_computer = False
else:
write_log('features_adding', 'db_adding_planes.log', url)
def _print_train_info(loss_function, bce_weights, epochs, lr, weights_are,
device_name, out_dir, log):
s = LINE + '\n' + f'Loss function: {loss_function} \n'
if bce_weights is not None:
s = s + f'Loss function channel weights: {bce_weights} \n'
s = s + 'Optimiser: Adam \n' + f'Learning rate: {lr} \n'
s = s + LINE + '\n'
s = s + f'Training {weights_are} U-net for {epochs} '
s = s + 'epochs with batch size 1 \n'
s = s + f'Device: {device_name} \n' + LINE
print(s)
if log:
write_log(LINE, out_dir)
write_log(s, out_dir)
def add_weapons_to_plane(plane, is_defensive, weapons, url):
'''
Adds defensive and offensive weapons to plane, based on is_defensive variable.
'''
if len(weapons) % 4 != 0:
write_log('weapons_adding', 'db_adding_planes.log', url)
else:
lst = (list_to_chunks(weapons, 4))
for weapon_list in lst:
weapon_id = get_id_or_create_weapon(weapon_list)
if is_defensive:
plane_weapon = PlaneDefensiveWeapon(quantity=weapon_list[0], rounds=weapon_list[2], weapon_id=weapon_id, plane_id=plane.plane_id)
else:
plane_weapon = PlaneOffensiveWeapon(quantity=weapon_list[0], rounds=weapon_list[2], weapon_id=weapon_id, plane_id=plane.plane_id)
db.session.add(plane_weapon)
def process_defensive_armament(lst, url):
if any(word in WORD_TO_REMOVE_PROCESSING for word in lst):
write_log(2, 'processing_planes.log', url)
lst = []
return lst
else:
lst = [
word.replace(' rounds', '').replace(' shots/min', '')
for word in lst
]
lst = [
int(word.replace(' ', '')) if re.match('[0-9 ]+$', word) else word
for word in lst
]
return separate_quantity_weapon(lst)
def validate_latest_batch(spe):
completed, incomplete = species_progress(spe)
for runid in incomplete:
if kallisto_processed(runid):
sweep(runid)
completed.append(runid)
incomplete.remove(runid)
log_path = helpers.initiate_logfile('progress', ['runid', 'status'],
spe=f"{spe}-")
for runid in completed:
to_write = f"{runid}\tcompleted\n"
helpers.write_log(to_write, log_path)
for runid in incomplete:
to_write = f"{runid}\tincomplete\n"
helpers.write_log(to_write, log_path)
return incomplete
def process_characteristics_table(lst, url):
if any(word in WORD_TO_REMOVE_PROCESSING
for word in lst) or len(lst) == 0 or len(lst[0]) != 8 or len(
lst[1]) not in [6, 8]:
write_log(6, 'processing_planes.log', url)
lst = []
return lst
elif len(lst[1]) == 6:
lst[1].insert(2, lst[0][2])
lst[1].insert(7, lst[0][7])
for i in range(0, 2):
lst[i] = [n.replace(' ', '').replace(',', '') for n in lst[i]]
lst[i] = [filter_characteristics(n) for n in lst[i]]
return lst
def test(self,
X: np.ndarray,
Y: np.ndarray,
feature_type: str,
plot_result=False):
"""
Test the network:
- Compute the MAE for each count in Y
- Compute MAE where y in [1, 10]
- Compute MAE where y in [1, 20]
- Compute the MAE over all labels
:param X: The test data set (list of files)
:param Y: The labels
:param feature_type: Feature type to use
:return MAE
"""
if self.__net is None:
write_log(
'Cannot test the network, as it is not initialized. Please train your model, or load it from filesystem',
True, True)
write_log('Testing network')
generator = TestSetGenerator(X, Y, self.batch_size, feature_type)
Y_hat = self.__net.predict(generator)
# Convert predictions to int: take median of poisson distribution
predictions = np.array(
[int(poisson(y_hat[0]).median()) for y_hat in Y_hat])
errors = {}
for speaker_count in range(min(Y), max(Y) + 1):
indices_with_count = np.argwhere(Y == speaker_count)
y_current = Y[indices_with_count]
predictions_current = predictions[indices_with_count]
error = mean_absolute_error(y_current, predictions_current)
errors[speaker_count] = error
for max_count in [10, 20]:
indices = np.argwhere(np.logical_and(Y >= 1, Y <= max_count))
errors[f'1_to_{max_count}'] = mean_absolute_error(
Y[indices], predictions[indices])
errors['mean'] = mean_absolute_error(Y, predictions)
if plot_result:
self.__plot_test_results(errors)
return errors
def process_suspended_armament(lst, url):
if any(word in WORD_TO_REMOVE_PROCESSING for word in lst):
write_log(4, 'processing_planes.log', url)
lst = []
return lst
else:
new_list = []
for word in lst:
setup = list(filter(None, (re.split('(\d{1,3} x )', word))))
for i in range(len(setup)):
if i % 2 == 0:
setup[i] = int(setup[i].replace(' x ', ''))
setup = [x for x in setup if not isinstance(x, int)]
new_list.append(setup)
new_list = flatten_list(new_list)
res = []
[res.append(x) for x in new_list if x not in res]
return res
def process_flight_characteristics(lst, url):
if any(word in WORD_TO_REMOVE_PROCESSING for word in lst) or len(lst) != 6:
write_log(1, 'processing_planes.log', url)
lst = []
return lst
else:
engines = lst[1].split(' х ')
lst[0] = int(lst[0].replace(' m', '').replace(' ', ''))
lst[4] = int(lst[4].replace(' km/h', '').replace(' ',
'').split('.')[0])
lst[5] = int(lst[5].replace(' km/h', '').replace(' ',
'').split('.')[0])
if len(engines) == 1:
lst[1] = 1
lst.insert(2, engines[0])
else:
lst[1] = int(engines[0])
lst.insert(2, engines[1])
return lst
def process_optimal_velocities_table(lst, url):
if len(lst) == 0:
write_log(9, 'processing_planes.log', url)
return lst
else:
new_list = []
for word in lst:
if '_' in word:
new_list.append('0')
else:
new_list.append(
word.replace('<',
'').replace('N/A',
'0').replace('>',
'').replace(' ', ''))
new_list = [word for word in new_list if word not in WORDS_TO_REMOVE]
new_list = [None if word == '0' else int(word) for word in new_list]
return new_list
def add_characteristics_to_plane(plane, characteristics, url):
if len(characteristics) != 0:
plane.max_speed_stock_ab = characteristics[0][0]
plane.max_speed_upgraded_ab = characteristics[1][0]
plane.max_speed_stock_rb = characteristics[0][1]
plane.max_speed_upgraded_rb = characteristics[1][1]
if characteristics[0][2] == characteristics[1][2] and characteristics[0][2] is not None:
if plane.max_alt is None:
plane.max_alt = characteristics[0][2]
else:
write_log('max_alt_adding', 'db_adding_planes.log', url)
plane.turn_stock_ab = characteristics[0][3]
plane.turn_upgraded_ab = characteristics[1][3]
plane.turn_stock_rb = characteristics[0][4]
plane.turn_upgraded_rb = characteristics[1][4]
plane.roc_stock_ab = characteristics[0][5]
plane.roc_upgraded_ab = characteristics[1][5]
plane.roc_stock_rb = characteristics[0][6]
plane.roc_upgraded_rb = characteristics[1][6]
if characteristics[0][7] == characteristics[1][7] and characteristics[0][7] is not None:
plane.take_off_run = characteristics[0][7]
else:
plane.take_off_run = None
write_log('take_off_run_adding', 'db_adding_planes.log', url)
else:
write_log('characteristics_adding', 'db_adding_planes.log', url)
def _validate(v_xs, v_ys, v_ids, device, unet, v_loss, progress, log, out_dir,
validation_dict, e, batch_no):
validation_loss = 0.0
with torch.no_grad():
v_y_hats = []
for i in range(len(v_xs)):
v_x, v_y = _prep_x_y(v_xs[i], v_ys[i], device)
v_y_hat = unet(v_x.float())
v_y_hats.append(v_y_hat)
vl = v_loss(v_y_hat, v_y)
validation_loss += vl.item()
validation_dict['epoch'].append(e)
validation_dict['validation_loss'].append(vl.item())
validation_dict['data_id'].append(v_ids[i])
validation_dict['batch_id'].append(batch_no)
progress.update(1)
score = validation_loss / len(v_xs)
s = f'Epoch {e} - validation loss: {score}'
print(s)
if log:
write_log(s, out_dir)
return v_y_hats
def _train_loop(no_iter, epochs, xs, ys, ids, device, unet, out_dir, optimiser,
loss, loss_dict, validate, v_xs, v_ys, v_ids, validation_dict,
v_loss, update_every, log, suffix, channels):
v_y_hats = None
# loop over training data
unet = unet.to(device=device, dtype=torch.float32)
with tqdm(total=no_iter, desc='unet training') as progress:
for e in range(epochs):
_set_epoch_if_epoch_weighted(loss, e)
if validate and e == 0:
_set_epoch_if_epoch_weighted(v_loss, e, verbose=False)
if e == 0:
# first validation at the start of the first epoch
v_y_hats = _validate(v_xs, v_ys, v_ids, device, unet,
v_loss, progress, log, out_dir,
validation_dict, e, 0)
running_loss = 0.0
y_hats = []
for i in range(len(xs)):
l = _train_step(i, xs, ys, ids, device, unet, optimiser,
y_hats, loss, loss_dict, e, channels)
optimiser.step()
running_loss += l.item()
progress.update(1)
if i % update_every == (update_every - 1):
s = f'Epoch {e} - running loss: '
s = s + f'{running_loss / update_every}'
print(s)
if log:
write_log(s, out_dir)
running_loss = 0.0
if validate:
# validation at the end of the epoch
batch_no = ((e + 1) * len(xs))
v_y_hats = _validate(v_xs, v_ys, v_ids, device, unet, v_loss,
progress, log, out_dir, validation_dict,
e, batch_no)
_save_checkpoint(unet.state_dict(), out_dir, f'{suffix}_epoch-{e}')
return y_hats, v_y_hats
def process_detailed_engine_info_table(lst, url):
new_list = []
is_info_found_flag = False
for _list in lst:
if _list[0] == 'Stationary':
new_list = [
word.replace(',', '').replace('kgf', '').replace(' ', '')
for word in _list[1:3]
]
is_info_found_flag = True
if 'Compressor' in _list[0]:
temp = lst[lst.index(_list) + 2]
new_list = [
word.replace(',', '').replace('hp', '').replace(' ', '')
for word in temp[1::]
]
is_info_found_flag = True
if is_info_found_flag is not True:
write_log(10, 'processing_planes.log', url)
return new_list
def process_economy(lst, url):
if any(word in WORD_TO_REMOVE_PROCESSING
for word in lst) or len(lst) == 0 or len(lst) != 12:
write_log(5, 'processing_planes.log', url)
lst = []
return lst
else:
if lst[0] in ['Premium', 'premium']:
new_list = [
word.replace(' %', '').replace(' ', '').replace('×2', '')
for word in lst
]
new_list[0] = 0
if 'or' in new_list[1]:
new_list[1] = 0
else:
new_list = [
word.replace(' %', '').replace(' ', '') for word in lst
]
if 'free' not in lst:
repair_sb = new_list[2].partition('/')
repair_rb = new_list[3].partition('/')
repair_ab = new_list[4].partition('/')
new_list[2] = repair_sb[0]
new_list[3] = repair_rb[0]
new_list[4] = repair_ab[0]
new_list.insert(3, repair_sb[2])
new_list.insert(5, repair_rb[2])
new_list.insert(7, repair_ab[2])
else:
new_list = [word.replace('free', '0') for word in new_list]
for i in range(3, 9):
new_list.insert(i, '0')
new_list = [int(word) for word in new_list]
return new_list
def __parse_all__(content_lines=("",), debug_mode=False, show_error=True):
# init variable
prev_type_index = 0 # this is the type index of the previous line(see helpers.TYPE_MAP for more)
cur_city = "" # this is the current city at the point we are parsing
cur_hospital = "" # this is the current hospital that we are parsing
cur_info = "" # this is the current info that we are parsing (used in info_subitem)
info_dict = {} # the return variable that contain all the
indentation_list = [] # this is the list that tracks the parent item indentation
# calculation
for content_line in content_lines:
if not content_line.isspace(): # if this line is not space
content_line = content_line.replace('\t', ' ') # replace tab with four spaces
if debug_mode:
helpers.write_log()
helpers.write_log('this is the current line we are processing:')
helpers.write_log(content_line)
# it is a title, which means this is a city
if content_line.startswith('#'):
parsed_line = __parse_city__(content_line)
if debug_mode:
helpers.write_log('this is a city line')
if parsed_line == '':
if show_error:
print('we encounter a warning while parsing readme: ')
print('this line will be parsed as empty:')
print(content_line[:-1])
print('we will not add this line to the result')
if debug_mode:
helpers.write_log('WARNING: this line contains illegal string, that will parse as empty')
else:
print('you can run program in debug mode for further information')
else:
info_dict.update({parsed_line: {}})
cur_city = parsed_line
indentation_list = []
else:
# determine the type of information of this line
cur_indentation = __get_indent__(content_line)
# see if the current indentation in the indent list
# (smaller or equal to at least on number in the indentation_list)
in_indent_list = False
for type_index, indent in enumerate(indentation_list):
if cur_indentation < indent:
cur_type_index = type_index + 1
indentation_list = indentation_list[:type_index - 1]
in_indent_list = True
break
elif cur_indentation == indent:
cur_type_index = type_index
indentation_list = indentation_list[:type_index]
in_indent_list = True
break
# larger than all the type index in the list
if not in_indent_list:
cur_type_index = len(indentation_list)
# parse the current line into the info_dict
# this line is a hospital
if cur_type_index == 0:
parsed_line = __parse_hospital__(content_line)
if parsed_line == '':
if show_error:
print('we encounter a warning while parsing readme: ')
print('this line will be parsed as empty:')
print('"' + content_line[:-1] + '"')
print('we will not add this line to the result')
if debug_mode:
helpers.write_log('WARNING: this line contains illegal string, that will parse as empty')
else:
print('you can run program in debug mode for further information')
else:
info_dict[cur_city].update({parsed_line: {}})
cur_hospital = parsed_line
if debug_mode:
helpers.write_log('the city is:', cur_city)
helpers.write_log('hospital is:', cur_hospital)
# this is a info line
elif cur_type_index == 1:
# value is a list, to indicate whether there is a value
key, value = __parse_hospital_info__(content_line)
try:
if value in info_dict[cur_city][cur_hospital][key]: # key in info dict and value duplicated
if show_error:
print('we encounter a warning will parsing readme:')
print('this line is a duplicate info of previous info')
print('"' + content_line[:-1] + '"')
print('the hospital is', cur_hospital)
print('the city is', cur_city)
if debug_mode:
helpers.write_log(
'WARNING: this line is a duplicate of info')
else:
print('you can run program in debug mode for further information')
else: # key in info dict and value not duplicated
info_dict[cur_city][cur_hospital][key].append(value[0])
except KeyError: # key not in info dict
info_dict[cur_city][cur_hospital][key] = value
cur_info = key
if debug_mode:
helpers.write_log('the current city is:', cur_city)
helpers.write_log('the current hospital is:', cur_hospital)
helpers.write_log('the current info name is:', cur_info)
helpers.write_log('the current info info is:', info_dict[cur_city][cur_hospital][cur_info])
# this is a info suitem line
elif cur_type_index == 2:
parsed_line = __parse_info_subitem__(content_line)
if parsed_line == '':
if show_error:
print('we encounter a warning while parsing readme: ')
print('this line will be parsed as empty:')
print('"' + content_line[:-1] + '"')
print('we will not add this line to the result')
if debug_mode:
helpers.write_log('WARNING: this line contains illegal string, that will parse as empty')
else:
print('you can run program in debug mode for further information')
else:
info_dict[cur_city][cur_hospital][cur_info].append(parsed_line)
if debug_mode:
helpers.write_log('the current city is:', cur_city)
helpers.write_log('the current hospital is:', cur_hospital)
helpers.write_log('the current info name is:', cur_info)
helpers.write_log('the current info info is:', info_dict[cur_city][cur_hospital][cur_info])
else:
if show_error:
print('we cannot recognize the type of the line')
print('here is some information you can send to the issue:')
print(' this is the line')
print(' ', "'" + content_line[:-1] + "'")
print(' previous line has type index:', prev_type_index)
print(' here is the indentation list:', indentation_list)
print(' current line has', cur_indentation, 'leading white spaces')
input('press <enter> to continue')
# just pass the previous info to this line to disregard this line
cur_type_index = prev_type_index
indentation_list.append(cur_indentation)
if debug_mode:
helpers.write_log('we cannot recognize the type of the line')
helpers.write_log('here is some information you can send to the issue:')
helpers.write_log(' this is the line')
helpers.write_log(' ', "'" + content_line[:-1] + "'")
helpers.write_log(' here is the indentation list:', indentation_list)
helpers.write_log(' current line has', cur_indentation, 'leading white spaces')
# update all the previous variable for the next round
prev_type_index = cur_type_index
indentation_list.append(cur_indentation)
if debug_mode:
helpers.write_log('the current line has', cur_indentation, 'leading white space')
helpers.write_log('the current line has type index of', cur_type_index)
helpers.write_log('the current indentation list is:', indentation_list)
print('parse all finished')
if debug_mode:
helpers.write_log()
helpers.write_log('here is a json of the output: ')
helpers.write_log(json.dumps(info_dict, ensure_ascii=False, sort_keys=True, indent=4))
return info_dict
def log_city_hospital():
if debug_mode:
helpers.write_log('the current city is:', cur_city)
helpers.write_log('the current hospital is:', cur_hospital)
def __parse_all__(content_lines=("",), debug_mode=False):
# init variable
prev_leading_space_num = 0 # this is the number of leading white space in the previous line
prev_type_index = 0 # this is the type index of the previous line(see helpers.TYPE_MAP for more)
cur_city = "" # this is the current city at the point we are parsing
cur_hospital = "" # this is the current hospital that we are parsing
info_dict = {} # the return variable that contain all the
# calculation
for content_line in content_lines:
if not content_line.isspace(): # if this line is not space
content_line = content_line.replace('\t', ' ') # replace tab with four spaces
if debug_mode:
helpers.write_log('')
helpers.write_log('this is the current line we are processing:')
helpers.write_log(content_line)
# it is a title, which means this is a city
if content_line.startswith('#'):
parsed_line = __parse_city__(content_line)
if debug_mode:
helpers.write_log('this is a city line')
if parsed_line == '':
print('we encounter a warning while parsing readme: ')
print('this line will be parsed as empty:')
print(content_line[:-1])
print('we will not add this line to the result')
if debug_mode:
helpers.write_log('WARNING: this line contains illegal string, that will parse as empty')
else:
print('you can run program in debug mode for further information')
else:
info_dict.update({parsed_line: {}})
cur_city = parsed_line
else:
# determine the type of information of this line
cur_leading_space_num = __get_leading_space_num__(content_line)
if cur_leading_space_num == prev_leading_space_num:
cur_type_index = prev_type_index
elif cur_leading_space_num < prev_leading_space_num:
cur_type_index = prev_type_index - 1
else:
cur_type_index = prev_type_index + 1
# parse the current line into the info_dict
# this line is a hospital
if cur_type_index == 0:
parsed_line = __parse_hospital__(content_line)
if parsed_line == '':
print('we encounter a warning while parsing readme: ')
print('this line will be parsed as empty:')
print('"' + content_line[:-1] + '"')
print('we will not add this line to the result')
if debug_mode:
helpers.write_log('WARNING: this line contains illegal string, that will parse as empty')
else:
print('you can run program in debug mode for further information')
else:
info_dict[cur_city].update({parsed_line: {}})
cur_hospital = parsed_line
# this is a info line
elif cur_type_index == 1:
key, value = __parse_hospital_info__(content_line)
try:
if value in info_dict[cur_city][cur_hospital][key]: # key in info dict and value duplicated
print('we encounter a warning will parsing readme:')
print('this line is a duplicate info of previous info')
print('"' + content_line[:-1] + '"')
print('the hospital is', cur_hospital)
print('the city is', cur_city)
if debug_mode:
helpers.write_log(
'WARNING: this line is a duplicate of info')
else:
print('you can run program in debug mode for further information')
else: # key in info dict and value not duplicated
info_dict[cur_city][cur_hospital][key].append(value)
except KeyError: # key not in info dict
info_dict[cur_city][cur_hospital][key] = [value, ]
else:
print('we cannot recognize the type of the line')
print('here is some information you can send to the issue:')
print(' this is the line')
print(' ', "'" + content_line[:-1] + "'")
print(' previous line has', prev_leading_space_num, 'leading white spaces')
print(' current line has', cur_leading_space_num, 'leading white spaces')
input('press <enter> to continue')
# just pass the previous info to this line to disregard this line
cur_type_index = prev_type_index
cur_leading_space_num = prev_leading_space_num
if debug_mode:
helpers.write_log('we cannot recognize the type of the line')
helpers.write_log('here is some information you can send to the issue:')
helpers.write_log(' this is the line')
helpers.write_log(' ', "'" + content_line[:-1] + "'")
helpers.write_log(' previous line has', prev_leading_space_num, 'leading white spaces')
helpers.write_log(' current line has', cur_leading_space_num, 'leading white spaces')
# update all the previous variable for the next round
prev_type_index = cur_type_index
prev_leading_space_num = cur_leading_space_num
if debug_mode:
helpers.write_log('the current line has', cur_leading_space_num, 'leading white space')
helpers.write_log('the current line has type index of', cur_type_index)
print('parse all finished')
if debug_mode:
helpers.write_log()
helpers.write_log('here is a json of the output: ')
helpers.write_log(json.dumps(info_dict, ensure_ascii=False, sort_keys=True, indent=4))
return info_dict
def log_city_hospital_info():
if debug_mode:
log_city_hospital()
helpers.write_log('the current info name is:', cur_info)
helpers.write_log('the current info info is:', info_dict[cur_city][cur_hospital][cur_info])
