def test_extra_facets_should_be_set_from_start_urls_variables_with_multiple_tags(self, monkeypatch):
from .mocked_init import MockedInit
monkeypatch.setattr("selenium.webdriver.Firefox", lambda: MockedInit())
monkeypatch.setattr("time.sleep", lambda x: "")
config({
"start_urls": [
{
"url": "https://test.com/doc/(?P<type_of_content>.*?)/(?P<version>.*?)",
"variables": {
"type_of_content": ["book", "bundles", "reference", "components", "cookbook", "best_practices"],
"version": ["1.0", "2.0"]
}
},
{
"url": "https://test.com/doc/(?P<type_of_content>.*?)/",
"variables": {
"type_of_content": ["test"]
}
}
]
})
actual = ConfigLoader()
extra_facets = actual.get_extra_facets()
assert "version" in extra_facets
assert "type_of_content" in extra_facets
def test_no_error_with_complete_config(self):
"""
A valid config should be returned when nothing is missing.
"""
json.load = MagicMock(return_value=self.valid_config)
config_loader = ConfigLoader(json)
self.assertEqual(config_loader.getConfig(), self.valid_config)
def test__config_loader__correct_response():
config_test_path = 'tests/resources/config.yaml'
config = ConfigLoader()
config.load(config_test_path)
assert config['string_key'] == 'value'
assert config['int_key'] == 1
assert config['float_key'] == 1.5
assert config['list_key'] == [1, 'value2', 3]
assert list(config['model_parameters'].keys()) == ['C', 'penalty']
assert config['model_parameters']['C']['uniform'] == [0.2, 1]
assert config['model_parameters']['penalty']['choice'] == ['l1', 'l2']
assert config.get('non_existent_key') is None
def test_initialization(self):
"""
Correct initialization should proceed without raising an exception.
"""
config_loader = ConfigLoader(json)
self.assertIsNotNone(config_loader)
def generate(config_path, show_city=False, show_time=False, show_stats=False):
if show_time:
t = time.process_time()
# Step 0: Load config.
config = ConfigLoader(config_path)
# Step 1: Grow road network.
road_network, vertex_dict = generate_road_network(config)
# Step 2: Compute polygons based on road network.
polys = polygons.get_polygons(vertex_dict)
del polys[
0] # We delete the first polygon as this corresponds to the outer area.
# Step 3: Determine land usages.
land_usages = land_usage.get_land_usage(polys, config)
major_roads = []
for road in road_network:
if (road.is_minor_road == False):
major_roads.append(road)
# Step 4: Dump to .json.
city_to_json(road_network, major_roads, list(vertex_dict.keys()),
land_usages)
if show_time:
print('Time:', time.process_time() - t)
if show_stats:
orientation_histogram = compute_orientation_histogram(road_network)
entropy = compute_orientation_entropy(orientation_histogram)
orientation_order = compute_orientation_order(entropy)
avg_node_degree = compute_average_node_degree(vertex_dict)
proportion_dead_ends = compute_proportion_dead_ends(vertex_dict)
proportion_3way_intersections = compute_proportion_3way_intersections(
vertex_dict)
proportion_4way_intersections = compute_proportion_4way_intersections(
vertex_dict)
intersection_count = compute_intersection_count(vertex_dict)
total_road_length = compute_total_road_length(road_network,
config=config)
print('Entropy:', entropy)
print('Orientation-Order:', orientation_order)
print('Average Node Degree:', avg_node_degree)
print('Proportion Dead-Ends:', proportion_dead_ends)
print('Proportion 3-way Intersections', proportion_3way_intersections)
print('Proportion 4-way Intersections', proportion_4way_intersections)
print('Intersection Count:', intersection_count)
print('Total Road Length:', total_road_length)
if show_city:
visualise(config.water_map_array,
road_network,
land_usages=land_usages)
if show_stats:
show_orientation_histogram(orientation_histogram)
def test_empty_config_object(self):
"""
If, after loading from a file, an empty json object is returned
an exception should be raised.
"""
json.load = MagicMock(return_value={})
config_loader = ConfigLoader(json)
self.assertRaises(MissingConfigValuesError, config_loader.getConfig)
def test_missing_coordinate(self):
"""
Even if a coordinate is only partially missing an exception shouldd be raised.
"""
incomplete_config = self.valid_config
del incomplete_config['northwest']['x']
json.load = MagicMock(return_value=incomplete_config)
config_loader = ConfigLoader(json)
self.assertRaises(MissingConfigValuesError, config_loader.getConfig)
def test_missing_corner_config_value(self):
"""
If an entry for a corner is missing an exception should be raised.
"""
incomplete_config = self.valid_config
del incomplete_config['northwest']
json.load = MagicMock(return_value=incomplete_config)
config_loader = ConfigLoader(json)
self.assertRaises(MissingConfigValuesError, config_loader.getConfig)
class MyTestCase(unittest.TestCase):
DIR_NAME = os.path.dirname(os.path.abspath(__file__))
config_loader = ConfigLoader(DIR_NAME + '/test_config.ini')
def test_configure_threads(self):
class_parameters = {
'reader': {
'log_path': '/path/to/test'
},
'displayer': {
'display_interval': 99,
'activate_color': True
},
'alert_system': {
'max_requests_per_second': 14,
'alert_interval': 112
},
'log_simulator': {
'file_to_write': '/path/to/test/test',
'hostname': 'ryuk',
'sections': ['/', '/section1', '/section2']
}
}
self.assertEqual(self.config_loader.configure_threads(), class_parameters)
def test_configure_threads_no_sim(self):
self.config_loader.config_file = self.DIR_NAME + '/test_config_no_sim.ini'
class_parameters = {
'reader': {
'log_path': '/path/to/test'
},
'displayer': {
'display_interval': 99,
'activate_color': True
},
'alert_system': {
'max_requests_per_second': 14,
'alert_interval': 112
},
'log_simulator': None
}
self.assertEqual(self.config_loader.configure_threads(), class_parameters)
to the database.
"""
import sys
import os
import json
import logging
from src.raspberry_factory import RaspberryFactory
from src.audio_feedback import AudioFeedback
from src.config_loader import ConfigLoader
from src.scale_reader import ScaleReader
from src.database import Database
from src.weight_monitor import WeightMonitor
logging.basicConfig(level=logging.INFO, format='%(levelname)s:%(message)s')
os.system('clear')
logging.info('Preparing...')
dry_run = "-d" in sys.argv or "--dry-run" in sys.argv
monitor = WeightMonitor(scale_reader=ScaleReader(ConfigLoader(json)),
audio_feedback=AudioFeedback(),
database=Database,
raspberry_factory=RaspberryFactory(),
delay=7,
dry_run=dry_run)
monitor.run()
from sklearn.pipeline import make_pipeline, make_union
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report
import mlflow
import mlflow.sklearn
from mlflow.exceptions import MlflowException
from src.logging_config import setup_logging
from src.config_loader import ConfigLoader
from src.transformers import SelectDtypeColumns, CountThresholder, CategoricalEncoder
from src.bayes_hyperopt import BayesOpt
from src.utils import persist_local_artifact
setup_logging()
CONFIG = ConfigLoader()
CONFIG.load('config/config.yaml')
class Trainer:
"""Train machine learning models and save artifacts using MLflow"""
# TODO: Log MLflow metrics
def __init__(self, config, seed=0):
self._config = config
self._seed = seed
np.random.seed(self._seed)
self.data = None
self.categorical_columns = None
self.numerical_columns = None
self.df_train = None
def test_extra_facets_should_be_empty_by_default(self):
config()
actual = ConfigLoader()
assert actual.get_extra_facets() == []
import os
import sys
from src.config_loader import ConfigLoader
from queue import Queue
from src.log_simulator import LogSimulator
from src.reader import Reader
from src.displayer import Displayer
from src.alert_system import AlertSystem
DIR_NAME = os.path.dirname(os.path.abspath(__file__))
if __name__ == '__main__':
try:
config = ConfigLoader(DIR_NAME + '/config.ini')
parameters = config.configure_threads()
read_line_queue = Queue()
traffic_queue = Queue()
alert_content = {
'type': AlertSystem.ALERT_RECOVER_TYPE,
'to_display': False
}
reader = Reader(input_queue=read_line_queue,
input_traffic_queue=traffic_queue,
**parameters['reader'])
displayer = Displayer(output_queue=read_line_queue,
alert_content=alert_content,
**parameters['displayer'])
from src.road_network.segment import Segment
from src.config_loader import ConfigLoader
from citygenerator import visualise
# Import and re-use statistic functions.
from src.stats import compute_total_road_length
from src.stats import compute_intersection_count
from src.stats import compute_proportion_dead_ends
from src.stats import compute_proportion_3way_intersections
from src.stats import compute_proportion_4way_intersections
from src.stats import compute_orientation_histogram
from src.stats import compute_orientation_order
from src.stats import compute_orientation_entropy
from src.stats import show_orientation_histogram
config = ConfigLoader('input/configs/auckland.json')
ox.config(log_console=True, use_cache=True)
weight_by_length = False
NUM_BINS = 36
# Grab Auckland matching the bbox we are using.
Auckland = ox.graph_from_bbox(-36.83, -36.94, 174.82, 174.68, network_type='drive')
# Extract road_segments and vertex_dict.
vertex_dict = {}
road_segments = []
for edge in Auckland.edges:
start_id, end_id, _ = edge
start_vert = Vertex(np.array((Auckland.nodes[start_id]["x"], Auckland.nodes[start_id]["y"])))
end_vert = Vertex(np.array((Auckland.nodes[end_id]["x"], Auckland.nodes[end_id]["y"])))
def config():
config_test_path = 'tests/resources/config.yaml'
config = ConfigLoader()
config.load(config_test_path)
return config