def main() -> None:
"""Execute this script to update or create Socrata datasets."""
command_line_args: Namespace = parse_command_line_args()
start_time: DateTime = DateTime.now()
if not command_line_args.envs:
command_line_args.envs = ENVS
envs_summary = ", ".join(command_line_args.envs)
date: Date = command_line_args.date
plan_year: str = command_line_args.plan_year
if not plan_year:
raise Exception("Missing required argument --plan-year")
# Create untracked datasets if --create-untracked-datasets is specified
if command_line_args.create_untracked_datasets is True:
logger.info("Not updating datasets because --create-untracked-datasets was specified")
if command_line_args.only_file:
logger.info("Ignoring --only-file because --create-untracked-datasets was specified")
logger.info(f"Creating untracked datasets using data from {date}, envs: {envs_summary}")
for env in command_line_args.envs:
logger.info(f"Loading env: {env}")
loader = Loader(env)
loader.create_all_datasets(plan_year, only_untracked=True)
# Otherwise, just update all datasets (default behavior)
else:
logger.info(f"Updating datasets for {date}, envs: {envs_summary}")
for env in command_line_args.envs:
logger.info(f"Loading env: {env}")
loader = Loader(env)
loader.update_all_datasets(plan_year, date, only_file=command_line_args.only_file)
time_elapsed: Duration = DateTime.now() - start_time
logger.info(f"Finished! Time elapsed: {time_elapsed.in_words()}")
def test_balancing_market_negative_offer_trade(
market=BalancingMarket(time_slot=DateTime.now())): # NOQA
offer = market.balancing_offer(20, -10, 'A', 'A')
now = DateTime.now(tz=TIME_ZONE)
trade = market.accept_offer(offer, 'B', time=now, energy=-10)
assert trade
assert trade == market.trades[0]
assert trade.id
assert trade.time == now
assert trade.offer is offer
assert trade.seller == 'A'
assert trade.buyer == 'B'
def fetch_access_token(self) -> None:
"""Fetch an access token to obtain Plan Finder data."""
# Construct request
url: str = TOKEN_URLS[self.env]
username: str = USERNAMES[self.env]
key_id, key_secret = API_KEYS[self.env]
body = {
"userName": username,
"scopes": "mpfpe_pde_full",
"keyId": key_id,
"keySecret": key_secret,
}
params = {}
if self.env in ACS_PARAMS:
params["ACS"] = ACS_PARAMS[self.env]
# Submit HTTP POST request to obtain token
logger.info(f"Fetching {self.env} access token")
response: Response = requests.post(url, json=body, params=params)
if response.status_code != 200:
logger.error(Loader._request_details(response))
raise RuntimeError(f"Failed to fetch token: HTTP status {response.status_code}")
# Extract token from response
response_json: dict = response.json()
access_token: str = response_json["accessToken"]
expires: int = response_json["expires"]
self.access_token = access_token
self.access_token_expires = DateTime.now() + Duration(seconds=expires)
logger.info(f"Fetched {self.env} access token; expires {self.access_token_expires}")
def fetch_zip_file(self, plan_year: str, date: Date = Date.today()) -> Path:
"""Download a Plan Finder zip file for a given date."""
# If we don't have a current access token, fetch one
no_access_token = self.access_token is None
if no_access_token or DateTime.now() > (self.access_token_expires - Duration(minutes=5)):
self.fetch_access_token()
# Construct request
url = DATA_URL
headers = {
"X-API-CONSUMER-ID": API_KEYS[self.env][0],
"Authorization": f"Bearer {self.access_token}",
}
params = {"fileName": f"{plan_year}_{date.to_date_string()}"}
# Submit GET request to download file
logger.info(f"Fetching {self.env} zip file for plan year {plan_year} and date {date}")
response = requests.get(url, headers=headers, params=params)
if not response.status_code == 200:
raise RuntimeError(
"Failed to fetch zip file (this may be expected for dates with no data): HTTP "
f"status {response.status_code}"
)
# Save zip file to disk and return its path
zip_bytes: bytes = response.content
zip_file_path = DATA_DIR_PATH / f"{self.env}_{date}.zip"
with open(zip_file_path, "wb") as zip_file:
zip_file.write(zip_bytes)
logger.info(f"Fetched {self.env} zip file: {zip_file_path}")
return zip_file_path
def due_now(self) -> bool:
"""
Are we in the range [start_datetime, end_datetime)?
"""
if not self.start_datetime or not self.end_datetime:
return False
return self.start_datetime <= Pendulum.now() < self.end_datetime
def __init__(self, name):
self.name = name
self.current_tick = 10
self.bc = False
self.now = DateTime.now()
DeviceRegistry.REGISTRY = device_registry_dict
ConstSettings.BalancingSettings.ENABLE_BALANCING_MARKET = True
def run(self, resume=False) -> (Period, duration):
self.sim_status = "running"
if resume:
log.critical("Resuming simulation")
self._info()
self.is_stopped = False
while True:
if resume:
# FIXME: Fix resume time calculation
if self.run_start is None or self.paused_time is None:
raise RuntimeError("Can't resume without saved state")
slot_resume, tick_resume = divmod(
self.area.current_tick,
self.simulation_config.ticks_per_slot)
else:
self.run_start = DateTime.now(tz=TIME_ZONE)
self.paused_time = 0
slot_resume = tick_resume = 0
try:
self._run_cli_execute_cycle(slot_resume, tick_resume) \
if self._started_from_cli \
else self._execute_simulation(slot_resume, tick_resume)
except KeyboardInterrupt:
break
except SimulationResetException:
break
else:
break
def test_market_trade_by_id(market, offer, accept_offer):
e_offer = getattr(market, offer)(20, 10, 'A', 'A')
now = DateTime.now(tz=TIME_ZONE)
trade = getattr(market, accept_offer)(offer_or_id=e_offer.id,
buyer='B',
energy=10,
time=now)
assert trade
def test_market_trade_bid_not_found(
market=TwoSidedPayAsBid(time_slot=DateTime.now())):
bid = market.bid(20, 10, 'A', 'B', 'A')
assert market.accept_bid(bid, 10, 'B', trade_offer_info=[2, 2, 1, 1, 2])
with pytest.raises(BidNotFound):
market.accept_bid(bid, 10, 'B', trade_offer_info=[2, 2, 1, 1, 2])
def test_market_accept_bid_yields_partial_bid_trade(
market=TwoSidedPayAsBid(time_slot=DateTime.now())):
bid = market.bid(2.0, 4, 'buyer', 'seller', 'buyer')
trade = market.accept_bid(bid,
energy=1,
seller='seller',
trade_offer_info=[2, 2, 1, 1, 2])
assert trade.offer.id == bid.id and trade.offer.energy == 1
def run(setup_module_name, settings_file, duration, slot_length, tick_length,
market_count, cloud_coverage, compare_alt_pricing,
enable_external_connection, start_date, pause_at, slot_length_realtime,
**kwargs):
# Force the multiprocessing start method to be 'fork' on macOS.
if platform.system() == 'Darwin':
multiprocessing.set_start_method('fork')
try:
if settings_file is not None:
simulation_settings, advanced_settings = read_settings_from_file(
settings_file)
update_advanced_settings(advanced_settings)
validate_global_settings(simulation_settings)
simulation_settings["external_connection_enabled"] = False
simulation_config = SimulationConfig(**simulation_settings)
else:
global_settings = {
"sim_duration": duration,
"slot_length": slot_length,
"tick_length": tick_length,
"cloud_coverage": cloud_coverage,
"market_count": market_count
}
validate_global_settings(global_settings)
simulation_config = \
SimulationConfig(duration, slot_length, tick_length, market_count,
cloud_coverage, start_date=start_date,
external_connection_enabled=enable_external_connection)
if compare_alt_pricing is True:
ConstSettings.IAASettings.AlternativePricing.COMPARE_PRICING_SCHEMES = True
# we need the seconds in the export dir name
kwargs["export_subdir"] = DateTime.now(
tz=TIME_ZONE).format(f"{DATE_TIME_FORMAT}:ss")
processes = []
for pricing_scheme in range(0, 4):
kwargs["pricing_scheme"] = pricing_scheme
p = Process(target=run_simulation,
args=(setup_module_name, simulation_config, None,
None, None, slot_length_realtime, kwargs))
p.start()
processes.append(p)
for p in processes:
p.join()
else:
if pause_at is not None:
kwargs["pause_after"] = convert_str_to_pause_after_interval(
start_date, pause_at)
run_simulation(setup_module_name, simulation_config, None, None,
None, slot_length_realtime, kwargs)
except D3AException as ex:
raise click.BadOptionUsage(ex.args[0])
def test_market_trade_partial_bid_invalid(
energy, market=TwoSidedPayAsBid(time_slot=DateTime.now())):
bid = market.bid(20, 20, 'A', 'B', 'A')
with pytest.raises(InvalidTrade):
market.accept_bid(bid,
energy=energy,
seller='A',
trade_offer_info=[1, 1, 1, 1, 1])
def __init__(self,
setup_module_name: str,
simulation_config: SimulationConfig = None,
simulation_events: str = None,
slowdown: int = 0,
seed=None,
paused: bool = False,
pause_after: duration = None,
repl: bool = False,
no_export: bool = False,
export_path: str = None,
export_subdir: str = None,
redis_job_id=None,
enable_bc=False):
self.initial_params = dict(slowdown=slowdown,
seed=seed,
paused=paused,
pause_after=pause_after)
self.progress_info = SimulationProgressInfo()
self.simulation_config = simulation_config
self.use_repl = repl
self.export_on_finish = not no_export
self.export_path = export_path
self.sim_status = "initializing"
self.is_timed_out = False
if export_subdir is None:
self.export_subdir = \
DateTime.now(tz=TIME_ZONE).format(f"{DATE_TIME_FORMAT}:ss")
else:
self.export_subdir = export_subdir
self.setup_module_name = setup_module_name
self.use_bc = enable_bc
self.is_stopped = False
self.live_events = LiveEvents(self.simulation_config)
self.redis_connection = RedisSimulationCommunication(
self, redis_job_id, self.live_events)
self._simulation_id = redis_job_id
self._started_from_cli = redis_job_id is None
self.run_start = None
self.paused_time = None
self._load_setup_module()
self._init(**self.initial_params, redis_job_id=redis_job_id)
deserialize_events_to_areas(simulation_events, self.area)
validate_const_settings_for_simulation()
if self.export_on_finish and not self.redis_connection.is_enabled():
self.export = ExportAndPlot(self.area, self.export_path,
self.export_subdir,
self.endpoint_buffer)
def save_state(self):
save_dir = Path('.d3a')
save_dir.mkdir(exist_ok=True)
save_file_name = save_dir.joinpath(
"saved-state_{:%Y%m%dT%H%M%S}.pickle".format(
DateTime.now(tz=TIME_ZONE)))
with save_file_name.open('wb') as save_file:
dill.dump(self, save_file, protocol=HIGHEST_PROTOCOL)
log.critical("Saved state to %s", save_file_name.resolve())
return save_file_name
def test_market_acct_simple(market=OneSidedMarket(time_slot=DateTime.now())):
offer = market.offer(20, 10, 'A', 'A')
market.accept_offer(offer, 'B')
assert market.traded_energy['A'] == offer.energy
assert market.traded_energy['B'] == -offer.energy
assert market.bought_energy('A') == 0
assert market.bought_energy('B') == offer.energy
assert market.sold_energy('A') == offer.energy
assert market.sold_energy('B') == 0
def test_market_issuance_acct_reverse(last_offer_size, traded_energy):
market = OneSidedMarket(time_slot=DateTime.now())
offer1 = market.offer(10, 20, 'A', 'A')
offer2 = market.offer(10, 10, 'A', 'A')
offer3 = market.offer(10, last_offer_size, 'D', 'D')
market.accept_offer(offer1, 'B')
market.accept_offer(offer2, 'C')
market.accept_offer(offer3, 'A')
assert market.traded_energy['A'] == traded_energy
def now(self) -> DateTime:
"""
Return the 'current time' as a `DateTime` object.
Can be overridden in subclasses to change the meaning of 'now'.
In this default implementation 'current time' is defined by the number of ticks that
have passed.
"""
return DateTime.now(tz=TIME_ZONE).start_of('day') + (
self.config.tick_length * self.current_tick)
def __init__(self, name, transfer_fee_pct=1):
self.name = name
self.current_tick = 10
self.bc = False
self.now = DateTime.now()
self.transfer_fee_pct = transfer_fee_pct
self.transfer_fee_const = 0
self.config = GlobalConfig
DeviceRegistry.REGISTRY = device_registry_dict
ConstSettings.BalancingSettings.ENABLE_BALANCING_MARKET = True
def test_market_accept_bid_always_updates_trade_stats(
called, market_method, market=TwoSidedPayAsBid(time_slot=DateTime.now())):
setattr(market, market_method, called)
bid = market.bid(20, 20, 'A', 'B', 'A')
trade = market.accept_bid(bid,
energy=5,
seller='B',
trade_offer_info=[1, 1, 1, 1, 1])
assert trade
assert len(getattr(market, market_method).calls) == 1
def test_market_trade(market, offer, accept_offer):
e_offer = getattr(market, offer)(20, 10, "A", "A")
now = DateTime.now(tz=TIME_ZONE)
trade = getattr(market, accept_offer)(offer_or_id=e_offer, buyer="B",
energy=10, time=now)
assert trade
assert trade == market.trades[0]
assert trade.id
assert trade.time == now
assert trade.offer_bid == e_offer
assert trade.seller == "A"
assert trade.buyer == "B"
def test_balancing_market_negative_offer_trade(market=BalancingMarket(
bc=NonBlockchainInterface(str(uuid4())), time_slot=now())): # NOQA
offer = market.balancing_offer(20, -10, "A", "A")
now = DateTime.now(tz=TIME_ZONE)
trade = market.accept_offer(offer, "B", time=now, energy=-10)
assert trade
assert trade == market.trades[0]
assert trade.id
assert trade.time == now
assert trade.offer_bid is offer
assert trade.seller == "A"
assert trade.buyer == "B"
def _update_progress_info(self, slot_no, slot_count):
run_duration = (DateTime.now(tz=TIME_ZONE) - self.run_start -
duration(seconds=self.paused_time))
self.progress_info.eta = (run_duration /
(slot_no + 1) * slot_count) - run_duration
self.progress_info.elapsed_time = run_duration
self.progress_info.percentage_completed = (slot_no +
1) / slot_count * 100
self.progress_info.current_slot_str = get_market_slot_time_str(
slot_no, self.simulation_config)
self.progress_info.next_slot_str = get_market_slot_time_str(
slot_no + 1, self.simulation_config)
def test_market_acct_multiple(market=OneSidedMarket(time_slot=DateTime.now())):
offer1 = market.offer(10, 20, 'A', 'A')
offer2 = market.offer(10, 10, 'A', 'A')
market.accept_offer(offer1, 'B')
market.accept_offer(offer2, 'C')
assert market.traded_energy['A'] == offer1.energy + offer2.energy == 30
assert market.traded_energy['B'] == -offer1.energy == -20
assert market.traded_energy['C'] == -offer2.energy == -10
assert market.bought_energy('A') == 0
assert market.sold_energy('A') == offer1.energy + offer2.energy == 30
assert market.bought_energy('B') == offer1.energy == 20
assert market.bought_energy('C') == offer2.energy == 10
def test_flex_time_now(tzinfo, granularity):
"""Test now for FlexTime class"""
gval = granularity.value
if granularity is FlexTime.MAX_GRANULARITY:
base_now_before = DateTime.now(tz=tzinfo)
flex_now = FlexTime.now(tz=tzinfo, granularity=gval)
base_now_after = DateTime.now(tz=tzinfo)
assert base_now_before < flex_now < base_now_after
base_tz_name = FlexTime.timezone_name(base_now_before.tzinfo)
flex_tz_name = FlexTime.timezone_name(flex_now.tzinfo)
assert flex_tz_name == base_tz_name
else:
success_count = 0
for i in range(2):
base_now = DateTime.now(tz=tzinfo)
flex_now = FlexTime.now(tz=tzinfo, granularity=gval)
# If they span a second change, next time they will not
if FlexTime.cast(base_now, granularity=gval) == flex_now:
success_count += 1
assert success_count > 0
def _fetch_all_tasks_without_index(self, parallel: bool = False) -> None:
"""
Fetch all tasks from the database.
"""
# AVOID parallel=True; see notes above.
if DEBUG_QUERY_TIMING:
start_time = Pendulum.now()
if parallel:
# Deprecated parallel fetch
threads = [] # type: List[FetchThread]
for task_class in self._filter.task_classes:
thread = FetchThread(self.req, task_class, self)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
if thread.error:
raise ValueError("Multithreaded fetch failed")
else:
# Fetch all tasks, classwise.
for task_class in self._filter.task_classes:
self._fetch_task_class(task_class)
if DEBUG_QUERY_TIMING:
end_time = Pendulum.now()
# noinspection PyUnboundLocalVariable
time_taken = end_time - start_time
log.info("_fetch_all_tasks took {}", time_taken)
# Build our joint task list
self._all_tasks = [] # type: List[Task]
for single_task_list in self._tasks_by_class.values():
self._all_tasks += single_task_list
sort_tasks_in_place(self._all_tasks, self._sort_method_global)
def test_hl7core_func(self) -> None:
self.announce("test_hl7core_func")
pitlist = [
HL7PatientIdentifier(pid="1",
id_type="TT",
assigning_authority="AA")
]
# noinspection PyTypeChecker
dob = Date.today() # type: Date
now = Pendulum.now()
task = self.dbsession.query(Phq9).first()
assert task, "Missing Phq9 in demo database!"
self.assertIsInstance(get_mod11_checkdigit("12345"), str)
self.assertIsInstance(get_mod11_checkdigit("badnumber"), str)
self.assertIsInstance(get_mod11_checkdigit("None"), str)
self.assertIsInstance(make_msh_segment(now, "control_id"), hl7.Segment)
self.assertIsInstance(
make_pid_segment(
forename="fname",
surname="sname",
dob=dob,
sex="M",
address="Somewhere",
patient_id_list=pitlist,
),
hl7.Segment,
)
self.assertIsInstance(make_obr_segment(task), hl7.Segment)
for task_format in (FileType.PDF, FileType.HTML, FileType.XML):
for comments in (True, False):
export_options = TaskExportOptions(
xml_include_comments=comments,
xml_with_header_comments=comments,
)
self.assertIsInstance(
make_obx_segment(
req=self.req,
task=task,
task_format=task_format,
observation_identifier="obs_id",
observation_datetime=now,
responsible_observer="responsible_observer",
export_options=export_options,
),
hl7.Segment,
)
self.assertIsInstance(escape_hl7_text("blahblah"), str)
def make_from_idnum(cls, idnum: PatientIdNum) -> "PatientIdNumIndexEntry":
"""
Returns an ID index entry for the specified
:class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum`. The
returned index requires inserting into a database session.
"""
# noinspection PyProtectedMember
assert idnum._current, "Only index current PatientIdNum objects"
index = cls()
index.idnum_pk = idnum.pk
index.patient_pk = idnum.get_patient_server_pk()
index.which_idnum = idnum.which_idnum
index.idnum_value = idnum.idnum_value
index.indexed_at_utc = Pendulum.now()
return index
def __init__(self, bids=[], m_id=123, time_slot=DateTime.now()):
super().__init__(transfer_fees=transfer_fees, time_slot=time_slot)
self.id = m_id
self._bids = bids
self.offer_count = 0
self.bid_count = 0
self.forwarded_offer_id = 'fwd'
self.forwarded_bid_id = 'fwd_bid_id'
self.calls_energy = []
self.calls_energy_bids = []
self.calls_offers = []
self.calls_bids = []
self.calls_bids_price = []
self.mcp_update_point = 20
self.current_tick = 19
def test_market_bid_trade(market=TwoSidedPayAsBid(time_slot=DateTime.now())):
bid = market.bid(20, 10, 'A', 'B', 'A', original_bid_price=20)
trade = market.accept_bid(bid,
energy=10,
seller='B',
trade_offer_info=[2, 2, 0.5, 0.5, 2])
assert trade
assert trade.id == market.trades[0].id
assert trade.id
assert trade.offer.price == bid.price
assert trade.offer.energy == bid.energy
assert trade.seller == 'B'
assert trade.buyer == 'A'
assert not trade.residual
def test_market_accept_bid_emits_bid_split_on_partial_bid(
called, market=TwoSidedPayAsBid(time_slot=DateTime.now())):
market.add_listener(called)
bid = market.bid(20, 20, 'A', 'B', 'A')
trade = market.accept_bid(bid, energy=1, trade_offer_info=[1, 1, 1, 1, 1])
assert all([
ev != repr(MarketEvent.BID_DELETED) for c in called.calls
for ev in c[0]
])
assert len(called.calls) == 2
assert called.calls[0][0] == (repr(MarketEvent.BID_SPLIT), )
assert called.calls[1][0] == (repr(MarketEvent.BID_TRADED), )
assert called.calls[1][1] == {
'market_id': repr(market.id),
'bid_trade': repr(trade),
}
