async def cache_data(self, date):
"""Cache the latest date into the database."""
# The WHERE condition is not required as there is always only one row
# in the `latest_date` table.
async with self.pool.acquire() as conn:
result = await conn.execute(
"UPDATE latest_date SET latest = $1, last_check = DEFAULT;",
str_to_date(date),
)
rows_updated = int(result.split()[1])
if rows_updated == 1:
self.logger.info("Successfully updated latest date in cache")
return
elif rows_updated > 1:
raise RuntimeError(
'The "latest_date" table has more than one row, '
"i.e. this table is corrupt")
# No rows were updated, so the "latest_date" table must be empty. This
# should only happen if this table was cleared manually, or this is the
# first run of this code on this database.
self.logger.info(
"Couldn't update latest date in cache; trying to insert it")
async with self.pool.acquire() as conn:
await conn.execute(
"INSERT INTO latest_date (latest) VALUES ($1);",
str_to_date(date),
)
def create(cls, dict_result):
financial = cls(edinet_code=dict_result['edinet_code'],
start_period=utils.str_to_date(dict_result['年度開始日']),
end_period=utils.str_to_date(dict_result['年度終了日']),
company_name=dict_result['会社名'],
submit_paper=dict_result['提出書類'],
submit_date=utils.str_to_date(dict_result['提出日']),
shared_num=dict_result['発行済み株式数'],
operating_cash=dict_result['営業CF'],
financial_cash=dict_result['財務CF'],
investment_cash=dict_result['投資CF'],
net_income=dict_result['純利益'],
sales_amt=dict_result['売上高'],
bs_cash_amt_ttl=dict_result['BS現金預金'],
bs_liabilities_amt_ttl=dict_result['BS負債合計'],
securities_code=dict_result['証券コード'],
industry=dict_result['業種'],
revision=dict_result['訂正'],
annual=dict_result['annual'],
file_name=dict_result['file_name'],
eps=0.0)
try:
with session_scope() as session:
session.add(financial)
return financial
except IntegrityError:
return False
def create_xbrl_url_json(self):
result_dict = {}
target_date = utils.str_to_date(settings.since)
count = 0
while True:
response_string = self.get_link_info_str(utils.date_to_str(target_date))
target_list = self.__json_parse(response_string)
if not target_list:
target_date = target_date + timedelta(days=1)
continue
info_dict = self.get_link(target_list)
result_dict.update(info_dict)
time.sleep(1)
target_date = target_date + timedelta(days=1)
count += 1
logger.info(f'action=create_xbrl_url_json count={count}')
if target_date >= utils.str_to_date(settings.until):
break
return result_dict
async def cache_data(self, data, date):
"""Cache the comic data into the database."""
# The given date can be invalid (i.e. we may have been redirected to a
# comic with a different date), hence get the correct date from the
# scraped data.
date = date_to_str(str_to_date(data["dateStr"], fmt=ALT_DATE_FMT))
# This lock ensures that the no. of rows in the cache doesn't increase.
# This can happen, as the code involves first clearing excess rows,
# then adding a new row. Therefore, the following can increase the no.
# of rows:
# 1. Coroutine 1 clears excess rows
# 2. Coroutine 2 clears no excess rows, as coroutine 1 did them
# 3. Coroutine 1 adds its row
# 4. Coroutine 2 adds its row
async with Lock():
try:
await self._clean_cache()
except Exception as ex:
# This crash means that there can be some extra rows in the
# cache. As the row limit is a little conservative, this should
# not be a big issue.
self.logger.error(f"Failed to clean cache: {ex}")
self.logger.debug("", exc_info=True)
date_obj = str_to_date(date)
try:
async with self.pool.acquire() as conn:
await conn.execute(
"""INSERT INTO comic_cache (comic, img_url, title)
VALUES ($1, $2, $3);""",
date_obj,
data["imgURL"],
data["title"],
)
except UniqueViolationError:
# This comic date exists, so some other coroutine has already
# cached this date in parallel. So we can simply update
# `last_used` later (outside the lock).
self.logger.warn(
f"Trying to cache date {date}, which is already cached."
)
else:
return # succeeded in caching data, so exit
# This only executes if caching data led to a UniqueViolation error.
# The lock isn't needed here, as this command cannot increase the no.
# of rows in the cache.
self.logger.info("Now trying to update `last_used` in cache.")
async with self.pool.acquire() as conn:
await conn.execute(
"UPDATE comic_cache SET last_used = DEFAULT WHERE comic = $1;",
date_obj,
)
def _flatten(self, item):
"""
Flatten a raw commit fetched by Perceval into a flat dictionary.
A list with a single flat directory will be returned.
That dictionary will have the elements we need for computing metrics.
The list may be empty, if for some reason the commit should not
be considered.
:param item: raw item fetched by Perceval (dictionary)
:returns: list of a single flat dictionary
"""
creation_date = utils.str_to_date(item['data']['AuthorDate'])
if self.since and (self.since > creation_date):
return []
if self.until and (self.until < creation_date):
return []
code_files = [
file['file'] for file in item['data']['files'] if all(
condition.check(file['file']) for condition in self.is_code)
]
if len(code_files) > 0:
flat = {
'repo': item['origin'],
'hash': item['data']['commit'],
'author': item['data']['Author'],
'category': "commit",
'created_date': creation_date,
'committer': item['data']['Commit'],
'commit_date': utils.str_to_date(item['data']['CommitDate']),
'files_no': len(item['data']['files']),
'refs': item['data']['refs'],
'parents': item['data']['parents'],
'files': item['data']['files']
}
actions = 0
for file in item['data']['files']:
if 'action' in file:
actions += 1
flat['files_action'] = actions
if 'Merge' in item['data']:
flat['merge'] = True
else:
flat['merge'] = False
return [flat]
else:
return []
async def comic_page(year: int, month: int, day: int) -> Union[str, Response]:
"""Serve the requested comic from the given URL."""
# This depends on the format given by `DATE_FMT` from constants.py
date = f"{year:04d}-{month:02d}-{day:02d}"
# Check to see if the date is invalid
try:
str_to_date(date)
except ValueError:
# Replicates the behaviour of "dilbert.com" by redirecting to the
# homepage.
return redirect("/")
return await serve_comic(date)
def _flatten(self, item):
"""
Flatten a raw pull_request fetched by Perceval into a flat dictionary.
A list with a single flat directory will be returned.
That dictionary will have the elements we need for computing metrics.
The list may be empty, if for some reason the pull_request should not
be considered.
:param item: raw item fetched by Perceval (dictionary)
:returns: list of a single flat dictionary
"""
creation_date = utils.str_to_date(item['data']['created_at'])
if self.since and (self.since > creation_date):
return []
if self.until and (self.until < creation_date):
return []
flat = {
'repo': item['origin'],
'hash': item['data']['id'],
'category': "pull_request",
'author': item['data']['user']['login'],
'created_date': creation_date,
'current_status': item['data']['state'],
'merged': item['data']['merged']
}
return [flat]
def get_status_time_distribution(self):
'''
Get the amount of time spent in each status
'''
distribution = {}
changelog = self.get_changelog_for_field_only('status')
from_date = self.created
to_date = self.created
if changelog:
# historical statuses
for change in changelog:
status = change['items'][0]['fromString']
to_date = str_to_date(change['created'])
if status not in distribution:
distribution[status] = []
distribution[status].append((from_date, to_date))
from_date = to_date
# final/current status
status = changelog[-1]['items'][0]['toString']
if status not in distribution:
distribution[status] = []
distribution[status].append(
(from_date, datetime.datetime.now(from_date.tzinfo)))
else:
distribution[self.status] = [
(from_date, datetime.datetime.now(from_date.tzinfo))
]
return distribution
def __init__(self, ticker_obj, expiry, strike, put):
self.obj = ticker_obj
self.strike = strike
self.put = put # bool
if isinstance(expiry, str):
expiry = utils.str_to_date(expiry)
# options data
self.price = 0
self.dte = 0
# price
self.collateral = self.strike * 100
assert (isinstance(ticker_obj, type(
yf.Ticker("AMD")))), "Option error: invalid ticker_obj"
# assert(date_to_str(expiry) in ticker_obj.options), "Option error: invalid expiry"
try:
expiration = utils.nearest_expiry(self.obj.options, expiry)
self.expiry = expiration
except:
self.expiry = False
raise ValueError("No expiry found within 3 days of date")
self.update()
def load_raids_and_raid_days():
with get_db() as db:
db_rows = {
table_name: {
row['id']: dict(row)
for row in db.execute(f'SELECT * FROM {table_name}')
}
for table_name in ('raid', 'raid_days', 'bosses')
}
raids = {
id: Raid(id, row['name'], row['short_name'])
for id, row in db_rows['raid'].items()
}
for row in db_rows['bosses'].values():
raids[row['raid_id']].bosses.append(row['name'])
raid_days = {
id: RaidDay(id, str_to_date(row['date']), row['name'],
raids[row['raid_id']])
for id, row in db_rows['raid_days'].items()
}
return raids, raid_days
async def random_comic() -> Response:
"""Serve a random comic."""
first = str_to_date(FIRST_COMIC)
latest = curr_date()
rand_date = date_to_str(random.uniform(first, latest)) # type: ignore
# If there is no comic for this date yet, "dilbert.com" will auto-redirect
# to the latest comic.
return redirect(f"/{rand_date}")
async def serve_comic(
date: str, allow_redirect: bool = True
) -> Union[str, Response]:
"""Serve the requested comic.
Args:
date: The date of the requested comic, in the format used by
"dilbert.com"
allow_redirect: If there is no comic found for this date, then
whether to redirect to the correct date
Returns:
The rendered template for the comic page
"""
# Execute both in parallel, as they are independent of each other
data, latest_comic = await asyncio.gather(
app.comic_scraper.get_data(date), app.latest_date_scraper.get_data(),
)
# This date differs from the input date if the input is invalid (i.e.
# "dilbert.com" would redirect to a comic with a different date).
actual_date_obj = str_to_date(data["dateStr"], fmt=ALT_DATE_FMT)
actual_date = date_to_str(actual_date_obj)
# Replicates the behaviour of "dilbert.com" by redirecting to the correct
# date.
if allow_redirect and actual_date != date:
return redirect(f"/{actual_date}")
# This will contain awaitables for caching data (if required) and rendering
# the template. They are both independent of each other, and thus can be
# run in parallel.
todos = []
# The date of the latest comic is often retrieved from the cache. If
# "dilbert.com" has redirected to a date which is newer than the cached
# value, then there is a new "latest comic". So cache the answer of
# "dilbert.com".
if str_to_date(latest_comic) < actual_date_obj:
latest_comic = actual_date
todos.append(app.latest_date_scraper.cache_data(actual_date))
todos.append(_serve_template(actual_date, data, latest_comic))
results = await asyncio.gather(*todos)
return results[-1] # this is the rendered template
def get_context_end_date(self, context_id):
if context_id not in self._context_end_dates:
node = self.getNode("//xbrli:context[@id='" + context_id + "']/xbrli:period/xbrli:endDate")
dt = None
if node is not None and node.text:
#dt = date(*map(int, node.text.split('-')))
dt = utils.str_to_date(node.text)
self._context_end_dates[context_id] = dt
return self._context_end_dates[context_id]
async def scrape_data(self):
"""Scrape the date of the latest comic from "dilbert.com"."""
# If there is no comic for this date yet, "dilbert.com" will
# auto-redirect to the latest comic.
latest = date_to_str(curr_date())
url = SRC_PREFIX + latest
async with self.sess.get(url) as resp:
self.logger.debug(f"Got response for latest date: {resp.status}")
date = resp.url.path.split("/")[-1]
# Check to see if the date is invalid
try:
str_to_date(date)
except ValueError:
raise ScrapingException(
"Error in scraping the latest date from the URL")
return date
async def _serve_template(date: str, data: dict, latest_comic: str) -> str:
"""Serve the HTML given scraped data.
Both input dates must be in the format used by "dilbert.com".
Args:
date: The (possibly corrected) date of the comic
data: The scraped comic data
latest_comic: The date of the latest comic
Returns:
The rendered template for the comic page
"""
date_obj = str_to_date(date)
# Links to previous and next comics
previous_comic = date_to_str(
max(str_to_date(FIRST_COMIC), date_obj - timedelta(days=1))
)
next_comic = date_to_str(
min(str_to_date(latest_comic), date_obj + timedelta(days=1))
)
# Whether to disable left/right navigation buttons
disable_left_nav = date == FIRST_COMIC
disable_right_nav = date == latest_comic
# Link to original strip on "dilbert.com"
permalink = SRC_PREFIX + date
return await render_template(
"layout.html",
data=data,
date=date,
first_comic=FIRST_COMIC,
previous_comic=previous_comic,
next_comic=next_comic,
disable_left_nav=disable_left_nav,
disable_right_nav=disable_right_nav,
permalink=permalink,
repo=REPO,
)
def convert_str_value_to_correct_type(param_value, old_value, use_timedelta=False):
"""Convert param_value to the same type as old_value."""
for primitive_type in [bool, int, float]:
if isinstance(old_value, primitive_type):
return primitive_type(param_value)
if isinstance(old_value, datetime.date):
if use_timedelta:
return datetime.timedelta(days=int(param_value))
return str_to_date(param_value)
raise NotImplementedError(f"Unknown type for value: {type(old_value)}")
def publish_data( keyvals, msg_dbg=''):
if keyvals != None and 'driver' in keyvals and 'device_id' in keyvals:
driver = keyvals['driver']
device_id = keyvals['device_id']
devdef = utils.read_device(driver)
if devdef == None:
print "Device definition for %s not found"%driver
return
## Custom device drivers here
if driver == 'custom!@#!@$!$#':
pass
else: # generic driver: try to match keys to feeds
if 'timestamp' in keyvals:
dev = publisher.find_device( device_id, create_new=True, device_type=driver, devdef=devdef )
ts = utils.date_to_unix(utils.str_to_date(keyvals['timestamp']))
datapoints = []
feednums = []
for key,val in keyvals.iteritems():
if key in ['driver','device_id','timestamp']:
continue
try:
f = float(val);
except:
print "Skipping Key-Value pair",key,"/",val,"as it is non-numeric"
continue;
if key in dev.feed_names:
idx = dev.feed_names.index(key)
feednums.append(idx)
datapoints.append(f)
else:
feednums.append(len(dev.feed_names))
datapoints.append(f)
dev.feed_names.append(key)
try:
if (len(datapoints) > 0):
publisher.publish_data( device_id, ts, datapoints, feednum=feednums, devdef=devdef, device_type=driver, dev=dev)
except:
import traceback
traceback.print_exc();
else:
print "Data Line '%s' did not have a timestamp field (for generic driver)"%msg_dbg
else:
print "Data Line '%s' did not contain a key for driver and/or device_id"%msg_dbg
async def get_cached_data(self, date):
"""Get the cached comic data from the database."""
async with self.pool.acquire() as conn:
# The other columns in the table are: `comic`, `last_used`. `comic`
# is not required here, as we already have the date as a function
# argument. In case the date given here is invalid (i.e. it would
# redirect to a comic with a different date), we cannot retrieve
# the correct date from the cache, as we aren't caching the mapping
# of incorrect:correct dates. `last_used` will be updated later.
row = await conn.fetchrow(
"SELECT img_url, title FROM comic_cache WHERE comic = $1;",
str_to_date(date),
)
if row is None:
# This means that the comic for this date wasn't cached, or the
# date is invalid (i.e. it would redirect to a comic with a
# different date).
return None
data = {
"title": row[1],
"dateStr": date_to_str(str_to_date(date), fmt=ALT_DATE_FMT),
"imgURL": row[0],
}
# Update `last_used`, so that this comic isn't accidently de-cached. We
# want to keep the most recently used comics in the cache, and we are
# currently using this comic.
self.logger.info("Updating `last_used` for data in cache")
async with self.pool.acquire() as conn:
await conn.execute(
"UPDATE comic_cache SET last_used = DEFAULT WHERE comic = $1;",
str_to_date(date),
)
return data
def __init__(self, data):
self.id = get_nested_key(data, 'key')
self.type = get_nested_key(data, 'fields.issuetype.name')
self.priority = get_nested_key(data, 'fields.priority.name')
self.created = str_to_date(get_nested_key(data, 'fields.created'))
self.status = get_nested_key(data, 'fields.status.name')
self.creator = get_nested_key(data, 'fields.creator.displayName')
self.assignee = get_nested_key(data, 'fields.assignee.displayName')
self.description = get_nested_key(data, 'fields.description', '')
self.acceptance_criteria = get_nested_key(data,
'fields.customfield_10741',
'')
self.how_to_test = get_nested_key(data, 'fields.customfield_10693', '')
self.comments = get_nested_key(data, 'fields.comment.comments')
self.changelog = get_nested_key(data, 'changelog.histories')
def price_by_date(ticker_obj,
date,
time="Close"): # time: {Open, High, Low, Close}
today = datetime.date.today()
if type(date) is str:
date = utils.str_to_date(date)
delta = (today - date).days + 2
if delta < 0:
return "Invalid date"
delta = str(delta) + "d"
hist = ticker_obj.history(period=delta)
try:
hist = hist.loc[date]
return hist[time]
except:
return "Non trading day"
def get_comments_by_status_distribution(self):
'''
Get the number of comments posted in each status
'''
distribution = {}
for comment in self.comments:
status = self.get_status_at_point_in_time(
str_to_date(comment['created']))
if status not in distribution:
distribution[status] = 0
distribution[status] += 1
return distribution
def get_field_update_by_status_distribution(self, field):
'''
Get the number of times a field has been updated per status
'''
distribution = {}
changelog = self.get_changelog_for_field_only(field)
for change in changelog:
status = self.get_status_at_point_in_time(
str_to_date(change['created']))
if status not in distribution:
distribution[status] = 0
distribution[status] += 1
return distribution
def data_by_ticker_and_pct_otm(
ticker, pct_otm
): # return list of (expiry, nearest strike, put option cost, DTE, ROI) for each expiry in ticker.options
recent_price = utils.moving_average(ticker, 3)
print(
str(ticker.info['symbol']) + " 3-day MA: %s; OTM %s pct" %
(str(recent_price), str(pct_otm * 100)))
today = datetime.date.today()
data = []
for date in ticker.options:
strike = nearest_strike_by_pct_otm(ticker, pct_otm, date)
option_chain = ticker.option_chain(date).puts
option_price = option_chain.loc[option_chain['strike'] ==
strike]['lastPrice'].iloc[0]
dte = (utils.str_to_date(date) - today).days
roi = round(option_price / recent_price, 4)
data.append((date, strike, option_price, dte, roi))
return data
def convert_mean_params_to_params_dict(mean_params):
"""Convert list of [param_name, param_value_raw] pairs to dict of param_name to param_value.
We also convert string dates to datetime objects
Parameters
----------
mean_params : list
list of [param_name, param_value_raw] pairs
"""
params_dict = {}
for param_name, param_value_raw in mean_params:
try:
# attempt to convert to datetime.date object if it is a date
params_dict[param_name] = str_to_date(param_value_raw)
except (TypeError, ValueError):
params_dict[param_name] = param_value_raw
return params_dict
def get_new_chapters(followed_issues, last_check):
paths = VOLUME_XPATH[1]
results = {}
for issue in followed_issues:
results[issue] = {}
uri = create_uri(issue)
response = requests.get(uri)
source = response.content
tree = html.fromstring(source)
volumes = tree.xpath(VOLUME_XPATH[0])
for volume in volumes:
title = volume.xpath(paths['title'])[0]
results[issue][title] = {'chapters': []}
chapters_data = volume.xpath(paths['chapters'][0])
for cd in chapters_data:
chapter = {
'title':
cd.xpath(paths['chapters'][1]['title'])[0],
'release':
str_to_date(cd.xpath(paths['chapters'][1]['release'])[0]),
'url':
cd.xpath(paths['chapters'][1]['url'])[0]
}
if is_new(chapter['release'], last_check):
results[issue][title]['chapters'].append(chapter)
if len(results[issue][title]['chapters']) == 0:
del (results[issue][title])
if len(results[issue]) == 0:
del (results[issue])
return results
def set_dates(self, data):
self.date_onset = str_to_date(data['date_onset'])
self.date_diagnosis = str_to_date(data['date_diagnosis'])
def set_dates(self, data):
self.date_prescribed = str_to_date(data['date_prescribed'])
self.date_started_taking = str_to_date(data['date_started_taking'])
self.date_stopped_taking = str_to_date(data['date_stopped_taking'])
optimizer = RMSprop()
loss = keras.losses.mean_squared_error # for regression
#loss = keras.losses.categorical_crossentropy # for classifying
model.compile( loss=loss, optimizer=optimizer )
model.summary()
#
return model
# MAIN
show_graphs=False
# Load data
df = pandas.read_csv( 'meteo_daily.csv', sep=';' )
fechas = utils.str_to_date( numpy.array( df['fecha'] ) )
T = numpy.zeros( [ len(fechas), 3 ] ) # Daily temperatures
T[:,0] = df['minima']
T[:,1] = df['media']
T[:,2] = df['maxima']
previous_days = 3
days_to_predict = 2
size_X = (len(T)-previous_days-days_to_predict+1)//1
X = numpy.zeros( [ size_X, T.shape[1]*previous_days ] )
Y = numpy.zeros( [ size_X, T.shape[1]*days_to_predict ] )
for t in range(len(X)):
i=t*1
Restaurant( \
id=int(current_rest[0]), \
name=current_rest[1], \
boro=current_rest[2], \
building=current_rest[3], \
street=current_rest[4], \
zip=int_or_none(current_rest[5]), \
phone=current_rest[6], \
cuisine=current_rest[7], \
) \
)
session.add( \
Inspection( \
rest_id=int(current_insp[0]), \
inspection_date=str_to_date(current_insp[1]), \
action=current_insp[2], \
violation_code=current_insp[3], \
violation_desc=current_insp[4], \
is_critical=current_insp[5] == 'Critical', \
score=int_or_none(current_insp[6]), \
grade=grade_or_none(current_insp[7]), \
grade_date=str_to_date(current_insp[8]), \
record_date=str_to_date(current_insp[9]), \
inspection_type=current_insp[10], \
) \
)
# commit changes to the database
session.commit()
session.close()
def get_devices_invalid_measurement_values(self):
devices = self.airqo_api.get_devices(tenant='airqo', active=True)
print(devices)
errors = []
for device in devices:
device_data = dict(device)
try:
current_measurements = dict(
self.airqo_api.get_airqo_device_current_measurements(
device_number=device_data["device_number"]))
except Exception as ex:
error = dict({
"self_link":
f"{os.getenv('AIRQO_BASE_URL')}data/feeds/transform/recent?channel={device_data['device_number']}",
"channelID": device_data["device_number"]
})
errors.append(error)
continue
created_at = str_to_date(current_measurements.get("created_at"))
check_date = datetime.utcnow() - timedelta(days=30)
error = dict({
"channelID": device_data["device_number"],
"device": device_data["name"],
"isActive": device_data["isActive"],
"siteName": device_data["siteName"],
"created_at": f"{created_at}",
})
for key, value in current_measurements.items():
key_str = f"{key}".strip().lower()
if key_str == 'externaltemperature' or key_str == 'externalhumidity' or key_str == 'pm10' \
or key_str == 's2_pm2_5' or key_str == 's2_pm10' or key_str == 'internaltemperature' \
or key_str == 'internalhumidity':
has_error = False
if not is_valid_double(value=value):
has_error = True
if key_str == 'pm2_5' and not has_error:
value = float(value)
if value < 0 or value > 500.5:
has_error = True
if not has_error and (key_str == 'pm2_5' or key_str
== 's2_pm2_5' or key_str == 's2_pm10'
or key_str == 'pm10'):
value = float(value)
if value < 0 or value > 500.5:
has_error = True
if not has_error and (key_str == 'externaltemperature'
or key_str == 'internalhumidity'
or key_str == 'internaltemperature'
or key_str == 'externalhumidity'):
value = float(value)
if value < 0 or value > 50:
has_error = True
if not has_error and (check_date > created_at):
has_error = True
if has_error:
error[key] = value
if len(error.keys()) > 5:
error[
"self_link"] = f"{os.getenv('AIRQO_BASE_URL')}data/feeds/transform/recent?channel={device_data['device_number']}"
errors.append(error)
self.__print(data=errors)
def main(args):
country = args.country
region = args.region
subregion = args.subregion
skip_hospitalizations = args.skip_hospitalizations
quarantine_perc = args.quarantine_perc
quarantine_effectiveness = args.quarantine_effectiveness
verbose = args.verbose
if country != 'US' and not region:
region = 'ALL'
best_params_type = args.best_params_type
assert best_params_type in ['mean', 'median', 'top',
'top10'], best_params_type
if args.best_params_dir:
# Load parameters from file
best_params = load_best_params_from_file(args.best_params_dir, country,
region, subregion)
simulation_start_date = str_to_date(best_params['first_date'])
simulation_create_date = str_to_date(best_params['date'])
simulation_end_date = str_to_date(best_params['projection_end_date'])
region_params = {'population': best_params['population']}
# mean_params, median_params, top_params, or top10_params
params_type_name = f'{best_params_type}_params'
if verbose:
print('best params type:', best_params_type)
params_dict = convert_mean_params_to_params_dict(
best_params[params_type_name])
else:
"""
You can hard code your own parameters if you do not want to use the preset parameters.
This can be especially useful for regions/countries where we do not have projections.
Then simply run `python run_simulation.py -v` to use these parameters.
"""
simulation_start_date = datetime.date(2020, 2, 1)
simulation_create_date = datetime.date.today(
) # not used so can also be None
simulation_end_date = datetime.date(2020, 10, 1)
region_params = {'population': 332000000}
params_dict = {
'INITIAL_R_0': 2.24,
'LOCKDOWN_R_0': 0.9,
'INFLECTION_DAY': datetime.date(2020, 3, 18),
'RATE_OF_INFLECTION': 0.25,
'LOCKDOWN_FATIGUE': 1.,
'DAILY_IMPORTS': 500,
'MORTALITY_RATE': 0.01,
'REOPEN_DATE': datetime.date(2020, 5, 20),
'REOPEN_SHIFT_DAYS': 0,
'REOPEN_R': 1.2,
'REOPEN_INFLECTION': 0.3,
'POST_REOPEN_EQUILIBRIUM_R': 1.,
'FALL_R_MULTIPLIER': 1.001,
}
if args.simulation_start_date:
simulation_start_date = str_to_date(args.simulation_start_date)
if args.simulation_end_date:
simulation_end_date = str_to_date(args.simulation_end_date)
if args.set_param:
print('---------------------------------------')
print('Overwriting params from command line...')
for param_name, param_value in args.set_param:
assert param_name in params_dict, f'Unrecognized param: {param_name}'
old_value = params_dict[param_name]
new_value = convert_str_value_to_correct_type(
param_value, old_value)
print(f'Setting {param_name} to: {new_value}')
params_dict[param_name] = new_value
if args.change_param:
print('---------------------------------------')
print('Changing params from command line...')
for param_name, value_change in args.change_param:
assert param_name in params_dict, f'Unrecognized param: {param_name}'
old_value = params_dict[param_name]
new_value = old_value + convert_str_value_to_correct_type(
value_change, old_value, use_timedelta=True)
print(f'Changing {param_name} from {old_value} to {new_value}')
params_dict[param_name] = new_value
region_model = RegionModel(
country,
region,
subregion,
simulation_start_date,
simulation_create_date,
simulation_end_date,
region_params,
compute_hospitalizations=(not skip_hospitalizations))
if quarantine_perc > 0:
print(f'Quarantine percentage: {quarantine_perc:.0%}')
print(f'Quarantine effectiveness: {quarantine_effectiveness:.0%}')
assert quarantine_effectiveness in [0.025, 0.1, 0.25, 0.5], \
('must specify --quarantine_effectiveness percentage.'
' Possible values: [0.025, 0.1, 0.25, 0.5]')
quarantine_effectiveness_to_reduction_idx = {
0.025: 0,
0.1: 1,
0.25: 2,
0.5: 3
}
region_model.quarantine_fraction = quarantine_perc
region_model.reduction_idx = \
quarantine_effectiveness_to_reduction_idx[quarantine_effectiveness]
if verbose:
print('================================')
print(region_model)
print('================================')
print('Parameters:')
for param_name, param_value in params_dict.items():
print(f'{param_name:<25s} : {param_value}')
# Add params to region_model
params_tups = tuple(params_dict.items())
region_model.init_params(params_tups)
if verbose:
print('--------------------------')
print('Running simulation...')
print('--------------------------')
# Run simulation
dates, infections, hospitalizations, deaths = run(region_model)
"""
The following are lists with length N, where N is the number of days from
simulation_start_date to simulation_end_date.
dates : datetime.date objects representing day i
infections : number of new infections on day i
hospitalizations : occupied hospital beds on day i
deaths : number of new deaths on day i
"""
assert len(dates) == len(infections) == len(hospitalizations) == len(
deaths)
assert dates[0] == simulation_start_date
assert dates[-1] == simulation_end_date
if verbose:
infections_total = infections.cumsum()
deaths_total = deaths.cumsum()
for i in range(len(dates)):
hospitalization_str = ''
if not skip_hospitalizations:
hospitalization_str = f'Hospital beds in use: {hospitalizations[i]:,.0f} - '
daily_str = (
f'{dates[i]} - '
f'New / total infections: {infections[i]:,.0f} / {infections_total[i]:,.0f} - '
f'{hospitalization_str}'
f'New / total deaths: {deaths[i]:,.2f} / {deaths_total[i]:,.1f} - '
f'Mean R: {region_model.effective_r_arr[i]:.3f} - '
f'IFR: {region_model.ifr_arr[i]:.2%}')
print(daily_str) # comment out to spare console buffer
print('-------------------------------------')
print(f'End of simulation : {region_model.projection_end_date}')
print(f'Total infections : {infections.sum():,.0f}')
if not skip_hospitalizations:
print(f'Peak hospital beds used : {hospitalizations.max():,.0f}')
print(f'Total deaths : {deaths.sum():,.0f}')
if args.save_csv_fname:
dates_str = np.array(list(map(str, dates)))
combined_arr = np.vstack((dates_str, infections, hospitalizations,
deaths, region_model.effective_r_arr)).T
headers = 'dates,infections,hospitalizations,deaths,mean_r_t'
np.savetxt(args.save_csv_fname,
combined_arr,
'%s',
delimiter=',',
header=headers)
print('----------\nSaved file to:', args.save_csv_fname)
for g in gparm:
print "\t\tState %-3d: N=%-7d mean=%20.10e variance=%20.10e"%(g.state_id,g.counts,g.mean,g.variance)
print "\n";
elif (op == 'insert'):
import devicedb,postgresops
devicedb.connect();
fromtime = None
totime = None
dtype = "GHMM"
ret,vals,sys.argv = utils.check_arg(sys.argv,'--from',1)
if ret:
try:
fromtime = utils.str_to_date(vals[0])
except ValueError, msg:
print "Bad from time: "+str(msg)
ret,vals,sys.argv = utils.check_arg(sys.argv,'--to',1)
if ret:
try:
totime = utils.str_to_date(vals[0])
except ValueError, msg:
print "Bad to time: "+str(msg)
ret,vals,sys.argv = utils.check_arg(sys.argv,'--type',1)
if ret:
dtype = vals[0]
pretend,j,sys.argv = utils.check_arg(sys.argv,'--pretend')
def main(args):
country = args.country
region = args.region
subregion = args.subregion
skip_hospitalizations = args.skip_hospitalizations
quarantine_perc = args.quarantine_perc
quarantine_effectiveness = args.quarantine_effectiveness
verbose = args.verbose
if country != "US" and not region:
region = "ALL"
best_params_type = args.best_params_type
assert best_params_type in ["mean", "median", "top", "top10"], best_params_type
if args.best_params_dir:
# Load parameters from file
best_params = load_best_params_from_file(
current_path / args.best_params_dir, country, region, subregion
)
simulation_start_date = str_to_date(best_params["first_date"])
simulation_create_date = str_to_date(best_params["date"])
simulation_end_date = str_to_date(best_params["projection_end_date"])
region_params = {"population": best_params["population"]}
# mean_params, median_params, top_params, or top10_params
params_type_name = f"{best_params_type}_params"
if verbose:
print("best params type:", best_params_type)
params_dict = convert_mean_params_to_params_dict(best_params[params_type_name])
else:
"""
You can hard code your own parameters if you do not want to use the preset parameters.
This can be especially useful for regions/countries where we do not have projections.
Then simply run `python run_simulation.py -v` to use these parameters.
"""
simulation_start_date = datetime.date(2020, 2, 1)
simulation_create_date = datetime.date.today() # not used so can also be None
simulation_end_date = datetime.date(2020, 10, 1)
region_params = {"population": 332000000}
params_dict = {
"INITIAL_R_0": 2.24,
"LOCKDOWN_R_0": 0.9,
"INFLECTION_DAY": datetime.date(2020, 3, 18),
"RATE_OF_INFLECTION": 0.25,
"LOCKDOWN_FATIGUE": 1.0,
"DAILY_IMPORTS": 500,
"MORTALITY_RATE": 0.01,
"REOPEN_DATE": datetime.date(2020, 5, 20),
"REOPEN_SHIFT_DAYS": 0,
"REOPEN_R": 1.2,
"REOPEN_INFLECTION": 0.3,
"POST_REOPEN_EQUILIBRIUM_R": 1.0,
"FALL_R_MULTIPLIER": 1.001,
}
if args.simulation_start_date:
simulation_start_date = str_to_date(args.simulation_start_date)
if args.simulation_end_date:
simulation_end_date = str_to_date(args.simulation_end_date)
if args.set_param:
print("---------------------------------------")
print("Overwriting params from command line...")
for param_name, param_value in args.set_param:
assert param_name in params_dict, f"Unrecognized param: {param_name}"
old_value = params_dict[param_name]
new_value = convert_str_value_to_correct_type(param_value, old_value)
print(f"Setting {param_name} to: {new_value}")
params_dict[param_name] = new_value
if args.change_param:
print("---------------------------------------")
print("Changing params from command line...")
for param_name, value_change in args.change_param:
assert param_name in params_dict, f"Unrecognized param: {param_name}"
old_value = params_dict[param_name]
new_value = old_value + convert_str_value_to_correct_type(
value_change, old_value, use_timedelta=True
)
print(f"Changing {param_name} from {old_value} to {new_value}")
params_dict[param_name] = new_value
region_model = RegionModel(
country,
region,
subregion,
simulation_start_date,
simulation_create_date,
simulation_end_date,
region_params,
compute_hospitalizations=(not skip_hospitalizations),
)
if quarantine_perc > 0:
print(f"Quarantine percentage: {quarantine_perc:.0%}")
print(f"Quarantine effectiveness: {quarantine_effectiveness:.0%}")
assert quarantine_effectiveness in [0.025, 0.1, 0.25, 0.5], (
"must specify --quarantine_effectiveness percentage."
" Possible values: [0.025, 0.1, 0.25, 0.5]"
)
quarantine_effectiveness_to_reduction_idx = {0.025: 0, 0.1: 1, 0.25: 2, 0.5: 3}
region_model.quarantine_fraction = quarantine_perc
region_model.reduction_idx = quarantine_effectiveness_to_reduction_idx[
quarantine_effectiveness
]
if verbose:
print("================================")
print(region_model)
print("================================")
print("Parameters:")
for param_name, param_value in params_dict.items():
print(f"{param_name:<25s} : {param_value}")
real_death = []
real_death_all = []
with open(
current_path / "../data/timeseries_prov/mortality_timeseries_prov.csv", "r"
) as csvfile:
# reader = csv.reader(csvfile)
reader = csv.DictReader(csvfile)
for row in reader:
if row["province"] == args.subregion:
real_death.append(float(row["deaths"]))
real_death_all.append(float(row["cumulative_deaths"]))
real_date = datetime.date(
int(row["date_death_report"][-4:]),
int(row["date_death_report"][3:5]),
int(row["date_death_report"][:2]),
)
# rows= [row for row in reader]
# Add params to region_model
# for params_dict['REOPEN_SHIFT_DAYS'] in [12,3]:
# params_dict['MORTALITY_RATE']=0.013
# params_dict['INITIAL_R_0']=1.79
# params_dict['LOCKDOWN_R_0']=0.76
# params_dict['DAILY_IMPORTS']=160
if verbose:
print("--------------------------")
print("Running simulation...")
print("--------------------------")
# Run simulation
t = time.time()
params_tups = tuple(params_dict.items())
# print(params_dict)
region_model.init_params(params_tups)
dates, infections, hospitalizations, deaths = run(region_model)
deaths_total = deaths.cumsum()
ind = np.where(dates == real_date)[0].item()
deaths_proj = deaths_total[: ind + 1]
if len(deaths_total[: ind + 1]) >= len(real_death_all):
pad = len(deaths_proj) - len(real_death_all)
real_death_all = np.array([0] * pad + real_death_all)
if len(real_death_all) > len(deaths_total[: ind + 1]):
real_death_all = np.array(real_death_all)
pad = len(real_death_all) - len(deaths_total[: ind + 1])
deaths_proj = np.zeros(pad).extend(deaths_proj)
days = len(real_death_all)
best_error = 1 / days * sum((real_death_all - deaths_proj) ** 2)
MR = params_dict["MORTALITY_RATE"] # =0.013
IR = params_dict["INITIAL_R_0"] # =1.79
LR = params_dict["LOCKDOWN_R_0"] # =0.76
DI = params_dict["DAILY_IMPORTS"] # =160
for params_dict["INITIAL_R_0"] in np.linspace(IR * 0.7, IR):
for params_dict["LOCKDOWN_R_0"] in np.linspace(LR * 0.95, LR * 1.05, 20):
for params_dict["DAILY_IMPORTS"] in np.linspace(DI, DI * 1.35):
for params_dict["MORTALITY_RATE"] in np.linspace(
MR * 1.5, MR * 2.5, 20
):
params_tups = tuple(params_dict.items())
region_model.init_params(params_tups)
dates, infections, hospitalizations, deaths = run(region_model)
deaths_total = deaths.cumsum()
deaths_proj = deaths_total[: ind + 1]
if len(real_death_all) > len(deaths_total[: ind + 1]):
real_death_all = np.array(real_death_all)
pad = len(real_death_all) - len(deaths_total[: ind + 1])
deaths_proj = np.zeros(pad).extend(deaths_total[: ind + 1])
error = 1 / days * sum((real_death_all - deaths_proj) ** 2)
if error <= best_error:
best_error = error
inf_proj, hosp_proj, death_proj = (
infections,
hospitalizations,
deaths,
)
best_parameters = {
"INITIAL_R_0": params_dict["INITIAL_R_0"],
"LOCKDOWN_R_0": params_dict["LOCKDOWN_R_0"],
"DAILY_IMPORTS": params_dict["DAILY_IMPORTS"],
"MORTALITY_RATE": params_dict["MORTALITY_RATE"],
}
print("Finding new optimum with error:", best_error)
print(best_parameters)
print(time.time() - t)
"""
The following are lists with length N, where N is the number of days from
simulation_start_date to simulation_end_date.
dates : datetime.date objects representing day i
infections : number of new infections on day i
hospitalizations : occupied hospital beds on day i
deaths : number of new deaths on day i
"""
infections, hospitalizations, deaths = inf_proj, hosp_proj, death_proj
assert len(dates) == len(infections) == len(hospitalizations) == len(deaths)
assert dates[0] == simulation_start_date
assert dates[-1] == simulation_end_date
if verbose:
infections_total = infections.cumsum()
deaths_total = deaths.cumsum()
for i in range(len(dates)):
hospitalization_str = ""
if not skip_hospitalizations:
hospitalization_str = (
f"Hospital beds in use: {hospitalizations[i]:,.0f} - "
)
daily_str = (
f"{i+1:<3} - {dates[i]} - "
f"New / total infections: {infections[i]:,.0f} / {infections_total[i]:,.0f} - "
f"{hospitalization_str}"
f"New / total deaths: {deaths[i]:,.2f} / {deaths_total[i]:,.1f} - "
f"Mean R: {region_model.effective_r_arr[i]:.3f} - "
f"IFR: {region_model.ifr_arr[i]:.2%}"
)
print(daily_str) # comment out to spare console buffer
print("-------------------------------------")
print(f"End of simulation : {region_model.projection_end_date}")
print(f"Total infections : {infections.sum():,.0f}")
if not skip_hospitalizations:
print(f"Peak hospital beds used : {hospitalizations.max():,.0f}")
print(f"Total deaths : {deaths.sum():,.0f}")
plt.plot(deaths_total, color="blue", linewidth=3.0, linestyle="-.", label="Proj.")
plt.plot(real_death_all, color="red", linewidth=3.0, linestyle="--", label="True")
plt.xlabel("Days")
plt.ylabel("Death")
plt.title("Death Proj. Result")
plt.legend(loc="best")
plt.savefig(
current_path / f"../output/Proj_{args.country}_{args.subregion}.png", dpi=1200
)
plt.close()
if args.save_csv_fname:
dates_str = np.array(list(map(str, dates)))
combined_arr = np.vstack(
(
dates_str,
infections,
hospitalizations,
deaths,
region_model.effective_r_arr,
)
).T
headers = "dates,infections,hospitalizations,deaths,mean_r_t"
np.savetxt(
args.save_csv_fname, combined_arr, "%s", delimiter=",", header=headers
)
print("----------\nSaved file to:", args.save_csv_fname)