async def cache_data(self, date): """Cache the latest date into the database.""" # The WHERE condition is not required as there is always only one row # in the `latest_date` table. async with self.pool.acquire() as conn: result = await conn.execute( "UPDATE latest_date SET latest = $1, last_check = DEFAULT;", str_to_date(date), ) rows_updated = int(result.split()[1]) if rows_updated == 1: self.logger.info("Successfully updated latest date in cache") return elif rows_updated > 1: raise RuntimeError( 'The "latest_date" table has more than one row, ' "i.e. this table is corrupt") # No rows were updated, so the "latest_date" table must be empty. This # should only happen if this table was cleared manually, or this is the # first run of this code on this database. self.logger.info( "Couldn't update latest date in cache; trying to insert it") async with self.pool.acquire() as conn: await conn.execute( "INSERT INTO latest_date (latest) VALUES ($1);", str_to_date(date), )
def create(cls, dict_result): financial = cls(edinet_code=dict_result['edinet_code'], start_period=utils.str_to_date(dict_result['年度開始日']), end_period=utils.str_to_date(dict_result['年度終了日']), company_name=dict_result['会社名'], submit_paper=dict_result['提出書類'], submit_date=utils.str_to_date(dict_result['提出日']), shared_num=dict_result['発行済み株式数'], operating_cash=dict_result['営業CF'], financial_cash=dict_result['財務CF'], investment_cash=dict_result['投資CF'], net_income=dict_result['純利益'], sales_amt=dict_result['売上高'], bs_cash_amt_ttl=dict_result['BS現金預金'], bs_liabilities_amt_ttl=dict_result['BS負債合計'], securities_code=dict_result['証券コード'], industry=dict_result['業種'], revision=dict_result['訂正'], annual=dict_result['annual'], file_name=dict_result['file_name'], eps=0.0) try: with session_scope() as session: session.add(financial) return financial except IntegrityError: return False
def create_xbrl_url_json(self): result_dict = {} target_date = utils.str_to_date(settings.since) count = 0 while True: response_string = self.get_link_info_str(utils.date_to_str(target_date)) target_list = self.__json_parse(response_string) if not target_list: target_date = target_date + timedelta(days=1) continue info_dict = self.get_link(target_list) result_dict.update(info_dict) time.sleep(1) target_date = target_date + timedelta(days=1) count += 1 logger.info(f'action=create_xbrl_url_json count={count}') if target_date >= utils.str_to_date(settings.until): break return result_dict
async def cache_data(self, data, date): """Cache the comic data into the database.""" # The given date can be invalid (i.e. we may have been redirected to a # comic with a different date), hence get the correct date from the # scraped data. date = date_to_str(str_to_date(data["dateStr"], fmt=ALT_DATE_FMT)) # This lock ensures that the no. of rows in the cache doesn't increase. # This can happen, as the code involves first clearing excess rows, # then adding a new row. Therefore, the following can increase the no. # of rows: # 1. Coroutine 1 clears excess rows # 2. Coroutine 2 clears no excess rows, as coroutine 1 did them # 3. Coroutine 1 adds its row # 4. Coroutine 2 adds its row async with Lock(): try: await self._clean_cache() except Exception as ex: # This crash means that there can be some extra rows in the # cache. As the row limit is a little conservative, this should # not be a big issue. self.logger.error(f"Failed to clean cache: {ex}") self.logger.debug("", exc_info=True) date_obj = str_to_date(date) try: async with self.pool.acquire() as conn: await conn.execute( """INSERT INTO comic_cache (comic, img_url, title) VALUES ($1, $2, $3);""", date_obj, data["imgURL"], data["title"], ) except UniqueViolationError: # This comic date exists, so some other coroutine has already # cached this date in parallel. So we can simply update # `last_used` later (outside the lock). self.logger.warn( f"Trying to cache date {date}, which is already cached." ) else: return # succeeded in caching data, so exit # This only executes if caching data led to a UniqueViolation error. # The lock isn't needed here, as this command cannot increase the no. # of rows in the cache. self.logger.info("Now trying to update `last_used` in cache.") async with self.pool.acquire() as conn: await conn.execute( "UPDATE comic_cache SET last_used = DEFAULT WHERE comic = $1;", date_obj, )
def _flatten(self, item): """ Flatten a raw commit fetched by Perceval into a flat dictionary. A list with a single flat directory will be returned. That dictionary will have the elements we need for computing metrics. The list may be empty, if for some reason the commit should not be considered. :param item: raw item fetched by Perceval (dictionary) :returns: list of a single flat dictionary """ creation_date = utils.str_to_date(item['data']['AuthorDate']) if self.since and (self.since > creation_date): return [] if self.until and (self.until < creation_date): return [] code_files = [ file['file'] for file in item['data']['files'] if all( condition.check(file['file']) for condition in self.is_code) ] if len(code_files) > 0: flat = { 'repo': item['origin'], 'hash': item['data']['commit'], 'author': item['data']['Author'], 'category': "commit", 'created_date': creation_date, 'committer': item['data']['Commit'], 'commit_date': utils.str_to_date(item['data']['CommitDate']), 'files_no': len(item['data']['files']), 'refs': item['data']['refs'], 'parents': item['data']['parents'], 'files': item['data']['files'] } actions = 0 for file in item['data']['files']: if 'action' in file: actions += 1 flat['files_action'] = actions if 'Merge' in item['data']: flat['merge'] = True else: flat['merge'] = False return [flat] else: return []
async def comic_page(year: int, month: int, day: int) -> Union[str, Response]: """Serve the requested comic from the given URL.""" # This depends on the format given by `DATE_FMT` from constants.py date = f"{year:04d}-{month:02d}-{day:02d}" # Check to see if the date is invalid try: str_to_date(date) except ValueError: # Replicates the behaviour of "dilbert.com" by redirecting to the # homepage. return redirect("/") return await serve_comic(date)
def _flatten(self, item): """ Flatten a raw pull_request fetched by Perceval into a flat dictionary. A list with a single flat directory will be returned. That dictionary will have the elements we need for computing metrics. The list may be empty, if for some reason the pull_request should not be considered. :param item: raw item fetched by Perceval (dictionary) :returns: list of a single flat dictionary """ creation_date = utils.str_to_date(item['data']['created_at']) if self.since and (self.since > creation_date): return [] if self.until and (self.until < creation_date): return [] flat = { 'repo': item['origin'], 'hash': item['data']['id'], 'category': "pull_request", 'author': item['data']['user']['login'], 'created_date': creation_date, 'current_status': item['data']['state'], 'merged': item['data']['merged'] } return [flat]
def get_status_time_distribution(self): ''' Get the amount of time spent in each status ''' distribution = {} changelog = self.get_changelog_for_field_only('status') from_date = self.created to_date = self.created if changelog: # historical statuses for change in changelog: status = change['items'][0]['fromString'] to_date = str_to_date(change['created']) if status not in distribution: distribution[status] = [] distribution[status].append((from_date, to_date)) from_date = to_date # final/current status status = changelog[-1]['items'][0]['toString'] if status not in distribution: distribution[status] = [] distribution[status].append( (from_date, datetime.datetime.now(from_date.tzinfo))) else: distribution[self.status] = [ (from_date, datetime.datetime.now(from_date.tzinfo)) ] return distribution
def __init__(self, ticker_obj, expiry, strike, put): self.obj = ticker_obj self.strike = strike self.put = put # bool if isinstance(expiry, str): expiry = utils.str_to_date(expiry) # options data self.price = 0 self.dte = 0 # price self.collateral = self.strike * 100 assert (isinstance(ticker_obj, type( yf.Ticker("AMD")))), "Option error: invalid ticker_obj" # assert(date_to_str(expiry) in ticker_obj.options), "Option error: invalid expiry" try: expiration = utils.nearest_expiry(self.obj.options, expiry) self.expiry = expiration except: self.expiry = False raise ValueError("No expiry found within 3 days of date") self.update()
def load_raids_and_raid_days(): with get_db() as db: db_rows = { table_name: { row['id']: dict(row) for row in db.execute(f'SELECT * FROM {table_name}') } for table_name in ('raid', 'raid_days', 'bosses') } raids = { id: Raid(id, row['name'], row['short_name']) for id, row in db_rows['raid'].items() } for row in db_rows['bosses'].values(): raids[row['raid_id']].bosses.append(row['name']) raid_days = { id: RaidDay(id, str_to_date(row['date']), row['name'], raids[row['raid_id']]) for id, row in db_rows['raid_days'].items() } return raids, raid_days
async def random_comic() -> Response: """Serve a random comic.""" first = str_to_date(FIRST_COMIC) latest = curr_date() rand_date = date_to_str(random.uniform(first, latest)) # type: ignore # If there is no comic for this date yet, "dilbert.com" will auto-redirect # to the latest comic. return redirect(f"/{rand_date}")
async def serve_comic( date: str, allow_redirect: bool = True ) -> Union[str, Response]: """Serve the requested comic. Args: date: The date of the requested comic, in the format used by "dilbert.com" allow_redirect: If there is no comic found for this date, then whether to redirect to the correct date Returns: The rendered template for the comic page """ # Execute both in parallel, as they are independent of each other data, latest_comic = await asyncio.gather( app.comic_scraper.get_data(date), app.latest_date_scraper.get_data(), ) # This date differs from the input date if the input is invalid (i.e. # "dilbert.com" would redirect to a comic with a different date). actual_date_obj = str_to_date(data["dateStr"], fmt=ALT_DATE_FMT) actual_date = date_to_str(actual_date_obj) # Replicates the behaviour of "dilbert.com" by redirecting to the correct # date. if allow_redirect and actual_date != date: return redirect(f"/{actual_date}") # This will contain awaitables for caching data (if required) and rendering # the template. They are both independent of each other, and thus can be # run in parallel. todos = [] # The date of the latest comic is often retrieved from the cache. If # "dilbert.com" has redirected to a date which is newer than the cached # value, then there is a new "latest comic". So cache the answer of # "dilbert.com". if str_to_date(latest_comic) < actual_date_obj: latest_comic = actual_date todos.append(app.latest_date_scraper.cache_data(actual_date)) todos.append(_serve_template(actual_date, data, latest_comic)) results = await asyncio.gather(*todos) return results[-1] # this is the rendered template
def get_context_end_date(self, context_id): if context_id not in self._context_end_dates: node = self.getNode("//xbrli:context[@id='" + context_id + "']/xbrli:period/xbrli:endDate") dt = None if node is not None and node.text: #dt = date(*map(int, node.text.split('-'))) dt = utils.str_to_date(node.text) self._context_end_dates[context_id] = dt return self._context_end_dates[context_id]
async def scrape_data(self): """Scrape the date of the latest comic from "dilbert.com".""" # If there is no comic for this date yet, "dilbert.com" will # auto-redirect to the latest comic. latest = date_to_str(curr_date()) url = SRC_PREFIX + latest async with self.sess.get(url) as resp: self.logger.debug(f"Got response for latest date: {resp.status}") date = resp.url.path.split("/")[-1] # Check to see if the date is invalid try: str_to_date(date) except ValueError: raise ScrapingException( "Error in scraping the latest date from the URL") return date
async def _serve_template(date: str, data: dict, latest_comic: str) -> str: """Serve the HTML given scraped data. Both input dates must be in the format used by "dilbert.com". Args: date: The (possibly corrected) date of the comic data: The scraped comic data latest_comic: The date of the latest comic Returns: The rendered template for the comic page """ date_obj = str_to_date(date) # Links to previous and next comics previous_comic = date_to_str( max(str_to_date(FIRST_COMIC), date_obj - timedelta(days=1)) ) next_comic = date_to_str( min(str_to_date(latest_comic), date_obj + timedelta(days=1)) ) # Whether to disable left/right navigation buttons disable_left_nav = date == FIRST_COMIC disable_right_nav = date == latest_comic # Link to original strip on "dilbert.com" permalink = SRC_PREFIX + date return await render_template( "layout.html", data=data, date=date, first_comic=FIRST_COMIC, previous_comic=previous_comic, next_comic=next_comic, disable_left_nav=disable_left_nav, disable_right_nav=disable_right_nav, permalink=permalink, repo=REPO, )
def convert_str_value_to_correct_type(param_value, old_value, use_timedelta=False): """Convert param_value to the same type as old_value.""" for primitive_type in [bool, int, float]: if isinstance(old_value, primitive_type): return primitive_type(param_value) if isinstance(old_value, datetime.date): if use_timedelta: return datetime.timedelta(days=int(param_value)) return str_to_date(param_value) raise NotImplementedError(f"Unknown type for value: {type(old_value)}")
def publish_data( keyvals, msg_dbg=''): if keyvals != None and 'driver' in keyvals and 'device_id' in keyvals: driver = keyvals['driver'] device_id = keyvals['device_id'] devdef = utils.read_device(driver) if devdef == None: print "Device definition for %s not found"%driver return ## Custom device drivers here if driver == 'custom!@#!@$!$#': pass else: # generic driver: try to match keys to feeds if 'timestamp' in keyvals: dev = publisher.find_device( device_id, create_new=True, device_type=driver, devdef=devdef ) ts = utils.date_to_unix(utils.str_to_date(keyvals['timestamp'])) datapoints = [] feednums = [] for key,val in keyvals.iteritems(): if key in ['driver','device_id','timestamp']: continue try: f = float(val); except: print "Skipping Key-Value pair",key,"/",val,"as it is non-numeric" continue; if key in dev.feed_names: idx = dev.feed_names.index(key) feednums.append(idx) datapoints.append(f) else: feednums.append(len(dev.feed_names)) datapoints.append(f) dev.feed_names.append(key) try: if (len(datapoints) > 0): publisher.publish_data( device_id, ts, datapoints, feednum=feednums, devdef=devdef, device_type=driver, dev=dev) except: import traceback traceback.print_exc(); else: print "Data Line '%s' did not have a timestamp field (for generic driver)"%msg_dbg else: print "Data Line '%s' did not contain a key for driver and/or device_id"%msg_dbg
async def get_cached_data(self, date): """Get the cached comic data from the database.""" async with self.pool.acquire() as conn: # The other columns in the table are: `comic`, `last_used`. `comic` # is not required here, as we already have the date as a function # argument. In case the date given here is invalid (i.e. it would # redirect to a comic with a different date), we cannot retrieve # the correct date from the cache, as we aren't caching the mapping # of incorrect:correct dates. `last_used` will be updated later. row = await conn.fetchrow( "SELECT img_url, title FROM comic_cache WHERE comic = $1;", str_to_date(date), ) if row is None: # This means that the comic for this date wasn't cached, or the # date is invalid (i.e. it would redirect to a comic with a # different date). return None data = { "title": row[1], "dateStr": date_to_str(str_to_date(date), fmt=ALT_DATE_FMT), "imgURL": row[0], } # Update `last_used`, so that this comic isn't accidently de-cached. We # want to keep the most recently used comics in the cache, and we are # currently using this comic. self.logger.info("Updating `last_used` for data in cache") async with self.pool.acquire() as conn: await conn.execute( "UPDATE comic_cache SET last_used = DEFAULT WHERE comic = $1;", str_to_date(date), ) return data
def __init__(self, data): self.id = get_nested_key(data, 'key') self.type = get_nested_key(data, 'fields.issuetype.name') self.priority = get_nested_key(data, 'fields.priority.name') self.created = str_to_date(get_nested_key(data, 'fields.created')) self.status = get_nested_key(data, 'fields.status.name') self.creator = get_nested_key(data, 'fields.creator.displayName') self.assignee = get_nested_key(data, 'fields.assignee.displayName') self.description = get_nested_key(data, 'fields.description', '') self.acceptance_criteria = get_nested_key(data, 'fields.customfield_10741', '') self.how_to_test = get_nested_key(data, 'fields.customfield_10693', '') self.comments = get_nested_key(data, 'fields.comment.comments') self.changelog = get_nested_key(data, 'changelog.histories')
def price_by_date(ticker_obj, date, time="Close"): # time: {Open, High, Low, Close} today = datetime.date.today() if type(date) is str: date = utils.str_to_date(date) delta = (today - date).days + 2 if delta < 0: return "Invalid date" delta = str(delta) + "d" hist = ticker_obj.history(period=delta) try: hist = hist.loc[date] return hist[time] except: return "Non trading day"
def get_comments_by_status_distribution(self): ''' Get the number of comments posted in each status ''' distribution = {} for comment in self.comments: status = self.get_status_at_point_in_time( str_to_date(comment['created'])) if status not in distribution: distribution[status] = 0 distribution[status] += 1 return distribution
def get_field_update_by_status_distribution(self, field): ''' Get the number of times a field has been updated per status ''' distribution = {} changelog = self.get_changelog_for_field_only(field) for change in changelog: status = self.get_status_at_point_in_time( str_to_date(change['created'])) if status not in distribution: distribution[status] = 0 distribution[status] += 1 return distribution
def data_by_ticker_and_pct_otm( ticker, pct_otm ): # return list of (expiry, nearest strike, put option cost, DTE, ROI) for each expiry in ticker.options recent_price = utils.moving_average(ticker, 3) print( str(ticker.info['symbol']) + " 3-day MA: %s; OTM %s pct" % (str(recent_price), str(pct_otm * 100))) today = datetime.date.today() data = [] for date in ticker.options: strike = nearest_strike_by_pct_otm(ticker, pct_otm, date) option_chain = ticker.option_chain(date).puts option_price = option_chain.loc[option_chain['strike'] == strike]['lastPrice'].iloc[0] dte = (utils.str_to_date(date) - today).days roi = round(option_price / recent_price, 4) data.append((date, strike, option_price, dte, roi)) return data
def convert_mean_params_to_params_dict(mean_params): """Convert list of [param_name, param_value_raw] pairs to dict of param_name to param_value. We also convert string dates to datetime objects Parameters ---------- mean_params : list list of [param_name, param_value_raw] pairs """ params_dict = {} for param_name, param_value_raw in mean_params: try: # attempt to convert to datetime.date object if it is a date params_dict[param_name] = str_to_date(param_value_raw) except (TypeError, ValueError): params_dict[param_name] = param_value_raw return params_dict
def get_new_chapters(followed_issues, last_check): paths = VOLUME_XPATH[1] results = {} for issue in followed_issues: results[issue] = {} uri = create_uri(issue) response = requests.get(uri) source = response.content tree = html.fromstring(source) volumes = tree.xpath(VOLUME_XPATH[0]) for volume in volumes: title = volume.xpath(paths['title'])[0] results[issue][title] = {'chapters': []} chapters_data = volume.xpath(paths['chapters'][0]) for cd in chapters_data: chapter = { 'title': cd.xpath(paths['chapters'][1]['title'])[0], 'release': str_to_date(cd.xpath(paths['chapters'][1]['release'])[0]), 'url': cd.xpath(paths['chapters'][1]['url'])[0] } if is_new(chapter['release'], last_check): results[issue][title]['chapters'].append(chapter) if len(results[issue][title]['chapters']) == 0: del (results[issue][title]) if len(results[issue]) == 0: del (results[issue]) return results
def set_dates(self, data): self.date_onset = str_to_date(data['date_onset']) self.date_diagnosis = str_to_date(data['date_diagnosis'])
def set_dates(self, data): self.date_prescribed = str_to_date(data['date_prescribed']) self.date_started_taking = str_to_date(data['date_started_taking']) self.date_stopped_taking = str_to_date(data['date_stopped_taking'])
optimizer = RMSprop() loss = keras.losses.mean_squared_error # for regression #loss = keras.losses.categorical_crossentropy # for classifying model.compile( loss=loss, optimizer=optimizer ) model.summary() # return model # MAIN show_graphs=False # Load data df = pandas.read_csv( 'meteo_daily.csv', sep=';' ) fechas = utils.str_to_date( numpy.array( df['fecha'] ) ) T = numpy.zeros( [ len(fechas), 3 ] ) # Daily temperatures T[:,0] = df['minima'] T[:,1] = df['media'] T[:,2] = df['maxima'] previous_days = 3 days_to_predict = 2 size_X = (len(T)-previous_days-days_to_predict+1)//1 X = numpy.zeros( [ size_X, T.shape[1]*previous_days ] ) Y = numpy.zeros( [ size_X, T.shape[1]*days_to_predict ] ) for t in range(len(X)): i=t*1
Restaurant( \ id=int(current_rest[0]), \ name=current_rest[1], \ boro=current_rest[2], \ building=current_rest[3], \ street=current_rest[4], \ zip=int_or_none(current_rest[5]), \ phone=current_rest[6], \ cuisine=current_rest[7], \ ) \ ) session.add( \ Inspection( \ rest_id=int(current_insp[0]), \ inspection_date=str_to_date(current_insp[1]), \ action=current_insp[2], \ violation_code=current_insp[3], \ violation_desc=current_insp[4], \ is_critical=current_insp[5] == 'Critical', \ score=int_or_none(current_insp[6]), \ grade=grade_or_none(current_insp[7]), \ grade_date=str_to_date(current_insp[8]), \ record_date=str_to_date(current_insp[9]), \ inspection_type=current_insp[10], \ ) \ ) # commit changes to the database session.commit() session.close()
def get_devices_invalid_measurement_values(self): devices = self.airqo_api.get_devices(tenant='airqo', active=True) print(devices) errors = [] for device in devices: device_data = dict(device) try: current_measurements = dict( self.airqo_api.get_airqo_device_current_measurements( device_number=device_data["device_number"])) except Exception as ex: error = dict({ "self_link": f"{os.getenv('AIRQO_BASE_URL')}data/feeds/transform/recent?channel={device_data['device_number']}", "channelID": device_data["device_number"] }) errors.append(error) continue created_at = str_to_date(current_measurements.get("created_at")) check_date = datetime.utcnow() - timedelta(days=30) error = dict({ "channelID": device_data["device_number"], "device": device_data["name"], "isActive": device_data["isActive"], "siteName": device_data["siteName"], "created_at": f"{created_at}", }) for key, value in current_measurements.items(): key_str = f"{key}".strip().lower() if key_str == 'externaltemperature' or key_str == 'externalhumidity' or key_str == 'pm10' \ or key_str == 's2_pm2_5' or key_str == 's2_pm10' or key_str == 'internaltemperature' \ or key_str == 'internalhumidity': has_error = False if not is_valid_double(value=value): has_error = True if key_str == 'pm2_5' and not has_error: value = float(value) if value < 0 or value > 500.5: has_error = True if not has_error and (key_str == 'pm2_5' or key_str == 's2_pm2_5' or key_str == 's2_pm10' or key_str == 'pm10'): value = float(value) if value < 0 or value > 500.5: has_error = True if not has_error and (key_str == 'externaltemperature' or key_str == 'internalhumidity' or key_str == 'internaltemperature' or key_str == 'externalhumidity'): value = float(value) if value < 0 or value > 50: has_error = True if not has_error and (check_date > created_at): has_error = True if has_error: error[key] = value if len(error.keys()) > 5: error[ "self_link"] = f"{os.getenv('AIRQO_BASE_URL')}data/feeds/transform/recent?channel={device_data['device_number']}" errors.append(error) self.__print(data=errors)
def main(args): country = args.country region = args.region subregion = args.subregion skip_hospitalizations = args.skip_hospitalizations quarantine_perc = args.quarantine_perc quarantine_effectiveness = args.quarantine_effectiveness verbose = args.verbose if country != 'US' and not region: region = 'ALL' best_params_type = args.best_params_type assert best_params_type in ['mean', 'median', 'top', 'top10'], best_params_type if args.best_params_dir: # Load parameters from file best_params = load_best_params_from_file(args.best_params_dir, country, region, subregion) simulation_start_date = str_to_date(best_params['first_date']) simulation_create_date = str_to_date(best_params['date']) simulation_end_date = str_to_date(best_params['projection_end_date']) region_params = {'population': best_params['population']} # mean_params, median_params, top_params, or top10_params params_type_name = f'{best_params_type}_params' if verbose: print('best params type:', best_params_type) params_dict = convert_mean_params_to_params_dict( best_params[params_type_name]) else: """ You can hard code your own parameters if you do not want to use the preset parameters. This can be especially useful for regions/countries where we do not have projections. Then simply run `python run_simulation.py -v` to use these parameters. """ simulation_start_date = datetime.date(2020, 2, 1) simulation_create_date = datetime.date.today( ) # not used so can also be None simulation_end_date = datetime.date(2020, 10, 1) region_params = {'population': 332000000} params_dict = { 'INITIAL_R_0': 2.24, 'LOCKDOWN_R_0': 0.9, 'INFLECTION_DAY': datetime.date(2020, 3, 18), 'RATE_OF_INFLECTION': 0.25, 'LOCKDOWN_FATIGUE': 1., 'DAILY_IMPORTS': 500, 'MORTALITY_RATE': 0.01, 'REOPEN_DATE': datetime.date(2020, 5, 20), 'REOPEN_SHIFT_DAYS': 0, 'REOPEN_R': 1.2, 'REOPEN_INFLECTION': 0.3, 'POST_REOPEN_EQUILIBRIUM_R': 1., 'FALL_R_MULTIPLIER': 1.001, } if args.simulation_start_date: simulation_start_date = str_to_date(args.simulation_start_date) if args.simulation_end_date: simulation_end_date = str_to_date(args.simulation_end_date) if args.set_param: print('---------------------------------------') print('Overwriting params from command line...') for param_name, param_value in args.set_param: assert param_name in params_dict, f'Unrecognized param: {param_name}' old_value = params_dict[param_name] new_value = convert_str_value_to_correct_type( param_value, old_value) print(f'Setting {param_name} to: {new_value}') params_dict[param_name] = new_value if args.change_param: print('---------------------------------------') print('Changing params from command line...') for param_name, value_change in args.change_param: assert param_name in params_dict, f'Unrecognized param: {param_name}' old_value = params_dict[param_name] new_value = old_value + convert_str_value_to_correct_type( value_change, old_value, use_timedelta=True) print(f'Changing {param_name} from {old_value} to {new_value}') params_dict[param_name] = new_value region_model = RegionModel( country, region, subregion, simulation_start_date, simulation_create_date, simulation_end_date, region_params, compute_hospitalizations=(not skip_hospitalizations)) if quarantine_perc > 0: print(f'Quarantine percentage: {quarantine_perc:.0%}') print(f'Quarantine effectiveness: {quarantine_effectiveness:.0%}') assert quarantine_effectiveness in [0.025, 0.1, 0.25, 0.5], \ ('must specify --quarantine_effectiveness percentage.' ' Possible values: [0.025, 0.1, 0.25, 0.5]') quarantine_effectiveness_to_reduction_idx = { 0.025: 0, 0.1: 1, 0.25: 2, 0.5: 3 } region_model.quarantine_fraction = quarantine_perc region_model.reduction_idx = \ quarantine_effectiveness_to_reduction_idx[quarantine_effectiveness] if verbose: print('================================') print(region_model) print('================================') print('Parameters:') for param_name, param_value in params_dict.items(): print(f'{param_name:<25s} : {param_value}') # Add params to region_model params_tups = tuple(params_dict.items()) region_model.init_params(params_tups) if verbose: print('--------------------------') print('Running simulation...') print('--------------------------') # Run simulation dates, infections, hospitalizations, deaths = run(region_model) """ The following are lists with length N, where N is the number of days from simulation_start_date to simulation_end_date. dates : datetime.date objects representing day i infections : number of new infections on day i hospitalizations : occupied hospital beds on day i deaths : number of new deaths on day i """ assert len(dates) == len(infections) == len(hospitalizations) == len( deaths) assert dates[0] == simulation_start_date assert dates[-1] == simulation_end_date if verbose: infections_total = infections.cumsum() deaths_total = deaths.cumsum() for i in range(len(dates)): hospitalization_str = '' if not skip_hospitalizations: hospitalization_str = f'Hospital beds in use: {hospitalizations[i]:,.0f} - ' daily_str = ( f'{dates[i]} - ' f'New / total infections: {infections[i]:,.0f} / {infections_total[i]:,.0f} - ' f'{hospitalization_str}' f'New / total deaths: {deaths[i]:,.2f} / {deaths_total[i]:,.1f} - ' f'Mean R: {region_model.effective_r_arr[i]:.3f} - ' f'IFR: {region_model.ifr_arr[i]:.2%}') print(daily_str) # comment out to spare console buffer print('-------------------------------------') print(f'End of simulation : {region_model.projection_end_date}') print(f'Total infections : {infections.sum():,.0f}') if not skip_hospitalizations: print(f'Peak hospital beds used : {hospitalizations.max():,.0f}') print(f'Total deaths : {deaths.sum():,.0f}') if args.save_csv_fname: dates_str = np.array(list(map(str, dates))) combined_arr = np.vstack((dates_str, infections, hospitalizations, deaths, region_model.effective_r_arr)).T headers = 'dates,infections,hospitalizations,deaths,mean_r_t' np.savetxt(args.save_csv_fname, combined_arr, '%s', delimiter=',', header=headers) print('----------\nSaved file to:', args.save_csv_fname)
for g in gparm: print "\t\tState %-3d: N=%-7d mean=%20.10e variance=%20.10e"%(g.state_id,g.counts,g.mean,g.variance) print "\n"; elif (op == 'insert'): import devicedb,postgresops devicedb.connect(); fromtime = None totime = None dtype = "GHMM" ret,vals,sys.argv = utils.check_arg(sys.argv,'--from',1) if ret: try: fromtime = utils.str_to_date(vals[0]) except ValueError, msg: print "Bad from time: "+str(msg) ret,vals,sys.argv = utils.check_arg(sys.argv,'--to',1) if ret: try: totime = utils.str_to_date(vals[0]) except ValueError, msg: print "Bad to time: "+str(msg) ret,vals,sys.argv = utils.check_arg(sys.argv,'--type',1) if ret: dtype = vals[0] pretend,j,sys.argv = utils.check_arg(sys.argv,'--pretend')
def main(args): country = args.country region = args.region subregion = args.subregion skip_hospitalizations = args.skip_hospitalizations quarantine_perc = args.quarantine_perc quarantine_effectiveness = args.quarantine_effectiveness verbose = args.verbose if country != "US" and not region: region = "ALL" best_params_type = args.best_params_type assert best_params_type in ["mean", "median", "top", "top10"], best_params_type if args.best_params_dir: # Load parameters from file best_params = load_best_params_from_file( current_path / args.best_params_dir, country, region, subregion ) simulation_start_date = str_to_date(best_params["first_date"]) simulation_create_date = str_to_date(best_params["date"]) simulation_end_date = str_to_date(best_params["projection_end_date"]) region_params = {"population": best_params["population"]} # mean_params, median_params, top_params, or top10_params params_type_name = f"{best_params_type}_params" if verbose: print("best params type:", best_params_type) params_dict = convert_mean_params_to_params_dict(best_params[params_type_name]) else: """ You can hard code your own parameters if you do not want to use the preset parameters. This can be especially useful for regions/countries where we do not have projections. Then simply run `python run_simulation.py -v` to use these parameters. """ simulation_start_date = datetime.date(2020, 2, 1) simulation_create_date = datetime.date.today() # not used so can also be None simulation_end_date = datetime.date(2020, 10, 1) region_params = {"population": 332000000} params_dict = { "INITIAL_R_0": 2.24, "LOCKDOWN_R_0": 0.9, "INFLECTION_DAY": datetime.date(2020, 3, 18), "RATE_OF_INFLECTION": 0.25, "LOCKDOWN_FATIGUE": 1.0, "DAILY_IMPORTS": 500, "MORTALITY_RATE": 0.01, "REOPEN_DATE": datetime.date(2020, 5, 20), "REOPEN_SHIFT_DAYS": 0, "REOPEN_R": 1.2, "REOPEN_INFLECTION": 0.3, "POST_REOPEN_EQUILIBRIUM_R": 1.0, "FALL_R_MULTIPLIER": 1.001, } if args.simulation_start_date: simulation_start_date = str_to_date(args.simulation_start_date) if args.simulation_end_date: simulation_end_date = str_to_date(args.simulation_end_date) if args.set_param: print("---------------------------------------") print("Overwriting params from command line...") for param_name, param_value in args.set_param: assert param_name in params_dict, f"Unrecognized param: {param_name}" old_value = params_dict[param_name] new_value = convert_str_value_to_correct_type(param_value, old_value) print(f"Setting {param_name} to: {new_value}") params_dict[param_name] = new_value if args.change_param: print("---------------------------------------") print("Changing params from command line...") for param_name, value_change in args.change_param: assert param_name in params_dict, f"Unrecognized param: {param_name}" old_value = params_dict[param_name] new_value = old_value + convert_str_value_to_correct_type( value_change, old_value, use_timedelta=True ) print(f"Changing {param_name} from {old_value} to {new_value}") params_dict[param_name] = new_value region_model = RegionModel( country, region, subregion, simulation_start_date, simulation_create_date, simulation_end_date, region_params, compute_hospitalizations=(not skip_hospitalizations), ) if quarantine_perc > 0: print(f"Quarantine percentage: {quarantine_perc:.0%}") print(f"Quarantine effectiveness: {quarantine_effectiveness:.0%}") assert quarantine_effectiveness in [0.025, 0.1, 0.25, 0.5], ( "must specify --quarantine_effectiveness percentage." " Possible values: [0.025, 0.1, 0.25, 0.5]" ) quarantine_effectiveness_to_reduction_idx = {0.025: 0, 0.1: 1, 0.25: 2, 0.5: 3} region_model.quarantine_fraction = quarantine_perc region_model.reduction_idx = quarantine_effectiveness_to_reduction_idx[ quarantine_effectiveness ] if verbose: print("================================") print(region_model) print("================================") print("Parameters:") for param_name, param_value in params_dict.items(): print(f"{param_name:<25s} : {param_value}") real_death = [] real_death_all = [] with open( current_path / "../data/timeseries_prov/mortality_timeseries_prov.csv", "r" ) as csvfile: # reader = csv.reader(csvfile) reader = csv.DictReader(csvfile) for row in reader: if row["province"] == args.subregion: real_death.append(float(row["deaths"])) real_death_all.append(float(row["cumulative_deaths"])) real_date = datetime.date( int(row["date_death_report"][-4:]), int(row["date_death_report"][3:5]), int(row["date_death_report"][:2]), ) # rows= [row for row in reader] # Add params to region_model # for params_dict['REOPEN_SHIFT_DAYS'] in [12,3]: # params_dict['MORTALITY_RATE']=0.013 # params_dict['INITIAL_R_0']=1.79 # params_dict['LOCKDOWN_R_0']=0.76 # params_dict['DAILY_IMPORTS']=160 if verbose: print("--------------------------") print("Running simulation...") print("--------------------------") # Run simulation t = time.time() params_tups = tuple(params_dict.items()) # print(params_dict) region_model.init_params(params_tups) dates, infections, hospitalizations, deaths = run(region_model) deaths_total = deaths.cumsum() ind = np.where(dates == real_date)[0].item() deaths_proj = deaths_total[: ind + 1] if len(deaths_total[: ind + 1]) >= len(real_death_all): pad = len(deaths_proj) - len(real_death_all) real_death_all = np.array([0] * pad + real_death_all) if len(real_death_all) > len(deaths_total[: ind + 1]): real_death_all = np.array(real_death_all) pad = len(real_death_all) - len(deaths_total[: ind + 1]) deaths_proj = np.zeros(pad).extend(deaths_proj) days = len(real_death_all) best_error = 1 / days * sum((real_death_all - deaths_proj) ** 2) MR = params_dict["MORTALITY_RATE"] # =0.013 IR = params_dict["INITIAL_R_0"] # =1.79 LR = params_dict["LOCKDOWN_R_0"] # =0.76 DI = params_dict["DAILY_IMPORTS"] # =160 for params_dict["INITIAL_R_0"] in np.linspace(IR * 0.7, IR): for params_dict["LOCKDOWN_R_0"] in np.linspace(LR * 0.95, LR * 1.05, 20): for params_dict["DAILY_IMPORTS"] in np.linspace(DI, DI * 1.35): for params_dict["MORTALITY_RATE"] in np.linspace( MR * 1.5, MR * 2.5, 20 ): params_tups = tuple(params_dict.items()) region_model.init_params(params_tups) dates, infections, hospitalizations, deaths = run(region_model) deaths_total = deaths.cumsum() deaths_proj = deaths_total[: ind + 1] if len(real_death_all) > len(deaths_total[: ind + 1]): real_death_all = np.array(real_death_all) pad = len(real_death_all) - len(deaths_total[: ind + 1]) deaths_proj = np.zeros(pad).extend(deaths_total[: ind + 1]) error = 1 / days * sum((real_death_all - deaths_proj) ** 2) if error <= best_error: best_error = error inf_proj, hosp_proj, death_proj = ( infections, hospitalizations, deaths, ) best_parameters = { "INITIAL_R_0": params_dict["INITIAL_R_0"], "LOCKDOWN_R_0": params_dict["LOCKDOWN_R_0"], "DAILY_IMPORTS": params_dict["DAILY_IMPORTS"], "MORTALITY_RATE": params_dict["MORTALITY_RATE"], } print("Finding new optimum with error:", best_error) print(best_parameters) print(time.time() - t) """ The following are lists with length N, where N is the number of days from simulation_start_date to simulation_end_date. dates : datetime.date objects representing day i infections : number of new infections on day i hospitalizations : occupied hospital beds on day i deaths : number of new deaths on day i """ infections, hospitalizations, deaths = inf_proj, hosp_proj, death_proj assert len(dates) == len(infections) == len(hospitalizations) == len(deaths) assert dates[0] == simulation_start_date assert dates[-1] == simulation_end_date if verbose: infections_total = infections.cumsum() deaths_total = deaths.cumsum() for i in range(len(dates)): hospitalization_str = "" if not skip_hospitalizations: hospitalization_str = ( f"Hospital beds in use: {hospitalizations[i]:,.0f} - " ) daily_str = ( f"{i+1:<3} - {dates[i]} - " f"New / total infections: {infections[i]:,.0f} / {infections_total[i]:,.0f} - " f"{hospitalization_str}" f"New / total deaths: {deaths[i]:,.2f} / {deaths_total[i]:,.1f} - " f"Mean R: {region_model.effective_r_arr[i]:.3f} - " f"IFR: {region_model.ifr_arr[i]:.2%}" ) print(daily_str) # comment out to spare console buffer print("-------------------------------------") print(f"End of simulation : {region_model.projection_end_date}") print(f"Total infections : {infections.sum():,.0f}") if not skip_hospitalizations: print(f"Peak hospital beds used : {hospitalizations.max():,.0f}") print(f"Total deaths : {deaths.sum():,.0f}") plt.plot(deaths_total, color="blue", linewidth=3.0, linestyle="-.", label="Proj.") plt.plot(real_death_all, color="red", linewidth=3.0, linestyle="--", label="True") plt.xlabel("Days") plt.ylabel("Death") plt.title("Death Proj. Result") plt.legend(loc="best") plt.savefig( current_path / f"../output/Proj_{args.country}_{args.subregion}.png", dpi=1200 ) plt.close() if args.save_csv_fname: dates_str = np.array(list(map(str, dates))) combined_arr = np.vstack( ( dates_str, infections, hospitalizations, deaths, region_model.effective_r_arr, ) ).T headers = "dates,infections,hospitalizations,deaths,mean_r_t" np.savetxt( args.save_csv_fname, combined_arr, "%s", delimiter=",", header=headers ) print("----------\nSaved file to:", args.save_csv_fname)