Пример #1
0
    async def cache_data(self, date):
        """Cache the latest date into the database."""
        # The WHERE condition is not required as there is always only one row
        # in the `latest_date` table.
        async with self.pool.acquire() as conn:
            result = await conn.execute(
                "UPDATE latest_date SET latest = $1, last_check = DEFAULT;",
                str_to_date(date),
            )

        rows_updated = int(result.split()[1])
        if rows_updated == 1:
            self.logger.info("Successfully updated latest date in cache")
            return
        elif rows_updated > 1:
            raise RuntimeError(
                'The "latest_date" table has more than one row, '
                "i.e. this table is corrupt")

        # No rows were updated, so the "latest_date" table must be empty. This
        # should only happen if this table was cleared manually, or this is the
        # first run of this code on this database.
        self.logger.info(
            "Couldn't update latest date in cache; trying to insert it")

        async with self.pool.acquire() as conn:
            await conn.execute(
                "INSERT INTO latest_date (latest) VALUES ($1);",
                str_to_date(date),
            )
Пример #2
0
    def create(cls, dict_result):
        financial = cls(edinet_code=dict_result['edinet_code'],
                        start_period=utils.str_to_date(dict_result['年度開始日']),
                        end_period=utils.str_to_date(dict_result['年度終了日']),
                        company_name=dict_result['会社名'],
                        submit_paper=dict_result['提出書類'],
                        submit_date=utils.str_to_date(dict_result['提出日']),
                        shared_num=dict_result['発行済み株式数'],
                        operating_cash=dict_result['営業CF'],
                        financial_cash=dict_result['財務CF'],
                        investment_cash=dict_result['投資CF'],
                        net_income=dict_result['純利益'],
                        sales_amt=dict_result['売上高'],
                        bs_cash_amt_ttl=dict_result['BS現金預金'],
                        bs_liabilities_amt_ttl=dict_result['BS負債合計'],
                        securities_code=dict_result['証券コード'],
                        industry=dict_result['業種'],
                        revision=dict_result['訂正'],
                        annual=dict_result['annual'],
                        file_name=dict_result['file_name'],
                        eps=0.0)

        try:
            with session_scope() as session:
                session.add(financial)
            return financial
        except IntegrityError:
            return False
Пример #3
0
    def create_xbrl_url_json(self):
        result_dict = {}
        target_date = utils.str_to_date(settings.since)

        count = 0

        while True:
            response_string = self.get_link_info_str(utils.date_to_str(target_date))
            target_list = self.__json_parse(response_string)
            if not target_list:
                target_date = target_date + timedelta(days=1)
                continue

            info_dict = self.get_link(target_list)
            result_dict.update(info_dict)

            time.sleep(1)
            target_date = target_date + timedelta(days=1)
            count += 1
            logger.info(f'action=create_xbrl_url_json count={count}')

            if target_date >= utils.str_to_date(settings.until):
                break

        return result_dict
Пример #4
0
    async def cache_data(self, data, date):
        """Cache the comic data into the database."""
        # The given date can be invalid (i.e. we may have been redirected to a
        # comic with a different date), hence get the correct date from the
        # scraped data.
        date = date_to_str(str_to_date(data["dateStr"], fmt=ALT_DATE_FMT))

        # This lock ensures that the no. of rows in the cache doesn't increase.
        # This can happen, as the code involves first clearing excess rows,
        # then adding a new row. Therefore, the following can increase the no.
        # of rows:
        #   1. Coroutine 1 clears excess rows
        #   2. Coroutine 2 clears no excess rows, as coroutine 1 did them
        #   3. Coroutine 1 adds its row
        #   4. Coroutine 2 adds its row
        async with Lock():
            try:
                await self._clean_cache()
            except Exception as ex:
                # This crash means that there can be some extra rows in the
                # cache. As the row limit is a little conservative, this should
                # not be a big issue.
                self.logger.error(f"Failed to clean cache: {ex}")
                self.logger.debug("", exc_info=True)

            date_obj = str_to_date(date)

            try:
                async with self.pool.acquire() as conn:
                    await conn.execute(
                        """INSERT INTO comic_cache (comic, img_url, title)
                        VALUES ($1, $2, $3);""",
                        date_obj,
                        data["imgURL"],
                        data["title"],
                    )
            except UniqueViolationError:
                # This comic date exists, so some other coroutine has already
                # cached this date in parallel. So we can simply update
                # `last_used` later (outside the lock).
                self.logger.warn(
                    f"Trying to cache date {date}, which is already cached."
                )
            else:
                return  # succeeded in caching data, so exit

        # This only executes if caching data led to a UniqueViolation error.
        # The lock isn't needed here, as this command cannot increase the no.
        # of rows in the cache.
        self.logger.info("Now trying to update `last_used` in cache.")
        async with self.pool.acquire() as conn:
            await conn.execute(
                "UPDATE comic_cache SET last_used = DEFAULT WHERE comic = $1;",
                date_obj,
            )
Пример #5
0
    def _flatten(self, item):
        """
        Flatten a raw commit fetched by Perceval into a flat dictionary.

        A list with a single flat directory will be returned.
        That dictionary will have the elements we need for computing metrics.
        The list may be empty, if for some reason the commit should not
        be considered.

        :param item: raw item fetched by Perceval (dictionary)
        :returns:    list of a single flat dictionary
        """

        creation_date = utils.str_to_date(item['data']['AuthorDate'])
        if self.since and (self.since > creation_date):
            return []

        if self.until and (self.until < creation_date):
            return []

        code_files = [
            file['file'] for file in item['data']['files'] if all(
                condition.check(file['file']) for condition in self.is_code)
        ]

        if len(code_files) > 0:
            flat = {
                'repo': item['origin'],
                'hash': item['data']['commit'],
                'author': item['data']['Author'],
                'category': "commit",
                'created_date': creation_date,
                'committer': item['data']['Commit'],
                'commit_date': utils.str_to_date(item['data']['CommitDate']),
                'files_no': len(item['data']['files']),
                'refs': item['data']['refs'],
                'parents': item['data']['parents'],
                'files': item['data']['files']
            }

            actions = 0
            for file in item['data']['files']:
                if 'action' in file:
                    actions += 1
            flat['files_action'] = actions

            if 'Merge' in item['data']:
                flat['merge'] = True
            else:
                flat['merge'] = False

            return [flat]
        else:
            return []
Пример #6
0
async def comic_page(year: int, month: int, day: int) -> Union[str, Response]:
    """Serve the requested comic from the given URL."""
    # This depends on the format given by `DATE_FMT` from constants.py
    date = f"{year:04d}-{month:02d}-{day:02d}"

    # Check to see if the date is invalid
    try:
        str_to_date(date)
    except ValueError:
        # Replicates the behaviour of "dilbert.com" by redirecting to the
        # homepage.
        return redirect("/")

    return await serve_comic(date)
Пример #7
0
    def _flatten(self, item):
        """
        Flatten a raw pull_request fetched by Perceval into a flat dictionary.

        A list with a single flat directory will be returned.
        That dictionary will have the elements we need for computing metrics.
        The list may be empty, if for some reason the pull_request should not
        be considered.

        :param item: raw item fetched by Perceval (dictionary)
        :returns:   list of a single flat dictionary
        """

        creation_date = utils.str_to_date(item['data']['created_at'])
        if self.since and (self.since > creation_date):
            return []

        if self.until and (self.until < creation_date):
            return []

        flat = {
            'repo': item['origin'],
            'hash': item['data']['id'],
            'category': "pull_request",
            'author': item['data']['user']['login'],
            'created_date': creation_date,
            'current_status': item['data']['state'],
            'merged': item['data']['merged']
        }

        return [flat]
Пример #8
0
    def get_status_time_distribution(self):
        '''
        Get the amount of time spent in each status
        '''
        distribution = {}

        changelog = self.get_changelog_for_field_only('status')
        from_date = self.created
        to_date = self.created

        if changelog:
            # historical statuses
            for change in changelog:
                status = change['items'][0]['fromString']
                to_date = str_to_date(change['created'])

                if status not in distribution:
                    distribution[status] = []

                distribution[status].append((from_date, to_date))
                from_date = to_date

            # final/current status
            status = changelog[-1]['items'][0]['toString']
            if status not in distribution:
                distribution[status] = []

            distribution[status].append(
                (from_date, datetime.datetime.now(from_date.tzinfo)))
        else:
            distribution[self.status] = [
                (from_date, datetime.datetime.now(from_date.tzinfo))
            ]

        return distribution
Пример #9
0
    def __init__(self, ticker_obj, expiry, strike, put):
        self.obj = ticker_obj
        self.strike = strike
        self.put = put  # bool
        if isinstance(expiry, str):
            expiry = utils.str_to_date(expiry)

        # options data
        self.price = 0
        self.dte = 0

        # price
        self.collateral = self.strike * 100

        assert (isinstance(ticker_obj, type(
            yf.Ticker("AMD")))), "Option error: invalid ticker_obj"
        # assert(date_to_str(expiry) in ticker_obj.options), "Option error: invalid expiry"

        try:
            expiration = utils.nearest_expiry(self.obj.options, expiry)
            self.expiry = expiration
        except:
            self.expiry = False
            raise ValueError("No expiry found within 3 days of date")

        self.update()
Пример #10
0
def load_raids_and_raid_days():
    with get_db() as db:
        db_rows = {
            table_name: {
                row['id']: dict(row)
                for row in db.execute(f'SELECT * FROM {table_name}')
            }
            for table_name in ('raid', 'raid_days', 'bosses')
        }

    raids = {
        id: Raid(id, row['name'], row['short_name'])
        for id, row in db_rows['raid'].items()
    }

    for row in db_rows['bosses'].values():
        raids[row['raid_id']].bosses.append(row['name'])

    raid_days = {
        id: RaidDay(id, str_to_date(row['date']), row['name'],
                    raids[row['raid_id']])
        for id, row in db_rows['raid_days'].items()
    }

    return raids, raid_days
Пример #11
0
async def random_comic() -> Response:
    """Serve a random comic."""
    first = str_to_date(FIRST_COMIC)
    latest = curr_date()
    rand_date = date_to_str(random.uniform(first, latest))  # type: ignore
    # If there is no comic for this date yet, "dilbert.com" will auto-redirect
    # to the latest comic.
    return redirect(f"/{rand_date}")
Пример #12
0
async def serve_comic(
    date: str, allow_redirect: bool = True
) -> Union[str, Response]:
    """Serve the requested comic.

    Args:
        date: The date of the requested comic, in the format used by
            "dilbert.com"
        allow_redirect: If there is no comic found for this date, then
            whether to redirect to the correct date

    Returns:
        The rendered template for the comic page

    """
    # Execute both in parallel, as they are independent of each other
    data, latest_comic = await asyncio.gather(
        app.comic_scraper.get_data(date), app.latest_date_scraper.get_data(),
    )

    # This date differs from the input date if the input is invalid (i.e.
    # "dilbert.com" would redirect to a comic with a different date).
    actual_date_obj = str_to_date(data["dateStr"], fmt=ALT_DATE_FMT)
    actual_date = date_to_str(actual_date_obj)

    # Replicates the behaviour of "dilbert.com" by redirecting to the correct
    # date.
    if allow_redirect and actual_date != date:
        return redirect(f"/{actual_date}")

    # This will contain awaitables for caching data (if required) and rendering
    # the template. They are both independent of each other, and thus can be
    # run in parallel.
    todos = []

    # The date of the latest comic is often retrieved from the cache. If
    # "dilbert.com" has redirected to a date which is newer than the cached
    # value, then there is a new "latest comic". So cache the answer of
    # "dilbert.com".
    if str_to_date(latest_comic) < actual_date_obj:
        latest_comic = actual_date
        todos.append(app.latest_date_scraper.cache_data(actual_date))

    todos.append(_serve_template(actual_date, data, latest_comic))
    results = await asyncio.gather(*todos)
    return results[-1]  # this is the rendered template
Пример #13
0
 def get_context_end_date(self, context_id):
     if context_id not in self._context_end_dates:
         node = self.getNode("//xbrli:context[@id='" + context_id + "']/xbrli:period/xbrli:endDate")
         dt = None
         if node is not None and node.text:
             #dt = date(*map(int, node.text.split('-')))
             dt = utils.str_to_date(node.text)
         self._context_end_dates[context_id] = dt
     return self._context_end_dates[context_id]
Пример #14
0
    async def scrape_data(self):
        """Scrape the date of the latest comic from "dilbert.com"."""
        # If there is no comic for this date yet, "dilbert.com" will
        # auto-redirect to the latest comic.
        latest = date_to_str(curr_date())
        url = SRC_PREFIX + latest

        async with self.sess.get(url) as resp:
            self.logger.debug(f"Got response for latest date: {resp.status}")
            date = resp.url.path.split("/")[-1]

        # Check to see if the date is invalid
        try:
            str_to_date(date)
        except ValueError:
            raise ScrapingException(
                "Error in scraping the latest date from the URL")

        return date
Пример #15
0
async def _serve_template(date: str, data: dict, latest_comic: str) -> str:
    """Serve the HTML given scraped data.

    Both input dates must be in the format used by "dilbert.com".

    Args:
        date: The (possibly corrected) date of the comic
        data: The scraped comic data
        latest_comic: The date of the latest comic

    Returns:
        The rendered template for the comic page

    """
    date_obj = str_to_date(date)

    # Links to previous and next comics
    previous_comic = date_to_str(
        max(str_to_date(FIRST_COMIC), date_obj - timedelta(days=1))
    )
    next_comic = date_to_str(
        min(str_to_date(latest_comic), date_obj + timedelta(days=1))
    )

    # Whether to disable left/right navigation buttons
    disable_left_nav = date == FIRST_COMIC
    disable_right_nav = date == latest_comic

    # Link to original strip on "dilbert.com"
    permalink = SRC_PREFIX + date

    return await render_template(
        "layout.html",
        data=data,
        date=date,
        first_comic=FIRST_COMIC,
        previous_comic=previous_comic,
        next_comic=next_comic,
        disable_left_nav=disable_left_nav,
        disable_right_nav=disable_right_nav,
        permalink=permalink,
        repo=REPO,
    )
def convert_str_value_to_correct_type(param_value, old_value, use_timedelta=False):
    """Convert param_value to the same type as old_value."""

    for primitive_type in [bool, int, float]:
        if isinstance(old_value, primitive_type):
            return primitive_type(param_value)

    if isinstance(old_value, datetime.date):
        if use_timedelta:
            return datetime.timedelta(days=int(param_value))
        return str_to_date(param_value)

    raise NotImplementedError(f"Unknown type for value: {type(old_value)}")
Пример #17
0
def publish_data( keyvals, msg_dbg=''):    
    if keyvals != None  and 'driver' in keyvals and 'device_id' in keyvals:
        driver = keyvals['driver']
        device_id = keyvals['device_id']
        devdef = utils.read_device(driver)
        if devdef == None:
            print "Device definition for %s not found"%driver
            return

        ## Custom device drivers here
        if driver == 'custom!@#!@$!$#':
            pass
        else: # generic driver: try to match keys to feeds
            if 'timestamp' in keyvals:
                dev = publisher.find_device( device_id, create_new=True, device_type=driver, devdef=devdef )
                
                ts = utils.date_to_unix(utils.str_to_date(keyvals['timestamp']))
                datapoints = []
                feednums = []
                
                for key,val in keyvals.iteritems():
                    if key in ['driver','device_id','timestamp']:
                        continue
                        
                    try:
                        f = float(val);
                    except:
                        print "Skipping Key-Value pair",key,"/",val,"as it is non-numeric"
                        continue;
                        
                    if key in dev.feed_names:
                        idx = dev.feed_names.index(key)
                        feednums.append(idx)
                        datapoints.append(f)
                    else:
                        feednums.append(len(dev.feed_names))
                        datapoints.append(f)
                        dev.feed_names.append(key)
                try:
                    if (len(datapoints) > 0):
                        publisher.publish_data( device_id, ts, datapoints, feednum=feednums, devdef=devdef, device_type=driver, dev=dev)
                except:
                    import traceback
                    traceback.print_exc();
            else:
                print "Data Line '%s' did not have a timestamp field (for generic driver)"%msg_dbg
        
        
    else:
        print "Data Line '%s' did not contain a key for driver and/or device_id"%msg_dbg
Пример #18
0
    async def get_cached_data(self, date):
        """Get the cached comic data from the database."""
        async with self.pool.acquire() as conn:
            # The other columns in the table are: `comic`, `last_used`. `comic`
            # is not required here, as we already have the date as a function
            # argument. In case the date given here is invalid (i.e. it would
            # redirect to a comic with a different date), we cannot retrieve
            # the correct date from the cache, as we aren't caching the mapping
            # of incorrect:correct dates. `last_used` will be updated later.
            row = await conn.fetchrow(
                "SELECT img_url, title FROM comic_cache WHERE comic = $1;",
                str_to_date(date),
            )

        if row is None:
            # This means that the comic for this date wasn't cached, or the
            # date is invalid (i.e. it would redirect to a comic with a
            # different date).
            return None

        data = {
            "title": row[1],
            "dateStr": date_to_str(str_to_date(date), fmt=ALT_DATE_FMT),
            "imgURL": row[0],
        }

        # Update `last_used`, so that this comic isn't accidently de-cached. We
        # want to keep the most recently used comics in the cache, and we are
        # currently using this comic.
        self.logger.info("Updating `last_used` for data in cache")
        async with self.pool.acquire() as conn:
            await conn.execute(
                "UPDATE comic_cache SET last_used = DEFAULT WHERE comic = $1;",
                str_to_date(date),
            )

        return data
Пример #19
0
 def __init__(self, data):
     self.id = get_nested_key(data, 'key')
     self.type = get_nested_key(data, 'fields.issuetype.name')
     self.priority = get_nested_key(data, 'fields.priority.name')
     self.created = str_to_date(get_nested_key(data, 'fields.created'))
     self.status = get_nested_key(data, 'fields.status.name')
     self.creator = get_nested_key(data, 'fields.creator.displayName')
     self.assignee = get_nested_key(data, 'fields.assignee.displayName')
     self.description = get_nested_key(data, 'fields.description', '')
     self.acceptance_criteria = get_nested_key(data,
                                               'fields.customfield_10741',
                                               '')
     self.how_to_test = get_nested_key(data, 'fields.customfield_10693', '')
     self.comments = get_nested_key(data, 'fields.comment.comments')
     self.changelog = get_nested_key(data, 'changelog.histories')
Пример #20
0
def price_by_date(ticker_obj,
                  date,
                  time="Close"):  # time: {Open, High, Low, Close}
    today = datetime.date.today()
    if type(date) is str:
        date = utils.str_to_date(date)
    delta = (today - date).days + 2
    if delta < 0:
        return "Invalid date"
    delta = str(delta) + "d"
    hist = ticker_obj.history(period=delta)
    try:
        hist = hist.loc[date]
        return hist[time]
    except:
        return "Non trading day"
Пример #21
0
    def get_comments_by_status_distribution(self):
        '''
        Get the number of comments posted in each status
        '''
        distribution = {}

        for comment in self.comments:
            status = self.get_status_at_point_in_time(
                str_to_date(comment['created']))

            if status not in distribution:
                distribution[status] = 0

            distribution[status] += 1

        return distribution
Пример #22
0
    def get_field_update_by_status_distribution(self, field):
        '''
        Get the number of times a field has been updated per status
        '''
        distribution = {}

        changelog = self.get_changelog_for_field_only(field)
        for change in changelog:
            status = self.get_status_at_point_in_time(
                str_to_date(change['created']))

            if status not in distribution:
                distribution[status] = 0

            distribution[status] += 1

        return distribution
Пример #23
0
def data_by_ticker_and_pct_otm(
    ticker, pct_otm
):  # return list of (expiry, nearest strike, put option cost, DTE, ROI) for each expiry in ticker.options
    recent_price = utils.moving_average(ticker, 3)
    print(
        str(ticker.info['symbol']) + " 3-day MA: %s; OTM %s pct" %
        (str(recent_price), str(pct_otm * 100)))
    today = datetime.date.today()
    data = []

    for date in ticker.options:
        strike = nearest_strike_by_pct_otm(ticker, pct_otm, date)
        option_chain = ticker.option_chain(date).puts
        option_price = option_chain.loc[option_chain['strike'] ==
                                        strike]['lastPrice'].iloc[0]
        dte = (utils.str_to_date(date) - today).days
        roi = round(option_price / recent_price, 4)
        data.append((date, strike, option_price, dte, roi))
    return data
Пример #24
0
def convert_mean_params_to_params_dict(mean_params):
    """Convert list of [param_name, param_value_raw] pairs to dict of param_name to param_value.

    We also convert string dates to datetime objects

    Parameters
    ----------
    mean_params : list
        list of [param_name, param_value_raw] pairs
    """

    params_dict = {}
    for param_name, param_value_raw in mean_params:
        try:
            # attempt to convert to datetime.date object if it is a date
            params_dict[param_name] = str_to_date(param_value_raw)
        except (TypeError, ValueError):
            params_dict[param_name] = param_value_raw

    return params_dict
Пример #25
0
def get_new_chapters(followed_issues, last_check):
    paths = VOLUME_XPATH[1]
    results = {}

    for issue in followed_issues:
        results[issue] = {}
        uri = create_uri(issue)
        response = requests.get(uri)
        source = response.content
        tree = html.fromstring(source)
        volumes = tree.xpath(VOLUME_XPATH[0])

        for volume in volumes:
            title = volume.xpath(paths['title'])[0]
            results[issue][title] = {'chapters': []}

            chapters_data = volume.xpath(paths['chapters'][0])

            for cd in chapters_data:
                chapter = {
                    'title':
                    cd.xpath(paths['chapters'][1]['title'])[0],
                    'release':
                    str_to_date(cd.xpath(paths['chapters'][1]['release'])[0]),
                    'url':
                    cd.xpath(paths['chapters'][1]['url'])[0]
                }

                if is_new(chapter['release'], last_check):
                    results[issue][title]['chapters'].append(chapter)

            if len(results[issue][title]['chapters']) == 0:
                del (results[issue][title])

        if len(results[issue]) == 0:
            del (results[issue])

    return results
Пример #26
0
 def set_dates(self, data):
     self.date_onset = str_to_date(data['date_onset'])
     self.date_diagnosis = str_to_date(data['date_diagnosis'])
Пример #27
0
 def set_dates(self, data):
     self.date_prescribed = str_to_date(data['date_prescribed'])
     self.date_started_taking = str_to_date(data['date_started_taking'])
     self.date_stopped_taking = str_to_date(data['date_stopped_taking'])
Пример #28
0
    optimizer = RMSprop()
    loss = keras.losses.mean_squared_error # for regression
    #loss = keras.losses.categorical_crossentropy # for classifying
    model.compile( loss=loss, optimizer=optimizer )
    model.summary()
    #
    return model


# MAIN

show_graphs=False

# Load data
df = pandas.read_csv( 'meteo_daily.csv', sep=';' )
fechas = utils.str_to_date( numpy.array( df['fecha'] ) )
T = numpy.zeros( [ len(fechas), 3 ] ) # Daily temperatures
T[:,0] = df['minima']
T[:,1] = df['media']
T[:,2] = df['maxima']


previous_days = 3
days_to_predict = 2
size_X = (len(T)-previous_days-days_to_predict+1)//1

X = numpy.zeros( [ size_X, T.shape[1]*previous_days ] )
Y = numpy.zeros( [ size_X, T.shape[1]*days_to_predict ] )

for t in range(len(X)):
    i=t*1
Пример #29
0
            Restaurant(                              \
                id=int(current_rest[0]),             \
                name=current_rest[1],                \
                boro=current_rest[2],                \
                building=current_rest[3],            \
                street=current_rest[4],              \
                zip=int_or_none(current_rest[5]),    \
                phone=current_rest[6],               \
                cuisine=current_rest[7],             \
            )                                        \
        )

    session.add(                                                \
        Inspection(                                             \
            rest_id=int(current_insp[0]),                       \
            inspection_date=str_to_date(current_insp[1]),       \
            action=current_insp[2],                             \
            violation_code=current_insp[3],                     \
            violation_desc=current_insp[4],                     \
            is_critical=current_insp[5] == 'Critical',          \
            score=int_or_none(current_insp[6]),                 \
            grade=grade_or_none(current_insp[7]),               \
            grade_date=str_to_date(current_insp[8]),            \
            record_date=str_to_date(current_insp[9]),           \
            inspection_type=current_insp[10],                   \
        )                                                       \
    )

# commit changes to the database
session.commit()
session.close()
Пример #30
0
    def get_devices_invalid_measurement_values(self):
        devices = self.airqo_api.get_devices(tenant='airqo', active=True)
        print(devices)

        errors = []
        for device in devices:
            device_data = dict(device)
            try:
                current_measurements = dict(
                    self.airqo_api.get_airqo_device_current_measurements(
                        device_number=device_data["device_number"]))
            except Exception as ex:
                error = dict({
                    "self_link":
                    f"{os.getenv('AIRQO_BASE_URL')}data/feeds/transform/recent?channel={device_data['device_number']}",
                    "channelID": device_data["device_number"]
                })
                errors.append(error)
                continue

            created_at = str_to_date(current_measurements.get("created_at"))
            check_date = datetime.utcnow() - timedelta(days=30)
            error = dict({
                "channelID": device_data["device_number"],
                "device": device_data["name"],
                "isActive": device_data["isActive"],
                "siteName": device_data["siteName"],
                "created_at": f"{created_at}",
            })

            for key, value in current_measurements.items():
                key_str = f"{key}".strip().lower()

                if key_str == 'externaltemperature' or key_str == 'externalhumidity' or key_str == 'pm10' \
                        or key_str == 's2_pm2_5' or key_str == 's2_pm10' or key_str == 'internaltemperature' \
                        or key_str == 'internalhumidity':

                    has_error = False

                    if not is_valid_double(value=value):
                        has_error = True

                    if key_str == 'pm2_5' and not has_error:
                        value = float(value)
                        if value < 0 or value > 500.5:
                            has_error = True

                    if not has_error and (key_str == 'pm2_5' or key_str
                                          == 's2_pm2_5' or key_str == 's2_pm10'
                                          or key_str == 'pm10'):
                        value = float(value)
                        if value < 0 or value > 500.5:
                            has_error = True

                    if not has_error and (key_str == 'externaltemperature'
                                          or key_str == 'internalhumidity'
                                          or key_str == 'internaltemperature'
                                          or key_str == 'externalhumidity'):
                        value = float(value)
                        if value < 0 or value > 50:
                            has_error = True

                    if not has_error and (check_date > created_at):
                        has_error = True

                    if has_error:
                        error[key] = value

            if len(error.keys()) > 5:
                error[
                    "self_link"] = f"{os.getenv('AIRQO_BASE_URL')}data/feeds/transform/recent?channel={device_data['device_number']}"
                errors.append(error)

        self.__print(data=errors)
Пример #31
0
def main(args):
    country = args.country
    region = args.region
    subregion = args.subregion
    skip_hospitalizations = args.skip_hospitalizations
    quarantine_perc = args.quarantine_perc
    quarantine_effectiveness = args.quarantine_effectiveness
    verbose = args.verbose

    if country != 'US' and not region:
        region = 'ALL'

    best_params_type = args.best_params_type
    assert best_params_type in ['mean', 'median', 'top',
                                'top10'], best_params_type

    if args.best_params_dir:
        # Load parameters from file
        best_params = load_best_params_from_file(args.best_params_dir, country,
                                                 region, subregion)
        simulation_start_date = str_to_date(best_params['first_date'])
        simulation_create_date = str_to_date(best_params['date'])
        simulation_end_date = str_to_date(best_params['projection_end_date'])

        region_params = {'population': best_params['population']}
        # mean_params, median_params, top_params, or top10_params
        params_type_name = f'{best_params_type}_params'
        if verbose:
            print('best params type:', best_params_type)
        params_dict = convert_mean_params_to_params_dict(
            best_params[params_type_name])
    else:
        """
        You can hard code your own parameters if you do not want to use the preset parameters.

        This can be especially useful for regions/countries where we do not have projections.

        Then simply run `python run_simulation.py -v` to use these parameters.
        """

        simulation_start_date = datetime.date(2020, 2, 1)
        simulation_create_date = datetime.date.today(
        )  # not used so can also be None
        simulation_end_date = datetime.date(2020, 10, 1)

        region_params = {'population': 332000000}
        params_dict = {
            'INITIAL_R_0': 2.24,
            'LOCKDOWN_R_0': 0.9,
            'INFLECTION_DAY': datetime.date(2020, 3, 18),
            'RATE_OF_INFLECTION': 0.25,
            'LOCKDOWN_FATIGUE': 1.,
            'DAILY_IMPORTS': 500,
            'MORTALITY_RATE': 0.01,
            'REOPEN_DATE': datetime.date(2020, 5, 20),
            'REOPEN_SHIFT_DAYS': 0,
            'REOPEN_R': 1.2,
            'REOPEN_INFLECTION': 0.3,
            'POST_REOPEN_EQUILIBRIUM_R': 1.,
            'FALL_R_MULTIPLIER': 1.001,
        }

    if args.simulation_start_date:
        simulation_start_date = str_to_date(args.simulation_start_date)
    if args.simulation_end_date:
        simulation_end_date = str_to_date(args.simulation_end_date)

    if args.set_param:
        print('---------------------------------------')
        print('Overwriting params from command line...')
        for param_name, param_value in args.set_param:
            assert param_name in params_dict, f'Unrecognized param: {param_name}'
            old_value = params_dict[param_name]
            new_value = convert_str_value_to_correct_type(
                param_value, old_value)
            print(f'Setting {param_name} to: {new_value}')
            params_dict[param_name] = new_value

    if args.change_param:
        print('---------------------------------------')
        print('Changing params from command line...')
        for param_name, value_change in args.change_param:
            assert param_name in params_dict, f'Unrecognized param: {param_name}'
            old_value = params_dict[param_name]
            new_value = old_value + convert_str_value_to_correct_type(
                value_change, old_value, use_timedelta=True)
            print(f'Changing {param_name} from {old_value} to {new_value}')
            params_dict[param_name] = new_value

    region_model = RegionModel(
        country,
        region,
        subregion,
        simulation_start_date,
        simulation_create_date,
        simulation_end_date,
        region_params,
        compute_hospitalizations=(not skip_hospitalizations))

    if quarantine_perc > 0:
        print(f'Quarantine percentage: {quarantine_perc:.0%}')
        print(f'Quarantine effectiveness: {quarantine_effectiveness:.0%}')
        assert quarantine_effectiveness in [0.025, 0.1, 0.25, 0.5], \
            ('must specify --quarantine_effectiveness percentage.'
                ' Possible values: [0.025, 0.1, 0.25, 0.5]')
        quarantine_effectiveness_to_reduction_idx = {
            0.025: 0,
            0.1: 1,
            0.25: 2,
            0.5: 3
        }
        region_model.quarantine_fraction = quarantine_perc
        region_model.reduction_idx = \
            quarantine_effectiveness_to_reduction_idx[quarantine_effectiveness]

    if verbose:
        print('================================')
        print(region_model)
        print('================================')
        print('Parameters:')
        for param_name, param_value in params_dict.items():
            print(f'{param_name:<25s} : {param_value}')

    # Add params to region_model
    params_tups = tuple(params_dict.items())
    region_model.init_params(params_tups)

    if verbose:
        print('--------------------------')
        print('Running simulation...')
        print('--------------------------')

    # Run simulation
    dates, infections, hospitalizations, deaths = run(region_model)
    """
    The following are lists with length N, where N is the number of days from
        simulation_start_date to simulation_end_date.

    dates            : datetime.date objects representing day i
    infections       : number of new infections on day i
    hospitalizations : occupied hospital beds on day i
    deaths           : number of new deaths on day i
    """
    assert len(dates) == len(infections) == len(hospitalizations) == len(
        deaths)
    assert dates[0] == simulation_start_date
    assert dates[-1] == simulation_end_date

    if verbose:
        infections_total = infections.cumsum()
        deaths_total = deaths.cumsum()
        for i in range(len(dates)):
            hospitalization_str = ''
            if not skip_hospitalizations:
                hospitalization_str = f'Hospital beds in use: {hospitalizations[i]:,.0f} - '
            daily_str = (
                f'{dates[i]} - '
                f'New / total infections: {infections[i]:,.0f} / {infections_total[i]:,.0f} - '
                f'{hospitalization_str}'
                f'New / total deaths: {deaths[i]:,.2f} / {deaths_total[i]:,.1f} - '
                f'Mean R: {region_model.effective_r_arr[i]:.3f} - '
                f'IFR: {region_model.ifr_arr[i]:.2%}')
            print(daily_str)  # comment out to spare console buffer
    print('-------------------------------------')
    print(f'End of simulation       : {region_model.projection_end_date}')
    print(f'Total infections        : {infections.sum():,.0f}')
    if not skip_hospitalizations:
        print(f'Peak hospital beds used : {hospitalizations.max():,.0f}')
    print(f'Total deaths            : {deaths.sum():,.0f}')

    if args.save_csv_fname:
        dates_str = np.array(list(map(str, dates)))
        combined_arr = np.vstack((dates_str, infections, hospitalizations,
                                  deaths, region_model.effective_r_arr)).T
        headers = 'dates,infections,hospitalizations,deaths,mean_r_t'
        np.savetxt(args.save_csv_fname,
                   combined_arr,
                   '%s',
                   delimiter=',',
                   header=headers)
        print('----------\nSaved file to:', args.save_csv_fname)
        for g in gparm:
            print "\t\tState %-3d: N=%-7d mean=%20.10e variance=%20.10e"%(g.state_id,g.counts,g.mean,g.variance)
        print "\n";
    
    
elif (op == 'insert'):
    import devicedb,postgresops
    devicedb.connect();
    fromtime = None
    totime = None
    dtype = "GHMM"
    
    ret,vals,sys.argv = utils.check_arg(sys.argv,'--from',1)
    if ret:
        try:
            fromtime = utils.str_to_date(vals[0])
        except ValueError, msg:
            print "Bad from time: "+str(msg)
    
    ret,vals,sys.argv = utils.check_arg(sys.argv,'--to',1)
    if ret:
        try:
            totime = utils.str_to_date(vals[0])
        except ValueError, msg:
            print "Bad to time: "+str(msg)
            
    ret,vals,sys.argv = utils.check_arg(sys.argv,'--type',1)
    if ret:
        dtype = vals[0]

    pretend,j,sys.argv = utils.check_arg(sys.argv,'--pretend')
def main(args):
    country = args.country
    region = args.region
    subregion = args.subregion
    skip_hospitalizations = args.skip_hospitalizations
    quarantine_perc = args.quarantine_perc
    quarantine_effectiveness = args.quarantine_effectiveness
    verbose = args.verbose

    if country != "US" and not region:
        region = "ALL"

    best_params_type = args.best_params_type
    assert best_params_type in ["mean", "median", "top", "top10"], best_params_type

    if args.best_params_dir:
        # Load parameters from file
        best_params = load_best_params_from_file(
            current_path / args.best_params_dir, country, region, subregion
        )
        simulation_start_date = str_to_date(best_params["first_date"])
        simulation_create_date = str_to_date(best_params["date"])
        simulation_end_date = str_to_date(best_params["projection_end_date"])

        region_params = {"population": best_params["population"]}
        # mean_params, median_params, top_params, or top10_params
        params_type_name = f"{best_params_type}_params"
        if verbose:
            print("best params type:", best_params_type)
        params_dict = convert_mean_params_to_params_dict(best_params[params_type_name])
    else:
        """
        You can hard code your own parameters if you do not want to use the preset parameters.

        This can be especially useful for regions/countries where we do not have projections.

        Then simply run `python run_simulation.py -v` to use these parameters.
        """

        simulation_start_date = datetime.date(2020, 2, 1)
        simulation_create_date = datetime.date.today()  # not used so can also be None
        simulation_end_date = datetime.date(2020, 10, 1)

        region_params = {"population": 332000000}
        params_dict = {
            "INITIAL_R_0": 2.24,
            "LOCKDOWN_R_0": 0.9,
            "INFLECTION_DAY": datetime.date(2020, 3, 18),
            "RATE_OF_INFLECTION": 0.25,
            "LOCKDOWN_FATIGUE": 1.0,
            "DAILY_IMPORTS": 500,
            "MORTALITY_RATE": 0.01,
            "REOPEN_DATE": datetime.date(2020, 5, 20),
            "REOPEN_SHIFT_DAYS": 0,
            "REOPEN_R": 1.2,
            "REOPEN_INFLECTION": 0.3,
            "POST_REOPEN_EQUILIBRIUM_R": 1.0,
            "FALL_R_MULTIPLIER": 1.001,
        }

    if args.simulation_start_date:
        simulation_start_date = str_to_date(args.simulation_start_date)
    if args.simulation_end_date:
        simulation_end_date = str_to_date(args.simulation_end_date)

    if args.set_param:
        print("---------------------------------------")
        print("Overwriting params from command line...")
        for param_name, param_value in args.set_param:
            assert param_name in params_dict, f"Unrecognized param: {param_name}"
            old_value = params_dict[param_name]
            new_value = convert_str_value_to_correct_type(param_value, old_value)
            print(f"Setting {param_name} to: {new_value}")
            params_dict[param_name] = new_value

    if args.change_param:
        print("---------------------------------------")
        print("Changing params from command line...")
        for param_name, value_change in args.change_param:
            assert param_name in params_dict, f"Unrecognized param: {param_name}"
            old_value = params_dict[param_name]
            new_value = old_value + convert_str_value_to_correct_type(
                value_change, old_value, use_timedelta=True
            )
            print(f"Changing {param_name} from {old_value} to {new_value}")
            params_dict[param_name] = new_value

    region_model = RegionModel(
        country,
        region,
        subregion,
        simulation_start_date,
        simulation_create_date,
        simulation_end_date,
        region_params,
        compute_hospitalizations=(not skip_hospitalizations),
    )

    if quarantine_perc > 0:
        print(f"Quarantine percentage: {quarantine_perc:.0%}")
        print(f"Quarantine effectiveness: {quarantine_effectiveness:.0%}")
        assert quarantine_effectiveness in [0.025, 0.1, 0.25, 0.5], (
            "must specify --quarantine_effectiveness percentage."
            " Possible values: [0.025, 0.1, 0.25, 0.5]"
        )
        quarantine_effectiveness_to_reduction_idx = {0.025: 0, 0.1: 1, 0.25: 2, 0.5: 3}
        region_model.quarantine_fraction = quarantine_perc
        region_model.reduction_idx = quarantine_effectiveness_to_reduction_idx[
            quarantine_effectiveness
        ]

    if verbose:
        print("================================")
        print(region_model)
        print("================================")
        print("Parameters:")
        for param_name, param_value in params_dict.items():
            print(f"{param_name:<25s} : {param_value}")

    real_death = []
    real_death_all = []
    with open(
        current_path / "../data/timeseries_prov/mortality_timeseries_prov.csv", "r"
    ) as csvfile:
        #        reader = csv.reader(csvfile)
        reader = csv.DictReader(csvfile)
        for row in reader:
            if row["province"] == args.subregion:
                real_death.append(float(row["deaths"]))
                real_death_all.append(float(row["cumulative_deaths"]))
        real_date = datetime.date(
            int(row["date_death_report"][-4:]),
            int(row["date_death_report"][3:5]),
            int(row["date_death_report"][:2]),
        )
    #        rows= [row for row in reader]

    # Add params to region_model
    #    for params_dict['REOPEN_SHIFT_DAYS'] in [12,3]:
    #    params_dict['MORTALITY_RATE']=0.013
    #    params_dict['INITIAL_R_0']=1.79

    #    params_dict['LOCKDOWN_R_0']=0.76
    #    params_dict['DAILY_IMPORTS']=160

    if verbose:
        print("--------------------------")
        print("Running simulation...")
        print("--------------------------")

    # Run simulation
    t = time.time()
    params_tups = tuple(params_dict.items())
    #    print(params_dict)
    region_model.init_params(params_tups)
    dates, infections, hospitalizations, deaths = run(region_model)

    deaths_total = deaths.cumsum()
    ind = np.where(dates == real_date)[0].item()
    deaths_proj = deaths_total[: ind + 1]

    if len(deaths_total[: ind + 1]) >= len(real_death_all):
        pad = len(deaths_proj) - len(real_death_all)
        real_death_all = np.array([0] * pad + real_death_all)

    if len(real_death_all) > len(deaths_total[: ind + 1]):
        real_death_all = np.array(real_death_all)
        pad = len(real_death_all) - len(deaths_total[: ind + 1])
        deaths_proj = np.zeros(pad).extend(deaths_proj)
    days = len(real_death_all)

    best_error = 1 / days * sum((real_death_all - deaths_proj) ** 2)
    MR = params_dict["MORTALITY_RATE"]  # =0.013
    IR = params_dict["INITIAL_R_0"]  # =1.79

    LR = params_dict["LOCKDOWN_R_0"]  # =0.76
    DI = params_dict["DAILY_IMPORTS"]  # =160

    for params_dict["INITIAL_R_0"] in np.linspace(IR * 0.7, IR):
        for params_dict["LOCKDOWN_R_0"] in np.linspace(LR * 0.95, LR * 1.05, 20):
            for params_dict["DAILY_IMPORTS"] in np.linspace(DI, DI * 1.35):
                for params_dict["MORTALITY_RATE"] in np.linspace(
                    MR * 1.5, MR * 2.5, 20
                ):
                    params_tups = tuple(params_dict.items())
                    region_model.init_params(params_tups)
                    dates, infections, hospitalizations, deaths = run(region_model)
                    deaths_total = deaths.cumsum()
                    deaths_proj = deaths_total[: ind + 1]

                    if len(real_death_all) > len(deaths_total[: ind + 1]):
                        real_death_all = np.array(real_death_all)
                        pad = len(real_death_all) - len(deaths_total[: ind + 1])
                        deaths_proj = np.zeros(pad).extend(deaths_total[: ind + 1])

                    error = 1 / days * sum((real_death_all - deaths_proj) ** 2)
                    if error <= best_error:
                        best_error = error
                        inf_proj, hosp_proj, death_proj = (
                            infections,
                            hospitalizations,
                            deaths,
                        )
                        best_parameters = {
                            "INITIAL_R_0": params_dict["INITIAL_R_0"],
                            "LOCKDOWN_R_0": params_dict["LOCKDOWN_R_0"],
                            "DAILY_IMPORTS": params_dict["DAILY_IMPORTS"],
                            "MORTALITY_RATE": params_dict["MORTALITY_RATE"],
                        }
                        print("Finding new optimum with error:", best_error)

    print(best_parameters)
    print(time.time() - t)

    """
    The following are lists with length N, where N is the number of days from
        simulation_start_date to simulation_end_date.

    dates            : datetime.date objects representing day i
    infections       : number of new infections on day i
    hospitalizations : occupied hospital beds on day i
    deaths           : number of new deaths on day i
    """
    infections, hospitalizations, deaths = inf_proj, hosp_proj, death_proj
    assert len(dates) == len(infections) == len(hospitalizations) == len(deaths)
    assert dates[0] == simulation_start_date
    assert dates[-1] == simulation_end_date

    if verbose:
        infections_total = infections.cumsum()
        deaths_total = deaths.cumsum()
        for i in range(len(dates)):
            hospitalization_str = ""
            if not skip_hospitalizations:
                hospitalization_str = (
                    f"Hospital beds in use: {hospitalizations[i]:,.0f} - "
                )
            daily_str = (
                f"{i+1:<3} - {dates[i]} - "
                f"New / total infections: {infections[i]:,.0f} / {infections_total[i]:,.0f} - "
                f"{hospitalization_str}"
                f"New / total deaths: {deaths[i]:,.2f} / {deaths_total[i]:,.1f} - "
                f"Mean R: {region_model.effective_r_arr[i]:.3f} - "
                f"IFR: {region_model.ifr_arr[i]:.2%}"
            )
            print(daily_str)  # comment out to spare console buffer
    print("-------------------------------------")
    print(f"End of simulation       : {region_model.projection_end_date}")
    print(f"Total infections        : {infections.sum():,.0f}")
    if not skip_hospitalizations:
        print(f"Peak hospital beds used : {hospitalizations.max():,.0f}")
    print(f"Total deaths            : {deaths.sum():,.0f}")

    plt.plot(deaths_total, color="blue", linewidth=3.0, linestyle="-.", label="Proj.")
    plt.plot(real_death_all, color="red", linewidth=3.0, linestyle="--", label="True")
    plt.xlabel("Days")
    plt.ylabel("Death")
    plt.title("Death Proj. Result")
    plt.legend(loc="best")
    plt.savefig(
        current_path / f"../output/Proj_{args.country}_{args.subregion}.png", dpi=1200
    )
    plt.close()

    if args.save_csv_fname:
        dates_str = np.array(list(map(str, dates)))
        combined_arr = np.vstack(
            (
                dates_str,
                infections,
                hospitalizations,
                deaths,
                region_model.effective_r_arr,
            )
        ).T
        headers = "dates,infections,hospitalizations,deaths,mean_r_t"
        np.savetxt(
            args.save_csv_fname, combined_arr, "%s", delimiter=",", header=headers
        )
        print("----------\nSaved file to:", args.save_csv_fname)