Esempio n. 1
0
def parse_date(row, idx, is_finish=False):
    date_str = row[idx].strip()
    try:
        if len(date_str) == 10:
            dt = utc_datetime_parse(date_str, "%Y-%m-%d")
            if is_finish:
                dt = dt + relativedelta(days=1) - HH
        else:
            dt = utc_datetime_parse(date_str, "%Y-%m-%d %H:%M")
        return validate_hh_start(dt)
    except ValueError as e:
        raise BadRequest(
            "Difficulty parsing a date in the row {row} at position "
            "{idx}: {e}".format(row=row, idx=idx, e=e))
Esempio n. 2
0
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check bank holidays")
                    contract = Contract.get_non_core_by_name(
                        sess, 'bank_holidays')
                    contract_props = contract.make_properties()

                    if contract_props.get('enabled', False):
                        url_str = contract_props['url']

                        self.log("Downloading from " + url_str + ".")
                        res = requests.get(url_str)
                        self.log(
                            ' '.join(
                                (
                                    "Received", str(res.status_code),
                                    res.reason)))
                        PREFIX = 'DTSTART;VALUE=DATE:'
                        hols = collections.defaultdict(list)
                        for line in res.text.splitlines():
                            if line.startswith(PREFIX):
                                dt = utc_datetime_parse(line[-8:], "%Y%m%d")
                                hols[dt.year].append(dt)

                        for year in sorted(hols.keys()):
                            year_start = utc_datetime(year, 1, 1)
                            year_finish = year_start + \
                                relativedelta(years=1) - HH
                            rs = sess.query(RateScript).filter(
                                RateScript.contract == contract,
                                RateScript.start_date == year_start).first()
                            if rs is None:
                                self.log(
                                    "Adding a new rate script starting at " +
                                    hh_format(year_start) + ".")

                                latest_rs = sess.query(RateScript).filter(
                                    RateScript.contract == contract).\
                                    order_by(RateScript.start_date.desc()). \
                                    first()

                                contract.update_rate_script(
                                    sess, latest_rs, latest_rs.start_date,
                                    year_finish, latest_rs.script)
                                rs = contract.insert_rate_script(
                                    sess, year_start, '')

                            script = {
                                'bank_holidays': [
                                    v.strftime("%Y-%m-%d")
                                    for v in hols[year]]}

                            self.log(
                                "Updating rate script starting at " +
                                hh_format(year_start) + ".")
                            contract.update_rate_script(
                                sess, rs, rs.start_date, rs.finish_date,
                                json.dumps(
                                    script, indent='    ', sort_keys=True))
                            sess.commit()
                    else:
                        self.log(
                            "The automatic importer is disabled. To "
                            "enable it, edit the contract properties to "
                            "set 'enabled' to True.")

                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    if sess is not None:
                        sess.close()
                    self.lock.release()
                    self.log("Finished checking bank holidays.")

            self.going.wait(24 * 60 * 60)
            self.going.clear()
Esempio n. 3
0
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check bank holidays")
                    contract = Contract.get_non_core_by_name(sess, "bank_holidays")
                    contract_props = contract.make_properties()

                    if contract_props.get("enabled", False):
                        url_str = contract_props["url"]

                        self.log("Downloading from " + url_str + ".")
                        res = requests.get(url_str)
                        self.log(
                            " ".join(("Received", str(res.status_code), res.reason))
                        )
                        PREFIX = "DTSTART;VALUE=DATE:"
                        hols = collections.defaultdict(list)
                        for line in res.text.splitlines():
                            if line.startswith(PREFIX):
                                dt = utc_datetime_parse(line[-8:], "%Y%m%d")
                                hols[dt.year].append(dt)

                        for year in sorted(hols.keys()):
                            year_start = utc_datetime(year, 1, 1)
                            year_finish = year_start + relativedelta(years=1) - HH
                            rs = (
                                sess.query(RateScript)
                                .filter(
                                    RateScript.contract == contract,
                                    RateScript.start_date == year_start,
                                )
                                .first()
                            )
                            if rs is None:
                                self.log(
                                    "Adding a new rate script starting at "
                                    + hh_format(year_start)
                                    + "."
                                )

                                latest_rs = (
                                    sess.query(RateScript)
                                    .filter(RateScript.contract == contract)
                                    .order_by(RateScript.start_date.desc())
                                    .first()
                                )

                                contract.update_rate_script(
                                    sess,
                                    latest_rs,
                                    latest_rs.start_date,
                                    year_finish,
                                    loads(latest_rs.script),
                                )
                                rs = contract.insert_rate_script(sess, year_start, {})

                            script = {
                                "bank_holidays": [
                                    v.strftime("%Y-%m-%d") for v in hols[year]
                                ]
                            }

                            contract.update_rate_script(
                                sess, rs, rs.start_date, rs.finish_date, script
                            )
                            sess.commit()
                            self.log(
                                "Updated rate script starting at "
                                + hh_format(year_start)
                                + "."
                            )
                    else:
                        self.log(
                            "The automatic importer is disabled. To "
                            "enable it, edit the contract properties to "
                            "set 'enabled' to True."
                        )

                except BaseException:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    if sess is not None:
                        sess.close()
                    self.lock.release()
                    self.log("Finished checking bank holidays.")

            self.going.wait(24 * 60 * 60)
            self.going.clear()
Esempio n. 4
0
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check RCRCs.")
                    contract = Contract.get_non_core_by_name(sess, "rcrc")
                    latest_rs = (
                        sess.query(RateScript)
                        .filter(RateScript.contract_id == contract.id)
                        .order_by(RateScript.start_date.desc())
                        .first()
                    )
                    latest_rs_id = latest_rs.id
                    latest_rs_start = latest_rs.start_date

                    month_start = latest_rs_start + relativedelta(months=1)
                    month_finish = month_start + relativedelta(months=1) - HH
                    now = utc_datetime_now()
                    if now > month_finish:
                        self.log(
                            "Checking to see if data is available from "
                            + hh_format(month_start)
                            + " to "
                            + hh_format(month_finish)
                            + " on Elexon Portal."
                        )
                        config = Contract.get_non_core_by_name(sess, "configuration")
                        props = config.make_properties()

                        scripting_key = props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
                        if scripting_key is None:
                            raise BadRequest(
                                "The property "
                                + ELEXON_PORTAL_SCRIPTING_KEY_KEY
                                + " cannot be found in the configuration "
                                "properties."
                            )

                        contract_props = contract.make_properties()
                        url_str = "".join(
                            (
                                contract_props["url"],
                                "file/download/RCRC_FILE?key=",
                                scripting_key,
                            )
                        )

                        r = requests.get(url_str, timeout=60)
                        parser = csv.reader(
                            (x.decode() for x in r.iter_lines()),
                            delimiter=",",
                            quotechar='"',
                        )
                        next(parser)
                        next(parser)
                        month_rcrcs = {}
                        for values in parser:
                            hh_date = utc_datetime_parse(values[0], "%d/%m/%Y")
                            hh_date += relativedelta(minutes=30 * int(values[2]))
                            if month_start <= hh_date <= month_finish:
                                month_rcrcs[key_format(hh_date)] = Decimal(values[3])
                        if key_format(month_finish) in month_rcrcs:
                            self.log("The whole month's data is there.")
                            script = {"rates": month_rcrcs}
                            contract = Contract.get_non_core_by_name(sess, "rcrc")
                            rs = RateScript.get_by_id(sess, latest_rs_id)
                            contract.update_rate_script(
                                sess, rs, rs.start_date, month_finish, loads(rs.script)
                            )
                            contract.insert_rate_script(sess, month_start, script)
                            sess.commit()
                            self.log(
                                "Added a new rate script starting at "
                                + hh_format(month_start)
                                + "."
                            )
                        else:
                            msg = "There isn't a whole month there yet."
                            if len(month_rcrcs) > 0:
                                msg += (
                                    " The last date is "
                                    + sorted(month_rcrcs.keys())[-1]
                                )
                            self.log(msg)
                except BaseException:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    self.lock.release()
                    self.log("Finished checking RCRC rates.")
                    if sess is not None:
                        sess.close()

            self.going.wait(30 * 60)
            self.going.clear()
Esempio n. 5
0
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check RCRCs.")
                    contract = Contract.get_non_core_by_name(sess, 'rcrc')
                    latest_rs = sess.query(RateScript).filter(
                        RateScript.contract_id == contract.id).order_by(
                        RateScript.start_date.desc()).first()
                    latest_rs_id = latest_rs.id
                    latest_rs_start = latest_rs.start_date

                    month_start = latest_rs_start + relativedelta(months=1)
                    month_finish = month_start + relativedelta(months=1) - HH
                    now = utc_datetime_now()
                    if now > month_finish:
                        self.log(
                            "Checking to see if data is available from " +
                            str(month_start) + " to " + str(month_finish) +
                            " on Elexon Portal.")
                        config = Contract.get_non_core_by_name(
                            sess, 'configuration')
                        props = config.make_properties()

                        scripting_key = props.get(
                            ELEXON_PORTAL_SCRIPTING_KEY_KEY)
                        if scripting_key is None:
                            raise BadRequest(
                                "The property " +
                                ELEXON_PORTAL_SCRIPTING_KEY_KEY +
                                " cannot be found in the configuration "
                                "properties.")

                        contract_props = contract.make_properties()
                        url_str = ''.join(
                            (
                                contract_props['url'],
                                'file/download/RCRC_FILE?key=',
                                scripting_key))

                        r = requests.get(url_str)
                        parser = csv.reader(
                            (l.decode() for l in r.iter_lines()),
                            delimiter=',', quotechar='"')
                        piterator = iter(parser)
                        values = next(piterator)
                        values = next(piterator)
                        month_rcrcs = {}
                        for values in piterator:
                            hh_date = utc_datetime_parse(values[0], "%d/%m/%Y")
                            hh_date += relativedelta(minutes=30*int(values[2]))
                            if month_start <= hh_date <= month_finish:
                                month_rcrcs[key_format(hh_date)] = values[3]

                        if key_format(month_finish) in month_rcrcs:
                            self.log("The whole month's data is there.")
                            script = "def rates():\n    return {\n" + \
                                ',\n'.join(
                                    "'" + k + "': " + month_rcrcs[k] for k in
                                    sorted(month_rcrcs.keys())) + "}"
                            contract = Contract.get_non_core_by_name(
                                sess, 'rcrc')
                            rs = RateScript.get_by_id(sess, latest_rs_id)
                            contract.update_rate_script(
                                sess, rs, rs.start_date, month_finish,
                                rs.script)
                            contract.insert_rate_script(
                                sess, month_start, script)
                            sess.commit()
                            self.log("Added new rate script.")
                        else:
                            msg = "There isn't a whole month there yet."
                            if len(month_rcrcs) > 0:
                                msg += " The last date is " + \
                                    sorted(month_rcrcs.keys())[-1]
                            self.log(msg)
                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    self.lock.release()
                    self.log("Finished checking RCRC rates.")
                    if sess is not None:
                        sess.close()

            self.going.wait(30 * 60)
            self.going.clear()