def extract(self, name, offset=0, interval=3600):
"""
Return generator of (datetime, array) tuples.
Yield arrays at times in the simulation accordin to given offset and
interval. It also yields at the last timestep in the period, because
animation (elsewhere in this repository) is only detected if there are
multiple frames to display.
:param name: name of netcdf variable to extract
:param offset: offset in seconds from start of calculation
:param interval: seconds between generated arrays
"""
start, stop = self.time.period
current = start + Timedelta(seconds=offset)
step = Timedelta(seconds=interval)
def result_for_time(time):
index, datetime = self.time.find(time)
array = self._read_snapshot(name=name, index=index)
return datetime, array
while current <= stop:
yield result_for_time(current)
current += step
yield result_for_time(stop)
def date_recv_binary(data: bytes, offset: int, length: int) -> date:
# 86400 seconds per day
seconds: float = i_unpack(data, offset)[0] * 86400
# Julian/Gregorian calendar cutoff point
if seconds < -12219292800: # October 4, 1582 -> October 15, 1582
seconds += 864000 # add 10 days worth of seconds
if seconds < -14825808000: # 1500-02-28 -> 1500-03-01
extraLeaps: float = (seconds + 14825808000) / 3155760000
extraLeaps -= 1
extraLeaps -= extraLeaps / 4
seconds += extraLeaps * 86400
microseconds: float = seconds * 1e6
try:
return (EPOCH + Timedelta(microseconds=microseconds)).date()
except OverflowError:
if Timedelta(microseconds=microseconds) < Timedelta(
seconds=EPOCH_SECONDS):
return date.min
else:
return date.max
except Exception as e:
raise e
def timestamptz_recv_integer(data: bytes, offset: int,
length: int) -> typing.Union[str, Datetime, int]:
micros: int = q_unpack(data, offset)[0]
try:
return EPOCH_TZ + Timedelta(microseconds=micros)
except OverflowError:
epoch_delta: Timedelta = Timedelta(seconds=EPOCH_SECONDS)
d_delta: Timedelta = Timedelta(microseconds=micros)
if d_delta < epoch_delta:
return Datetime.min
else:
return Datetime.max
def synchronize(self, keep, text, source, target):
# determine sources
dataset = Dataset(**source)
items = dataset.latest(Timedelta(**keep) // dataset.timedelta)
transfer = {}
for item in items:
filename = item["filename"]
transfer[item["datetime"].strftime(target["template"])] = filename
# list and inspect target dir
target_dir = target['dir']
threshold = Datetime.utcnow() - Timedelta(**keep)
for target_name_or_path in self.target.nlst(target_dir):
# some servers return names, others return paths
if target_name_or_path.startswith(target_dir):
target_path = target_name_or_path
target_name = basename(target_path)
else:
target_name = target_name_or_path
target_path = join(target_dir, target_name)
# remove from transfer dictionary if it is already present
if target_name in transfer:
del transfer[target_name]
# find old targets by name parsing and delete them
datetime = Datetime.strptime(
target_name[target['timestamp']], '%Y%m%d%H',
)
if datetime < threshold:
logger.info('Remove %s', target_name)
self.target.delete(target_path)
# transfer the rest
for target_name, source_name in transfer.items():
# read
data = io.BytesIO(dataset.retrieve(source_name))
logger.info('Retrieved %s', source_name)
# do not allow null characters in text format products
if text and b'\x00' in data.getvalue():
logger.info('Null characters found, skipping this one.')
continue
# write
target_path = join(target_dir, target_name)
target_path_in = target_path + '.in'
self.target.storbinary('STOR ' + target_path_in, data)
self.target.rename(target_path_in, target_path)
logger.info('Stored %s', target_name)
def test_interval_out(cursor):
retval = tuple(
cursor.execute(
"SELECT '1 month 16 days 12 hours 32 minutes 64 seconds'"
"::interval"))
expected_value = pg8000.Interval(
microseconds=(12 * 60 * 60 * 1000 * 1000) +
(32 * 60 * 1000 * 1000) + (64 * 1000 * 1000), days=16, months=1)
assert retval[0][0] == expected_value
retval = tuple(cursor.execute("select interval '30 seconds'"))
assert retval[0][0] == Timedelta(seconds=30)
retval = tuple(cursor.execute("select interval '12 days 30 seconds'"))
assert retval[0][0] == Timedelta(days=12, seconds=30)
def get_date_last_saturday():
### import with conventional class notation
from datetime import timedelta as Timedelta
from datetime import datetime as Datetime
from dateutil import relativedelta
today = Datetime.now()
if today.weekday() == 5:
weeks_back_in_time = 0 ### If today is saturday. we want "last saturday" to adtop today's value
else:
weeks_back_in_time = -1
print('weeks_back_in_time: {}'.format(weeks_back_in_time))
start = today - Timedelta((today.weekday() + 1) % 7)
dtm_last_saturday = start + relativedelta.relativedelta(
weekday=relativedelta.SA(weeks_back_in_time))
dt_last_saturday = dtm_last_saturday.date()
str_last_saturday = dt_last_saturday.strftime(format="%Y-%m-%d")
print('Last Saturday was {}'.format(str_last_saturday))
return str_last_saturday, dt_last_saturday
def parse_to_date(date_str):
if len(date_str) == 0:
return None
else:
dt = to_ct(Datetime.strptime(date_str, "%d/%m/%Y"))
dt += Timedelta(hours=23, minutes=30)
return to_utc(dt)
def interval_recv_integer(data: bytes, offset: int,
length: int) -> typing.Union[Timedelta, Interval]:
microseconds, days, months = typing.cast(typing.Tuple[int, ...],
qhh_unpack(data, offset))
seconds, micros = divmod(microseconds, 1e6)
if months != 0:
return Interval(microseconds, days, months)
else:
return Timedelta(days, seconds, micros)
def get_sources_and_requests(self, **request):
kwargs = self._snap_kwargs
start = request.get("start")
stop = request.get("stop")
mode = request["mode"]
start, stop = self._snap_to_resampled_labels(start, stop)
if start is None:
return [({"empty": True, "mode": mode}, None)]
# a time request does not involve a request to self.source
if mode == "time":
kwargs["mode"] = "time"
kwargs["start"] = start
kwargs["stop"] = stop
return [(kwargs, None)]
# vals or source requests do need a request to self.source
if self.frequency is None:
request["start"], request["stop"] = self.source.period
else:
if stop is None or start == stop:
# recover the period that is closest to start
start_period = stop_period = _label_to_period(start, **kwargs)
else:
# recover the period that has label >= start
start_period = _label_to_period(start, **kwargs)
# recover the period that has label <= stop
stop_period = _label_to_period(stop, **kwargs)
# snap request 'start' to the start of the first period
request["start"] = _ts_to_dt(start_period.start_time, self.timezone)
# snap request 'stop' to the end of the last period
request["stop"] = _ts_to_dt(stop_period.end_time, self.timezone)
if kwargs["closed"] != "left":
request["stop"] += Timedelta(microseconds=1)
# return sources and requests depending on the mode
kwargs["mode"] = request["mode"]
kwargs["start"] = start
kwargs["stop"] = stop
if mode == "vals":
kwargs["dtype"] = np.dtype(self.dtype).str
kwargs["statistic"] = self.statistic
time_request = {
"mode": "time",
"start": request["start"],
"stop": request["stop"],
}
# In case the data request dictates a temporal resolution,
# also set this in temporal request
if "time_resolution" in request:
time_request["time_resolution"] = request["time_resolution"]
return [(kwargs, None), (self.source, time_request), (self.source, request)]
def test_interval_roundtrip(cursor):
v = pg8000.Interval(microseconds=123456789, days=2, months=24)
cursor.execute("SELECT %s as f1", (v,))
retval = cursor.fetchall()
assert retval[0][0] == v
v = Timedelta(seconds=30)
cursor.execute("SELECT %s as f1", (v,))
retval = cursor.fetchall()
assert retval[0][0] == v
def testIntervalRoundtrip(self):
v = pg8000.Interval(microseconds=123456789, days=2, months=24)
self.cursor.execute("SELECT %s as f1", (v, ))
retval = self.cursor.fetchall()
self.assertEqual(retval[0][0], v)
v = Timedelta(seconds=30)
self.cursor.execute("SELECT %s as f1", (v, ))
retval = self.cursor.fetchall()
self.assertEqual(retval[0][0], v)
def test_interval_roundtrip(cursor):
v = nzpy.Interval(microseconds=123456789, days=2, months=24)
cursor.execute("SELECT '?' as f1", (v,))
retval = cursor.fetchall()
assert retval[0][0] == '<Interval 24 months 2 days 123456789 microseconds>'
v = Timedelta(seconds=30)
cursor.execute("SELECT '?' as f1", (v,))
retval = cursor.fetchall()
assert retval[0][0] == '0:00:30'
def __init__(self, dataset, version, step, pattern):
"""Represents a Dataplatform Dataset.
Args:
dataset (str): dataset name
version (str): dataset version
"""
self.url = self.URL.format(dataset=dataset, version=version)
self.timedelta = Timedelta(**step)
self.pattern = pattern
def timestamptz_recv_integer(data: bytes, offset: int,
length: int) -> typing.Union[str, Datetime, int]:
micros: int = q_unpack(data, offset)[0]
try:
return EPOCH_TZ + Timedelta(microseconds=micros)
except OverflowError:
if micros == INFINITY_MICROSECONDS:
return "infinity"
elif micros == MINUS_INFINITY_MICROSECONDS:
return "-infinity"
else:
return micros
def testIntervalOut(self):
self.cursor.execute(
"SELECT '1 month 16 days 12 hours 32 minutes 64 seconds'"
"::interval")
retval = self.cursor.fetchall()
expected_value = pg8000.Interval(
microseconds=(12 * 60 * 60 * 1000 * 1000) +
(32 * 60 * 1000 * 1000) + (64 * 1000 * 1000),
days=16,
months=1)
self.assertEqual(retval[0][0], expected_value)
self.cursor.execute("select interval '30 seconds'")
retval = self.cursor.fetchall()
expected_value = Timedelta(seconds=30)
self.assertEqual(retval[0][0], expected_value)
self.cursor.execute("select interval '12 days 30 seconds'")
retval = self.cursor.fetchall()
expected_value = Timedelta(days=12, seconds=30)
self.assertEqual(retval[0][0], expected_value)
def getWeekdayDates(date, weekday):
''' date: Date object - resets day to 1
weekday: int - 0=mon, 6=sun
returns: list of dates in month that match weekday
'''
date = date.replace(day=1)
delta = weekday - date.weekday()
if delta >= 0:
first_weekday = date + Timedelta(days=delta)
else:
first_weekday = date + Timedelta(days=delta + 7)
result = []
for i in range(5):
new_date = first_weekday + Timedelta(days=i * 7)
if new_date.month == date.month:
result.append(new_date)
else:
break
return result
def duration(self):
# type: () -> float
"""Compute the duration of the transfer, in seconds.
If the transfer has not started, this returns 0. If the transfer is
still ongoing, this returns how much time has elapsed so far.
"""
if not self.started_at:
timedelta = Timedelta()
else:
latest = self.stopped_at or Datetime.utcnow()
timedelta = latest - self.started_at
return timedelta.total_seconds()
def get_sources_and_requests(self, **request):
# a time request does not involve any resampling, so just propagate
if request["mode"] == "time":
return [({"mode": "time"}, None), (self.source, request)]
kwargs = self._snap_kwargs
start = request.get("start")
stop = request.get("stop")
mode = request["mode"]
# we need to now what times will be returned in order to figure out
# what times we need to compute the cumulative
time_data = self.source.get_data(mode="time", start=start, stop=stop)
if time_data is None or not time_data.get("time"):
# return early for an empty source
return [({"empty": True, "mode": mode}, None)]
# get the periods from the first and last timestamp
start = time_data["time"][0]
stop = time_data["time"][-1]
if self.frequency is None:
request["start"] = self.period[0]
request["stop"] = stop
else:
start_period = _get_bin_period(start, **kwargs)
# snap request 'start' to the start of the first period
request["start"] = _ts_to_dt(start_period.start_time,
self.timezone)
# snap request 'stop' to the last requested time
request["stop"] = stop
if kwargs["closed"] != "left":
request["stop"] += Timedelta(microseconds=1)
# return sources and requests depending on the mode
kwargs["mode"] = request["mode"]
kwargs["start"] = start
kwargs["stop"] = stop
if mode == "vals":
kwargs["dtype"] = np.dtype(self.dtype).str
kwargs["statistic"] = self.statistic
time_request = {
"mode": "time",
"start": request["start"],
"stop": request["stop"],
}
return [(kwargs, None), (self.source, time_request),
(self.source, request)]
def timedelta_in(data):
t = {}
curr_val = None
for k in data.split():
if ':' in k:
t['hours'], t['minutes'], t['seconds'] = map(float, k.split(':'))
else:
try:
curr_val = float(k)
except ValueError:
t[PGInterval.UNIT_MAP[k]] = curr_val
for n in ['weeks', 'months', 'years', 'decades', 'centuries', 'millennia']:
if n in t:
raise InterfaceError("Can't fit the interval " + str(t) +
" into a datetime.timedelta.")
return Timedelta(**t)
def interval_in(data):
t = {}
curr_val = None
for k in data.split():
if ":" in k:
t["hours"], t["minutes"], t["seconds"] = map(float, k.split(":"))
else:
try:
curr_val = float(k)
except ValueError:
t[PGInterval.UNIT_MAP[k]] = curr_val
for n in ["weeks", "months", "years", "decades", "centuries", "millennia"]:
if n in t:
raise InterfaceError(
f"Can't fit the interval {t} into a datetime.timedelta.")
return Timedelta(**t)
def hh(data_source):
ssp_rate_set = data_source.supplier_rate_sets['ssp-rate']
sbp_rate_set = data_source.supplier_rate_sets['sbp-rate']
for h in data_source.hh_data:
try:
sbp, ssp = data_source.caches['system_price'][h['start-date']]
except KeyError:
try:
system_price_cache = data_source.caches['system_price']
except KeyError:
system_price_cache = data_source.caches['system_price'] = {}
db_id = get_non_core_contract_id('system_price')
h_start = h['start-date']
rates = data_source.hh_rate(db_id, h_start)['gbp_per_nbp_mwh']
try:
try:
rdict = rates[key_format(h_start)]
except KeyError:
rdict = rates[key_format(h_start - Timedelta(days=3))]
sbp = float(rdict['sbp'] / 1000)
ssp = float(rdict['ssp'] / 1000)
system_price_cache[h_start] = (sbp, ssp)
except KeyError:
raise BadRequest("For the System Price rate script at " +
hh_format(h_start) +
" the rate cannot be found.")
except TypeError:
raise BadRequest(
"For the System Price rate script at " +
hh_format(h_start) +
" the rate 'rates_gbp_per_mwh' has the problem: " +
traceback.format_exc())
h['sbp'] = sbp
h['sbp-gbp'] = h['nbp-kwh'] * sbp
sbp_rate_set.add(sbp)
h['ssp'] = ssp
h['ssp-gbp'] = h['nbp-kwh'] * ssp
ssp_rate_set.add(ssp)
def hh(data_source):
for h in data_source.hh_data:
try:
sbp, ssp = data_source.caches["system_price"][h["start-date"]]
except KeyError:
try:
system_price_cache = data_source.caches["system_price"]
except KeyError:
system_price_cache = data_source.caches["system_price"] = {}
db_id = get_non_core_contract_id("system_price")
h_start = h["start-date"]
rates = data_source.hh_rate(db_id, h_start)["gbp_per_nbp_mwh"]
try:
try:
rdict = rates[key_format(h_start)]
except KeyError:
rdict = rates[key_format(h_start - Timedelta(days=3))]
sbp = float(rdict["sbp"] / 1000)
ssp = float(rdict["ssp"] / 1000)
system_price_cache[h_start] = (sbp, ssp)
except KeyError:
raise BadRequest(
"For the System Price rate script at "
+ hh_format(h_start)
+ " the rate cannot be found."
)
except TypeError:
raise BadRequest(
"For the System Price rate script at "
+ hh_format(h_start)
+ " the rate 'rates_gbp_per_mwh' has the problem: "
+ traceback.format_exc()
)
h["sbp"] = sbp
h["sbp-gbp"] = h["nbp-kwh"] * sbp
h["ssp"] = ssp
h["ssp-gbp"] = h["nbp-kwh"] * ssp
def test_timedelta_roundtrip(con):
v = Timedelta(seconds=30)
retval = con.run("SELECT cast(:v as interval)", v=v)
assert retval[0][0] == v
def time(self):
return Timedelta(milliseconds=self.args[1])
elif 1860 <= now_in_seconds < 3600:
starting_time = 3600 - now_in_seconds
i = 1
while True:
print("Iteration No. {}: scheduler starts in {} seconds.".format(
i, starting_time))
scheduler.enter(starting_time, 1, task)
scheduler.run()
starting_time = __period
if Datetime.now().second % 10 > 2:
starting_time -= 1
i += 1
def __to_seconds(t):
return t.hour * 3600 + t.minute * 60 + t.second
def __convert_to_local_time(t):
return (t - (get_time_zone_diff() * 3600)) % 86400
####################################################################
test_bot_token = "425426086:AAFtPbcx_YNjAzZgdudQyQ5yuQ48x2g6O6A"
main_bot_token = "413427401:AAEgcTahApxJLAPGHK43TfJAl40K7CdJ8pw"
journal_bot = Bot(main_bot_token)
scheduler = Scheduler()
__period = Timedelta(minutes=30).total_seconds()
def add_gigasecond(date):
assert type(date) == type(Date.today()), "Input must be a valid date object."
gigasecond = Timedelta(seconds=(10**9))
return date + gigasecond
def job_001(self, context):
"""这个作业的名称是 001
启动方式：$ python jobs.py start 001
"""
from iproxy import ProxyPoolContext, ProxyLoaderContext, ProxyValidatorContext, \
FatezeroProxySpider, IPValidator
from handler import HandlerContext, ProxyValidateHandler, MySQLStreamInserter
from datetime import timedelta as Timedelta
from database import MySQLOperation
## 0. 配置上下文，为各个组件提供全局环境
ctx = {
'job_name': context.job_name,
'job_time': context.job_time,
'logger': context.logger,
}
## 1. 创建代理池
pool = ProxyPool(context=ProxyPoolContext(**ctx))
## 2. 加载代理
# 创建代理加载器
loader = FatezeroProxySpider(
timeout=60, # 设置超时（可选）
num=5000, # 需要加载的代理数量
context=ProxyLoaderContext(**ctx),
)
# 执行加载
pool.load(loader)
## 3. 准备验证器
v = IPValidator(
**IPValidator.PLAN_IP138, # 指定验证方式，这里使用预定义方案
timeout=5, # 设置超时（可选）
context=ProxyValidatorContext(**ctx),
)
## 4. 准备过滤器
# 创建代理过滤器（以下参数都是可选的）
pf = SimpleProxyFilter(
# port_list=[80, 8080], # 端口号
# protocol_list=['http', 'https'], # 协议
# local_list=['home'], # 验证地区
# collected_timedelta=Timedelta(days=1),# 收录时间与当前时间的距离（最大值）
)
# 创建代理测试过滤器（以下参数都是可选的）
ptf = SimpleProxyTestFilter(
proxy_filter=pf, # 代理过滤器
response_elapsed_mean=6, # 平均响应时长（最大值，秒）
transfer_elapsed_mean=10, # 平级传输时长（最大值，秒）
timeout_exception_pr=0.34, # 超时异常概率（最大值）
proxy_exception_pr=0.34, # 代理异常概率（最大值）
valid_responses_pr=1, # 有效响应概率（最小值）
pre_tested_timedelta=Timedelta(days=1), # 每个测试的前置条件-测试时间与当前时间的距离（最大值）
pre_verification_ip=True, # 每个测试的前置条件-是否经过IP验证
# pre_valid_responses=True, # 每个测试的前置条件-测是否有效响应
)
## 5. 准备处理器
# 创建代理处理器
ph = MySQLStreamInserter(
buffer_size=50, # 缓冲区大小
concurrency=10, # 最大并发数量
context=HandlerContext(**ctx),
)
# 创建测试日志处理器
tlh = MySQLStreamInserter(
buffer_size=50, # 缓冲区大小
concurrency=10, # 最大并发数量
context=HandlerContext(**ctx),
)
# 创建验证结果处理器，负责达标代理的处理（如入库）
h = ProxyValidateHandler(
proxy_handler=ph, # 针对Proxy的子处理器（可选）
test_log_handler=tlh, # 针对TestLog的子处理器（可选）
proxy_test_filter=ptf, # 指定过滤器，筛选达标的代理（可选）
context=HandlerContext(**ctx),
)
## 6. 执行验证
MySQLOperation.init_pool()
pool.verify(
validator=v, # 验证器
handler=h, # 处理器
repeat=3, # 每个代理的重复验证次数
concurrency=10, # 最大并发数量
sleep=1, # 线程间歇（秒）
)
MySQLOperation.close_pool()
def test_interval_in_30_seconds(con):
retval = con.run("select interval '30 seconds'")
assert retval[0][0] == Timedelta(seconds=30)
def test_interval_in_12_days_30_seconds(con):
retval = con.run("select interval '12 days 30 seconds'")
assert retval[0][0] == Timedelta(days=12, seconds=30)
def https_handler(sess, log_f, properties, contract, now=None):
url_template_str = properties["url_template"]
url_values = properties.get("url_values", {})
download_days = properties["download_days"]
if now is None:
now = utc_datetime_now()
window_finish = utc_datetime(now.year, now.month, now.day) - HH
window_start = utc_datetime(now.year, now.month,
now.day) - Timedelta(days=download_days)
log_f(f"Window start: {hh_format(window_start)}")
log_f(f"Window finish: {hh_format(window_finish)}")
env = jinja2.Environment(autoescape=True, undefined=jinja2.StrictUndefined)
url_template = env.from_string(url_template_str)
for era in (sess.query(Era).filter(
Era.dc_contract == contract,
Era.start_date <= window_finish,
or_(Era.finish_date == null(), Era.finish_date >= window_start),
).distinct()):
chunk_start = hh_max(era.start_date, window_start)
chunk_finish = hh_min(era.finish_date, window_finish)
for mpan_core in (era.imp_mpan_core, era.exp_mpan_core):
if mpan_core is None:
continue
log_f(f"Looking at MPAN core {mpan_core}.")
vals = {"chunk_start": chunk_start, "chunk_finish": chunk_finish}
vals.update(url_values.get(mpan_core, {}))
try:
url = url_template.render(vals)
except jinja2.exceptions.UndefinedError as e:
raise BadRequest(
f"Problem rendering the URL template: {url_template_str}. "
f"The problem is: {e}. This can be fixed by editing the "
f"properties of this contract.")
log_f(f"Retrieving data from {url}.")
sess.rollback() # Avoid long transactions
res = requests.get(url, timeout=120)
res.raise_for_status()
result = requests.get(url, timeout=120).json()
if isinstance(result, dict):
result_data = result["DataPoints"]
elif isinstance(result, list):
result_data = result
else:
raise BadRequest(
f"Expecting a JSON object at the top level, but instead got "
f"{result}")
raw_data = []
for jdatum in result_data:
raw_data.append(
dict(
mpan_core=mpan_core,
start_date=utc_datetime(1, 1, 1) +
Timedelta(seconds=jdatum["Time"] / 10000000),
channel_type="ACTIVE",
value=jdatum["Value"],
status="A",
))
HhDatum.insert(sess, raw_data, contract)
sess.commit()
log_f("Finished loading.")
return False