def get_barcode_forecast(data_frame: DataFrame, now: Arrow,
for_date: Arrow) -> Series:
if not data_frame.empty:
last_data_date = arrow.get(data_frame.index.max())
beg_date = now.shift(months=-1)
end_date = for_date
real = create_df_with_zeroes(data_frame, beg_date.shift(weeks=-1),
end_date)
old = create_df_with_zeroes(data_frame, beg_date.shift(weeks=-1),
end_date, lambda a: a.shift(years=-1))
real_smoothed = smooth_df(real, beg_date, last_data_date)
old_smoothed = smooth_df(old, beg_date, end_date)
if not real_smoothed.empty:
_, diff = compare_df(real_smoothed, old_smoothed,
now.shift(days=1), end_date)
result_forecast = shift_df(old_smoothed, diff, now.shift(days=1),
end_date)
return result_forecast['quantity'][now.date():for_date.date()]
else:
raise Exception('Empty data frame')
else:
raise Exception('Empty data frame')
def make_time_list(terminus_times: list, current_time: arrow.Arrow,
data_pool: str):
"""
:param terminus_times:
:type terminus_times:
:param current_time:
:type current_time:
:param data_pool:
:type data_pool:
:return:
:rtype:
"""
time_list = []
previous_hour = int(current_time.shift(hours=-1).format('HH'))
current_hour = int(current_time.format('HH'))
next_hour = int(current_time.shift(hours=1).format('HH'))
prev_hr = find_hr(terminus_times, previous_hour)
curr_hr = find_hr(terminus_times, current_hour)
next_hr = find_hr(terminus_times, next_hour)
for hour_set in [prev_hr, curr_hr, next_hr]:
if hour_set:
hour = hour_set.get('hour')
minute_set = hour_set.get(data_pool)
for minutes in minute_set:
time_list.append(make_time(hour, minutes))
return time_list
def get_weekday_after(date: arrow.Arrow, iso_weekday: int) -> arrow.Arrow:
"""
Returns the first day with the given weekday that is equal to or greater than
the given date.
"""
new_date = date.shift(weekday=iso_weekday - 1)
return new_date
def DailyCallback(self, schedule_time: arrow.Arrow, fn: Callable, *args,
**kwargs) -> Job:
"""Schedules fn to be run once a day at schedule_time.
The actual scheduled time is perturbed randomly +/-30s unless the kwarg
'_jitter' is set to False.
Args:
schedule_time: An Arrow object specifying when to run fn.
fn: The function to be run.
*args: Arguments to pass to fn.
**kwargs: Keyworded arguments to pass to fn. Special kwargs listed below:
_jitter - {int} How many seconds to perturb scheduling time by, in
both directions. Defaults to 30s.
Returns:
APScheduler Job.
"""
if self._local_tz:
schedule_time = schedule_time.to(self._local_tz)
jitter = kwargs.get('_jitter', 30)
if jitter:
jitter_secs = random.randint(-jitter, jitter)
schedule_time = schedule_time.shift(seconds=jitter_secs)
kwargs.pop('_jitter', None)
# APScheduler 2.1.2 doesn't understand timezones.
return self._scheduler.add_interval_job(fn,
args=args,
kwargs=kwargs,
start_date=schedule_time.naive,
days=1)
def apply_weekday_offset(start_time: arrow.Arrow,
week_start: str) -> arrow.Arrow:
"""
Apply the offset required to move the start date `start_time` of a week
starting on Monday to that of a week starting on `week_start`.
"""
weekdays = dict(
zip(
[
"monday",
"tuesday",
"wednesday",
"thursday",
"friday",
"saturday",
"sunday",
],
range(0, 7),
))
new_start = week_start.lower()
if new_start not in weekdays:
return start_time
now = datetime.datetime.now()
offset = weekdays[new_start] - 7 * (weekdays[new_start] > now.weekday())
# -> Array.shift(days=)
return start_time.shift(days=offset)
def get_date_range_with_end_date(
statistical_interval: Optional[str],
end_date: arrow.Arrow) -> Tuple[arrow.Arrow, arrow.Arrow]:
# end_date = end_date.shift(days=1)
if statistical_interval:
if statistical_interval == "daily":
# start = end_date.shift(days=-1)
return end_date.floor('day'), end_date.ceil('day')
elif statistical_interval == "monthly":
start = end_date.shift(months=-1)
return start.floor('day'), end_date.ceil('day')
elif statistical_interval == "weekly":
start = end_date.shift(weeks=-1)
return start.floor('day'), end_date.ceil('day')
else:
# start = end_date.shift(days=-1)
return end_date.floor('day'), end_date.ceil('day')
def exam_dates(begin: Tuple[int, int]) -> Iterable[Tuple[int, int]]:
date = Arrow(2020, begin[0], begin[1], tzinfo=TIMEZONE)
while True:
yield date.date().month, date.date().day
while True:
date = date.shift(days=1)
if date.isoweekday() < 6:
break
def combine_date_time(date: arrow.Arrow, time: arrow.Arrow) -> arrow.Arrow:
"""
Returns an Arrow object with the date of the first argument and the time of the second.
"""
return date.shift(hours=time.hour,
minutes=time.minute,
seconds=time.second,
microseconds=time.microsecond)
def get_mean_forecast(data_frame: DataFrame, now: Arrow,
for_date: Arrow) -> Series:
tomorrow = now.shift(days=1)
beg = now.shift(days=-6)
arr = []
for day in Arrow.range('day', beg, now):
try:
value = data_frame.loc[day.date()].values[0]
except KeyError:
value = 0.0
arr.append([day.date(), value])
init_df = create_df_indexed_by_date(create_df(arr))
df = create_df_with_zeroes(init_df, beg, for_date)
for day in Arrow.range('day', tomorrow, for_date):
yesterday = day.shift(days=-1)
past_week = df[yesterday.shift(days=-6).date():yesterday.date()]
df.loc[day.date()] = past_week.mean()
return df['quantity'][tomorrow.date():for_date.date()]
def check_datadog(objective: models.Objective, query_datetime: arrow.Arrow) -> float:
response = api.Metric.query(
start=query_datetime.shift(days=-30).timestamp,
end=query_datetime.timestamp,
query=objective.indicator_query,
)
series = response["series"][0]
point = series["pointlist"][0]
_, point_value = point
return round(Decimal(point_value), 4)
def run_captain(
distance: float,
ship: OurShip,
days_spent: int,
date: Arrow,
initial_destination: str,
) -> str:
"""
Simulate the entire lifetime of a captain and their ship, returning a log of their endeavours
"""
log = ""
destination = initial_destination
while not ship.destroyed:
date_already_logged = False
travel_log, date_already_logged = do_travel(distance, ship, days_spent,
date)
log += travel_log
# We have arrived!
if ship.destroyed:
break
if not date_already_logged:
log += f"\n{get_date(date)}"
location_log = do_location(
ship,
destination,
days_spent,
data.place_names,
)
(
new_destination,
new_distance,
destination_log,
) = navigation.get_new_heading(
destination,
data.place_coords,
ship.visited,
)
log += location_log
log += destination_log
# Re-initialise variables
date = date.shift(days=+1)
days_spent = 0
destination = new_destination
distance = new_distance
date_already_logged = False
return log
def get_category_forecast(barcode_data_frame: DataFrame,
category_data_frame: DataFrame, now: Arrow,
for_date: Arrow) -> Series:
tomorrow = now.shift(days=1)
past_month = now.shift(months=-1)
barcode_df = create_df_with_zeroes(barcode_data_frame, past_month, now)
barcode_smoothed = smooth_df(barcode_df, past_month, now)
category_df = create_df_with_zeroes(category_data_frame, past_month,
for_date, lambda a: a.shift(years=-1))
category_smoothed = smooth_df(category_df, past_month, for_date)
barcode_series = barcode_smoothed['quantity'][past_month.date():now.date()]
category_series = category_smoothed['quantity'][past_month.date():now.date(
)]
percent, _ = compare_df(barcode_series.to_frame(),
category_series.to_frame(), past_month, now)
forecast = category_smoothed['quantity'][tomorrow.date():for_date.date()]
forecast_normalized = increase_df(forecast.to_frame(), percent, tomorrow,
for_date)
return forecast_normalized['quantity'][now.date():for_date.date()]
async def clear_raid(self, guild_id, raid_cooldown_end: arrow.Arrow):
"""
cleares the current raid ( resets redis stuff )
sets the cleared at for the current active raid in postgres ( used for the stat upload,
since that is bound to a specific raid )
"""
logger.debug("clear_raid: raid_cooldown_end: %s", raid_cooldown_end)
raid_config = await self.raid_dao.get_raid_configuration(guild_id)
spawn_timestamp = raid_config.get(RAID_SPAWN, 0)
cooldown_timestamp = raid_config.get(RAID_COOLDOWN, 0)
now = arrow.utcnow()
spawn = arrow.get(spawn_timestamp)
if not spawn_timestamp and not cooldown_timestamp:
raise NoRaidActive()
if cooldown_timestamp:
raise RaidAlreadyCleared()
if now < spawn:
raise RaidUnspawned()
delta_now_cd = raid_cooldown_end - now
logger.debug("clear_raid: time until raid cooldown ends from now %s",
delta_now_cd)
# ~2 secs for delays and stuff
raid_cooldown_start = raid_cooldown_end.shift(minutes=-59, seconds=-59)
time_needed_to_clear = raid_cooldown_start - spawn
logger.debug(
"clear_raid: raid cooldown started at %s, time needed to clear raid: %s",
raid_cooldown_start,
time_needed_to_clear,
)
if spawn > raid_cooldown_start:
logger.warning(
"clear_raid: something went wrong :o spawn %s is after raid cooldown start %s",
spawn,
raid_cooldown_start,
)
raise ValueError("Raid cooldown must be 60m after spawn")
await self.raid_postgres_dao.complete_last_raid_stat_entry(
guild_id, raid_cooldown_start)
await self._clear_current_raid_data(guild_id)
await self.raid_dao.set_raid_cooldown(guild_id,
raid_cooldown_end.timestamp)
return time_needed_to_clear
def get_metrics_for_signal(
cluster: str,
pool: str,
scheduler: str,
app: str,
metrics_client: ClustermanMetricsBotoClient,
required_metrics: List[MetricsConfigDict],
end_time: arrow.Arrow,
) -> MetricsValuesDict:
""" Get the metrics required for a signal """
metrics: MetricsValuesDict = defaultdict(list)
for metric_dict in required_metrics:
if metric_dict['type'] not in (SYSTEM_METRICS, APP_METRICS):
raise MetricsError(
f"Metrics of type {metric_dict['type']} cannot be queried by signals."
)
# Need to add the cluster/pool to get the right system metrics
# TODO (CLUSTERMAN-126) this should probably be cluster/pool/app eventually
# TODO (CLUSTERMAN-446) if a mesos pool and a k8s pool share the same app_name,
# APP_METRICS will be used for both
if metric_dict['type'] == SYSTEM_METRICS:
dims_list = [get_cluster_dimensions(cluster, pool, scheduler)]
if scheduler == 'mesos': # handle old (non-scheduler-aware) metrics
dims_list.insert(0,
get_cluster_dimensions(cluster, pool, None))
else:
dims_list = [{}]
# We only support regex expressions for APP_METRICS
if 'regex' not in metric_dict:
metric_dict['regex'] = False
start_time = end_time.shift(minutes=-metric_dict['minute_range'])
for dims in dims_list:
query_results = metrics_client.get_metric_values(
metric_dict['name'],
metric_dict['type'],
start_time.timestamp,
end_time.timestamp,
is_regex=metric_dict['regex'],
extra_dimensions=dims,
app_identifier=app,
)
for metric_name, timeseries in query_results.items():
metrics[metric_name].extend(timeseries)
# safeguard; the metrics _should_ already be sorted since we inserted the old
# (non-scheduler-aware) metrics before the new metrics above, so this should be fast
metrics[metric_name].sort()
return metrics
def __generate_search_url_by_day(query: str, date: Arrow):
"""
Returns a string with a url to ask twitter for a query in a day
:param query:str twitter advanced query string
:param date: date to query
:return: url for date
"""
search_url = '%s since:%s until:%s' % (query, date.format('YYYY-MM-DD'),
date.shift(
days=1).format('YYYY-MM-DD'))
search_url = 'https://mobile.twitter.com/search?q=' + urllib.parse.quote_plus(
search_url)
logger.debug(f"Generated url: {search_url}")
return search_url
def export_monthly(self, target_month: arrow.Arrow):
"""
导入一个月的数据
:param target_month: 要导入的月份
:return:
"""
batch_size = self.batch_size
start_i = 0
epoch = 0
df = pandas.DataFrame()
self._tmp_show_index = 0
total_num = 0
for batch_i in range((self.all_vins_len // batch_size) + 1):
# 选取新的vin号查询
end_i = (batch_i + 1) * batch_size
self._tmp_show_index = end_i
if end_i > self.all_vins_len:
end_i = self.all_vins_len
select_vins = self.all_vins[start_i:end_i]
# 月查询
dd, err_list = self.export(
select_vins, [(target_month, target_month.shift(months=1))])
df = df.append(dd)
# 错误vin号划分为日查询
if len(err_list) > 0:
df = df.append(self.export_daily2month(err_list, target_month))
# 后续处理
df_len = len(df)
start_i = end_i
if df_len >= self.max_per_batch or end_i == self.all_vins_len:
epoch += 1
total_num += df_len
file_path = f"{self.data_dir}{target_month.format('YYYY-MM')}/"
if not os.path.exists(file_path):
os.makedirs(file_path)
df.to_parquet(f"{file_path}{epoch}.snappy.parquet",
index=False)
df = pandas.DataFrame()
self._show_process(target_month, end_i, df_len)
logger.info(
f'completed:{target_month}_{epoch},len={df_len},,process={round(self._tmp_show_index / self.all_vins_len * 100, 4)}%'
)
self.record_batch_ok(target_month.format('YYYY-MM'),
select_vins)
logger.info(f'completed:{target_month},total_num={total_num}')
return total_num
def addTime(self, cur: arrow.Arrow, fore_unit: int) -> arrow.Arrow:
if fore_unit == 0:
cur = cur.shift(years=1)
elif fore_unit == 1:
cur = cur.shift(months=1)
elif fore_unit == 2:
cur = cur.shift(days=1)
elif fore_unit == 3:
cur = cur.shift(hours=1)
elif fore_unit == 4:
cur = cur.shift(minutes=1)
elif fore_unit == 5:
cur = cur.shift(seconds=1)
return cur
def export_hourly2day(self, target_day: arrow.Arrow):
"""
将一天拆成小时查询并合并
:param vins: 需要查询的vin
:param target_day: 需要查询的精确到天的时间
:return: 查询的vin的目标日的数据
"""
day_hours = list(
target_day.range('hour', target_day, target_day.shift(days=1)))
target_time_bucket = []
for i in range(len(day_hours) - 1):
target_time_bucket.append((day_hours[i], day_hours[i + 1]))
# 查询数据
df = pandas.DataFrame()
for t_t, d_t in target_time_bucket:
dd, err_list = self.export(t_t, d_t)
df = df.append(dd)
if len(err_list) > 0:
self.record_err(t_t.format('YYY-MM-DD_HH'))
return df
def export_daily2month(self, vins: list, target_month: arrow.Arrow):
"""
将目标vin号的一个月拆分为单天查询后组合返回
:param vins: 需要查询的vin
:param target_month: 需要查询的精确到月的时间
:return: 查询的vin的目标月的数据
"""
# 拼装一个月的单日时间范围
month_days = list(
target_month.range('day', target_month,
target_month.shift(months=1)))
target_time_bucket = []
for i in range(len(month_days) - 1):
target_time_bucket.append((month_days[i], month_days[i + 1]))
# 查询数据
df = pandas.DataFrame()
for t_t, d_t in target_time_bucket:
dd, err_list = self.export(vins, [(t_t, d_t)])
df = df.append(dd)
if len(err_list) > 0:
df = df.append(self.export_hourly2day(err_list, t_t))
return df
def check_pingdom(objective: models.Objective, query_datetime: arrow.Arrow) -> float:
api_url = (
f"https://api.pingdom.com/api/2.1/summary.average/{objective.indicator_query}"
)
response = requests.get(
api_url,
auth=(settings.PINGDOM_EMAIL, settings.PINGDOM_PASS),
headers={"App-Key": settings.PINGDOM_APP_KEY},
params={
"from": query_datetime.shift(days=-30).timestamp,
"to": query_datetime.timestamp,
"includeuptime": "true",
},
)
response.raise_for_status()
status = response.json()["summary"]["status"]
total_time = sum(status.values())
return round(Decimal(status["totalup"] / total_time), 4)
def do_travel(distance: float, ship: OurShip, days_spent: int,
date: Arrow) -> Tuple[str, bool]:
"""
Performs the travelling part of our journey, will mutate `date` as a side effect
Returns a log entry and a boolean denoting whether the current date has already been logged
"""
log = ""
date_already_logged = False
while distance > 0:
day_log_entry = "<span>"
sunk, provision_log = handle_provisions(ship)
day_log_entry += provision_log
if sunk:
log += day_log_entry
break
sunk, encounter_log = do_encounter(ship)
day_log_entry += encounter_log
if sunk:
log += day_log_entry
break
wind = navigation.check_wind(ship.captain)
distance_travelled = 125 * wind[0]
days_spent += 1
current_date = date.shift(days=+1)
distance -= distance_travelled
# Check if anything interesting happened and add it to the main log if so
if day_log_entry != "<span>":
log += f"\n{get_date(current_date)}"
log += f"{day_log_entry}</span>"
date_already_logged = True
return log, date_already_logged
def create_time_entry(date: arrow.Arrow, start_hour: int, start_minute: int,
end_hour: int, end_minute: int, hourly_rate: float):
if end_hour == 0 and end_minute == 0:
end_hour = 24
if end_hour < start_hour: # Create two separate time entries if the recorded time is winds back due to midnight
print("TWO ENTRIES FOR MIDNIGHT")
print(date)
print(str(start_hour) + ":" + str(start_minute))
print(str(end_hour) + ":" + str(end_minute))
return [
create_time_entry(date, start_hour, start_minute, 24, 0,
hourly_rate)[0],
create_time_entry(date.shift(days=1), 0, 0, end_hour, end_minute,
hourly_rate)[0]
]
start = start_hour + (start_minute / 60.0)
end = end_hour + (end_minute / 60.0)
#print(str(start_hour) + ":" + str(start_minute) + " -> " + str(end_hour) + ":" + str(end_minute))
return [TimeEntry(date, start, end, hourly_rate)]
def initialize_room(self, room: Room, now: Arrow) -> Room:
"""
部屋の情報を初期化する。次のバックログの見積もりを始める際に実施する。
:param now:
:param room: 部屋
:return:
"""
item = {
"room_id": room.room_id,
"opened": False,
"ttl": now.shift(days=1).int_timestamp,
}
for member in room.members:
item[f"mem_{member.member_id}"] = {
"nickname": member.nickname,
"point": None
}
self.table.put_item(Item=item)
new_room = self.query_room(room.room_id)
if new_room:
return new_room
raise Exception
def open_time(control_dist_km: float, brevet_dist_km: float,
brevet_start_time: Arrow):
"""
Args:
control_dist_km: number, control distance in kilometers
brevet_dist_km: number, nominal distance of the brevet
in kilometers, which must be one of 200, 300, 400, 600,
or 1000 (the only official ACP brevet distances)
brevet_start_time: A date object (arrow)
Returns:
A date object indicating the control open time.
This will be in the same time zone as the brevet start time.
"""
# note: times are rounded to the nearest minute
# special case: if control_dist_km == 0, open_time is brevet_start_time
if control_dist_km == 0:
open_time = brevet_start_time
# for distances below 200 - normal case
elif control_dist_km <= 200:
(hours, minutes) = h_m_at_speed(control_dist_km, TOP_SPEEDS['to200'])
open_time = brevet_start_time.shift(hours=hours,
minutes=round(minutes))
# special case: if brevet_dist == 200, controle distances can be up to 20% longer
# but the controle times remain fixed at their 200 values
elif (brevet_dist_km
== 200) and (control_dist_km <= round(200 + 200 * .2)):
open_time = brevet_start_time.shift(hours=5, minutes=53)
# special case: if brevet_dist == 300km, controle distances can be up to 20% longer
# but the controle times remain fixed at their 300km values
elif (brevet_dist_km == 300) and (control_dist_km <= round(300 + 300 * .2)
) and (control_dist_km >= 300):
open_time = brevet_start_time.shift(hours=9)
# for distances below 400
elif control_dist_km <= 400:
first_200: Tuple[int, float] = h_m_at_speed(200, TOP_SPEEDS['to200'])
remaining_distance = control_dist_km - 200
second_200: Tuple[int, float] = h_m_at_speed(remaining_distance,
TOP_SPEEDS['to400'])
(total_hours,
total_minutes) = (first_200[0] + second_200[0]), (first_200[1] +
second_200[1])
# summing minutes may result in more than 60, so we need to carry over into hour
if total_minutes >= 60:
(total_hours,
total_minutes) = carry_m_to_h(total_hours, total_minutes)
open_time = brevet_start_time.shift(hours=total_hours,
minutes=round(total_minutes))
# special case: if brevet_dist == 400km, controle distances can be up to 20% longer
# but the controle times remain fixed at their 400km values
elif (brevet_dist_km
== 400) and (control_dist_km <= round(400 + 400 * .2)):
open_time = brevet_start_time.shift(hours=12, minutes=8)
# for distances below 600
elif control_dist_km <= 600:
first_200: Tuple[int, float] = h_m_at_speed(200, TOP_SPEEDS['to200'])
second_200: Tuple[int, float] = h_m_at_speed(200, TOP_SPEEDS['to400'])
remaining_distance = control_dist_km - 400
third_200: Tuple[int, float] = h_m_at_speed(remaining_distance,
TOP_SPEEDS['to600'])
(total_hours,
total_minutes) = (first_200[0] + second_200[0]) + third_200[0], (
first_200[1] + second_200[1] + third_200[1])
# summing minutes may result in more than 60, so we need to carry over into hour
if total_minutes >= 60:
(total_hours,
total_minutes) = carry_m_to_h(total_hours, total_minutes)
open_time = brevet_start_time.shift(hours=total_hours,
minutes=round(total_minutes))
# special case: if brevet_dist == 600, controle distances can be up to 20% longer
# but the controle times remain fixed at their 600km values
elif (brevet_dist_km
== 600) and (control_dist_km <= round(600 + 600 * .2)):
open_time = brevet_start_time.shift(hours=18, minutes=48)
# a 1000km brevet may have a final controle up to 20% greater than 1000
elif control_dist_km < round(1000 + 1000 * .2):
if control_dist_km > 1000:
control_dist_km = 1000
first_200: Tuple[int, float] = h_m_at_speed(200, TOP_SPEEDS['to200'])
second_200: Tuple[int, float] = h_m_at_speed(200, TOP_SPEEDS['to400'])
third_200: Tuple[int, float] = h_m_at_speed(200, TOP_SPEEDS['to600'])
remaining_distance = control_dist_km - 600
final_400: Tuple[int, float] = h_m_at_speed(remaining_distance,
TOP_SPEEDS['to1000'])
(total_hours, total_minutes
) = (first_200[0] + second_200[0]) + third_200[0] + final_400[0], (
first_200[1] + second_200[1] + third_200[1] + final_400[1])
# summing minutes may result in more than 60, so we need to carry over into hour
if total_minutes >= 60:
(total_hours,
total_minutes) = carry_m_to_h(total_hours, total_minutes)
open_time = brevet_start_time.shift(hours=total_hours,
minutes=round(total_minutes))
# should never end up here
else:
# indicate error for now by returning datetime arrow with all 0's
open_time = arrow.get('0000-00-00 00:00:00', 'YYYY-MM-DD HH:mm:ss')
# open_time = None
return open_time