def _get_oai_identifiers(self,
sickle=None,
metadata_prefix=None,
set_=None,
identifiers_list: List[str] = None,
from_: arrow.Arrow = None):
if identifiers_list:
return [
self.sickle.GetRecord(
identifier=identifier,
metadataPrefix=self.metadata_prefix).header
for identifier in identifiers_list
]
if not sickle:
sickle = self.sickle
if not metadata_prefix:
metadata_prefix = self.metadata_prefix
if not set_:
set_ = self.set_
if not from_:
if self.from_:
from_ = self.from_
else:
return sickle.ListIdentifiers(metadataPrefix=metadata_prefix,
set=set_)
return sickle.ListIdentifiers(
**{
"metadataPrefix": metadata_prefix,
"set": set_,
"from": from_.format("YYYY-MM-DD")
})
def _format_dates(date: arrow.Arrow) -> list[str]:
"""
Return a list of date strings formatted according to the discord timestamp styles.
These are used in the description of each style in the dropdown
"""
date = date.to('utc')
formatted = [str(int(date.timestamp()))]
formatted += [
date.format(format[1]) for format in list(STYLES.values())[1:7]
]
formatted.append(date.humanize())
return formatted
def check_date_format(self, start_date_input: typing.Any,
date_today: arrow.Arrow) -> arrow.Arrow:
"""Check input date format.
:param start_date_input:
:param date_today:
:return:
"""
date_string = re.match(r"\d{4}-\d{2}-\d{2}", start_date_input)
if start_date_input == date_today.format("YYYY-MM-DD"):
return date_today
if date_string:
return arrow.get(start_date_input, "YYYY-MM-DD")
return self.wrong_date_format_loop(date_today=date_today,
start_date_input=start_date_input)
def __generate_search_url_by_day(query: str, date: Arrow):
"""
Returns a string with a url to ask twitter for a query in a day
:param query:str twitter advanced query string
:param date: date to query
:return: url for date
"""
search_url = '%s since:%s until:%s' % (query, date.format('YYYY-MM-DD'),
date.shift(
days=1).format('YYYY-MM-DD'))
search_url = 'https://mobile.twitter.com/search?q=' + urllib.parse.quote_plus(
search_url)
logger.debug(f"Generated url: {search_url}")
return search_url
def make_time_list(terminus_times: list, current_time: arrow.Arrow,
data_pool: str):
time_list = []
previous_hour = int(current_time.shift(hours=-1).format('HH'))
current_hour = int(current_time.format('HH'))
next_hour = int(current_time.shift(hours=1).format('HH'))
prev_hr = find_hr(terminus_times, previous_hour)
curr_hr = find_hr(terminus_times, current_hour)
next_hr = find_hr(terminus_times, next_hour)
for hour_set in [prev_hr, curr_hr, next_hr]:
if hour_set:
hour = hour_set.get('hour')
minute_set = hour_set.get(data_pool)
for minutes in minute_set:
time_list.append(make_time(hour, minutes))
return time_list
def get_day_lesson(self, dt: arrow.Arrow = arrow.now(), text: bool = False):
"""
:param dt: Date for show lesson
:param text: Return type is text
"""
c = Calendar(self.get_timetable_ics(2019, 3, self._data["gapsid"], 2))
if text:
ret = dt.format('*dddd D MMMM YYYY*\n', locale="fr")
for i in c.timeline.on(dt):
ret += "`" \
+ i.begin.to('Europe/Paris').format('HH:mm') \
+ "→" + i.end.to('Europe/Paris').format('HH:mm') \
+ "`: *" + i.location + "*: " + str(i.name) + "\n"
return ret
else:
return c.timeline.on(dt)
def export_monthly(self, target_month: arrow.Arrow):
"""
导入一个月的数据
:param target_month: 要导入的月份
:return:
"""
batch_size = self.batch_size
start_i = 0
epoch = 0
df = pandas.DataFrame()
self._tmp_show_index = 0
total_num = 0
for batch_i in range((self.all_vins_len // batch_size) + 1):
# 选取新的vin号查询
end_i = (batch_i + 1) * batch_size
self._tmp_show_index = end_i
if end_i > self.all_vins_len:
end_i = self.all_vins_len
select_vins = self.all_vins[start_i:end_i]
# 月查询
dd, err_list = self.export(
select_vins, [(target_month, target_month.shift(months=1))])
df = df.append(dd)
# 错误vin号划分为日查询
if len(err_list) > 0:
df = df.append(self.export_daily2month(err_list, target_month))
# 后续处理
df_len = len(df)
start_i = end_i
if df_len >= self.max_per_batch or end_i == self.all_vins_len:
epoch += 1
total_num += df_len
file_path = f"{self.data_dir}{target_month.format('YYYY-MM')}/"
if not os.path.exists(file_path):
os.makedirs(file_path)
df.to_parquet(f"{file_path}{epoch}.snappy.parquet",
index=False)
df = pandas.DataFrame()
self._show_process(target_month, end_i, df_len)
logger.info(
f'completed:{target_month}_{epoch},len={df_len},,process={round(self._tmp_show_index / self.all_vins_len * 100, 4)}%'
)
self.record_batch_ok(target_month.format('YYYY-MM'),
select_vins)
logger.info(f'completed:{target_month},total_num={total_num}')
return total_num
def _calendar_print(tasks: List[Tuple[Arrow, TaskDB]], date: Arrow) -> None:
cols = [Column(f"{d}", justify="left") for d in weekdays]
table = Table(*cols, title=date.format("MMMM"), box=box.MINIMAL, show_lines=True)
today = (
lambda d: "bold yellow"
if d.isocalendar() == date.isocalendar()
else "white"
if d.month == date.month
else "bright_black"
)
format = lambda date, db: Text("\n").join(
[Text(f"{date.day} ", style=today(date)), format_tdb(db, False, False)]
)
days = [format(date, db) for date, db in tasks]
weeks = [days[i : i + 7] for i in range(0, len(days), 7)]
for w in weeks:
table.add_row(*w)
print(table)
def export_hourly2day(self, vins: list, target_day: arrow.Arrow):
"""
将一天拆成小时查询并合并
:param vins: 需要查询的vin
:param target_day: 需要查询的精确到天的时间
:return: 查询的vin的目标日的数据
"""
day_hours = list(
target_day.range('hour', target_day, target_day.shift(days=1)))
target_time_bucket = []
for i in range(len(day_hours) - 1):
target_time_bucket.append((day_hours[i], day_hours[i + 1]))
# 查询数据
df = pandas.DataFrame()
for t_t, d_t in target_time_bucket:
dd, err_list = self.export(vins, [(t_t, d_t)])
df = df.append(dd)
if len(err_list) > 0:
for v in err_list:
self.record_one_err(target_day.format('YYY-MM-DD'), v, t_t)
return df
def wrong_date_format_loop(date_today: arrow.Arrow,
start_date_input: str) -> arrow.Arrow:
"""Wrong input date format loop.
:param start_date_input:
:param date_today:
"""
date_string = re.match(r"\d{4}-\d{2}-\d{2}", start_date_input)
while not date_string:
console.print(
f"[bold]{escape('[perry-bot]:')}[/bold] Please enter"
f" a valid start date in the format YYYY-MM-DD.",
style="default",
)
start_date_input_again = Prompt.ask(
default=date_today.format("YYYY-MM-DD"), show_default=True)
date_string_again = re.match(r"\d{4}-\d{2}-\d{2}",
start_date_input_again)
if date_string_again:
return arrow.get(start_date_input_again)
return arrow.get(start_date_input)
async def get_logs(self, date: arrow.Arrow = arrow.now()) -> str:
params = json_obj()
if date:
params.date = date.format('YYYY/MM/DD')
return (await self._get('logs', **params)).json()['result']
def arrow2grid(arrow_ins: Arrow) -> List[str]:
return arrow_ins.format("YYYY-M-D-H-m-s").split("-")
def ebay_date_format(date: arrow.Arrow) -> str:
"""Ebay requires the old form of ISO dates with 'Z' for zero offset instead of standard +0000."""
return date.format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z'
def format_date(date: arrow.Arrow) -> str:
datetime_format = "YYYY-MM-DD HH:mm:ss"
return date.format(datetime_format)
def fetch_data_entry(dt: arrow.Arrow):
params = {"to": dt.format(DATE_FORMAT)}
response = requests.get(DATA_URL, params=params)
if response.status_code == 200:
return response.json()
async def get_interval_unit_data(cls, meta_ids: list, start: arrow.Arrow,
end: arrow.Arrow, unit: str) -> list:
"""
获取
:param meta_ids:
:param start:
:param end:
:param unit:
:param db:
:return:
"""
"""
由于 tortoise-orm 的问题以下的语句不能使用，所以使用原生 sql 语句
from pypika import CustomFunction
from tortoise.expressions import F
from tortoise.functions import Function
class TruncTime(Function):
database_func = CustomFunction(
"DATE_FORMAT", ["name", "dt_format"])
unit_fns = {
'day': TruncTime('d_t', '%Y-%m-%d'),
'week': TruncTime('d_t', '%x-%v'),
'month': TruncTime('d_t', '%Y-%m'),
'year': TruncTime('d_t', '%Y'),
}
res = await StatisticsData.filter(
d_t__gte=start,
d_t__lte=end,
meta_id__in=meta_ids,
).annotate(
time=unit_fns.get(unit),
vals=Sum('val'),
).group_by('meta_id', 'time').values(
'meta_id',time='time', val='vals')
"""
units = {
'day': '%Y-%m-%d',
'week': '%x-%v',
'month': '%Y-%m',
'year': '%Y',
}
formatter = units.get(unit)
meta_ids = tuple(set(meta_ids))
start = start.format()
end = end.format()
conn = Tortoise.get_connection("default")
date_func = f"DATE_FORMAT( `d_t`, '{formatter}' ) `time`"
from app import debug
if debug:
date_func = f"strftime('{formatter}', `d_t` ) `time`"
sql = f"""
SELECT
`meta_id` `meta_id`,
{date_func},
SUM( `val` ) `val`
FROM
`statistics_data_v2`
WHERE
`d_t` >= '{start}'
AND `d_t` <= '{end}'
AND `meta_id` IN {meta_ids}
GROUP BY
`meta_id`,
`time`
"""
res = await conn.execute_query_dict(sql)
return res
