def _get_oai_identifiers(self, sickle=None, metadata_prefix=None, set_=None, identifiers_list: List[str] = None, from_: arrow.Arrow = None): if identifiers_list: return [ self.sickle.GetRecord( identifier=identifier, metadataPrefix=self.metadata_prefix).header for identifier in identifiers_list ] if not sickle: sickle = self.sickle if not metadata_prefix: metadata_prefix = self.metadata_prefix if not set_: set_ = self.set_ if not from_: if self.from_: from_ = self.from_ else: return sickle.ListIdentifiers(metadataPrefix=metadata_prefix, set=set_) return sickle.ListIdentifiers( **{ "metadataPrefix": metadata_prefix, "set": set_, "from": from_.format("YYYY-MM-DD") })
def _format_dates(date: arrow.Arrow) -> list[str]: """ Return a list of date strings formatted according to the discord timestamp styles. These are used in the description of each style in the dropdown """ date = date.to('utc') formatted = [str(int(date.timestamp()))] formatted += [ date.format(format[1]) for format in list(STYLES.values())[1:7] ] formatted.append(date.humanize()) return formatted
def check_date_format(self, start_date_input: typing.Any, date_today: arrow.Arrow) -> arrow.Arrow: """Check input date format. :param start_date_input: :param date_today: :return: """ date_string = re.match(r"\d{4}-\d{2}-\d{2}", start_date_input) if start_date_input == date_today.format("YYYY-MM-DD"): return date_today if date_string: return arrow.get(start_date_input, "YYYY-MM-DD") return self.wrong_date_format_loop(date_today=date_today, start_date_input=start_date_input)
def __generate_search_url_by_day(query: str, date: Arrow): """ Returns a string with a url to ask twitter for a query in a day :param query:str twitter advanced query string :param date: date to query :return: url for date """ search_url = '%s since:%s until:%s' % (query, date.format('YYYY-MM-DD'), date.shift( days=1).format('YYYY-MM-DD')) search_url = 'https://mobile.twitter.com/search?q=' + urllib.parse.quote_plus( search_url) logger.debug(f"Generated url: {search_url}") return search_url
def make_time_list(terminus_times: list, current_time: arrow.Arrow, data_pool: str): time_list = [] previous_hour = int(current_time.shift(hours=-1).format('HH')) current_hour = int(current_time.format('HH')) next_hour = int(current_time.shift(hours=1).format('HH')) prev_hr = find_hr(terminus_times, previous_hour) curr_hr = find_hr(terminus_times, current_hour) next_hr = find_hr(terminus_times, next_hour) for hour_set in [prev_hr, curr_hr, next_hr]: if hour_set: hour = hour_set.get('hour') minute_set = hour_set.get(data_pool) for minutes in minute_set: time_list.append(make_time(hour, minutes)) return time_list
def get_day_lesson(self, dt: arrow.Arrow = arrow.now(), text: bool = False): """ :param dt: Date for show lesson :param text: Return type is text """ c = Calendar(self.get_timetable_ics(2019, 3, self._data["gapsid"], 2)) if text: ret = dt.format('*dddd D MMMM YYYY*\n', locale="fr") for i in c.timeline.on(dt): ret += "`" \ + i.begin.to('Europe/Paris').format('HH:mm') \ + "→" + i.end.to('Europe/Paris').format('HH:mm') \ + "`: *" + i.location + "*: " + str(i.name) + "\n" return ret else: return c.timeline.on(dt)
def export_monthly(self, target_month: arrow.Arrow): """ 导入一个月的数据 :param target_month: 要导入的月份 :return: """ batch_size = self.batch_size start_i = 0 epoch = 0 df = pandas.DataFrame() self._tmp_show_index = 0 total_num = 0 for batch_i in range((self.all_vins_len // batch_size) + 1): # 选取新的vin号查询 end_i = (batch_i + 1) * batch_size self._tmp_show_index = end_i if end_i > self.all_vins_len: end_i = self.all_vins_len select_vins = self.all_vins[start_i:end_i] # 月查询 dd, err_list = self.export( select_vins, [(target_month, target_month.shift(months=1))]) df = df.append(dd) # 错误vin号划分为日查询 if len(err_list) > 0: df = df.append(self.export_daily2month(err_list, target_month)) # 后续处理 df_len = len(df) start_i = end_i if df_len >= self.max_per_batch or end_i == self.all_vins_len: epoch += 1 total_num += df_len file_path = f"{self.data_dir}{target_month.format('YYYY-MM')}/" if not os.path.exists(file_path): os.makedirs(file_path) df.to_parquet(f"{file_path}{epoch}.snappy.parquet", index=False) df = pandas.DataFrame() self._show_process(target_month, end_i, df_len) logger.info( f'completed:{target_month}_{epoch},len={df_len},,process={round(self._tmp_show_index / self.all_vins_len * 100, 4)}%' ) self.record_batch_ok(target_month.format('YYYY-MM'), select_vins) logger.info(f'completed:{target_month},total_num={total_num}') return total_num
def _calendar_print(tasks: List[Tuple[Arrow, TaskDB]], date: Arrow) -> None: cols = [Column(f"{d}", justify="left") for d in weekdays] table = Table(*cols, title=date.format("MMMM"), box=box.MINIMAL, show_lines=True) today = ( lambda d: "bold yellow" if d.isocalendar() == date.isocalendar() else "white" if d.month == date.month else "bright_black" ) format = lambda date, db: Text("\n").join( [Text(f"{date.day} ", style=today(date)), format_tdb(db, False, False)] ) days = [format(date, db) for date, db in tasks] weeks = [days[i : i + 7] for i in range(0, len(days), 7)] for w in weeks: table.add_row(*w) print(table)
def export_hourly2day(self, vins: list, target_day: arrow.Arrow): """ 将一天拆成小时查询并合并 :param vins: 需要查询的vin :param target_day: 需要查询的精确到天的时间 :return: 查询的vin的目标日的数据 """ day_hours = list( target_day.range('hour', target_day, target_day.shift(days=1))) target_time_bucket = [] for i in range(len(day_hours) - 1): target_time_bucket.append((day_hours[i], day_hours[i + 1])) # 查询数据 df = pandas.DataFrame() for t_t, d_t in target_time_bucket: dd, err_list = self.export(vins, [(t_t, d_t)]) df = df.append(dd) if len(err_list) > 0: for v in err_list: self.record_one_err(target_day.format('YYY-MM-DD'), v, t_t) return df
def wrong_date_format_loop(date_today: arrow.Arrow, start_date_input: str) -> arrow.Arrow: """Wrong input date format loop. :param start_date_input: :param date_today: """ date_string = re.match(r"\d{4}-\d{2}-\d{2}", start_date_input) while not date_string: console.print( f"[bold]{escape('[perry-bot]:')}[/bold] Please enter" f" a valid start date in the format YYYY-MM-DD.", style="default", ) start_date_input_again = Prompt.ask( default=date_today.format("YYYY-MM-DD"), show_default=True) date_string_again = re.match(r"\d{4}-\d{2}-\d{2}", start_date_input_again) if date_string_again: return arrow.get(start_date_input_again) return arrow.get(start_date_input)
async def get_logs(self, date: arrow.Arrow = arrow.now()) -> str: params = json_obj() if date: params.date = date.format('YYYY/MM/DD') return (await self._get('logs', **params)).json()['result']
def arrow2grid(arrow_ins: Arrow) -> List[str]: return arrow_ins.format("YYYY-M-D-H-m-s").split("-")
def ebay_date_format(date: arrow.Arrow) -> str: """Ebay requires the old form of ISO dates with 'Z' for zero offset instead of standard +0000.""" return date.format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z'
def format_date(date: arrow.Arrow) -> str: datetime_format = "YYYY-MM-DD HH:mm:ss" return date.format(datetime_format)
def fetch_data_entry(dt: arrow.Arrow): params = {"to": dt.format(DATE_FORMAT)} response = requests.get(DATA_URL, params=params) if response.status_code == 200: return response.json()
async def get_interval_unit_data(cls, meta_ids: list, start: arrow.Arrow, end: arrow.Arrow, unit: str) -> list: """ 获取 :param meta_ids: :param start: :param end: :param unit: :param db: :return: """ """ 由于 tortoise-orm 的问题以下的语句不能使用,所以使用原生 sql 语句 from pypika import CustomFunction from tortoise.expressions import F from tortoise.functions import Function class TruncTime(Function): database_func = CustomFunction( "DATE_FORMAT", ["name", "dt_format"]) unit_fns = { 'day': TruncTime('d_t', '%Y-%m-%d'), 'week': TruncTime('d_t', '%x-%v'), 'month': TruncTime('d_t', '%Y-%m'), 'year': TruncTime('d_t', '%Y'), } res = await StatisticsData.filter( d_t__gte=start, d_t__lte=end, meta_id__in=meta_ids, ).annotate( time=unit_fns.get(unit), vals=Sum('val'), ).group_by('meta_id', 'time').values( 'meta_id',time='time', val='vals') """ units = { 'day': '%Y-%m-%d', 'week': '%x-%v', 'month': '%Y-%m', 'year': '%Y', } formatter = units.get(unit) meta_ids = tuple(set(meta_ids)) start = start.format() end = end.format() conn = Tortoise.get_connection("default") date_func = f"DATE_FORMAT( `d_t`, '{formatter}' ) `time`" from app import debug if debug: date_func = f"strftime('{formatter}', `d_t` ) `time`" sql = f""" SELECT `meta_id` `meta_id`, {date_func}, SUM( `val` ) `val` FROM `statistics_data_v2` WHERE `d_t` >= '{start}' AND `d_t` <= '{end}' AND `meta_id` IN {meta_ids} GROUP BY `meta_id`, `time` """ res = await conn.execute_query_dict(sql) return res