def convert_date(raw_date: str) -> tuple: def _get_datetime_range(date: str) -> tuple: """ Makes one day period by received date. Parameters ---------- date: Date for report generating. Returns ------- Datetime range. """ date = datetime.strptime(date, "%Y-%m-%d") from_date = date.astimezone(utc) from_date = from_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z") to_date = (date.replace( hour=23, minute=59, second=59).astimezone(utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z")) return from_date, to_date parser = Calendar() date, status = parser.parse(raw_date) date = datetime(*date[:3]).strftime("%Y-%m-%d") return _get_datetime_range(date)
class CsvTypeParser(TabularTypeParser): """CSV type parser""" calendar: ClassVar[Calendar] = Calendar() """Calendar object used for date parsing""" def __post_init__(self): if self.parse is None: if issubclass(self.pytype, datetime): self.parse = self.parse_datetime elif issubclass(self.pytype, date): self.parse = self.parse_date super().__post_init__() @classmethod def parse_datetime(cls, value): """Parse datetime from CSV value""" timestamp, ret = cls.calendar.parseDT(value) if not ret: raise ValueError("Invalid date: '%s'" % value) return timestamp @classmethod def parse_date(cls, value): """Parse date from CSV value""" return cls.parse_datetime(value).date()
def str2date(a, verbose=False): struct, status = Calendar().parse(a) if status == 0: if verbose: print(f"Unable to recognize date in \"{a}\".") return date.today() return date(*struct[:3])
def cli(config, src, dst, when): """ commute.py helper CLI to compute possible commute options. """ if when is not None: when = int(time.mktime(Calendar().parseDT(when)[0].timetuple())) for rank, path in commute.get_all_paths(config, src, dst, when): click.echo(commute.format_path(rank, path)) click.echo("-" * 5)
async def poll(self): cal = Calendar() now = datetime.now(timezone("UTC")) then = cal.parseDT("1 minute", now) print("polling...") for r in Reminder.select().where(Reminder.time.between(now, then[0])): await self.bot.get_channel(r.channel_id).send(r.reminder_text)
def get_time(time_string): cal = Calendar() now = datetime.now(timezone("UTC")) if time_string == "": time = cal.parseDT("1 day", now) else: time = cal.parseDT(time_string, now) return time[0]
def _guess_game_date(datetime_str): # Try to turn a datetime string from FD into an actual datetime datetime_str.replace('Sun', 'Sunday') datetime_str.replace('Mon', 'Monday') datetime_str.replace('Tues', 'Tuesday') datetime_str.replace('Wed', 'Wednesday') datetime_str.replace('Thurs', 'Thursday') datetime_str.replace('Fri', 'Friday') datetime_str.replace('Sat', 'Saturday') cal = Calendar() dt, ret_code = cal.parseDT(datetime_str) return dt
def parse_mtime_single(value): """Convert a human readable time description to a """ if not HAS_PARSEDATETIME: return int(value) calendar = Calendar() guess, rc = calendar.parseDT(value) if rc == 0: LOGGER.warning('Could not parse date: %s', value) return int(value) return guess.timestamp()
async def convert(self, ctx: commands.Context, argument: str): cal = Calendar() utc = timezone("UTC") utcnow = datetime.utcnow().replace(microsecond=0, tzinfo=utc) parsed = cal.parseDT(argument, sourceTime=utcnow, tzinfo=utc)[0] if parsed == utcnow: raise commands.BadArgument(f'"{argument}" is not a valid time.') elif parsed < utcnow: raise commands.BadArgument(f'"{argument}" is in the past.') return parsed
def parse(self, response): # follow links to offer pages offers = response.xpath('//a[@data-cy="listing-ad-title"]') for offer in offers: yield response.follow(url=offer.attrib['href'], callback=self.parse_offer) latest_date = response.xpath('//small//span//i[@data-icon="clock"]/../text()').getall().pop() olx_req = OlxRequirements(req) cal = Calendar() required_date = cal.parse(' '.join([str(olx_req.number_of_days), 'days ago']))[0] parsed_date = latest_date.replace('\n', '').replace('\t', '') latest_date_processed = olx_convert_to_time_struct(parsed_date) if latest_date_processed < required_date: return yield response.follow(url=response.xpath('//a[@data-cy="page-link-next"]').attrib['href'], callback=self.parse)
def str_to_datetime(string, on_fail_return=None): try: from dateutil import parser as p output = p.parse(string) return output except: from parsedatetime import Calendar from datetime import datetime c = Calendar() output, flags = c.parse(string) if flags > 0: return datetime(*output[:6]) else: return None return None
def clean(self, value): """ Validates that the input can be converted to a time. Returns a Python datetime.time object. """ super(NaturalTimeField, self).clean(value) if value in EMPTY_VALUES: return None if isinstance(value, datetime.time): return value c = Calendar() parsed = c.parse(value) if parsed[1] == 2: return datetime.time(*parsed[0][3:6]) raise ValidationError(self.error_messages['invalid'])
def parse_contest_element(browser_elem): """ Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost :param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element :return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description """ match_groups = contest_element_id_regex.match( browser_elem.get_attribute("id")) fd_game_id = int(match_groups.group("gameid")) fd_table_id = int(match_groups.group("tableid")) game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace( '$', '').replace(',', '') game_time = browser_elem.find_element_by_xpath("div[2]/time").text cal = Calendar() fd_game_date, ret_code = cal.parseDT(game_time) entry_button_element = browser_elem.find_element_by_xpath("a") return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def test_olx_convert_to_time_struct_valid_case(): test_input = { 'dzisiaj 13:48': 'today 13:48', 'wczoraj 13:48': 'yesterday 13:48', '16 sty': '16 january', '16 wrz': '16 september', '16 lut': '16 february', '16 mar': '16 march', '16 kwi': '16 april', '16 maj': '16 may', '16 cze': '16 june', '16 lip': '16 july', '16 sie': '16 august', '16 paź': '16 october', '16 lis': '16 november', '16 gru': '16 december', } cal = Calendar() for k, v in test_input.items(): assert olx_convert_to_time_struct(k) == cal.parse(v)[0]
def olx_convert_to_time_struct(date_str): olx_date_mapper = { 'dzisiaj': 'today', 'wczoraj': 'yesterday', 'sty': 'january', 'wrz': 'september', 'lut': 'february', 'mar': 'march', 'kwi': 'april', 'maj': 'may', 'cze': 'june', 'lip': 'july', 'sie': 'august', 'paź': 'october', 'lis': 'november', 'gru': 'december', } for k, v in olx_date_mapper.items(): date_str = date_str.replace(k, v) cal = Calendar() return cal.parse(date_str)[0]
def parse_offer(self, response): """Extracts the details of the offer to search through.""" unprocessed_date = response.css( '#wrapper > div:nth-child(1) > div.vip-header-and-details > div.vip-details' ' > ul > li:nth-child(1) > div > span.value').get() (_, unprocessed_date, _) = unprocessed_date.split('>') (processed_date, _) = unprocessed_date.split('<') gumtree_req = GumtreeRequirements(req) cal = Calendar() offer_submission_date = cal.parse(processed_date) required_date = cal.parse(' '.join( [str(gumtree_req.number_of_days), 'days ago'])) if offer_submission_date < required_date: return spans = response.xpath("//span[@class='value']").getall() spans_text = get_processed_text(spans) p = response.xpath("//span[@class='pre']").getall() p_text = get_processed_text(p) text = ' '.join([spans_text, p_text]) if calculate_intersection(text, gumtree_req.tags) >= gumtree_req.threshold: append_to_file(path='scrapy-data/urls.txt', response=response)
def parse(self, response): # follow links to offer pages offers = response.xpath("//a[@class='href-link tile-title-text']") for offer in offers: yield response.follow(url=offer.attrib['href'], callback=self.parse_offer) # follow pagination link tmp_1 = response.xpath( "//a[@class='arrows icon-angle-right-gray icon-right-arrow']") tmp_2 = response.xpath( "//a[@class='arrows icon-right-arrow icon-angle-right-gray']") next_page = tmp_1 if 'href' in tmp_1.attrib else tmp_2 offer_date = response.xpath("//div[@class='creation-date']") parsed_date = extract_date(offer_date[len(offer_date) - 1].get()) gumtree_req = GumtreeRequirements(req) cal = Calendar() date = cal.parse(parsed_date) required_date = cal.parse(' '.join( [str(gumtree_req.number_of_days), 'days ago'])) if date < required_date: return yield response.follow(next_page.attrib['href'], self.parse)
def parse_offer(self, response): """Extracts the details of the offer to search through.""" # Extract date of offer submission # We don't want an offer which is too old unprocessed_date = response.xpath("//div[@class='css-lh1bxu']").get() (_, _, unprocessed_date) = unprocessed_date.partition(':') (processed_date, _, _) = unprocessed_date.partition('<') cal = Calendar() offer_submission_date = cal.parse(processed_date) otodom_req = OtodomRequirements(req) required_date = cal.parse(' '.join( [str(otodom_req.number_of_days), 'days ago'])) if offer_submission_date < required_date: return li = response.xpath("//li/text()").getall() li_text = get_processed_text(li) p = response.xpath("//p/text()").getall() p_text = get_processed_text(p) text = ' '.join([li_text, p_text]) if calculate_intersection(text, otodom_req.tags) >= otodom_req.threshold: append_to_file(path='scrapy-data/urls.txt', response=response)
def action(self): self._logger.info('Start viewing the log events...') def _exit(signum, frame): self._logger.info('Exit by code {} ...'.format(signum)) self._exit = True signal.signal(signal.SIGTERM, _exit) signal.signal(signal.SIGINT, _exit) start = time.time() if self._start is not None: time_struct, _ = Calendar().parse(self._start) start = mktime(time_struct) start = int(start * 1000) client = self.get_logs_client() function = self._config.get_function_name() event_ids = {} while self._exit is False: events = client.get_log_events(function, start, self._filter) for e in events: if e['eventId'] not in event_ids: event_ids[e['eventId']] = None print(e['message']) if e['timestamp'] > start: start = e['timestamp'] event_ids = {} if not self._follow: break time.sleep(self._interval)
def parse_time( time_str: str, seconds: int = True ) -> typing.Union[None, int, typing.Tuple[datetime.datetime, int]]: """ Parses a time. :param time_str: The time string to parse. :return: The total number of seconds between now and then. """ calendar = Calendar() t_struct, parse_status = calendar.parse( time_str, sourceTime=datetime.datetime.utcnow()) if parse_status == 0: return None dt = datetime.datetime(*t_struct[:6]) diff = np.ceil((dt - datetime.datetime.utcnow()).total_seconds()) if seconds: return diff else: return dt, diff
def clean(self, value): """ Validates that the input can be converted to a datetime. Returns a Python datetime.datetime object. """ super(NaturalDateTimeField, self).clean(value) if value in EMPTY_VALUES: return None if isinstance(value, datetime.datetime): return value if isinstance(value, datetime.date): return datetime.datetime(value.year, value.month, value.day) if isinstance(value, list): # Input comes from a SplitDateTimeWidget, for example. So, it's two # components: date and time. if len(value) != 2: raise ValidationError(self.error_messages['invalid']) value = '%s %s' % tuple(value) c = Calendar() parsed = c.parse(value) if parsed[1] == 3: return datetime.date(*parsed[0][:6]) raise ValidationError(self.error_messages['invalid'])
def parse(self, response): # follow links to offer pages offers = response.xpath("//a[@class='property_link property-url']") for offer in offers: yield response.follow(url=offer.attrib['href'], callback=self.parse_offer) latest_date = response.xpath("//span[@class='single-result__category single-result__category--date']")\ .getall()\ .pop() # the strong word appears in the rag when the date is in bold (and it's only bold when the offer has just been # added, hence there is no sense in finding out whether is't an old offer if 'strong' not in latest_date: morizon_req = MorizonRequirements(req) cal = Calendar() required_date = cal.parse(' '.join([str(morizon_req.number_of_days), 'days ago'])) parsed_date = extract_from_tag(latest_date) latest_date_processed = morizon_convert_to_time_struct(parsed_date) if latest_date_processed < required_date: return yield response.follow( url=response.xpath("//a[@class='mz-pagination-number__btn mz-pagination-number__btn--next']") .pop() .attrib['href'], callback=self.parse)
def process(input_wikitext): input_wikitext, encoded_wikilinks = encode_wikilinks(input_wikitext) old_ah_wikitext_search = ARTICLE_HISTORY.search(input_wikitext) if not old_ah_wikitext_search: return input_wikitext old_ah_wikitext = old_ah_wikitext_search.group(0) history = History(old_ah_wikitext) # For use in sorting parameters by_time = lambda x: datetime.datetime.fromtimestamp( mktime(Calendar().parse(x[0])[0])) lines_to_delete = [] if ITN.search(input_wikitext): itn_list = history.get_relevant_params("itn") for itn_result in ITN.finditer(input_wikitext): itn = itn_result.group(1) itn_params = itn.split("|")[1:] if "=" not in itn_params[0] and "=" not in itn_params[1]: # {{ITN talk|DD monthname|YYYY}} itn_list.append((itn_params[0] + " " + itn_params[1], "")) else: itn_list += [(x[x.find("=") + 1:], "") for x in itn.split("|")[1:] if "date" in x] lines_to_delete.append(itn_result.group(0)) itn_list.sort(key=by_time) # Update the article history template history.other_parameters["itndate"] = itn_list[0][0] if itn_list[0][1]: history.other_parameters["itnlink"] = itn_list[0][1] for i, item in enumerate(itn_list[1:], start=2): history.other_parameters["itn%ddate" % i] = item[0] if item[1]: history.other_parameters["itn%ditem" % i] = item[1] if OTD.search(input_wikitext): otd_list = history.get_relevant_params("otd") for otd_result in OTD.finditer(input_wikitext): otd = otd_result.group(1) otd_params = { x: y for x, y in [t.strip().split("=") for t in otd.split("|")[1:]] } for i in itertools.count(1): date_key = "date%d" % i if date_key in otd_params: otd_list.append((otd_params[date_key], otd_params.get("oldid%d" % i, ""), "")) else: break lines_to_delete.append(otd_result.group(0)) otd_list.sort(key=by_time) # Update the article history template history.other_parameters["otddate"], history.other_parameters[ "otdoldid"], _ = otd_list[0] if otd_list[0][2]: history.other_parameters["otdlink"] = otd_list[0][2] for i, item in enumerate(otd_list[1:], start=2): history.other_parameters["otd%ddate" % i], history.other_parameters["otd%doldid" % i], _ = item if item[2]: history.other_parameters["otd%dlink" % i] = item[2] dyk_search = DYK.search(input_wikitext) if dyk_search: dyk = dyk_search.group(1) dyk_params = dyk.split("|")[1:] history.other_parameters["dykentry"] = next( x for x in dyk_params if x.startswith("entry")).split("=")[1] positional_dyk_params = [x for x in dyk_params if "=" not in x] if len(positional_dyk_params) == 1: history.other_parameters["dykdate"] = positional_dyk_params[0] elif len(positional_dyk_params) == 2: for param in positional_dyk_params: if len(param) == 4: year = param else: month_day = param history.other_parameters["dykdate"] = month_day + " " + year # Delete the DYK template lines_to_delete.append(dyk_search.group(0)) # Delete the lines with only the delete comment on them result_text = "\n".join( ifilterfalse(lines_to_delete.__contains__, input_wikitext.splitlines())) result_text = result_text.replace(old_ah_wikitext, history.as_wikitext()) result_text = decode_wikilinks(result_text, encoded_wikilinks) return result_text
tokens.append(sent[span[0]:span[1]]) current = span[1] tags = pos_tag(tokens) new_tags = [] for word, tag in tags: if word[:2] == '__': new_tags.append((word[2:], tag_dict[word[2:]])) else: tag = [t[1] for t in original_tags if t[0] == word][0] # FIXED new_tags.append((word, tag)) return new_tags cal = Calendar() month2num = { "january": 1, "february": 2, "march": 3, "april": 4, "may": 5, "june": 6, "july": 7, "august": 8, "september": 9, "october": 10, "november": 11, "december": 12 } num2month = dict([(n, m) for (m, n) in month2num.items()])
def __init__(self): self.parser.add_argument('-s', '--status', action='store_true', help='show rule list with expirations') self.parser.add_argument('-c', '--clean', action='store_true', help='clean up expired rules') self.parser.add_argument('-r', '--rule', help='rule to be added to `ufw`') self.parser.add_argument('-p', '--position', default=1, help='position to add the rule') self.parser.add_argument('-t', '--ttl', default='30 days', help='time to live for the rule') args = self.parser.parse_args() # Our file names pid_file = '/var/run/' + __file__ + '.pid' rules_file = '/usr/local/share/' + __file__ + '/rules' tmp_rules_file = '/tmp/' + __file__ + '-rules' if args.status: if path.exists(rules_file): try: print("Expiration\t\tRule") print('=' * 80) # Loops through the rules lines for line in open(rules_file, 'r'): # Breaks apart line into expiration timestamp and rule timestamp, rule = line.strip("\n").split(' ', 1) print( str(datetime.fromtimestamp(float(timestamp))) + "\t" + rule) except IOError: self.error('unable to read from the rules file: ' + rules_file) else: self.error('there are no rules to display') elif args.clean: # Checks for PID file if path.exists(pid_file): self.error(__file__ + ' is already running') else: # Creates the PID file try: handle = open(pid_file, 'w') handle.write(str(getpid())) handle.close() except IOError: self.error('unable to create PID file: ' + pid_file) # Checks for the rules file if path.exists(rules_file): # Opens the temporary rules file try: handle = open(tmp_rules_file, 'a') except IOError: self.error('unable to write to the tmp rules file: ' + tmp_rules_file) try: current_time = time() # Loops through the rules lines for line in open(rules_file, 'r'): # Breaks apart line into expiration timestamp and rule timestamp, rule = line.strip("\n").split(' ', 1) # Checks if rule has expired if current_time < float(timestamp): handle.write(line) print( str(datetime.fromtimestamp(time())) + "\tskipped rule\t" + rule) else: try: self.ufw_execute('delete ' + rule) print( str(datetime.fromtimestamp(time())) + "\tdeleted rule\t" + rule) except CalledProcessError as error: self.ufw_error(error) handle.close() # Moves the tmp file to the rules file move(tmp_rules_file, rules_file) except IOError: self.error('unable to from the read rules file: ' + rules_file) # Removes the PID remove(pid_file) elif args.rule: rules_path = path.dirname(rules_file) if not path.exists(rules_path): makedirs(rules_path) # Converts the TTL to a timestamp cal = Calendar() timestamp = mktime(cal.parse(args.ttl)[0]) # Writes the rule to the rules file try: # TODO Check if rule already exists and update it instead of adding it again handle = open(rules_file, 'a') handle.write(str(timestamp) + ' ' + args.rule) handle.write("\n") handle.close() except IOError: self.error('unable to write to the rules file: ' + rules_file) # Attempts to add the rule to `ufw` try: self.ufw_execute('insert ' + str(args.position) + ' ' + args.rule) except CalledProcessError as error: # Catches an error when attempting to add a rule to an empty database if error.output == b"ERROR: Invalid position '1'\n": try: self.ufw_execute(args.rule) except CalledProcessError as error: self.ufw_error(error) else: self.ufw_error(error) else: self.error('no arguments specified')
import re from datetime import datetime import logging from parsedatetime import Calendar from pytz import timezone, utc from recurrent.constants import * pdt = Calendar() log = logging.getLogger('recurrent') RE_TIME = re.compile( r'(?P<hour>\d{1,2}):?(?P<minute>\d{2})?\s?(?P<mod>am|pm)?(oclock)?') RE_AT_TIME = re.compile(r'at\s%s' % RE_TIME.pattern) RE_AT_TIME_END = re.compile(r'at\s%s$' % RE_TIME.pattern) RE_STARTING = re.compile(r'start(?:s|ing)?') RE_ENDING = re.compile(r'(?:\bend|until)(?:s|ing)?') RE_REPEAT = re.compile(r'(?:every|each|\bon\b|repeat(s|ing)?)') RE_START = r'(%s)\s(?P<starting>.*)' % RE_STARTING.pattern RE_EVENT = r'(?P<event>(?:every|each|\bon\b|repeat|%s|%s)(?:s|ing)?(.*))' % ( RE_DAILY.pattern, RE_PLURAL_WEEKDAY.pattern) RE_END = r'%s(?P<ending>.*)' % RE_ENDING.pattern RE_START_EVENT = re.compile(r'%s\s%s' % (RE_START, RE_EVENT)) RE_EVENT_START = re.compile(r'%s\s%s' % (RE_EVENT, RE_START)) RE_FROM_TO = re.compile( r'(?P<event>.*)from(?P<starting>.*)(to|through|thru)(?P<ending>.*)') RE_START_END = re.compile(r'%s\s%s' % (RE_START, RE_END)) RE_OTHER_END = re.compile(r'(?P<other>.*)\s%s' % RE_END) RE_SEP = re.compile( r'(from|to|through|thru|on|at|of|in|a|an|the|and|or|both)$')
def parse_message(text): c = Calendar() t_s, p_s = c.parse(text) time = datetime(*t_s[:6]) td = time - datetime.now() return None if td.days < 0 else td.total_seconds()
def parsedatetime_calendar(): from parsedatetime import Calendar, Constants return Calendar(parsedatetime_constants())
def __init__(self, bot: Bot): self.bot = bot self.cal = Calendar() self.remind_loop.start()
def __init__(self) -> None: self.cal: Calendar = Calendar()