def convert_date(raw_date: str) -> tuple: def _get_datetime_range(date: str) -> tuple: """ Makes one day period by received date. Parameters ---------- date: Date for report generating. Returns ------- Datetime range. """ date = datetime.strptime(date, "%Y-%m-%d") from_date = date.astimezone(utc) from_date = from_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z") to_date = (date.replace( hour=23, minute=59, second=59).astimezone(utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z")) return from_date, to_date parser = Calendar() date, status = parser.parse(raw_date) date = datetime(*date[:3]).strftime("%Y-%m-%d") return _get_datetime_range(date)
def process_datetime(date_string): cal = Calendar() try: dt = date_parse(date_string) except TypeError: dt = cal.nlp(date_string)[0][0] return dt
async def poll(self): cal = Calendar() now = datetime.now(timezone("UTC")) then = cal.parseDT("1 minute", now) print("polling...") for r in Reminder.select().where(Reminder.time.between(now, then[0])): await self.bot.get_channel(r.channel_id).send(r.reminder_text)
def get_time(time_string): cal = Calendar() now = datetime.now(timezone("UTC")) if time_string == "": time = cal.parseDT("1 day", now) else: time = cal.parseDT(time_string, now) return time[0]
def _guess_game_date(datetime_str): # Try to turn a datetime string from FD into an actual datetime datetime_str.replace('Sun', 'Sunday') datetime_str.replace('Mon', 'Monday') datetime_str.replace('Tues', 'Tuesday') datetime_str.replace('Wed', 'Wednesday') datetime_str.replace('Thurs', 'Thursday') datetime_str.replace('Fri', 'Friday') datetime_str.replace('Sat', 'Saturday') cal = Calendar() dt, ret_code = cal.parseDT(datetime_str) return dt
def parse_mtime_single(value): """Convert a human readable time description to a """ if not HAS_PARSEDATETIME: return int(value) calendar = Calendar() guess, rc = calendar.parseDT(value) if rc == 0: LOGGER.warning('Could not parse date: %s', value) return int(value) return guess.timestamp()
def parse_mtime_single(value): """Convert a human readable time description to a """ if not HAS_PARSEDATETIME: return int(value) calendar = Calendar() guess, rc = calendar.parseDT(value) if rc is 0: LOGGER.warning("Could not parse date: %s", value) return int(value) return guess.timestamp()
async def convert(self, ctx: commands.Context, argument: str): cal = Calendar() utc = timezone("UTC") utcnow = datetime.utcnow().replace(microsecond=0, tzinfo=utc) parsed = cal.parseDT(argument, sourceTime=utcnow, tzinfo=utc)[0] if parsed == utcnow: raise commands.BadArgument(f'"{argument}" is not a valid time.') elif parsed < utcnow: raise commands.BadArgument(f'"{argument}" is in the past.') return parsed
def parse(self, response): # follow links to offer pages offers = response.xpath('//a[@data-cy="listing-ad-title"]') for offer in offers: yield response.follow(url=offer.attrib['href'], callback=self.parse_offer) latest_date = response.xpath('//small//span//i[@data-icon="clock"]/../text()').getall().pop() olx_req = OlxRequirements(req) cal = Calendar() required_date = cal.parse(' '.join([str(olx_req.number_of_days), 'days ago']))[0] parsed_date = latest_date.replace('\n', '').replace('\t', '') latest_date_processed = olx_convert_to_time_struct(parsed_date) if latest_date_processed < required_date: return yield response.follow(url=response.xpath('//a[@data-cy="page-link-next"]').attrib['href'], callback=self.parse)
class CsvTypeParser(TabularTypeParser): """CSV type parser""" calendar: ClassVar[Calendar] = Calendar() """Calendar object used for date parsing""" def __post_init__(self): if self.parse is None: if issubclass(self.pytype, datetime): self.parse = self.parse_datetime elif issubclass(self.pytype, date): self.parse = self.parse_date super().__post_init__() @classmethod def parse_datetime(cls, value): """Parse datetime from CSV value""" timestamp, ret = cls.calendar.parseDT(value) if not ret: raise ValueError("Invalid date: '%s'" % value) return timestamp @classmethod def parse_date(cls, value): """Parse date from CSV value""" return cls.parse_datetime(value).date()
def parse_contest_element(browser_elem): """ Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost :param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element :return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description """ match_groups = contest_element_id_regex.match(browser_elem.get_attribute("id")) fd_game_id = int(match_groups.group("gameid")) fd_table_id = int(match_groups.group("tableid")) game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace('$','').replace(',','') game_time = browser_elem.find_element_by_xpath("div[2]/time").text cal = Calendar() fd_game_date, ret_code = cal.parseDT(game_time) entry_button_element = browser_elem.find_element_by_xpath("a") return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def clean(self, value): """ Validates that the input can be converted to a time. Returns a Python datetime.time object. """ super(NaturalTimeField, self).clean(value) if value in EMPTY_VALUES: return None if isinstance(value, datetime.time): return value c = Calendar() parsed = c.parse(value) if parsed[1] == 2: return datetime.time(*parsed[0][3:6]) raise ValidationError(self.error_messages['invalid'])
def str_to_datetime(string, on_fail_return=None): try: from dateutil import parser as p output = p.parse(string) return output except: from parsedatetime import Calendar from datetime import datetime c = Calendar() output, flags = c.parse(string) if flags > 0: return datetime(*output[:6]) else: return None return None
def str2date(a, verbose=False): struct, status = Calendar().parse(a) if status == 0: if verbose: print(f"Unable to recognize date in \"{a}\".") return date.today() return date(*struct[:3])
def parse_contest_element(browser_elem): """ Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost :param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element :return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description """ match_groups = contest_element_id_regex.match( browser_elem.get_attribute("id")) fd_game_id = int(match_groups.group("gameid")) fd_table_id = int(match_groups.group("tableid")) game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace( '$', '').replace(',', '') game_time = browser_elem.find_element_by_xpath("div[2]/time").text cal = Calendar() fd_game_date, ret_code = cal.parseDT(game_time) entry_button_element = browser_elem.find_element_by_xpath("a") return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def cli(config, src, dst, when): """ commute.py helper CLI to compute possible commute options. """ if when is not None: when = int(time.mktime(Calendar().parseDT(when)[0].timetuple())) for rank, path in commute.get_all_paths(config, src, dst, when): click.echo(commute.format_path(rank, path)) click.echo("-" * 5)
def test_olx_convert_to_time_struct_valid_case(): test_input = { 'dzisiaj 13:48': 'today 13:48', 'wczoraj 13:48': 'yesterday 13:48', '16 sty': '16 january', '16 wrz': '16 september', '16 lut': '16 february', '16 mar': '16 march', '16 kwi': '16 april', '16 maj': '16 may', '16 cze': '16 june', '16 lip': '16 july', '16 sie': '16 august', '16 paź': '16 october', '16 lis': '16 november', '16 gru': '16 december', } cal = Calendar() for k, v in test_input.items(): assert olx_convert_to_time_struct(k) == cal.parse(v)[0]
def parse_datetime(date_string): """Parse a timetime object from a natural language string.""" cal = Calendar() unaware_dt = cal.nlp(date_string) if not unaware_dt: raise DateParseException("Due date '%s' unclear" % date_string) else: code = unaware_dt[0][1] unaware_dt = unaware_dt[0][0] # If no time is supplied, assume 6pm if code == 1: unaware_dt = unaware_dt.replace(hour=18, minute=0, second=0, microsecond=0) if os.environ.get('HABIT_TZ'): localtz = pytz.timezone(os.environ.get('HABIT_TZ')) else: localtz = get_localzone() aware_dt = localtz.localize(unaware_dt) return aware_dt
def olx_convert_to_time_struct(date_str): olx_date_mapper = { 'dzisiaj': 'today', 'wczoraj': 'yesterday', 'sty': 'january', 'wrz': 'september', 'lut': 'february', 'mar': 'march', 'kwi': 'april', 'maj': 'may', 'cze': 'june', 'lip': 'july', 'sie': 'august', 'paź': 'october', 'lis': 'november', 'gru': 'december', } for k, v in olx_date_mapper.items(): date_str = date_str.replace(k, v) cal = Calendar() return cal.parse(date_str)[0]
class Reminders(commands.Cog): """Set reminders.""" def __init__(self, bot: Bot): self.bot = bot self.cal = Calendar() self.remind_loop.start() @commands.command(name="remind", aliases=["remindme"]) async def remind(self, ctx: Context, t: str, *, message: str): """Set a reminder.""" parsed, success = self.cal.parse(t) if not success: return await ctx.reply("Please use a valid time string, such as 3d6h.") time = mktime(parsed) dt = datetime.fromtimestamp(time) td = dt - datetime.utcnow() total_diff = td.total_seconds() if total_diff < 5: return await ctx.reply("I can't create reminders less than 5 seconds in the future!") id = await self.bot.db.create_reminder(ctx.author.id, ctx.guild.id, ctx.channel.id, ctx.message.id, dt, message) await ctx.reply(f"Reminder created with ID {id} set to remind you at <t:{int(dt.timestamp())}:F>.") @tasks.loop(seconds=10) async def remind_loop(self): await self.bot.wait_until_ready() rs = await self.bot.db.get_expired_reminders() for reminder in rs: if reminder["expires"] > datetime.now(): continue logger.info(f"Actioning reminder ID {reminder['id']} from {reminder['userid']}") await self.bot.db.mark_reminder_completed(reminder["id"]) channel = self.bot.get_channel(reminder["cid"]) if not channel: continue jump = f"https://discord.com/channels/{reminder['gid']}/{reminder['cid']}/{reminder['mid']}" await channel.send( f"<@{reminder['userid']}> reminder:\n{reminder['content']}\n\nOriginal message: {jump}", allowed_mentions=AllowedMentions(users=True), )
def parse_offer(self, response): """Extracts the details of the offer to search through.""" # Extract date of offer submission # We don't want an offer which is too old unprocessed_date = response.xpath("//div[@class='css-lh1bxu']").get() (_, _, unprocessed_date) = unprocessed_date.partition(':') (processed_date, _, _) = unprocessed_date.partition('<') cal = Calendar() offer_submission_date = cal.parse(processed_date) otodom_req = OtodomRequirements(req) required_date = cal.parse(' '.join( [str(otodom_req.number_of_days), 'days ago'])) if offer_submission_date < required_date: return li = response.xpath("//li/text()").getall() li_text = get_processed_text(li) p = response.xpath("//p/text()").getall() p_text = get_processed_text(p) text = ' '.join([li_text, p_text]) if calculate_intersection(text, otodom_req.tags) >= otodom_req.threshold: append_to_file(path='scrapy-data/urls.txt', response=response)
def parse(self, response): # follow links to offer pages offers = response.xpath("//a[@class='href-link tile-title-text']") for offer in offers: yield response.follow(url=offer.attrib['href'], callback=self.parse_offer) # follow pagination link tmp_1 = response.xpath( "//a[@class='arrows icon-angle-right-gray icon-right-arrow']") tmp_2 = response.xpath( "//a[@class='arrows icon-right-arrow icon-angle-right-gray']") next_page = tmp_1 if 'href' in tmp_1.attrib else tmp_2 offer_date = response.xpath("//div[@class='creation-date']") parsed_date = extract_date(offer_date[len(offer_date) - 1].get()) gumtree_req = GumtreeRequirements(req) cal = Calendar() date = cal.parse(parsed_date) required_date = cal.parse(' '.join( [str(gumtree_req.number_of_days), 'days ago'])) if date < required_date: return yield response.follow(next_page.attrib['href'], self.parse)
def parse_offer(self, response): """Extracts the details of the offer to search through.""" unprocessed_date = response.css( '#wrapper > div:nth-child(1) > div.vip-header-and-details > div.vip-details' ' > ul > li:nth-child(1) > div > span.value').get() (_, unprocessed_date, _) = unprocessed_date.split('>') (processed_date, _) = unprocessed_date.split('<') gumtree_req = GumtreeRequirements(req) cal = Calendar() offer_submission_date = cal.parse(processed_date) required_date = cal.parse(' '.join( [str(gumtree_req.number_of_days), 'days ago'])) if offer_submission_date < required_date: return spans = response.xpath("//span[@class='value']").getall() spans_text = get_processed_text(spans) p = response.xpath("//span[@class='pre']").getall() p_text = get_processed_text(p) text = ' '.join([spans_text, p_text]) if calculate_intersection(text, gumtree_req.tags) >= gumtree_req.threshold: append_to_file(path='scrapy-data/urls.txt', response=response)
def clean(self, value): """ Validates that the input can be converted to a datetime. Returns a Python datetime.datetime object. """ super(NaturalDateTimeField, self).clean(value) if value in EMPTY_VALUES: return None if isinstance(value, datetime.datetime): return value if isinstance(value, datetime.date): return datetime.datetime(value.year, value.month, value.day) if isinstance(value, list): # Input comes from a SplitDateTimeWidget, for example. So, it's two # components: date and time. if len(value) != 2: raise ValidationError(self.error_messages['invalid']) value = '%s %s' % tuple(value) c = Calendar() parsed = c.parse(value) if parsed[1] == 3: return datetime.date(*parsed[0][:6]) raise ValidationError(self.error_messages['invalid'])
def parse_time( time_str: str, seconds: int = True ) -> typing.Union[None, int, typing.Tuple[datetime.datetime, int]]: """ Parses a time. :param time_str: The time string to parse. :return: The total number of seconds between now and then. """ calendar = Calendar() t_struct, parse_status = calendar.parse( time_str, sourceTime=datetime.datetime.utcnow()) if parse_status == 0: return None dt = datetime.datetime(*t_struct[:6]) diff = np.ceil((dt - datetime.datetime.utcnow()).total_seconds()) if seconds: return diff else: return dt, diff
def parse(self, response): # follow links to offer pages offers = response.xpath("//a[@class='property_link property-url']") for offer in offers: yield response.follow(url=offer.attrib['href'], callback=self.parse_offer) latest_date = response.xpath("//span[@class='single-result__category single-result__category--date']")\ .getall()\ .pop() # the strong word appears in the rag when the date is in bold (and it's only bold when the offer has just been # added, hence there is no sense in finding out whether is't an old offer if 'strong' not in latest_date: morizon_req = MorizonRequirements(req) cal = Calendar() required_date = cal.parse(' '.join([str(morizon_req.number_of_days), 'days ago'])) parsed_date = extract_from_tag(latest_date) latest_date_processed = morizon_convert_to_time_struct(parsed_date) if latest_date_processed < required_date: return yield response.follow( url=response.xpath("//a[@class='mz-pagination-number__btn mz-pagination-number__btn--next']") .pop() .attrib['href'], callback=self.parse)
def action(self): self._logger.info('Start viewing the log events...') def _exit(signum, frame): self._logger.info('Exit by code {} ...'.format(signum)) self._exit = True signal.signal(signal.SIGTERM, _exit) signal.signal(signal.SIGINT, _exit) start = time.time() if self._start is not None: time_struct, _ = Calendar().parse(self._start) start = mktime(time_struct) start = int(start * 1000) client = self.get_logs_client() function = self._config.get_function_name() event_ids = {} while self._exit is False: events = client.get_log_events(function, start, self._filter) for e in events: if e['eventId'] not in event_ids: event_ids[e['eventId']] = None print(e['message']) if e['timestamp'] > start: start = e['timestamp'] event_ids = {} if not self._follow: break time.sleep(self._interval)
def process(input_wikitext): input_wikitext, encoded_wikilinks = encode_wikilinks(input_wikitext) old_ah_wikitext_search = ARTICLE_HISTORY.search(input_wikitext) if not old_ah_wikitext_search: return input_wikitext old_ah_wikitext = old_ah_wikitext_search.group(0) history = History(old_ah_wikitext) # For use in sorting parameters by_time = lambda x: datetime.datetime.fromtimestamp( mktime(Calendar().parse(x[0])[0])) lines_to_delete = [] if ITN.search(input_wikitext): itn_list = history.get_relevant_params("itn") for itn_result in ITN.finditer(input_wikitext): itn = itn_result.group(1) itn_params = itn.split("|")[1:] if "=" not in itn_params[0] and "=" not in itn_params[1]: # {{ITN talk|DD monthname|YYYY}} itn_list.append((itn_params[0] + " " + itn_params[1], "")) else: itn_list += [(x[x.find("=") + 1:], "") for x in itn.split("|")[1:] if "date" in x] lines_to_delete.append(itn_result.group(0)) itn_list.sort(key=by_time) # Update the article history template history.other_parameters["itndate"] = itn_list[0][0] if itn_list[0][1]: history.other_parameters["itnlink"] = itn_list[0][1] for i, item in enumerate(itn_list[1:], start=2): history.other_parameters["itn%ddate" % i] = item[0] if item[1]: history.other_parameters["itn%ditem" % i] = item[1] if OTD.search(input_wikitext): otd_list = history.get_relevant_params("otd") for otd_result in OTD.finditer(input_wikitext): otd = otd_result.group(1) otd_params = { x: y for x, y in [t.strip().split("=") for t in otd.split("|")[1:]] } for i in itertools.count(1): date_key = "date%d" % i if date_key in otd_params: otd_list.append((otd_params[date_key], otd_params.get("oldid%d" % i, ""), "")) else: break lines_to_delete.append(otd_result.group(0)) otd_list.sort(key=by_time) # Update the article history template history.other_parameters["otddate"], history.other_parameters[ "otdoldid"], _ = otd_list[0] if otd_list[0][2]: history.other_parameters["otdlink"] = otd_list[0][2] for i, item in enumerate(otd_list[1:], start=2): history.other_parameters["otd%ddate" % i], history.other_parameters["otd%doldid" % i], _ = item if item[2]: history.other_parameters["otd%dlink" % i] = item[2] dyk_search = DYK.search(input_wikitext) if dyk_search: dyk = dyk_search.group(1) dyk_params = dyk.split("|")[1:] history.other_parameters["dykentry"] = next( x for x in dyk_params if x.startswith("entry")).split("=")[1] positional_dyk_params = [x for x in dyk_params if "=" not in x] if len(positional_dyk_params) == 1: history.other_parameters["dykdate"] = positional_dyk_params[0] elif len(positional_dyk_params) == 2: for param in positional_dyk_params: if len(param) == 4: year = param else: month_day = param history.other_parameters["dykdate"] = month_day + " " + year # Delete the DYK template lines_to_delete.append(dyk_search.group(0)) # Delete the lines with only the delete comment on them result_text = "\n".join( ifilterfalse(lines_to_delete.__contains__, input_wikitext.splitlines())) result_text = result_text.replace(old_ah_wikitext, history.as_wikitext()) result_text = decode_wikilinks(result_text, encoded_wikilinks) return result_text
tokens.append(sent[span[0]:span[1]]) current = span[1] tags = pos_tag(tokens) new_tags = [] for word, tag in tags: if word[:2] == '__': new_tags.append((word[2:], tag_dict[word[2:]])) else: tag = [t[1] for t in original_tags if t[0] == word][0] # FIXED new_tags.append((word, tag)) return new_tags cal = Calendar() month2num = { "january": 1, "february": 2, "march": 3, "april": 4, "may": 5, "june": 6, "july": 7, "august": 8, "september": 9, "october": 10, "november": 11, "december": 12 } num2month = dict([(n, m) for (m, n) in month2num.items()])
def __init__(self): self.parser.add_argument('-s', '--status', action='store_true', help='show rule list with expirations') self.parser.add_argument('-c', '--clean', action='store_true', help='clean up expired rules') self.parser.add_argument('-r', '--rule', help='rule to be added to `ufw`') self.parser.add_argument('-p', '--position', default=1, help='position to add the rule') self.parser.add_argument('-t', '--ttl', default='30 days', help='time to live for the rule') args = self.parser.parse_args() # Our file names pid_file = '/var/run/' + __file__ + '.pid' rules_file = '/usr/local/share/' + __file__ + '/rules' tmp_rules_file = '/tmp/' + __file__ + '-rules' if args.status: if path.exists(rules_file): try: print("Expiration\t\tRule") print('=' * 80) # Loops through the rules lines for line in open(rules_file, 'r'): # Breaks apart line into expiration timestamp and rule timestamp, rule = line.strip("\n").split(' ', 1) print( str(datetime.fromtimestamp(float(timestamp))) + "\t" + rule) except IOError: self.error('unable to read from the rules file: ' + rules_file) else: self.error('there are no rules to display') elif args.clean: # Checks for PID file if path.exists(pid_file): self.error(__file__ + ' is already running') else: # Creates the PID file try: handle = open(pid_file, 'w') handle.write(str(getpid())) handle.close() except IOError: self.error('unable to create PID file: ' + pid_file) # Checks for the rules file if path.exists(rules_file): # Opens the temporary rules file try: handle = open(tmp_rules_file, 'a') except IOError: self.error('unable to write to the tmp rules file: ' + tmp_rules_file) try: current_time = time() # Loops through the rules lines for line in open(rules_file, 'r'): # Breaks apart line into expiration timestamp and rule timestamp, rule = line.strip("\n").split(' ', 1) # Checks if rule has expired if current_time < float(timestamp): handle.write(line) print( str(datetime.fromtimestamp(time())) + "\tskipped rule\t" + rule) else: try: self.ufw_execute('delete ' + rule) print( str(datetime.fromtimestamp(time())) + "\tdeleted rule\t" + rule) except CalledProcessError as error: self.ufw_error(error) handle.close() # Moves the tmp file to the rules file move(tmp_rules_file, rules_file) except IOError: self.error('unable to from the read rules file: ' + rules_file) # Removes the PID remove(pid_file) elif args.rule: rules_path = path.dirname(rules_file) if not path.exists(rules_path): makedirs(rules_path) # Converts the TTL to a timestamp cal = Calendar() timestamp = mktime(cal.parse(args.ttl)[0]) # Writes the rule to the rules file try: # TODO Check if rule already exists and update it instead of adding it again handle = open(rules_file, 'a') handle.write(str(timestamp) + ' ' + args.rule) handle.write("\n") handle.close() except IOError: self.error('unable to write to the rules file: ' + rules_file) # Attempts to add the rule to `ufw` try: self.ufw_execute('insert ' + str(args.position) + ' ' + args.rule) except CalledProcessError as error: # Catches an error when attempting to add a rule to an empty database if error.output == b"ERROR: Invalid position '1'\n": try: self.ufw_execute(args.rule) except CalledProcessError as error: self.ufw_error(error) else: self.ufw_error(error) else: self.error('no arguments specified')
import re from datetime import datetime import logging from parsedatetime import Calendar from pytz import timezone, utc from recurrent.constants import * pdt = Calendar() log = logging.getLogger('recurrent') RE_TIME = re.compile( r'(?P<hour>\d{1,2}):?(?P<minute>\d{2})?\s?(?P<mod>am|pm)?(oclock)?') RE_AT_TIME = re.compile(r'at\s%s' % RE_TIME.pattern) RE_AT_TIME_END = re.compile(r'at\s%s$' % RE_TIME.pattern) RE_STARTING = re.compile(r'start(?:s|ing)?') RE_ENDING = re.compile(r'(?:\bend|until)(?:s|ing)?') RE_REPEAT = re.compile(r'(?:every|each|\bon\b|repeat(s|ing)?)') RE_START = r'(%s)\s(?P<starting>.*)' % RE_STARTING.pattern RE_EVENT = r'(?P<event>(?:every|each|\bon\b|repeat|%s|%s)(?:s|ing)?(.*))' % ( RE_DAILY.pattern, RE_PLURAL_WEEKDAY.pattern) RE_END = r'%s(?P<ending>.*)' % RE_ENDING.pattern RE_START_EVENT = re.compile(r'%s\s%s' % (RE_START, RE_EVENT)) RE_EVENT_START = re.compile(r'%s\s%s' % (RE_EVENT, RE_START)) RE_FROM_TO = re.compile( r'(?P<event>.*)from(?P<starting>.*)(to|through|thru)(?P<ending>.*)') RE_START_END = re.compile(r'%s\s%s' % (RE_START, RE_END)) RE_OTHER_END = re.compile(r'(?P<other>.*)\s%s' % RE_END) RE_SEP = re.compile( r'(from|to|through|thru|on|at|of|in|a|an|the|and|or|both)$')
def handle(self, *args, **options): if len(args) != 1: raise CommandError('Must pass exactly one json file to import') same, new, updated = 0, 0, 0 # Object counts calendar = Calendar() # date parser # Load data datasource = DataSource(args[0], encoding='iso-8859-1') layer = datasource[0] for feature in layer: # Gather and transform attributes attr = { 'object_id': feature['OBJECTID'].value, 'case_id': loads('"%s"' % feature['Case_'].value), 'status': feature['Status'].value, 'link': loads('"%s"' % feature['Link'].value), } raw_location = feature['Location'].value if '\r\n' in raw_location: self.stderr.write('Invalid Location %s' % repr(raw_location)) raw_location = raw_location.split('\r\n', 1)[0] attr['location'] = loads('"%s"' % raw_location) raw_type = feature['Type'].value if '\r\n' in raw_type: self.stderr.write('Invalid Type %s' % repr(raw_type)) raw_type = raw_type.split('\r\n', 1)[0] attr['case_type'] = loads('"%s"' % raw_type) raw_date = loads('"%s"' % feature['Date_'].value) timestamp = mktime(calendar.parseDateText(raw_date)) attr['hearing_date'] = datetime.fromtimestamp(timestamp).date() attr['geom'] = GEOSGeometry(feature.geom.wkt) # Is there a case with this object ID? case_query = Case.objects.filter( object_id=attr['object_id'], domain=Case.DOMAIN_TMAPC) if case_query.exists(): case = case_query.get() # Did the data change? different = [] for name, new_value in attr.items(): existing = getattr(case, name) if existing != new_value: different.append((name, existing, new_value)) if different: updated += 1 diff_message = ', '.join( ["%s:'%s' -> '%s'" % x for x in different]) msg = 'Updated case "%s" (%s)' % ( attr['case_id'], diff_message) self.stdout.write(msg) else: same += 1 self.stdout.write( 'No change to case "%s"' % attr['case_id']) else: # Create new Case case = Case( object_id=attr['object_id'], domain=Case.DOMAIN_TMAPC) self.stdout.write('New case "%s"' % attr['case_id']) new += 1 different = True if different: # Save changes for name, value in attr.items(): setattr(case, name, value) case.save() self.stdout.write( 'Imported %d BOA cases (%d new, %d changed, %d unchanged)' % ( new + updated + same, new, updated, same))
def str_ftime(arg, format): cal = Calendar() return strftime(format, cal.parse(str(arg))[0])
print("Counts down until the specified time. May be specified as a") print("relative time (for example, '5 minutes' or 'tomorrow') or an") print("actual time (for example, '8am').") if error is not None: print("\n\n") print (error) sys.exit(1) if len(sys.argv) == 1: helptext() from parsedatetime import Calendar from time import mktime, sleep from datetime import datetime, timedelta p = Calendar() specifiedtime = p.parse(" ".join(sys.argv[1:])) timeleft = datetime.fromtimestamp(mktime(specifiedtime[0])) - datetime.now() length = 0 while timeleft > timedelta(microseconds=-1): interval = 10 if timeleft < timedelta(hours=1): interval = 1 if timeleft < timedelta(minutes=10): interval = 0.1 if timeleft < timedelta(minutes=1): interval = 0.001 sys.stdout.write("\b" * length) outline = str(timeleft)
def human_time_to_datetime(s): cal = Calendar() t, _ = cal.parse(s) return datetime(*t[:6])
def parsedatetime_calendar(): from parsedatetime import Calendar, Constants return Calendar(parsedatetime_constants())
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Must pass exactly one json file to import") same, new, updated = 0, 0, 0 # Object counts calendar = Calendar() # date parser # Load data datasource = DataSource(args[0], encoding="iso-8859-1") layer = datasource[0] for feature in layer: # Gather and transform attributes attr = { "object_id": feature["OBJECTID"].value, "case_id": loads('"%s"' % feature["Case_"].value), "status": feature["Status"].value, "location": loads('"%s"' % feature["Location"].value), "link": loads('"%s"' % feature["Link"].value), } raw_date = loads('"%s"' % feature["Date_"].value) timestamp = mktime(calendar.parseDateText(raw_date)) attr["hearing_date"] = datetime.fromtimestamp(timestamp).date() attr["geom"] = GEOSGeometry(feature.geom.wkt) # Is there a case with this object ID? case_query = Case.objects.filter(object_id=attr["object_id"], domain=Case.DOMAIN_BOA) if case_query.exists(): case = case_query.get() # Did the data change? different = [] for name, new_value in attr.items(): existing = getattr(case, name) if existing != new_value: different.append((name, existing, new_value)) if different: updated += 1 diff_message = ", ".join(["%s:'%s' -> '%s'" % x for x in different]) msg = 'Updated case "%s" (%s)' % (attr["case_id"], diff_message) self.stdout.write(msg) else: same += 1 self.stdout.write('No change to case "%s"' % attr["case_id"]) else: # Create new Case case = Case(object_id=attr["object_id"], domain=Case.DOMAIN_BOA, case_type="Board of Adjustment") self.stdout.write('New case "%s"' % attr["case_id"]) new += 1 different = True if different: # Save changes for name, value in attr.items(): setattr(case, name, value) case.save() self.stdout.write( "Imported %d BOA cases (%d new, %d changed, %d unchanged)" % (new + updated + same, new, updated, same) )
def __init__(self) -> None: self.cal: Calendar = Calendar()
def __init__(self, bot: Bot): self.bot = bot self.cal = Calendar() self.remind_loop.start()
def parse_datetime(value): calendar = Calendar() raw_date = loads('"%s"' % value) timestamp = mktime(calendar.parseDateText(raw_date)) return datetime.fromtimestamp(timestamp).date()
def scrape_single_game(sport, browser, debug=False): # Since we are separating contests by their start time now, we don't currently try to choose a # best contest to scrape. We should certainly upgrade this in the future! contest_element_id_regex = re.compile(r"contest_(?P<gameid>\d+)-(?P<tableid>\d+)") browser_elem = browser.find_element_by_class_name('contest-list-item') match_groups = contest_element_id_regex.match(browser_elem.get_attribute("id")) fd_game_id = int(match_groups.group("gameid")) fd_table_id = int(match_groups.group("tableid")) fd_game_title = browser_elem.find_element_by_class_name("contest-name-text").text entry_fee = browser_elem.find_element_by_class_name("entry-fee-cell").text.replace('$','').replace(',','') game_time = browser_elem.find_element_by_class_name("startdate-cell").text cal = Calendar() new_parsed_dt, ret_code = cal.parseDT(game_time) # If it is between 9 PM and midnight locally (Pacific), we'll have an issue: # FD shows us the game time as e.g. "7 PM" but we'll interpret that as today. # Workaround: if the game time is in the past, its probably tomorrow. Add 1 day. # Break if it's not between 9PM and midnight so we know this happened if new_parsed_dt < datetime.datetime.now(): if datetime.datetime.now().hour < 21: print 'Parsed time of FanDuel game start is in the past, but this isnt a TZ issue!' IPython.embed() else: new_parsed_dt += datetime.timedelta(days=1) assert new_parsed_dt > datetime.datetime.now() browser_elem.click() print 'Scraping this game:' print ' ', fd_game_title print ' on', new_parsed_dt.isoformat() game_entry_url = 'https://www.fanduel.com/games/{game_id}/contests/{game_id}-{table_id}/enter'.format(game_id=fd_game_id, table_id=fd_table_id) # Go to the details for the game, to find list of eligible players for this game. browser.get(game_entry_url) time.sleep(1) # Get the salary cap for the game directly salary_text = browser.find_element_by_xpath('//*[@id="ui-skeleton"]/div/section/div[2]/div[4]/div[2]/section/header/' 'remaining-salary/div/div[1]/figure').text cap = int(salary_text.replace('$','').replace(',','')) # Download the master player / salary list player_list_link = browser.find_element_by_link_text("Download players list") # Sometimes this doesn't work the first time... download_filename = get_csv_file(sport, new_parsed_dt, fd_game_id) attempts = 1 while not os.path.exists(download_filename) and attempts <= 10: print '...trying to download file (attempt %d)' % attempts time.sleep(3) player_list_link.click() time.sleep(3) attempts += 1 if attempts > 5: print "Problem downloading player list -- not saving anything" return False else: print '...success!' # Get player lineups if sport == 'mlb': print "Accessing lineups..." lineups_link = browser.find_element_by_link_text("Lineup Info") # You can't just click it, it opens in a new tab :( So we visit it with get() and then go back() lineup_url = lineups_link.get_attribute('href') browser.get(lineup_url) lineup_info = parse_lineup_page(browser.page_source) browser.back() print " %d / %d lineups submitted (%d players)" % (len(lineup_info.loaded_teams), len(lineup_info.loaded_teams) + len(lineup_info.unloaded_teams), len(lineup_info.parsed_players)) else: # Should consider grabbing injury / GTD status here lineup_info = None # Except that doesn't work too well # parse_nba_player_list(browser.page_source) add_game_info_to_db(sport, fd_game_id, cap, entry_fee, new_parsed_dt, fd_table_id, game_title=fd_game_title, lineup=lineup_info) # Return to the main lobby page with back() browser.back() return True
def fuzzydtparse(request): """ Attempts to parse a natural-language date/time string and return a formatted representation of the parsed date/time. The returned representations are formatted using django's php-style datetime formatting facilities. See http://docs.djangoproject.com/en/dev/ref/templates/builtins/#now for a full specification GET/POST arguments: dtstring *(required)* The natural-language date and/or time string to parse. dateformat (optional) A format specifier string used to format the returned representation if it is determined to be a date instance. The default format is 'l, F jS Y', which produces values like 'Thursday, October 1st 2009'. timeformat (optional) A format specifier string used to format the returned representation if it is determined to be a time instance. The default format is 'P', which produces values like '6:26 p.m.'. dtformat (optional) A format specifier string used to format the returned representation if it is determined to be a datetime instance. The default format is 'l, F jS Y, P', which produces values like 'Thursday, October 1st 2009, 6:26 p.m.'. require (optional) One of 'date', 'time', 'datetime'. If the parsed value is not of the specified type, the view will return 400/Bad Request callback (optional) JSONP callback. Returns a json dictionary containing two values: type An string indicating the kind of representation returned. One of 'date', 'time', or 'datetime' parsed A string representation of the parsed datetime. Invalid / unparsable datetimes will return 400/Bad Request. """ try: dtstring = request.REQUEST['dtstring'] except KeyError: return HttpResponseBadRequest() kind = { 'date': 1, 'time': 2, 'datetime': 3 } dateformat = request.REQUEST.get('dateformat', 'l, F jS Y' ) timeformat = request.REQUEST.get('timeformat', 'P') dtformat = request.REQUEST.get('dtformat', 'l, F jS Y, P') require = request.REQUEST.get('require', None) callback = request.REQUEST.get('callback', None) if require and require not in kind: return HttpResponseBadRequest() c = Calendar() # TODO: possible security hole? parsed = c.parse(dtstring) if parsed[1] == 0: return HttpResponseBadRequest() parsed_dt = datetime.datetime(*parsed[0][:6]) response_dict = {} if require and parsed[1] != kind[require]: return HttpResponseBadRequest() try: if parsed[1] == 1: response_dict['type'] = 'date' response_dict['parsed'] = format(parsed_dt, dateformat) elif parsed[1] == 2: response_dict['type'] = 'time' response_dict['parsed'] = format(parsed_dt, timeformat) elif parsed[1] == 3: response_dict['type'] = 'datetime' response_dict['parsed'] = format(parsed_dt, dtformat) else: #should never be here return HttpResponseBadRequest() except: return HttpResponseBadRequest() if callback: resp = "%s(%s)" % (callback, json.dumps(response_dict)) else: resp = json.dumps(response_dict) return HttpResponse(resp, mimetype='application/javascript')