def convert_date(raw_date: str) -> tuple:
def _get_datetime_range(date: str) -> tuple:
""" Makes one day period by received date.
Parameters
----------
date:
Date for report generating.
Returns
-------
Datetime range.
"""
date = datetime.strptime(date, "%Y-%m-%d")
from_date = date.astimezone(utc)
from_date = from_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
to_date = (date.replace(
hour=23, minute=59,
second=59).astimezone(utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z"))
return from_date, to_date
parser = Calendar()
date, status = parser.parse(raw_date)
date = datetime(*date[:3]).strftime("%Y-%m-%d")
return _get_datetime_range(date)
class CsvTypeParser(TabularTypeParser):
"""CSV type parser"""
calendar: ClassVar[Calendar] = Calendar()
"""Calendar object used for date parsing"""
def __post_init__(self):
if self.parse is None:
if issubclass(self.pytype, datetime):
self.parse = self.parse_datetime
elif issubclass(self.pytype, date):
self.parse = self.parse_date
super().__post_init__()
@classmethod
def parse_datetime(cls, value):
"""Parse datetime from CSV value"""
timestamp, ret = cls.calendar.parseDT(value)
if not ret:
raise ValueError("Invalid date: '%s'" % value)
return timestamp
@classmethod
def parse_date(cls, value):
"""Parse date from CSV value"""
return cls.parse_datetime(value).date()
def str2date(a, verbose=False):
struct, status = Calendar().parse(a)
if status == 0:
if verbose:
print(f"Unable to recognize date in \"{a}\".")
return date.today()
return date(*struct[:3])
def cli(config, src, dst, when):
"""
commute.py helper CLI to compute possible commute options.
"""
if when is not None:
when = int(time.mktime(Calendar().parseDT(when)[0].timetuple()))
for rank, path in commute.get_all_paths(config, src, dst, when):
click.echo(commute.format_path(rank, path))
click.echo("-" * 5)
async def poll(self):
cal = Calendar()
now = datetime.now(timezone("UTC"))
then = cal.parseDT("1 minute", now)
print("polling...")
for r in Reminder.select().where(Reminder.time.between(now, then[0])):
await self.bot.get_channel(r.channel_id).send(r.reminder_text)
def get_time(time_string):
cal = Calendar()
now = datetime.now(timezone("UTC"))
if time_string == "":
time = cal.parseDT("1 day", now)
else:
time = cal.parseDT(time_string, now)
return time[0]
def _guess_game_date(datetime_str):
# Try to turn a datetime string from FD into an actual datetime
datetime_str.replace('Sun', 'Sunday')
datetime_str.replace('Mon', 'Monday')
datetime_str.replace('Tues', 'Tuesday')
datetime_str.replace('Wed', 'Wednesday')
datetime_str.replace('Thurs', 'Thursday')
datetime_str.replace('Fri', 'Friday')
datetime_str.replace('Sat', 'Saturday')
cal = Calendar()
dt, ret_code = cal.parseDT(datetime_str)
return dt
def parse_mtime_single(value):
"""Convert a human readable time description to a """
if not HAS_PARSEDATETIME:
return int(value)
calendar = Calendar()
guess, rc = calendar.parseDT(value)
if rc == 0:
LOGGER.warning('Could not parse date: %s', value)
return int(value)
return guess.timestamp()
async def convert(self, ctx: commands.Context, argument: str):
cal = Calendar()
utc = timezone("UTC")
utcnow = datetime.utcnow().replace(microsecond=0, tzinfo=utc)
parsed = cal.parseDT(argument, sourceTime=utcnow, tzinfo=utc)[0]
if parsed == utcnow:
raise commands.BadArgument(f'"{argument}" is not a valid time.')
elif parsed < utcnow:
raise commands.BadArgument(f'"{argument}" is in the past.')
return parsed
def parse(self, response):
# follow links to offer pages
offers = response.xpath('//a[@data-cy="listing-ad-title"]')
for offer in offers:
yield response.follow(url=offer.attrib['href'], callback=self.parse_offer)
latest_date = response.xpath('//small//span//i[@data-icon="clock"]/../text()').getall().pop()
olx_req = OlxRequirements(req)
cal = Calendar()
required_date = cal.parse(' '.join([str(olx_req.number_of_days), 'days ago']))[0]
parsed_date = latest_date.replace('\n', '').replace('\t', '')
latest_date_processed = olx_convert_to_time_struct(parsed_date)
if latest_date_processed < required_date:
return
yield response.follow(url=response.xpath('//a[@data-cy="page-link-next"]').attrib['href'], callback=self.parse)
def str_to_datetime(string, on_fail_return=None):
try:
from dateutil import parser as p
output = p.parse(string)
return output
except:
from parsedatetime import Calendar
from datetime import datetime
c = Calendar()
output, flags = c.parse(string)
if flags > 0:
return datetime(*output[:6])
else:
return None
return None
def clean(self, value):
"""
Validates that the input can be converted to a time. Returns a Python
datetime.time object.
"""
super(NaturalTimeField, self).clean(value)
if value in EMPTY_VALUES:
return None
if isinstance(value, datetime.time):
return value
c = Calendar()
parsed = c.parse(value)
if parsed[1] == 2:
return datetime.time(*parsed[0][3:6])
raise ValidationError(self.error_messages['invalid'])
def parse_contest_element(browser_elem):
"""
Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost
:param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element
:return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description
"""
match_groups = contest_element_id_regex.match(
browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text
game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace(
'$', '').replace(',', '')
game_time = browser_elem.find_element_by_xpath("div[2]/time").text
cal = Calendar()
fd_game_date, ret_code = cal.parseDT(game_time)
entry_button_element = browser_elem.find_element_by_xpath("a")
return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def test_olx_convert_to_time_struct_valid_case():
test_input = {
'dzisiaj 13:48': 'today 13:48',
'wczoraj 13:48': 'yesterday 13:48',
'16 sty': '16 january',
'16 wrz': '16 september',
'16 lut': '16 february',
'16 mar': '16 march',
'16 kwi': '16 april',
'16 maj': '16 may',
'16 cze': '16 june',
'16 lip': '16 july',
'16 sie': '16 august',
'16 paź': '16 october',
'16 lis': '16 november',
'16 gru': '16 december',
}
cal = Calendar()
for k, v in test_input.items():
assert olx_convert_to_time_struct(k) == cal.parse(v)[0]
def olx_convert_to_time_struct(date_str):
olx_date_mapper = {
'dzisiaj': 'today',
'wczoraj': 'yesterday',
'sty': 'january',
'wrz': 'september',
'lut': 'february',
'mar': 'march',
'kwi': 'april',
'maj': 'may',
'cze': 'june',
'lip': 'july',
'sie': 'august',
'paź': 'october',
'lis': 'november',
'gru': 'december',
}
for k, v in olx_date_mapper.items():
date_str = date_str.replace(k, v)
cal = Calendar()
return cal.parse(date_str)[0]
def parse_offer(self, response):
"""Extracts the details of the offer to search through."""
unprocessed_date = response.css(
'#wrapper > div:nth-child(1) > div.vip-header-and-details > div.vip-details'
' > ul > li:nth-child(1) > div > span.value').get()
(_, unprocessed_date, _) = unprocessed_date.split('>')
(processed_date, _) = unprocessed_date.split('<')
gumtree_req = GumtreeRequirements(req)
cal = Calendar()
offer_submission_date = cal.parse(processed_date)
required_date = cal.parse(' '.join(
[str(gumtree_req.number_of_days), 'days ago']))
if offer_submission_date < required_date:
return
spans = response.xpath("//span[@class='value']").getall()
spans_text = get_processed_text(spans)
p = response.xpath("//span[@class='pre']").getall()
p_text = get_processed_text(p)
text = ' '.join([spans_text, p_text])
if calculate_intersection(text,
gumtree_req.tags) >= gumtree_req.threshold:
append_to_file(path='scrapy-data/urls.txt', response=response)
def parse(self, response):
# follow links to offer pages
offers = response.xpath("//a[@class='href-link tile-title-text']")
for offer in offers:
yield response.follow(url=offer.attrib['href'],
callback=self.parse_offer)
# follow pagination link
tmp_1 = response.xpath(
"//a[@class='arrows icon-angle-right-gray icon-right-arrow']")
tmp_2 = response.xpath(
"//a[@class='arrows icon-right-arrow icon-angle-right-gray']")
next_page = tmp_1 if 'href' in tmp_1.attrib else tmp_2
offer_date = response.xpath("//div[@class='creation-date']")
parsed_date = extract_date(offer_date[len(offer_date) - 1].get())
gumtree_req = GumtreeRequirements(req)
cal = Calendar()
date = cal.parse(parsed_date)
required_date = cal.parse(' '.join(
[str(gumtree_req.number_of_days), 'days ago']))
if date < required_date:
return
yield response.follow(next_page.attrib['href'], self.parse)
def parse_offer(self, response):
"""Extracts the details of the offer to search through."""
# Extract date of offer submission
# We don't want an offer which is too old
unprocessed_date = response.xpath("//div[@class='css-lh1bxu']").get()
(_, _, unprocessed_date) = unprocessed_date.partition(':')
(processed_date, _, _) = unprocessed_date.partition('<')
cal = Calendar()
offer_submission_date = cal.parse(processed_date)
otodom_req = OtodomRequirements(req)
required_date = cal.parse(' '.join(
[str(otodom_req.number_of_days), 'days ago']))
if offer_submission_date < required_date:
return
li = response.xpath("//li/text()").getall()
li_text = get_processed_text(li)
p = response.xpath("//p/text()").getall()
p_text = get_processed_text(p)
text = ' '.join([li_text, p_text])
if calculate_intersection(text,
otodom_req.tags) >= otodom_req.threshold:
append_to_file(path='scrapy-data/urls.txt', response=response)
def action(self):
self._logger.info('Start viewing the log events...')
def _exit(signum, frame):
self._logger.info('Exit by code {} ...'.format(signum))
self._exit = True
signal.signal(signal.SIGTERM, _exit)
signal.signal(signal.SIGINT, _exit)
start = time.time()
if self._start is not None:
time_struct, _ = Calendar().parse(self._start)
start = mktime(time_struct)
start = int(start * 1000)
client = self.get_logs_client()
function = self._config.get_function_name()
event_ids = {}
while self._exit is False:
events = client.get_log_events(function, start, self._filter)
for e in events:
if e['eventId'] not in event_ids:
event_ids[e['eventId']] = None
print(e['message'])
if e['timestamp'] > start:
start = e['timestamp']
event_ids = {}
if not self._follow:
break
time.sleep(self._interval)
def parse_time(
time_str: str,
seconds: int = True
) -> typing.Union[None, int, typing.Tuple[datetime.datetime, int]]:
"""
Parses a time.
:param time_str: The time string to parse.
:return: The total number of seconds between now and then.
"""
calendar = Calendar()
t_struct, parse_status = calendar.parse(
time_str, sourceTime=datetime.datetime.utcnow())
if parse_status == 0:
return None
dt = datetime.datetime(*t_struct[:6])
diff = np.ceil((dt - datetime.datetime.utcnow()).total_seconds())
if seconds:
return diff
else:
return dt, diff
def clean(self, value):
"""
Validates that the input can be converted to a datetime. Returns a
Python datetime.datetime object.
"""
super(NaturalDateTimeField, self).clean(value)
if value in EMPTY_VALUES:
return None
if isinstance(value, datetime.datetime):
return value
if isinstance(value, datetime.date):
return datetime.datetime(value.year, value.month, value.day)
if isinstance(value, list):
# Input comes from a SplitDateTimeWidget, for example. So, it's two
# components: date and time.
if len(value) != 2:
raise ValidationError(self.error_messages['invalid'])
value = '%s %s' % tuple(value)
c = Calendar()
parsed = c.parse(value)
if parsed[1] == 3:
return datetime.date(*parsed[0][:6])
raise ValidationError(self.error_messages['invalid'])
def parse(self, response):
# follow links to offer pages
offers = response.xpath("//a[@class='property_link property-url']")
for offer in offers:
yield response.follow(url=offer.attrib['href'], callback=self.parse_offer)
latest_date = response.xpath("//span[@class='single-result__category single-result__category--date']")\
.getall()\
.pop()
# the strong word appears in the rag when the date is in bold (and it's only bold when the offer has just been
# added, hence there is no sense in finding out whether is't an old offer
if 'strong' not in latest_date:
morizon_req = MorizonRequirements(req)
cal = Calendar()
required_date = cal.parse(' '.join([str(morizon_req.number_of_days), 'days ago']))
parsed_date = extract_from_tag(latest_date)
latest_date_processed = morizon_convert_to_time_struct(parsed_date)
if latest_date_processed < required_date:
return
yield response.follow(
url=response.xpath("//a[@class='mz-pagination-number__btn mz-pagination-number__btn--next']")
.pop()
.attrib['href'],
callback=self.parse)
def process(input_wikitext):
input_wikitext, encoded_wikilinks = encode_wikilinks(input_wikitext)
old_ah_wikitext_search = ARTICLE_HISTORY.search(input_wikitext)
if not old_ah_wikitext_search:
return input_wikitext
old_ah_wikitext = old_ah_wikitext_search.group(0)
history = History(old_ah_wikitext)
# For use in sorting parameters
by_time = lambda x: datetime.datetime.fromtimestamp(
mktime(Calendar().parse(x[0])[0]))
lines_to_delete = []
if ITN.search(input_wikitext):
itn_list = history.get_relevant_params("itn")
for itn_result in ITN.finditer(input_wikitext):
itn = itn_result.group(1)
itn_params = itn.split("|")[1:]
if "=" not in itn_params[0] and "=" not in itn_params[1]:
# {{ITN talk|DD monthname|YYYY}}
itn_list.append((itn_params[0] + " " + itn_params[1], ""))
else:
itn_list += [(x[x.find("=") + 1:], "")
for x in itn.split("|")[1:] if "date" in x]
lines_to_delete.append(itn_result.group(0))
itn_list.sort(key=by_time)
# Update the article history template
history.other_parameters["itndate"] = itn_list[0][0]
if itn_list[0][1]:
history.other_parameters["itnlink"] = itn_list[0][1]
for i, item in enumerate(itn_list[1:], start=2):
history.other_parameters["itn%ddate" % i] = item[0]
if item[1]:
history.other_parameters["itn%ditem" % i] = item[1]
if OTD.search(input_wikitext):
otd_list = history.get_relevant_params("otd")
for otd_result in OTD.finditer(input_wikitext):
otd = otd_result.group(1)
otd_params = {
x: y
for x, y in [t.strip().split("=") for t in otd.split("|")[1:]]
}
for i in itertools.count(1):
date_key = "date%d" % i
if date_key in otd_params:
otd_list.append((otd_params[date_key],
otd_params.get("oldid%d" % i, ""), ""))
else:
break
lines_to_delete.append(otd_result.group(0))
otd_list.sort(key=by_time)
# Update the article history template
history.other_parameters["otddate"], history.other_parameters[
"otdoldid"], _ = otd_list[0]
if otd_list[0][2]:
history.other_parameters["otdlink"] = otd_list[0][2]
for i, item in enumerate(otd_list[1:], start=2):
history.other_parameters["otd%ddate" %
i], history.other_parameters["otd%doldid"
%
i], _ = item
if item[2]:
history.other_parameters["otd%dlink" % i] = item[2]
dyk_search = DYK.search(input_wikitext)
if dyk_search:
dyk = dyk_search.group(1)
dyk_params = dyk.split("|")[1:]
history.other_parameters["dykentry"] = next(
x for x in dyk_params if x.startswith("entry")).split("=")[1]
positional_dyk_params = [x for x in dyk_params if "=" not in x]
if len(positional_dyk_params) == 1:
history.other_parameters["dykdate"] = positional_dyk_params[0]
elif len(positional_dyk_params) == 2:
for param in positional_dyk_params:
if len(param) == 4:
year = param
else:
month_day = param
history.other_parameters["dykdate"] = month_day + " " + year
# Delete the DYK template
lines_to_delete.append(dyk_search.group(0))
# Delete the lines with only the delete comment on them
result_text = "\n".join(
ifilterfalse(lines_to_delete.__contains__,
input_wikitext.splitlines()))
result_text = result_text.replace(old_ah_wikitext, history.as_wikitext())
result_text = decode_wikilinks(result_text, encoded_wikilinks)
return result_text
tokens.append(sent[span[0]:span[1]])
current = span[1]
tags = pos_tag(tokens)
new_tags = []
for word, tag in tags:
if word[:2] == '__':
new_tags.append((word[2:], tag_dict[word[2:]]))
else:
tag = [t[1] for t in original_tags if t[0] == word][0] # FIXED
new_tags.append((word, tag))
return new_tags
cal = Calendar()
month2num = {
"january": 1,
"february": 2,
"march": 3,
"april": 4,
"may": 5,
"june": 6,
"july": 7,
"august": 8,
"september": 9,
"october": 10,
"november": 11,
"december": 12
}
num2month = dict([(n, m) for (m, n) in month2num.items()])
def __init__(self):
self.parser.add_argument('-s',
'--status',
action='store_true',
help='show rule list with expirations')
self.parser.add_argument('-c',
'--clean',
action='store_true',
help='clean up expired rules')
self.parser.add_argument('-r',
'--rule',
help='rule to be added to `ufw`')
self.parser.add_argument('-p',
'--position',
default=1,
help='position to add the rule')
self.parser.add_argument('-t',
'--ttl',
default='30 days',
help='time to live for the rule')
args = self.parser.parse_args()
# Our file names
pid_file = '/var/run/' + __file__ + '.pid'
rules_file = '/usr/local/share/' + __file__ + '/rules'
tmp_rules_file = '/tmp/' + __file__ + '-rules'
if args.status:
if path.exists(rules_file):
try:
print("Expiration\t\tRule")
print('=' * 80)
# Loops through the rules lines
for line in open(rules_file, 'r'):
# Breaks apart line into expiration timestamp and rule
timestamp, rule = line.strip("\n").split(' ', 1)
print(
str(datetime.fromtimestamp(float(timestamp))) +
"\t" + rule)
except IOError:
self.error('unable to read from the rules file: ' +
rules_file)
else:
self.error('there are no rules to display')
elif args.clean:
# Checks for PID file
if path.exists(pid_file):
self.error(__file__ + ' is already running')
else:
# Creates the PID file
try:
handle = open(pid_file, 'w')
handle.write(str(getpid()))
handle.close()
except IOError:
self.error('unable to create PID file: ' + pid_file)
# Checks for the rules file
if path.exists(rules_file):
# Opens the temporary rules file
try:
handle = open(tmp_rules_file, 'a')
except IOError:
self.error('unable to write to the tmp rules file: ' +
tmp_rules_file)
try:
current_time = time()
# Loops through the rules lines
for line in open(rules_file, 'r'):
# Breaks apart line into expiration timestamp and rule
timestamp, rule = line.strip("\n").split(' ', 1)
# Checks if rule has expired
if current_time < float(timestamp):
handle.write(line)
print(
str(datetime.fromtimestamp(time())) +
"\tskipped rule\t" + rule)
else:
try:
self.ufw_execute('delete ' + rule)
print(
str(datetime.fromtimestamp(time())) +
"\tdeleted rule\t" + rule)
except CalledProcessError as error:
self.ufw_error(error)
handle.close()
# Moves the tmp file to the rules file
move(tmp_rules_file, rules_file)
except IOError:
self.error('unable to from the read rules file: ' +
rules_file)
# Removes the PID
remove(pid_file)
elif args.rule:
rules_path = path.dirname(rules_file)
if not path.exists(rules_path):
makedirs(rules_path)
# Converts the TTL to a timestamp
cal = Calendar()
timestamp = mktime(cal.parse(args.ttl)[0])
# Writes the rule to the rules file
try:
# TODO Check if rule already exists and update it instead of adding it again
handle = open(rules_file, 'a')
handle.write(str(timestamp) + ' ' + args.rule)
handle.write("\n")
handle.close()
except IOError:
self.error('unable to write to the rules file: ' + rules_file)
# Attempts to add the rule to `ufw`
try:
self.ufw_execute('insert ' + str(args.position) + ' ' +
args.rule)
except CalledProcessError as error:
# Catches an error when attempting to add a rule to an empty database
if error.output == b"ERROR: Invalid position '1'\n":
try:
self.ufw_execute(args.rule)
except CalledProcessError as error:
self.ufw_error(error)
else:
self.ufw_error(error)
else:
self.error('no arguments specified')
import re
from datetime import datetime
import logging
from parsedatetime import Calendar
from pytz import timezone, utc
from recurrent.constants import *
pdt = Calendar()
log = logging.getLogger('recurrent')
RE_TIME = re.compile(
r'(?P<hour>\d{1,2}):?(?P<minute>\d{2})?\s?(?P<mod>am|pm)?(oclock)?')
RE_AT_TIME = re.compile(r'at\s%s' % RE_TIME.pattern)
RE_AT_TIME_END = re.compile(r'at\s%s$' % RE_TIME.pattern)
RE_STARTING = re.compile(r'start(?:s|ing)?')
RE_ENDING = re.compile(r'(?:\bend|until)(?:s|ing)?')
RE_REPEAT = re.compile(r'(?:every|each|\bon\b|repeat(s|ing)?)')
RE_START = r'(%s)\s(?P<starting>.*)' % RE_STARTING.pattern
RE_EVENT = r'(?P<event>(?:every|each|\bon\b|repeat|%s|%s)(?:s|ing)?(.*))' % (
RE_DAILY.pattern, RE_PLURAL_WEEKDAY.pattern)
RE_END = r'%s(?P<ending>.*)' % RE_ENDING.pattern
RE_START_EVENT = re.compile(r'%s\s%s' % (RE_START, RE_EVENT))
RE_EVENT_START = re.compile(r'%s\s%s' % (RE_EVENT, RE_START))
RE_FROM_TO = re.compile(
r'(?P<event>.*)from(?P<starting>.*)(to|through|thru)(?P<ending>.*)')
RE_START_END = re.compile(r'%s\s%s' % (RE_START, RE_END))
RE_OTHER_END = re.compile(r'(?P<other>.*)\s%s' % RE_END)
RE_SEP = re.compile(
r'(from|to|through|thru|on|at|of|in|a|an|the|and|or|both)$')
def parse_message(text):
c = Calendar()
t_s, p_s = c.parse(text)
time = datetime(*t_s[:6])
td = time - datetime.now()
return None if td.days < 0 else td.total_seconds()
def parsedatetime_calendar():
from parsedatetime import Calendar, Constants
return Calendar(parsedatetime_constants())
def __init__(self, bot: Bot):
self.bot = bot
self.cal = Calendar()
self.remind_loop.start()
def __init__(self) -> None:
self.cal: Calendar = Calendar()
