def convert_date(raw_date: str) -> tuple:
def _get_datetime_range(date: str) -> tuple:
""" Makes one day period by received date.
Parameters
----------
date:
Date for report generating.
Returns
-------
Datetime range.
"""
date = datetime.strptime(date, "%Y-%m-%d")
from_date = date.astimezone(utc)
from_date = from_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
to_date = (date.replace(
hour=23, minute=59,
second=59).astimezone(utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z"))
return from_date, to_date
parser = Calendar()
date, status = parser.parse(raw_date)
date = datetime(*date[:3]).strftime("%Y-%m-%d")
return _get_datetime_range(date)
def process_datetime(date_string):
cal = Calendar()
try:
dt = date_parse(date_string)
except TypeError:
dt = cal.nlp(date_string)[0][0]
return dt
async def poll(self):
cal = Calendar()
now = datetime.now(timezone("UTC"))
then = cal.parseDT("1 minute", now)
print("polling...")
for r in Reminder.select().where(Reminder.time.between(now, then[0])):
await self.bot.get_channel(r.channel_id).send(r.reminder_text)
def get_time(time_string):
cal = Calendar()
now = datetime.now(timezone("UTC"))
if time_string == "":
time = cal.parseDT("1 day", now)
else:
time = cal.parseDT(time_string, now)
return time[0]
def _guess_game_date(datetime_str):
# Try to turn a datetime string from FD into an actual datetime
datetime_str.replace('Sun', 'Sunday')
datetime_str.replace('Mon', 'Monday')
datetime_str.replace('Tues', 'Tuesday')
datetime_str.replace('Wed', 'Wednesday')
datetime_str.replace('Thurs', 'Thursday')
datetime_str.replace('Fri', 'Friday')
datetime_str.replace('Sat', 'Saturday')
cal = Calendar()
dt, ret_code = cal.parseDT(datetime_str)
return dt
def _guess_game_date(datetime_str):
# Try to turn a datetime string from FD into an actual datetime
datetime_str.replace('Sun', 'Sunday')
datetime_str.replace('Mon', 'Monday')
datetime_str.replace('Tues', 'Tuesday')
datetime_str.replace('Wed', 'Wednesday')
datetime_str.replace('Thurs', 'Thursday')
datetime_str.replace('Fri', 'Friday')
datetime_str.replace('Sat', 'Saturday')
cal = Calendar()
dt, ret_code = cal.parseDT(datetime_str)
return dt
def parse_mtime_single(value):
"""Convert a human readable time description to a """
if not HAS_PARSEDATETIME:
return int(value)
calendar = Calendar()
guess, rc = calendar.parseDT(value)
if rc == 0:
LOGGER.warning('Could not parse date: %s', value)
return int(value)
return guess.timestamp()
def parse_mtime_single(value):
"""Convert a human readable time description to a """
if not HAS_PARSEDATETIME:
return int(value)
calendar = Calendar()
guess, rc = calendar.parseDT(value)
if rc is 0:
LOGGER.warning("Could not parse date: %s", value)
return int(value)
return guess.timestamp()
async def convert(self, ctx: commands.Context, argument: str):
cal = Calendar()
utc = timezone("UTC")
utcnow = datetime.utcnow().replace(microsecond=0, tzinfo=utc)
parsed = cal.parseDT(argument, sourceTime=utcnow, tzinfo=utc)[0]
if parsed == utcnow:
raise commands.BadArgument(f'"{argument}" is not a valid time.')
elif parsed < utcnow:
raise commands.BadArgument(f'"{argument}" is in the past.')
return parsed
def parse(self, response):
# follow links to offer pages
offers = response.xpath('//a[@data-cy="listing-ad-title"]')
for offer in offers:
yield response.follow(url=offer.attrib['href'], callback=self.parse_offer)
latest_date = response.xpath('//small//span//i[@data-icon="clock"]/../text()').getall().pop()
olx_req = OlxRequirements(req)
cal = Calendar()
required_date = cal.parse(' '.join([str(olx_req.number_of_days), 'days ago']))[0]
parsed_date = latest_date.replace('\n', '').replace('\t', '')
latest_date_processed = olx_convert_to_time_struct(parsed_date)
if latest_date_processed < required_date:
return
yield response.follow(url=response.xpath('//a[@data-cy="page-link-next"]').attrib['href'], callback=self.parse)
class CsvTypeParser(TabularTypeParser):
"""CSV type parser"""
calendar: ClassVar[Calendar] = Calendar()
"""Calendar object used for date parsing"""
def __post_init__(self):
if self.parse is None:
if issubclass(self.pytype, datetime):
self.parse = self.parse_datetime
elif issubclass(self.pytype, date):
self.parse = self.parse_date
super().__post_init__()
@classmethod
def parse_datetime(cls, value):
"""Parse datetime from CSV value"""
timestamp, ret = cls.calendar.parseDT(value)
if not ret:
raise ValueError("Invalid date: '%s'" % value)
return timestamp
@classmethod
def parse_date(cls, value):
"""Parse date from CSV value"""
return cls.parse_datetime(value).date()
def parse_contest_element(browser_elem):
"""
Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost
:param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element
:return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description
"""
match_groups = contest_element_id_regex.match(browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text
game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace('$','').replace(',','')
game_time = browser_elem.find_element_by_xpath("div[2]/time").text
cal = Calendar()
fd_game_date, ret_code = cal.parseDT(game_time)
entry_button_element = browser_elem.find_element_by_xpath("a")
return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def clean(self, value):
"""
Validates that the input can be converted to a time. Returns a Python
datetime.time object.
"""
super(NaturalTimeField, self).clean(value)
if value in EMPTY_VALUES:
return None
if isinstance(value, datetime.time):
return value
c = Calendar()
parsed = c.parse(value)
if parsed[1] == 2:
return datetime.time(*parsed[0][3:6])
raise ValidationError(self.error_messages['invalid'])
def str_to_datetime(string, on_fail_return=None):
try:
from dateutil import parser as p
output = p.parse(string)
return output
except:
from parsedatetime import Calendar
from datetime import datetime
c = Calendar()
output, flags = c.parse(string)
if flags > 0:
return datetime(*output[:6])
else:
return None
return None
def clean(self, value):
"""
Validates that the input can be converted to a time. Returns a Python
datetime.time object.
"""
super(NaturalTimeField, self).clean(value)
if value in EMPTY_VALUES:
return None
if isinstance(value, datetime.time):
return value
c = Calendar()
parsed = c.parse(value)
if parsed[1] == 2:
return datetime.time(*parsed[0][3:6])
raise ValidationError(self.error_messages['invalid'])
def str2date(a, verbose=False):
struct, status = Calendar().parse(a)
if status == 0:
if verbose:
print(f"Unable to recognize date in \"{a}\".")
return date.today()
return date(*struct[:3])
def str_to_datetime(string, on_fail_return=None):
try:
from dateutil import parser as p
output = p.parse(string)
return output
except:
from parsedatetime import Calendar
from datetime import datetime
c = Calendar()
output, flags = c.parse(string)
if flags > 0:
return datetime(*output[:6])
else:
return None
return None
def parse_contest_element(browser_elem):
"""
Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost
:param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element
:return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description
"""
match_groups = contest_element_id_regex.match(
browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text
game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace(
'$', '').replace(',', '')
game_time = browser_elem.find_element_by_xpath("div[2]/time").text
cal = Calendar()
fd_game_date, ret_code = cal.parseDT(game_time)
entry_button_element = browser_elem.find_element_by_xpath("a")
return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def cli(config, src, dst, when):
"""
commute.py helper CLI to compute possible commute options.
"""
if when is not None:
when = int(time.mktime(Calendar().parseDT(when)[0].timetuple()))
for rank, path in commute.get_all_paths(config, src, dst, when):
click.echo(commute.format_path(rank, path))
click.echo("-" * 5)
def test_olx_convert_to_time_struct_valid_case():
test_input = {
'dzisiaj 13:48': 'today 13:48',
'wczoraj 13:48': 'yesterday 13:48',
'16 sty': '16 january',
'16 wrz': '16 september',
'16 lut': '16 february',
'16 mar': '16 march',
'16 kwi': '16 april',
'16 maj': '16 may',
'16 cze': '16 june',
'16 lip': '16 july',
'16 sie': '16 august',
'16 paź': '16 october',
'16 lis': '16 november',
'16 gru': '16 december',
}
cal = Calendar()
for k, v in test_input.items():
assert olx_convert_to_time_struct(k) == cal.parse(v)[0]
def parse_datetime(date_string):
"""Parse a timetime object from a natural language string."""
cal = Calendar()
unaware_dt = cal.nlp(date_string)
if not unaware_dt:
raise DateParseException("Due date '%s' unclear" % date_string)
else:
code = unaware_dt[0][1]
unaware_dt = unaware_dt[0][0]
# If no time is supplied, assume 6pm
if code == 1:
unaware_dt = unaware_dt.replace(hour=18,
minute=0,
second=0,
microsecond=0)
if os.environ.get('HABIT_TZ'):
localtz = pytz.timezone(os.environ.get('HABIT_TZ'))
else:
localtz = get_localzone()
aware_dt = localtz.localize(unaware_dt)
return aware_dt
def olx_convert_to_time_struct(date_str):
olx_date_mapper = {
'dzisiaj': 'today',
'wczoraj': 'yesterday',
'sty': 'january',
'wrz': 'september',
'lut': 'february',
'mar': 'march',
'kwi': 'april',
'maj': 'may',
'cze': 'june',
'lip': 'july',
'sie': 'august',
'paź': 'october',
'lis': 'november',
'gru': 'december',
}
for k, v in olx_date_mapper.items():
date_str = date_str.replace(k, v)
cal = Calendar()
return cal.parse(date_str)[0]
class Reminders(commands.Cog):
"""Set reminders."""
def __init__(self, bot: Bot):
self.bot = bot
self.cal = Calendar()
self.remind_loop.start()
@commands.command(name="remind", aliases=["remindme"])
async def remind(self, ctx: Context, t: str, *, message: str):
"""Set a reminder."""
parsed, success = self.cal.parse(t)
if not success:
return await ctx.reply("Please use a valid time string, such as 3d6h.")
time = mktime(parsed)
dt = datetime.fromtimestamp(time)
td = dt - datetime.utcnow()
total_diff = td.total_seconds()
if total_diff < 5:
return await ctx.reply("I can't create reminders less than 5 seconds in the future!")
id = await self.bot.db.create_reminder(ctx.author.id, ctx.guild.id, ctx.channel.id, ctx.message.id, dt, message)
await ctx.reply(f"Reminder created with ID {id} set to remind you at <t:{int(dt.timestamp())}:F>.")
@tasks.loop(seconds=10)
async def remind_loop(self):
await self.bot.wait_until_ready()
rs = await self.bot.db.get_expired_reminders()
for reminder in rs:
if reminder["expires"] > datetime.now():
continue
logger.info(f"Actioning reminder ID {reminder['id']} from {reminder['userid']}")
await self.bot.db.mark_reminder_completed(reminder["id"])
channel = self.bot.get_channel(reminder["cid"])
if not channel:
continue
jump = f"https://discord.com/channels/{reminder['gid']}/{reminder['cid']}/{reminder['mid']}"
await channel.send(
f"<@{reminder['userid']}> reminder:\n{reminder['content']}\n\nOriginal message: {jump}",
allowed_mentions=AllowedMentions(users=True),
)
def parse_offer(self, response):
"""Extracts the details of the offer to search through."""
# Extract date of offer submission
# We don't want an offer which is too old
unprocessed_date = response.xpath("//div[@class='css-lh1bxu']").get()
(_, _, unprocessed_date) = unprocessed_date.partition(':')
(processed_date, _, _) = unprocessed_date.partition('<')
cal = Calendar()
offer_submission_date = cal.parse(processed_date)
otodom_req = OtodomRequirements(req)
required_date = cal.parse(' '.join(
[str(otodom_req.number_of_days), 'days ago']))
if offer_submission_date < required_date:
return
li = response.xpath("//li/text()").getall()
li_text = get_processed_text(li)
p = response.xpath("//p/text()").getall()
p_text = get_processed_text(p)
text = ' '.join([li_text, p_text])
if calculate_intersection(text,
otodom_req.tags) >= otodom_req.threshold:
append_to_file(path='scrapy-data/urls.txt', response=response)
def parse(self, response):
# follow links to offer pages
offers = response.xpath("//a[@class='href-link tile-title-text']")
for offer in offers:
yield response.follow(url=offer.attrib['href'],
callback=self.parse_offer)
# follow pagination link
tmp_1 = response.xpath(
"//a[@class='arrows icon-angle-right-gray icon-right-arrow']")
tmp_2 = response.xpath(
"//a[@class='arrows icon-right-arrow icon-angle-right-gray']")
next_page = tmp_1 if 'href' in tmp_1.attrib else tmp_2
offer_date = response.xpath("//div[@class='creation-date']")
parsed_date = extract_date(offer_date[len(offer_date) - 1].get())
gumtree_req = GumtreeRequirements(req)
cal = Calendar()
date = cal.parse(parsed_date)
required_date = cal.parse(' '.join(
[str(gumtree_req.number_of_days), 'days ago']))
if date < required_date:
return
yield response.follow(next_page.attrib['href'], self.parse)
def parse_offer(self, response):
"""Extracts the details of the offer to search through."""
unprocessed_date = response.css(
'#wrapper > div:nth-child(1) > div.vip-header-and-details > div.vip-details'
' > ul > li:nth-child(1) > div > span.value').get()
(_, unprocessed_date, _) = unprocessed_date.split('>')
(processed_date, _) = unprocessed_date.split('<')
gumtree_req = GumtreeRequirements(req)
cal = Calendar()
offer_submission_date = cal.parse(processed_date)
required_date = cal.parse(' '.join(
[str(gumtree_req.number_of_days), 'days ago']))
if offer_submission_date < required_date:
return
spans = response.xpath("//span[@class='value']").getall()
spans_text = get_processed_text(spans)
p = response.xpath("//span[@class='pre']").getall()
p_text = get_processed_text(p)
text = ' '.join([spans_text, p_text])
if calculate_intersection(text,
gumtree_req.tags) >= gumtree_req.threshold:
append_to_file(path='scrapy-data/urls.txt', response=response)
def clean(self, value):
"""
Validates that the input can be converted to a datetime. Returns a
Python datetime.datetime object.
"""
super(NaturalDateTimeField, self).clean(value)
if value in EMPTY_VALUES:
return None
if isinstance(value, datetime.datetime):
return value
if isinstance(value, datetime.date):
return datetime.datetime(value.year, value.month, value.day)
if isinstance(value, list):
# Input comes from a SplitDateTimeWidget, for example. So, it's two
# components: date and time.
if len(value) != 2:
raise ValidationError(self.error_messages['invalid'])
value = '%s %s' % tuple(value)
c = Calendar()
parsed = c.parse(value)
if parsed[1] == 3:
return datetime.date(*parsed[0][:6])
raise ValidationError(self.error_messages['invalid'])
def parse_time(
time_str: str,
seconds: int = True
) -> typing.Union[None, int, typing.Tuple[datetime.datetime, int]]:
"""
Parses a time.
:param time_str: The time string to parse.
:return: The total number of seconds between now and then.
"""
calendar = Calendar()
t_struct, parse_status = calendar.parse(
time_str, sourceTime=datetime.datetime.utcnow())
if parse_status == 0:
return None
dt = datetime.datetime(*t_struct[:6])
diff = np.ceil((dt - datetime.datetime.utcnow()).total_seconds())
if seconds:
return diff
else:
return dt, diff
def parse(self, response):
# follow links to offer pages
offers = response.xpath("//a[@class='property_link property-url']")
for offer in offers:
yield response.follow(url=offer.attrib['href'], callback=self.parse_offer)
latest_date = response.xpath("//span[@class='single-result__category single-result__category--date']")\
.getall()\
.pop()
# the strong word appears in the rag when the date is in bold (and it's only bold when the offer has just been
# added, hence there is no sense in finding out whether is't an old offer
if 'strong' not in latest_date:
morizon_req = MorizonRequirements(req)
cal = Calendar()
required_date = cal.parse(' '.join([str(morizon_req.number_of_days), 'days ago']))
parsed_date = extract_from_tag(latest_date)
latest_date_processed = morizon_convert_to_time_struct(parsed_date)
if latest_date_processed < required_date:
return
yield response.follow(
url=response.xpath("//a[@class='mz-pagination-number__btn mz-pagination-number__btn--next']")
.pop()
.attrib['href'],
callback=self.parse)
def clean(self, value):
"""
Validates that the input can be converted to a datetime. Returns a
Python datetime.datetime object.
"""
super(NaturalDateTimeField, self).clean(value)
if value in EMPTY_VALUES:
return None
if isinstance(value, datetime.datetime):
return value
if isinstance(value, datetime.date):
return datetime.datetime(value.year, value.month, value.day)
if isinstance(value, list):
# Input comes from a SplitDateTimeWidget, for example. So, it's two
# components: date and time.
if len(value) != 2:
raise ValidationError(self.error_messages['invalid'])
value = '%s %s' % tuple(value)
c = Calendar()
parsed = c.parse(value)
if parsed[1] == 3:
return datetime.date(*parsed[0][:6])
raise ValidationError(self.error_messages['invalid'])
def action(self):
self._logger.info('Start viewing the log events...')
def _exit(signum, frame):
self._logger.info('Exit by code {} ...'.format(signum))
self._exit = True
signal.signal(signal.SIGTERM, _exit)
signal.signal(signal.SIGINT, _exit)
start = time.time()
if self._start is not None:
time_struct, _ = Calendar().parse(self._start)
start = mktime(time_struct)
start = int(start * 1000)
client = self.get_logs_client()
function = self._config.get_function_name()
event_ids = {}
while self._exit is False:
events = client.get_log_events(function, start, self._filter)
for e in events:
if e['eventId'] not in event_ids:
event_ids[e['eventId']] = None
print(e['message'])
if e['timestamp'] > start:
start = e['timestamp']
event_ids = {}
if not self._follow:
break
time.sleep(self._interval)
def process(input_wikitext):
input_wikitext, encoded_wikilinks = encode_wikilinks(input_wikitext)
old_ah_wikitext_search = ARTICLE_HISTORY.search(input_wikitext)
if not old_ah_wikitext_search:
return input_wikitext
old_ah_wikitext = old_ah_wikitext_search.group(0)
history = History(old_ah_wikitext)
# For use in sorting parameters
by_time = lambda x: datetime.datetime.fromtimestamp(
mktime(Calendar().parse(x[0])[0]))
lines_to_delete = []
if ITN.search(input_wikitext):
itn_list = history.get_relevant_params("itn")
for itn_result in ITN.finditer(input_wikitext):
itn = itn_result.group(1)
itn_params = itn.split("|")[1:]
if "=" not in itn_params[0] and "=" not in itn_params[1]:
# {{ITN talk|DD monthname|YYYY}}
itn_list.append((itn_params[0] + " " + itn_params[1], ""))
else:
itn_list += [(x[x.find("=") + 1:], "")
for x in itn.split("|")[1:] if "date" in x]
lines_to_delete.append(itn_result.group(0))
itn_list.sort(key=by_time)
# Update the article history template
history.other_parameters["itndate"] = itn_list[0][0]
if itn_list[0][1]:
history.other_parameters["itnlink"] = itn_list[0][1]
for i, item in enumerate(itn_list[1:], start=2):
history.other_parameters["itn%ddate" % i] = item[0]
if item[1]:
history.other_parameters["itn%ditem" % i] = item[1]
if OTD.search(input_wikitext):
otd_list = history.get_relevant_params("otd")
for otd_result in OTD.finditer(input_wikitext):
otd = otd_result.group(1)
otd_params = {
x: y
for x, y in [t.strip().split("=") for t in otd.split("|")[1:]]
}
for i in itertools.count(1):
date_key = "date%d" % i
if date_key in otd_params:
otd_list.append((otd_params[date_key],
otd_params.get("oldid%d" % i, ""), ""))
else:
break
lines_to_delete.append(otd_result.group(0))
otd_list.sort(key=by_time)
# Update the article history template
history.other_parameters["otddate"], history.other_parameters[
"otdoldid"], _ = otd_list[0]
if otd_list[0][2]:
history.other_parameters["otdlink"] = otd_list[0][2]
for i, item in enumerate(otd_list[1:], start=2):
history.other_parameters["otd%ddate" %
i], history.other_parameters["otd%doldid"
%
i], _ = item
if item[2]:
history.other_parameters["otd%dlink" % i] = item[2]
dyk_search = DYK.search(input_wikitext)
if dyk_search:
dyk = dyk_search.group(1)
dyk_params = dyk.split("|")[1:]
history.other_parameters["dykentry"] = next(
x for x in dyk_params if x.startswith("entry")).split("=")[1]
positional_dyk_params = [x for x in dyk_params if "=" not in x]
if len(positional_dyk_params) == 1:
history.other_parameters["dykdate"] = positional_dyk_params[0]
elif len(positional_dyk_params) == 2:
for param in positional_dyk_params:
if len(param) == 4:
year = param
else:
month_day = param
history.other_parameters["dykdate"] = month_day + " " + year
# Delete the DYK template
lines_to_delete.append(dyk_search.group(0))
# Delete the lines with only the delete comment on them
result_text = "\n".join(
ifilterfalse(lines_to_delete.__contains__,
input_wikitext.splitlines()))
result_text = result_text.replace(old_ah_wikitext, history.as_wikitext())
result_text = decode_wikilinks(result_text, encoded_wikilinks)
return result_text
tokens.append(sent[span[0]:span[1]])
current = span[1]
tags = pos_tag(tokens)
new_tags = []
for word, tag in tags:
if word[:2] == '__':
new_tags.append((word[2:], tag_dict[word[2:]]))
else:
tag = [t[1] for t in original_tags if t[0] == word][0] # FIXED
new_tags.append((word, tag))
return new_tags
cal = Calendar()
month2num = {
"january": 1,
"february": 2,
"march": 3,
"april": 4,
"may": 5,
"june": 6,
"july": 7,
"august": 8,
"september": 9,
"october": 10,
"november": 11,
"december": 12
}
num2month = dict([(n, m) for (m, n) in month2num.items()])
def __init__(self):
self.parser.add_argument('-s',
'--status',
action='store_true',
help='show rule list with expirations')
self.parser.add_argument('-c',
'--clean',
action='store_true',
help='clean up expired rules')
self.parser.add_argument('-r',
'--rule',
help='rule to be added to `ufw`')
self.parser.add_argument('-p',
'--position',
default=1,
help='position to add the rule')
self.parser.add_argument('-t',
'--ttl',
default='30 days',
help='time to live for the rule')
args = self.parser.parse_args()
# Our file names
pid_file = '/var/run/' + __file__ + '.pid'
rules_file = '/usr/local/share/' + __file__ + '/rules'
tmp_rules_file = '/tmp/' + __file__ + '-rules'
if args.status:
if path.exists(rules_file):
try:
print("Expiration\t\tRule")
print('=' * 80)
# Loops through the rules lines
for line in open(rules_file, 'r'):
# Breaks apart line into expiration timestamp and rule
timestamp, rule = line.strip("\n").split(' ', 1)
print(
str(datetime.fromtimestamp(float(timestamp))) +
"\t" + rule)
except IOError:
self.error('unable to read from the rules file: ' +
rules_file)
else:
self.error('there are no rules to display')
elif args.clean:
# Checks for PID file
if path.exists(pid_file):
self.error(__file__ + ' is already running')
else:
# Creates the PID file
try:
handle = open(pid_file, 'w')
handle.write(str(getpid()))
handle.close()
except IOError:
self.error('unable to create PID file: ' + pid_file)
# Checks for the rules file
if path.exists(rules_file):
# Opens the temporary rules file
try:
handle = open(tmp_rules_file, 'a')
except IOError:
self.error('unable to write to the tmp rules file: ' +
tmp_rules_file)
try:
current_time = time()
# Loops through the rules lines
for line in open(rules_file, 'r'):
# Breaks apart line into expiration timestamp and rule
timestamp, rule = line.strip("\n").split(' ', 1)
# Checks if rule has expired
if current_time < float(timestamp):
handle.write(line)
print(
str(datetime.fromtimestamp(time())) +
"\tskipped rule\t" + rule)
else:
try:
self.ufw_execute('delete ' + rule)
print(
str(datetime.fromtimestamp(time())) +
"\tdeleted rule\t" + rule)
except CalledProcessError as error:
self.ufw_error(error)
handle.close()
# Moves the tmp file to the rules file
move(tmp_rules_file, rules_file)
except IOError:
self.error('unable to from the read rules file: ' +
rules_file)
# Removes the PID
remove(pid_file)
elif args.rule:
rules_path = path.dirname(rules_file)
if not path.exists(rules_path):
makedirs(rules_path)
# Converts the TTL to a timestamp
cal = Calendar()
timestamp = mktime(cal.parse(args.ttl)[0])
# Writes the rule to the rules file
try:
# TODO Check if rule already exists and update it instead of adding it again
handle = open(rules_file, 'a')
handle.write(str(timestamp) + ' ' + args.rule)
handle.write("\n")
handle.close()
except IOError:
self.error('unable to write to the rules file: ' + rules_file)
# Attempts to add the rule to `ufw`
try:
self.ufw_execute('insert ' + str(args.position) + ' ' +
args.rule)
except CalledProcessError as error:
# Catches an error when attempting to add a rule to an empty database
if error.output == b"ERROR: Invalid position '1'\n":
try:
self.ufw_execute(args.rule)
except CalledProcessError as error:
self.ufw_error(error)
else:
self.ufw_error(error)
else:
self.error('no arguments specified')
import re
from datetime import datetime
import logging
from parsedatetime import Calendar
from pytz import timezone, utc
from recurrent.constants import *
pdt = Calendar()
log = logging.getLogger('recurrent')
RE_TIME = re.compile(
r'(?P<hour>\d{1,2}):?(?P<minute>\d{2})?\s?(?P<mod>am|pm)?(oclock)?')
RE_AT_TIME = re.compile(r'at\s%s' % RE_TIME.pattern)
RE_AT_TIME_END = re.compile(r'at\s%s$' % RE_TIME.pattern)
RE_STARTING = re.compile(r'start(?:s|ing)?')
RE_ENDING = re.compile(r'(?:\bend|until)(?:s|ing)?')
RE_REPEAT = re.compile(r'(?:every|each|\bon\b|repeat(s|ing)?)')
RE_START = r'(%s)\s(?P<starting>.*)' % RE_STARTING.pattern
RE_EVENT = r'(?P<event>(?:every|each|\bon\b|repeat|%s|%s)(?:s|ing)?(.*))' % (
RE_DAILY.pattern, RE_PLURAL_WEEKDAY.pattern)
RE_END = r'%s(?P<ending>.*)' % RE_ENDING.pattern
RE_START_EVENT = re.compile(r'%s\s%s' % (RE_START, RE_EVENT))
RE_EVENT_START = re.compile(r'%s\s%s' % (RE_EVENT, RE_START))
RE_FROM_TO = re.compile(
r'(?P<event>.*)from(?P<starting>.*)(to|through|thru)(?P<ending>.*)')
RE_START_END = re.compile(r'%s\s%s' % (RE_START, RE_END))
RE_OTHER_END = re.compile(r'(?P<other>.*)\s%s' % RE_END)
RE_SEP = re.compile(
r'(from|to|through|thru|on|at|of|in|a|an|the|and|or|both)$')
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError('Must pass exactly one json file to import')
same, new, updated = 0, 0, 0 # Object counts
calendar = Calendar() # date parser
# Load data
datasource = DataSource(args[0], encoding='iso-8859-1')
layer = datasource[0]
for feature in layer:
# Gather and transform attributes
attr = {
'object_id': feature['OBJECTID'].value,
'case_id': loads('"%s"' % feature['Case_'].value),
'status': feature['Status'].value,
'link': loads('"%s"' % feature['Link'].value),
}
raw_location = feature['Location'].value
if '\r\n' in raw_location:
self.stderr.write('Invalid Location %s' % repr(raw_location))
raw_location = raw_location.split('\r\n', 1)[0]
attr['location'] = loads('"%s"' % raw_location)
raw_type = feature['Type'].value
if '\r\n' in raw_type:
self.stderr.write('Invalid Type %s' % repr(raw_type))
raw_type = raw_type.split('\r\n', 1)[0]
attr['case_type'] = loads('"%s"' % raw_type)
raw_date = loads('"%s"' % feature['Date_'].value)
timestamp = mktime(calendar.parseDateText(raw_date))
attr['hearing_date'] = datetime.fromtimestamp(timestamp).date()
attr['geom'] = GEOSGeometry(feature.geom.wkt)
# Is there a case with this object ID?
case_query = Case.objects.filter(
object_id=attr['object_id'], domain=Case.DOMAIN_TMAPC)
if case_query.exists():
case = case_query.get()
# Did the data change?
different = []
for name, new_value in attr.items():
existing = getattr(case, name)
if existing != new_value:
different.append((name, existing, new_value))
if different:
updated += 1
diff_message = ', '.join(
["%s:'%s' -> '%s'" % x for x in different])
msg = 'Updated case "%s" (%s)' % (
attr['case_id'], diff_message)
self.stdout.write(msg)
else:
same += 1
self.stdout.write(
'No change to case "%s"' % attr['case_id'])
else:
# Create new Case
case = Case(
object_id=attr['object_id'], domain=Case.DOMAIN_TMAPC)
self.stdout.write('New case "%s"' % attr['case_id'])
new += 1
different = True
if different:
# Save changes
for name, value in attr.items():
setattr(case, name, value)
case.save()
self.stdout.write(
'Imported %d BOA cases (%d new, %d changed, %d unchanged)' % (
new + updated + same, new, updated, same))
def str_ftime(arg, format):
cal = Calendar()
return strftime(format, cal.parse(str(arg))[0])
print("Counts down until the specified time. May be specified as a")
print("relative time (for example, '5 minutes' or 'tomorrow') or an")
print("actual time (for example, '8am').")
if error is not None:
print("\n\n")
print (error)
sys.exit(1)
if len(sys.argv) == 1:
helptext()
from parsedatetime import Calendar
from time import mktime, sleep
from datetime import datetime, timedelta
p = Calendar()
specifiedtime = p.parse(" ".join(sys.argv[1:]))
timeleft = datetime.fromtimestamp(mktime(specifiedtime[0])) - datetime.now()
length = 0
while timeleft > timedelta(microseconds=-1):
interval = 10
if timeleft < timedelta(hours=1):
interval = 1
if timeleft < timedelta(minutes=10):
interval = 0.1
if timeleft < timedelta(minutes=1):
interval = 0.001
sys.stdout.write("\b" * length)
outline = str(timeleft)
def human_time_to_datetime(s):
cal = Calendar()
t, _ = cal.parse(s)
return datetime(*t[:6])
def parsedatetime_calendar():
from parsedatetime import Calendar, Constants
return Calendar(parsedatetime_constants())
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Must pass exactly one json file to import")
same, new, updated = 0, 0, 0 # Object counts
calendar = Calendar() # date parser
# Load data
datasource = DataSource(args[0], encoding="iso-8859-1")
layer = datasource[0]
for feature in layer:
# Gather and transform attributes
attr = {
"object_id": feature["OBJECTID"].value,
"case_id": loads('"%s"' % feature["Case_"].value),
"status": feature["Status"].value,
"location": loads('"%s"' % feature["Location"].value),
"link": loads('"%s"' % feature["Link"].value),
}
raw_date = loads('"%s"' % feature["Date_"].value)
timestamp = mktime(calendar.parseDateText(raw_date))
attr["hearing_date"] = datetime.fromtimestamp(timestamp).date()
attr["geom"] = GEOSGeometry(feature.geom.wkt)
# Is there a case with this object ID?
case_query = Case.objects.filter(object_id=attr["object_id"], domain=Case.DOMAIN_BOA)
if case_query.exists():
case = case_query.get()
# Did the data change?
different = []
for name, new_value in attr.items():
existing = getattr(case, name)
if existing != new_value:
different.append((name, existing, new_value))
if different:
updated += 1
diff_message = ", ".join(["%s:'%s' -> '%s'" % x for x in different])
msg = 'Updated case "%s" (%s)' % (attr["case_id"], diff_message)
self.stdout.write(msg)
else:
same += 1
self.stdout.write('No change to case "%s"' % attr["case_id"])
else:
# Create new Case
case = Case(object_id=attr["object_id"], domain=Case.DOMAIN_BOA, case_type="Board of Adjustment")
self.stdout.write('New case "%s"' % attr["case_id"])
new += 1
different = True
if different:
# Save changes
for name, value in attr.items():
setattr(case, name, value)
case.save()
self.stdout.write(
"Imported %d BOA cases (%d new, %d changed, %d unchanged)" % (new + updated + same, new, updated, same)
)
def __init__(self) -> None:
self.cal: Calendar = Calendar()
def __init__(self, bot: Bot):
self.bot = bot
self.cal = Calendar()
self.remind_loop.start()
def parse_datetime(value):
calendar = Calendar()
raw_date = loads('"%s"' % value)
timestamp = mktime(calendar.parseDateText(raw_date))
return datetime.fromtimestamp(timestamp).date()
def scrape_single_game(sport, browser, debug=False):
# Since we are separating contests by their start time now, we don't currently try to choose a
# best contest to scrape. We should certainly upgrade this in the future!
contest_element_id_regex = re.compile(r"contest_(?P<gameid>\d+)-(?P<tableid>\d+)")
browser_elem = browser.find_element_by_class_name('contest-list-item')
match_groups = contest_element_id_regex.match(browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
fd_game_title = browser_elem.find_element_by_class_name("contest-name-text").text
entry_fee = browser_elem.find_element_by_class_name("entry-fee-cell").text.replace('$','').replace(',','')
game_time = browser_elem.find_element_by_class_name("startdate-cell").text
cal = Calendar()
new_parsed_dt, ret_code = cal.parseDT(game_time)
# If it is between 9 PM and midnight locally (Pacific), we'll have an issue:
# FD shows us the game time as e.g. "7 PM" but we'll interpret that as today.
# Workaround: if the game time is in the past, its probably tomorrow. Add 1 day.
# Break if it's not between 9PM and midnight so we know this happened
if new_parsed_dt < datetime.datetime.now():
if datetime.datetime.now().hour < 21:
print 'Parsed time of FanDuel game start is in the past, but this isnt a TZ issue!'
IPython.embed()
else:
new_parsed_dt += datetime.timedelta(days=1)
assert new_parsed_dt > datetime.datetime.now()
browser_elem.click()
print 'Scraping this game:'
print ' ', fd_game_title
print ' on', new_parsed_dt.isoformat()
game_entry_url = 'https://www.fanduel.com/games/{game_id}/contests/{game_id}-{table_id}/enter'.format(game_id=fd_game_id, table_id=fd_table_id)
# Go to the details for the game, to find list of eligible players for this game.
browser.get(game_entry_url)
time.sleep(1)
# Get the salary cap for the game directly
salary_text = browser.find_element_by_xpath('//*[@id="ui-skeleton"]/div/section/div[2]/div[4]/div[2]/section/header/'
'remaining-salary/div/div[1]/figure').text
cap = int(salary_text.replace('$','').replace(',',''))
# Download the master player / salary list
player_list_link = browser.find_element_by_link_text("Download players list")
# Sometimes this doesn't work the first time...
download_filename = get_csv_file(sport, new_parsed_dt, fd_game_id)
attempts = 1
while not os.path.exists(download_filename) and attempts <= 10:
print '...trying to download file (attempt %d)' % attempts
time.sleep(3)
player_list_link.click()
time.sleep(3)
attempts += 1
if attempts > 5:
print "Problem downloading player list -- not saving anything"
return False
else:
print '...success!'
# Get player lineups
if sport == 'mlb':
print "Accessing lineups..."
lineups_link = browser.find_element_by_link_text("Lineup Info")
# You can't just click it, it opens in a new tab :( So we visit it with get() and then go back()
lineup_url = lineups_link.get_attribute('href')
browser.get(lineup_url)
lineup_info = parse_lineup_page(browser.page_source)
browser.back()
print " %d / %d lineups submitted (%d players)" % (len(lineup_info.loaded_teams),
len(lineup_info.loaded_teams) + len(lineup_info.unloaded_teams),
len(lineup_info.parsed_players))
else:
# Should consider grabbing injury / GTD status here
lineup_info = None
# Except that doesn't work too well
# parse_nba_player_list(browser.page_source)
add_game_info_to_db(sport,
fd_game_id,
cap,
entry_fee,
new_parsed_dt,
fd_table_id,
game_title=fd_game_title,
lineup=lineup_info)
# Return to the main lobby page with back()
browser.back()
return True
def fuzzydtparse(request):
"""
Attempts to parse a natural-language date/time string and return a
formatted representation of the parsed date/time.
The returned representations are formatted using django's php-style
datetime formatting facilities. See
http://docs.djangoproject.com/en/dev/ref/templates/builtins/#now for a
full specification
GET/POST arguments:
dtstring *(required)*
The natural-language date and/or time string to parse.
dateformat (optional)
A format specifier string used to format the returned representation
if it is determined to be a date instance. The default format is
'l, F jS Y', which produces values like 'Thursday, October 1st 2009'.
timeformat (optional)
A format specifier string used to format the returned representation
if it is determined to be a time instance. The default format is
'P', which produces values like '6:26 p.m.'.
dtformat (optional)
A format specifier string used to format the returned representation
if it is determined to be a datetime instance. The default format is
'l, F jS Y, P', which produces values like
'Thursday, October 1st 2009, 6:26 p.m.'.
require (optional)
One of 'date', 'time', 'datetime'. If the parsed value is not of the
specified type, the view will return 400/Bad Request
callback (optional)
JSONP callback.
Returns a json dictionary containing two values:
type
An string indicating the kind of representation returned.
One of 'date', 'time', or 'datetime'
parsed
A string representation of the parsed datetime.
Invalid / unparsable datetimes will return 400/Bad Request.
"""
try:
dtstring = request.REQUEST['dtstring']
except KeyError:
return HttpResponseBadRequest()
kind = {
'date': 1,
'time': 2,
'datetime': 3
}
dateformat = request.REQUEST.get('dateformat', 'l, F jS Y' )
timeformat = request.REQUEST.get('timeformat', 'P')
dtformat = request.REQUEST.get('dtformat', 'l, F jS Y, P')
require = request.REQUEST.get('require', None)
callback = request.REQUEST.get('callback', None)
if require and require not in kind:
return HttpResponseBadRequest()
c = Calendar()
# TODO: possible security hole?
parsed = c.parse(dtstring)
if parsed[1] == 0:
return HttpResponseBadRequest()
parsed_dt = datetime.datetime(*parsed[0][:6])
response_dict = {}
if require and parsed[1] != kind[require]:
return HttpResponseBadRequest()
try:
if parsed[1] == 1:
response_dict['type'] = 'date'
response_dict['parsed'] = format(parsed_dt, dateformat)
elif parsed[1] == 2:
response_dict['type'] = 'time'
response_dict['parsed'] = format(parsed_dt, timeformat)
elif parsed[1] == 3:
response_dict['type'] = 'datetime'
response_dict['parsed'] = format(parsed_dt, dtformat)
else:
#should never be here
return HttpResponseBadRequest()
except:
return HttpResponseBadRequest()
if callback:
resp = "%s(%s)" % (callback, json.dumps(response_dict))
else:
resp = json.dumps(response_dict)
return HttpResponse(resp, mimetype='application/javascript')
