def add_arguments(self):
self.arg_parser.add_argument("--num_lines",
type=int,
dest="num_lines",
default=100,
metavar="100")
self.arg_parser.add_argument(
"--from",
type=str,
dest="from_",
metavar="'{0}'".format(utcnow().naive().strftime(DATE_FORMAT)))
self.arg_parser.add_argument(
"--to",
type=str,
dest="to_",
metavar="'{0}'".format(utcnow().naive().strftime(DATE_FORMAT)))
def parse_query_value(query_str):
""" Return value for the query string """
try:
query_str = str(query_str).strip('"\' ')
if query_str == 'now':
d = utcnow()
elif query_str.startswith('tz'):
global _offset
# get correction for local time and DST
_offset = local_time_offset()
# get current time
epoch_time = time.time()
# create delorean with local time
d = epoch(float(epoch_time + _offset))
else:
# Parse datetime string or timestamp
try:
d = epoch(float(query_str))
except ValueError:
d = parse(str(query_str))
except (TypeError, ValueError):
d = None
return d
def test_list_entry_annotations(self, monkeypatch):
def getentry_request(self):
return Response(
200, (
'{"id": 1, "title": "title",'
'"content": "<h1>head</h1>content", "url": "url",'
'"is_archived": 0, "is_starred": 1,'
'"annotations": [{'
'"user": "******", "annotator_schema_version":'
' "v1.0", "id": 1, "text": "content", '
'"created_at": "2020-10-28T10:50:51+0000", '
'"updated_at": "2020-10-28T10:50:51+0000", '
'"quote": "quote", "ranges": '
'[{"start": "/div[1]/p[1]", "startOffset": "23", '
'"end": "/div[1]/p[1]", "endOffset": "49"}]}]}'))
monkeypatch.setattr(GetEntry, 'request', getentry_request)
params = AnnoCommandParams()
params.entry_id = 1
result = AnnoCommand(self.config, params).execute()
assert result[0]
past = delorean.utcnow() - delorean.parse('2020-10-28T10:50:51+0000')
assert result[1] == f'1. quote ({humanize.naturaltime(past)}) [7]'
def __get_anno_full(self, anno):
if self.params.anno_id:
if self.params.anno_id != int(anno['id']):
return ""
past = delorean.utcnow() - delorean.parse(anno['updated_at'])
return (f"{anno['id']}. {anno['quote']} "
f"({humanize.naturaltime(past)}):\n\n\t{anno['text']}\n")
def _load_my_session(self) -> Dict:
sessions = self._load()
session = sessions.get(self.__session_id, {})
expired_at_s = session.get("_expired_at")
if expired_at_s:
expired_at = delorean.parse(expired_at_s)
if expired_at <= delorean.utcnow():
return {}
return session
def run(self, session, args):
num_lines = min(MAX_NUM_LINES, args.num_lines)
from_ = None
to_ = utcnow()
if args.from_:
from_ = parse(args.from_)
if args.to_:
to_ = parse(args.to_)
result = []
for history_item in self.get_history(session, num_lines, to_, from_=from_):
result.append(self.format_message(history_item))
return result
def start_requests(self):
# We only parse today and yesterday because at the end of the day this
# already produces a lot of requests and feed readers cache previous
# days (i.e. old contents of our feed) anyways.
# It's not enough to parse only today because we might miss shows that
# aired just before midnight but were streamed after midnight
# (see also https://github.com/nblock/feeds/issues/27)
today = delorean.utcnow().shift(self._timezone)
for day in [today, today - timedelta(days=1)]:
yield Request(
'https://api-tvthek.orf.at/api/v3/schedule/{}?limit=1000'.
format(day.format_datetime('Y-MM-dd')),
meta={'dont_cache': True})
def parse_query_value(query_str):
""" Return value for the query string """
global tz
try:
query_str = str(query_str).strip('"\' ')
if query_str == 'now':
d = utcnow().shift(tz)
elif 'now' in query_str and '@' in query_str:
tz = query_str.split('@')[1].strip()
d = utcnow().shift(tz)
else:
# Parse datetime string or timestamp
if '@' in query_str:
datas = query_str.split('@')
query_str = datas[0].strip()
tz = datas[1].strip()
try:
d = epoch(float(query_str)).shift(tz)
except ValueError:
d = parse(str(query_str)).shift(tz)
except (TypeError, ValueError):
d = None
return d
def finish_setup():
meta = get_twbot_meta(c)
if meta["first_run_step"] != 2:
return
finish_time = dmc.utcnow().shift(UK_TZ).format_datetime()
api.update_profile(description="Twitter Quiz Robot since " + finish_time)
api.update_profile_image("static/img/twbot.png")
# -1 means that the first run phase has been completed
c.execute("UPDATE meta SET first_run_step = -1;")
conn.commit()
b.redirect("/")
def run(self, session, args):
num_lines = min(MAX_NUM_LINES, args.num_lines)
from_ = None
to_ = utcnow()
if args.from_:
from_ = parse(args.from_)
if args.to_:
to_ = parse(args.to_)
result = []
for history_item in self.get_history(session,
num_lines,
to_,
from_=from_):
result.append(self.format_message(history_item))
return result
def parse_query_value(query_str):
""" Return value for the query string """
try:
query_str = str(query_str).strip('"\' ')
if query_str == "" or query_str == 'now':
d = utcnow()
else:
# Parse datetime string or timestamp
try:
d = epoch(float(query_str))
except ValueError:
d = parse(str(query_str))
except (TypeError, ValueError):
d = None
return d
def send_notification(entry, send, config):
if not send:
return entry
# Route each of the notifications
METHOD = {
'PHONE': send_phone_notification,
'EMAIL': send_email_notification,
}
method = METHOD.get(entry['Contact Method'], invalid_method)
result = method(entry, config)
entry['Timestamp'] = delorean.utcnow().datetime.isoformat()
entry['Status'] = result
return entry
def parse_query_value(query_str):
""" Return value for the query string """
try:
query_str = str(query_str).strip('"\' ')
if query_str == 'now':
d = utcnow()
else:
# Parse datetime string or timestamp
try:
d = epoch(float(query_str))
except ValueError:
d = parse(str(query_str))
except (TypeError, ValueError):
d = None
return d
def parse_query_value(query_str):
""" Return value for the query string """
try:
query_str = str(query_str).strip('"\' ')
if query_str == 'now':
d = utcnow()
elif query_str == 'yesterday' or query_str == '1 day ago':
d = utcnow().last_day()
elif ' days ago' in query_str:
count = count_from_query(query_str)
d = utcnow().last_day(count)
elif query_str == 'last week' or query_str == '1 week ago':
d = utcnow().last_week()
elif ' weeks ago' in query_str:
count = count_from_query(query_str)
d = utcnow().last_week(count)
elif query_str == 'last month' or query_str == '1 month ago':
d = utcnow().last_month()
elif ' months ago' in query_str:
count = count_from_query(query_str)
d = utcnow().last_month(count)
elif query_str == 'last year' or query_str == '1 year ago':
d = utcnow().last_year()
elif ' years ago' in query_str:
count = count_from_query(query_str)
d = utcnow().last_year(count)
else:
# Parse datetime string or timestamp
try:
d = epoch(float(query_str))
except ValueError:
d = parse(str(query_str))
except (TypeError, ValueError):
d = None
return d
def test_put(user_factory, post_model, delete_on_exit):
h = os.urandom(4).hex()
atm = utcnow().datetime
user = user_factory(username=f"user_{h}")
delete_on_exit(user)
post_id = randint(1000, 2000)
obj_url = f"/api/blog/post/{post_id}/"
expected = Post(
author_id=user.id,
content=f"post_content_{h}",
created_at=atm,
image=f"post_image_{h}",
title=f"post_title_{h}",
)
response = client.put(
obj_url,
data=expected.json(),
headers={"Content-Type": "application/vnd.api+json"},
)
obj = post_model.objects.get(id=post_id)
delete_on_exit(obj)
expected.id = obj.id
assert response.status_code == status.HTTP_201_CREATED
validate_content_location(response, obj_url)
got = Post.parse_obj(response.json()["data"])
assert got == expected
expected.title = f"{expected.title}_put2"
response = client.put(
obj_url,
data=expected.json(),
headers={"Content-Type": "application/vnd.api+json"},
)
assert response.status_code == status.HTTP_204_NO_CONTENT
assert not response.content
validate_content_location(response, obj_url)
obj = post_model.objects.get(id=post_id)
delete_on_exit(obj)
def parse_query_value(query_str):
""" Return value for the query string """
try:
query_str = str(query_str).strip('"\' ')
if query_str == 'now':
d = utcnow()
else:
# Parse datetime string or timestamp
try:
if query_str.isdigit() and len(query_str) == 13:
query_str = query_str[:10] + '.' + query_str[10:]
d = epoch(float(query_str))
except ValueError:
d = parse(str(query_str))
except (TypeError, ValueError):
d = None
return d
def send_notification(entry, send, config):
if not send:
return entry
# Route each of the notifications
METHOD = {
'PHONE': send_phone_notification,
'EMAIL': send_email_notification,
}
try:
method = METHOD[entry['Contact Method']]
result = method(entry, config)
except KeyError:
result = 'INVALID_METHOD'
entry['Timestamp'] = delorean.utcnow().datetime.isoformat()
entry['Status'] = result
return entry
def parse_query_value(query_str):
""" Return value for the query string """
try:
query_str = str(query_str).strip('"\' ')
if query_str == 'now':
d = utcnow()
else:
# Parse datetime string or timestamp
if len(query_str) == 13:
query_str = int(query_str) / int('1000')
try:
d = epoch(float(query_str))
d = d.shift("Asia/Shanghai")
except ValueError:
d = parse(str(query_str))
except (TypeError, ValueError):
d = None
return d
def parse_query_value(query_str):
""" Return value for the query string """
try:
query_str = str(query_str).strip('"\' ')
if query_str == 'now':
d = utcnow()
else:
# Parse datetime string or timestamp
try:
if query_str.isdigit() and len(query_str) == 13:
query_str = query_str[:10] + '.' + query_str[10:]
if query_str.isdigit() and len(query_str) == 19:
query_str = query_str[:10] + '.' + query_str[10:]
d = epoch(float(query_str))
except ValueError:
d = parse(str(query_str))
except (TypeError, ValueError):
d = None
return d
def parse_node(self, response, node):
il = FeedEntryItemLoader(response=response,
base_url='https://{}'.format(self.name),
dayfirst=False)
updated = node.xpath('dc:date/text()').extract_first()
if (delorean.parse(updated) + timedelta(days=self._num_days) <
delorean.utcnow()):
self.logger.debug(('Skipping item from {} since older than {} '
'days').format(updated, self._num_days))
return
il.add_value('updated', updated)
title = node.xpath('rss:title/text()').extract_first()
paywalled = title.startswith('[$]')
if paywalled:
title = title.replace('[$] ', '')
il.add_value('category', 'paywalled')
link = node.xpath('rss:link/text()').extract_first()
link = link.replace('rss', '')
link = link.replace('http://', 'https://')
meta = {'il': il}
if paywalled and not self._subscribed:
il.add_value('title', title)
il.add_value('author_name',
node.xpath('dc:creator/text()').extract_first())
il.add_value('content_text',
node.xpath('rss:description/text()').extract_first())
il.add_value('link', link)
yield il.load_item()
else:
if 'LWN.net Weekly Edition for' in title:
meta['updated'] = updated
callback = self._parse_weekly_edition
link += 'bigpage'
else:
callback = self._parse_article
# Don't include link yet, we will use the subscriber link later.
# So subscriber articles can be shared from the feed reader and
# read in browser without logging in.
yield scrapy.Request(link, callback, meta=meta)
def parse_program(self, response):
if not response.css('.jsb_video\/FlashPlayer'):
return
data = (
json.loads(response.css('.jsb_video\/FlashPlayer').xpath(
'@data-jsb').extract()[0])
)
data = (
data['config']['initial_video']['parts'][0]['tracking']['nurago']
)
il = FeedEntryItemLoader(response=response,
base_url='http://{}'.format(self.name),
timezone=self._timezone,
dayfirst=True)
il.add_value('link', data['clipurl'])
il.add_value('title', data['programname'])
il.add_value('updated', data['airdate'])
il.add_xpath('content_html', '//p[@class="plot_summary"]')
item = il.load_item()
# Only include videos posted in the last 7 days.
if (item['updated'] + self._timerange >
delorean.utcnow().shift(self._timezone)):
yield item
def create_summary_brief(summary, temp_file):
'''
Write a PDF page with the summary information, in the specified temp_file
'''
document = fpdf.FPDF()
document.set_font('Times', '', 12)
document.add_page()
TEMPLATE = '''
Report generated at {now}
Covering data from {start_time} to {end_time}
Summary
-------
TOTAL INCOME: $ {income}
TOTAL UNIT: {units} units
AVERAGE DISCOUNT: {discount}%
'''
def format_full_tmp(timestamp):
return timestamp.datetime.isoformat()
def format_brief_tmp(timestamp):
return timestamp.datetime.strftime('%d %b')
text = TEMPLATE.format(now=format_full_tmp(delorean.utcnow()),
start_time=format_brief_tmp(summary['start_time']),
end_time=format_brief_tmp(summary['end_time']),
income=summary['total_income'],
units=summary['units'],
discount=summary['average_discount'])
document.multi_cell(0, 6, text)
document.ln()
document.output(temp_file)
return temp_file
def _delorean():
return delorean.utcnow()
def __get_anno_string(self, anno):
past = delorean.utcnow() - delorean.parse(anno['updated_at'])
return (f"{anno['id']}. {anno['quote']} "
f"({humanize.naturaltime(past)}) [{len(anno['text'])}]")
def _utcnow() -> datetime:
return utcnow().datetime
def add_arguments(self):
self.arg_parser.add_argument("--num_lines", type=int, dest="num_lines", default=100, metavar="100")
self.arg_parser.add_argument("--from", type=str, dest="from_",
metavar="'{0}'".format(utcnow().naive().strftime(DATE_FORMAT)))
self.arg_parser.add_argument("--to", type=str, dest="to_",
metavar="'{0}'".format(utcnow().naive().strftime(DATE_FORMAT)))
def start_requests(self):
yield self.build_request(delorean.utcnow(), self._max_days,
self._max_days_limit)
def do_work():
# calls to the Twitter API will be wrapped in "##"s for visibility
conn = sqlite.connect("twbot.db")
conn.row_factory = sqlite.Row
c = conn.cursor()
meta = get_twbot_meta(c)
print("\ndoing work..")
if meta["first_run_step"] == -1:
api = get_twitter_api(c)
print("\ndoing work..")
try:
questions = c.execute(
"SELECT * FROM questions WHERE is_asked = 0 AND ask_time <= ?",
(int(dmc.utcnow().epoch),),
).fetchall()
# QUESTIONS
for question in questions:
print("\ngot question:", question["question"], "to be asked at", question["ask_time"])
## statuses/update ##
tweet = api.update_status(status="Question " + str(question["id"]) + ": " + question["question"])
print("tweeted question with status id", tweet.id)
c.execute("UPDATE questions SET is_asked = 1 WHERE id = ?;", (question["id"],))
conn.commit()
if question["possible_answer_id"]:
# multiple choice
answers = c.execute(
"SELECT * FROM possible_answers WHERE question_id = ?;",
(question["id"],)
)
# sort the answers in alphabetical order
answers = sorted(answers, key=lambda answer: answer["letter"])
for answer in answers:
print("got answer", answer["letter"], answer["answer"])
# reply to question tweet with possible answer
## statuses/update ##
answer_tweet = api.update_status(
status="@" + api.me().screen_name + " Is it " + answer["letter"].upper() + ": " + answer["answer"] + "?",
in_reply_to_status_id=tweet.id,
)
print("tweeted answer with id", answer_tweet.id)
helper_text = HELPER_TEXT_MULTIPLE_CHOICE.format(number=question["id"])
## statuses/update ##
api.update_status(
status="@" + api.me().screen_name + " " + helper_text,
in_reply_to_status_id=tweet.id,
)
print("tweeted helper text")
else:
# open-ended question
helper_text = HELPER_TEXT_OPEN_ENDED.format(number=question["id"])
api.update_status(
status="@" + api.me().screen_name + " " + helper_text,
in_reply_to_status_id=tweet.id,
)
# MENTIONS
## statuses/mentions_timeline ##
mentions = api.mentions_timeline(count=200)
for mention in mentions:
print("\ngot mention from", mention.user.screen_name, "body:", mention.text)
results = c.execute("SELECT * FROM mentions WHERE id = ?;", (mention.id,)).fetchall()
if results:
print("already handled mention..")
continue
c.execute("INSERT INTO mentions (id) VALUES (?);", (mention.id,))
print("inserted mention")
conn.commit()
if mention.user.screen_name == api.me().screen_name:
print("mention from twbot user, ignoring..")
continue
# this tweet has not been handled
print("new mention!")
players = c.execute("SELECT * FROM players WHERE id = ?;", (mention.user.id,)).fetchall()
if not players:
# follow user, allowing them to direct message
print("following user..")
## friendships/create ##
api.create_friendship(user_id=mention.user.id)
# reply to the tweet
print("replying to mention..", mention.id)
## statuses/update ##
api.update_status(
status="@" + mention.user.screen_name +
" Welcome! Follow me to be notified of questions!",
in_reply_to_status_id=mention.id,
)
c.execute("INSERT INTO players (id) VALUES (?);", (mention.user.id,))
print("inserted created new player")
conn.commit()
# DIRECT MESSAGES
## direct_messages ##
dms = api.direct_messages(count=200)
for dm in dms:
print("\ndirect messaged by", dm.sender.screen_name, "body:", dm.text)
results = c.execute("SELECT * FROM dms WHERE id = ?;", (dm.id,)).fetchall()
if results:
print("already handled dm..")
continue
c.execute("INSERT INTO dms (id) VALUES (?);", (dm.id,))
conn.commit()
# this dm has not been handled
print("new dm!")
answer_parts = dm.text.split(":")
if len(answer_parts) < 2:
# this player is not adhering to the answer format
print("badly formatted dm..")
## direct_messages/new ##
api.send_direct_message(
user_id=dm.sender.id,
text="I don't understand... Please refer to the instructions given with the question",
)
continue
if len(answer_parts) > 2:
# have only two list elements, with the second being the full answer
# e.g. ["1", "Lorem", " Ipsum", " Dolor"] -> ["1", "Lorem Ipsum Dolor"]
answer_parts[1] = "".join(answer_parts[1:])
# remove leading and trailing whitespace with .strip()
question_id = int(answer_parts[0].strip())
answer = answer_parts[1].strip()
print("got answer to question id", question_id, "with answer", answer)
questions = c.execute("SELECT * FROM questions WHERE id = ?;", (question_id,)).fetchall()
if not questions:
print("no question with id", question_id)
api.send_direct_message(user_id=dm.sender.id, text="No such question!")
continue
question = questions[0]
print("found question:", question)
player = c.execute("SELECT * FROM players WHERE id = ?;", (dm.sender.id,)).fetchone()
answers = c.execute(
"SELECT * FROM answers WHERE player_id = ? AND question_id = ?;",
(dm.sender.id, question["id"]),
).fetchall()
if answers:
print("player already answered this..")
api.send_direct_message(user_id=dm.sender.id, text="You already answered that question!")
print("sent tweet")
continue
dm_time = dmc.Delorean(dm.created_at, "UTC")
print("dm was posted at", dm_time)
if question["possible_answer_id"]:
# multiple choice question
print("multiple choice")
answer = answer.lower()
# get a list of possible answers to the question
possible_answers = c.execute(
"SELECT * FROM possible_answers WHERE question_id = ?;",
(question_id,),
).fetchall()
possible_answer_match = None
for possible_answer in possible_answers:
# loop through each possible answer to see if the player's answer exists
if possible_answer["letter"] == answer:
possible_answer_match = possible_answer["id"]
break
if not possible_answer_match:
print("no answer with letter", answer)
api.send_direct_message(user_id=dm.sender.id, text="No such answer with that letter!")
continue
correct_answer = c.execute(
"SELECT * FROM possible_answers WHERE id = ?;",
(question["possible_answer_id"],),
).fetchone()
is_correct = True if answer == correct_answer["letter"] else False
score = 10 if is_correct else 0
if is_correct:
message = "Correct! +" + str(score) + " points! "
else:
message = \
"Incorrect! The correct answer was " + \
correct_answer["letter"].upper()
api.send_direct_message(user_id=dm.sender.id, text=message)
c.execute("""INSERT INTO answers (
player_id,
question_id,
possible_answer_id,
answer,
time,
score,
dm_id,
messaged_score
) VALUES (
?, -- player_id
?, -- question_id
?, -- possible_answer_id
?, -- answer
?, -- time
?, -- score
?, -- dm_id
? -- messaged_score
);""", (
player["id"],
question["id"],
possible_answer_match,
answer,
dm_time.epoch,
score,
dm.id,
1
))
conn.commit()
else:
# non-multiple choice question
c.execute("""INSERT INTO answers (
player_id,
question_id,
answer,
time,
dm_id,
messaged_score
) VALUES (
?, -- player_id
?, -- question_id
?, -- answer
?, -- time
?, -- dm_id
? -- messaged_score
);""", (
player["id"],
question["id"],
answer,
dm_time.epoch,
dm.id,
False
))
conn.commit()
answers = c.execute("SELECT * FROM answers;").fetchall()
for answer in answers:
answer = dict(answer)
if (answer.get("score") is not None) and (not answer["messaged_score"]):
# in this case, the non-multiple choice answer has been scored by the administrator, and the player needs to be notified of their score
print("notifying user of score..")
api.send_direct_message(user_id=dm.sender.id, text="You got +" + str(answer["score"]) + " points for your answer!")
c.execute("UPDATE answers SET messaged_score = 1;")
conn.commit()
except t.TweepError as err:
# an exception will crash this timer and stop it from re-running, so it
# would be best to log it and carry on for the user's sake
print("ACHTUNG, got error", err)
else:
print("first run not completed..")
c.close()
# run every 10 minutes
timer = threading.Timer(60 * 10, do_work)
timer.daemon = True
timer.start()
print("\nscheduled next timer..")
def alfred_items_for_value(value):
"""
Given a delorean datetime object, return a list of
alfred items for each of the results
"""
index = 0
results = []
# First item as timestamp
unixtime = calendar.timegm(value.utctimetuple())
results.append(alfred.Item(
title=str(unixtime),
subtitle=u'UTC Timestamp',
attributes={
# 'uid': alfred.uid(index),
'arg': unixtime,
},
icon='icon.png',
))
index += 1
# Add support for UTC timestamps in millisecond format // Warning: Millisecond precision is lost during conversion
results.append(alfred.Item(
title=str(int(unixtime)*int('1000')),
subtitle=u'UTC Timestamp (Milliseconds)',
attributes={
# 'uid': alfred.uid(index),
'arg': int(unixtime)*int('1000'),
},
icon='icon.png',
))
index += 1
# Add ISO 8601 UTC
utctime = utcnow().datetime.strftime("%Y-%m-%dT%H:%M:%SZ")
results.append(alfred.Item(
title=str(utctime),
subtitle='ISO 8601 (UTC)',
attributes={
# 'uid': alfred.uid(index),
'arg': utctime,
},
icon='icon.png',
))
index += 1
# Local Time Hack
almost_time = datetime.fromtimestamp(unixtime)
localtime = almost_time.replace(tzinfo=tzlocal())
item_value = localtime.strftime("%Y-%m-%dT%H:%M:%S%z")
results.append(alfred.Item(
title=str(item_value),
subtitle='ISO 8601 (Local)',
attributes={
# 'uid': alfred.uid(index),
'arg': item_value,
},
icon='icon.png',
))
# Various formats
formats = [
# Sun, 19 May 2002 15:21:36
("%a, %d %b %Y %H:%M:%S %Z (%z)", 'RFC1123'),
# 2018-W13
("%Y-W%W", 'Week of Year'),
# 2018-W13-1
("%Y-W%W-%w", 'Week of Year with Day'),
# 2018-211
("%Y-%j", 'Day of Year'),
]
for format, description in formats:
item_value = localtime.strftime(format)
results.append(alfred.Item(
title=str(item_value),
subtitle=description,
attributes={
# 'uid': alfred.uid(index),
'arg': item_value,
},
icon='icon.png',
))
index += 1
return results
def _now():
return delorean.utcnow().datetime
