def handle_gh_issue_comment(self, evt):
if evt.author == cfg.github.account.login:
return
self.bot.say('[%s] %s commented on #%s (%s): %s' %
(Tags.UnderlinePink(evt.repo),
self.format_nickname(evt.author), evt.id, evt.title,
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def handle_gh_pull_request(self, evt):
self.bot.say(
'[%s] %s %s pull request #%d: %s (%s...%s): %s' %
(Tags.UnderlinePink(evt.repo), self.format_nickname(
evt.author), evt.action, evt.id, evt.title,
Tags.Purple(evt.base_ref_name), Tags.Purple(evt.head_ref_name),
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def handle_gh_pull_request_review(self, evt):
# GitHub sends a review event in addition to a review comment event
# when someone replies to an earlier comment or adds a single comment,
# even when they didn't really submit a pull request review.
# To prevent useless notifications, we skip any "review" which only has
# one comment, since there is already a comment notification for those.
if len(evt.comments) == 1 and \
evt.comments[0].created_at == evt.comments[0].updated_at:
return
if evt.state == 'pending':
return
if evt.action == 'submitted' and evt.state == 'approved':
action = 'approved'
if len(evt.comments) != 0:
action += ' and commented on'
elif evt.action == 'submitted' and evt.state == 'commented':
action = 'reviewed and commented on'
elif evt.action == 'submitted' and evt.state == 'changes_requested':
action = 'requested changes to'
elif evt.state == 'dismissed':
action = 'dismissed their review on'
else:
action = '%s review on' % evt.action
self.bot.say(
'[%s] %s %s pull request #%s (%s): %s' %
(Tags.UnderlinePink(evt.repo), self.format_nickname(
evt.author), action, evt.pr_id, evt.pr_title,
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def shorten():
""" Return a shortened version of the original url """
original_url = request.args.get('url')
if original_url.startswith('www'):
original_url = '{}{}'.format('https://', original_url)
if not original_url:
raise BadRequest()
elif not valid_url(original_url):
return render_template('error.html')
existing_url = db.session.query(Url).filter(Url.original == original_url).first()
if existing_url:
# get shortened from db
shortened_url = existing_url.shortened
hits = existing_url.hits
else:
shortened_url = shorten_url(original_url)
hits = 0
# add to db
db.session.add(Url(original_url, shortened_url))
db.session.commit()
# Render the result page
short_url = "http://{}/{}".format(request.host, shortened_url)
return render_template('shortened.html', short_url=short_url, hits=hits)
def export(self, url):
file_name = 'output/products/' + shorten_url(url) + '.json'
if os.path.exists(file_name):
return
response = requests.get(url)
soup = make_soup(response.text)
if self.get_product_name(soup) is None:
print('*** Error:', url, '***')
return
if self.get_specifications(soup) is None:
print('*** Error Table:', url, '***')
return
if self.get_applications(soup) is None:
print('*** Error Application:', url, '***')
return
details = {
'product_name': self.get_product_name(soup),
'specifications': self.get_specifications(soup),
'applications': self.get_applications(soup),
'image_url': self.get_image_url(soup)
}
save_json(details, file_name)
print(file_name)
def handle_gh_pull_request_comment(self, evt):
if evt.is_part_of_review or evt.action != 'created':
return
self.bot.say('[%s] %s commented on #%s %s: %s' %
(Tags.UnderlinePink(evt.repo),
self.format_nickname(evt.author), evt.id, evt.hash[:6],
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def url():
#if post request, generate the shortened url
if request.method == "POST":
shortened_url = shorten_url(request.form['url'], request.url_root)
return render_template('url.html', shortened_url=shortened_url)
else:
return render_template('url.html')
def handle_gh_pull_request_review(self, evt):
# GitHub sends a review event in addition to a review comment event
# when someone replies to an earlier comment or adds a single comment,
# even when they didn't really submit a pull request review.
# To prevent useless notifications, we skip any "review" which only has
# one comment, since there is already a comment notification for those.
if len(evt.comments) == 1 and \
evt.comments[0].created_at == evt.comments[0].updated_at:
return
if evt.state == 'pending':
return
if evt.action == 'submitted' and evt.state == 'approved':
action = 'approved'
if len(evt.comments) != 0:
action += ' and commented on'
elif evt.action == 'submitted' and evt.state == 'commented':
action = 'reviewed and commented on'
elif evt.action == 'submitted' and evt.state == 'changes_requested':
action = 'requested changes to'
elif evt.state == 'dismissed':
action = 'dismissed their review on'
else:
action = '%s review on' % evt.action
self.bot.say('[%s] %s %s pull request #%s (%s): %s' %
(Tags.UnderlinePink(evt.repo),
self.format_nickname(evt.author), action,
evt.pr_id, evt.pr_title,
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def get_product_details(url):
file_name = 'output/products/' + shorten_url(url) + '.json'
if os.path.exists(file_name):
return load_json(file_name)
else:
print('\nFile not found', file_name)
return None
def load_shorten_android_download_url(game, reload=False):
if game.android_download_url is None:
return None
android_download_shorten_url = cache.get(_get_game_android_shorten_download_url_key(game))
if android_download_shorten_url is None or reload:
android_download_shorten_url = shorten_url(game.android_download_url)
cache.set(_get_game_android_shorten_download_url_key(game), android_download_shorten_url, CACHE_TIMEOUT_WEEK)
return android_download_shorten_url
def load_shorten_ios_download_url(game, reload=False):
if game.iOS_download_url is None:
return None
ios_download_shorten_url = cache.get(_get_game_ios_shorten_download_url_key(game))
if ios_download_shorten_url is None or reload:
ios_download_shorten_url = shorten_url(game.iOS_download_url)
cache.set(_get_game_ios_shorten_download_url_key(game), ios_download_shorten_url, CACHE_TIMEOUT_WEEK)
return ios_download_shorten_url
def handle_gh_issue_comment(self, evt):
if evt.author == cfg.github.account.login:
return
if evt.action == 'created':
action = 'commented on'
else:
action = '%s a comment on' % evt.action
self.bot.say('[%s] %s %s #%s (%s): %s' % (
Tags.UnderlinePink(evt.repo), self.format_nickname(evt.author),
action, evt.id, evt.title, Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def handle_gcode_issue(self, evt):
"""Sends an IRC message notifying of a new GCode issue update."""
author = self.format_nickname(evt.author)
url = Tags.UnderlineBlue(utils.shorten_url(evt.url))
if evt.new:
msg = 'Issue %d created: "%s" by %s - %s'
msg = msg % (evt.issue, evt.title, author, url)
else:
msg = 'Update %d to issue %d ("%s") by %s - %s'
msg = msg % (evt.update, evt.issue, evt.title, author, url)
self.bot.say(msg)
def load_shorten_ios_download_url(game, reload=False):
if game.iOS_download_url is None:
return None
ios_download_shorten_url = cache.get(
_get_game_ios_shorten_download_url_key(game))
if ios_download_shorten_url is None or reload:
ios_download_shorten_url = shorten_url(game.iOS_download_url)
cache.set(_get_game_ios_shorten_download_url_key(game),
ios_download_shorten_url, CACHE_TIMEOUT_WEEK)
return ios_download_shorten_url
def load_shorten_android_download_url(game, reload=False):
if game.android_download_url is None:
return None
android_download_shorten_url = cache.get(
_get_game_android_shorten_download_url_key(game))
if android_download_shorten_url is None or reload:
android_download_shorten_url = shorten_url(game.android_download_url)
cache.set(_get_game_android_shorten_download_url_key(game),
android_download_shorten_url, CACHE_TIMEOUT_WEEK)
return android_download_shorten_url
def handle_gcode_issue(self, evt):
"""Sends an IRC message notifying of a new GCode issue update."""
author = self.format_nickname(evt.author)
url = Tags.UnderlineBlue(utils.shorten_url(evt.url))
if evt.new:
msg = 'Issue %d created: "%s" by %s - %s'
msg = msg % (evt.issue, evt.title, author, url)
else:
msg = 'Update %d to issue %d ("%s") by %s - %s'
msg = msg % (evt.update, evt.issue, evt.title, author, url)
self.bot.say(msg)
def handle_gh_pull_request(self, evt):
action = evt.action
if action == 'synchronize':
action = 'synchronized'
elif action == 'review_requested':
action = 'requested a review from %s for' % ', '.join([user.login for user in evt.requested_reviewers])
elif action == 'review_request_removed':
action = 'dismissed a review request on'
elif action == 'closed' and evt.merged:
action = 'merged'
self.bot.say('[%s] %s %s pull request #%d: %s (%s...%s): %s' % (
Tags.UnderlinePink(evt.repo), self.format_nickname(evt.author),
action, evt.id, evt.title, Tags.Purple(evt.base_ref_name),
Tags.Purple(evt.head_ref_name),
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def send_daily_summary(self, correlation_id=-1):
"""Sends the generated summary story for its recipient"""
# look up a summary
tod = today()
query = StoryCell.objects.filter(is_aggregation=True, recipients__pk=self.user.id, last_update__gte=tod)
if query.count() > 0:
summary_story = query[0]
# if found, send a link to it the user
log_event("notify", "Agent", self.id, "Sending summary story to user %s" % self.user.user_name, correlation_id)
link = summary_story.get_absolute_url()
shortened_url = shorten_url(link)
message = "Here's your daily summary for %s: %s" % (tod, shortened_url)
if send_twitter_direct_message(self, self.user.user_name, message, correlation_id):
self.last_summary_delivered_at = datetime.datetime.now()
self.save()
def handle_build_status_settled(self, evts):
per_shortrev = {}
for evt in evts:
per_shortrev.setdefault(evt.shortrev, []).append(evt)
for shortrev, evts in per_shortrev.items():
builders = [evt.service for evt in evts]
builders.sort()
evt = evts[0]
if evt.pr is not None:
shortrev = '#%s' % evt.pr
self.bot.say('[%s] build for %s %s on builders [%s]: %s' %
(Tags.UnderlinePink(evt.repo), shortrev,
Tags.Red('failed'), ', '.join(builders),
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def get_url_and_display_variant(update, context):
user = context.chat_data['user']
search_url = utils.extract_url(update, context)
if search_url is not None:
logger.info("Bot extracted URL: %s", search_url)
channel = utils.extract_domain(search_url)
if channel in SUPPORTED_CHANNELS:
update.message.reply_text(f"Brb! I'm learning more about this product on {channel}.")
item_dict, variants_dict, variants_display_dict = utils.get_item_information(channel, search_url)
update.message.reply_markdown(f"Hurray! Ive found \n\n{item_dict['item_name']}\n\n"
'Which of these product variations would you like to track?',
reply_markup=ReplyKeyboardMarkup.from_column(variants_display_dict,
one_time_keyboard=True))
logger.info(f"BOT: prompted {user.first_name} for variant choice")
# Store in context
context_store_item(item_dict, context)
context.chat_data['item_url'] = utils.shorten_url([search_url])[0]
context.chat_data['channel'] = utils.extract_domain(search_url)
context.chat_data['variants'] = variants_dict
logger.info(context.chat_data['variants'])
context.chat_data['variants_displayed'] = variants_display_dict
# context.chat_data['item'] = item_dict
logger.info(f"CONTEXT: Stored channel, variants, display, url for item {item_dict['item_name']}")
return CHOOSE_THRESHOLD
else:
update.message.reply_text(f"Oops, I do not support {channel} yet. Let's try again.",
reply_markup=ReplyKeyboardMarkup(start_reply_keyboard, one_time_keyboard=True))
return INITIAL_CHOICE
else:
update.message.reply_text("Oops, you did not key in a valid URL. Let's try again.",
reply_markup=ReplyKeyboardMarkup(start_reply_keyboard, one_time_keyboard=True))
return INITIAL_CHOICE
def scrape(self):
product_list = []
with os.scandir('output/data') as it1:
for make_entry in it1:
if os.path.isdir(make_entry):
with os.scandir(make_entry.path) as it2:
for model_entry in it2:
if os.path.isdir(model_entry):
with os.scandir(model_entry.path) as it3:
for year_entry in it3:
if year_entry.name.endswith('.json'):
data = load_json(year_entry.path)
for item in data:
if item['front_pad_url'] != '':
product_list.append(
item['front_pad_url'])
if item['rear_pad_url'] != '':
product_list.append(
item['rear_pad_url'])
product_list = list(dict.fromkeys(product_list))
temp = product_list
product_list = []
for item in temp:
file_name = 'output/products/' + shorten_url(item) + '.json'
if os.path.exists(file_name):
continue
product_list.append(item)
print('===', len(product_list), '===')
# self.export(product_list[0])
pool = Pool(50)
pool.map(self.export, product_list)
def handle_gh_commit_comment(self, evt):
self.bot.say('[%s] %s commented on commit %s: %s' % (
Tags.UnderlinePink(evt.repo), self.format_nickname(evt.author),
evt.commit, Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def authorization_url(self, http=None, **kwargs):
"""Gets a shortened authorization URL."""
long_url, state = super(_ShortURLFlow,
self).authorization_url(**kwargs)
short_url = utils.shorten_url(long_url)
return short_url, state
def handle_gh_push(self, evt):
fmt_url = Tags.UnderlineBlue
fmt_repo_name = Tags.UnderlinePink
fmt_ref = Tags.Purple
fmt_hash = lambda h: Tags.Grey(h[:6])
commits = [utils.ObjectLike(c) for c in evt.commits]
distinct_commits = [c for c in commits
if c.distinct and c.message.strip()]
num_commits = len(distinct_commits)
parts = []
parts.append('[' + fmt_repo_name(evt.repo) + ']')
parts.append(self.format_nickname(evt.pusher))
if evt.created:
if evt.ref_type == 'tags':
parts.append('tagged ' + fmt_ref(evt.ref_name) + ' at')
parts.append(fmt_ref(evt.base_ref_name)
if evt.base_ref_name else fmt_hash(evt.after_sha))
else:
parts.append('created ' + fmt_ref(evt.ref_name))
if evt.base_ref_name:
parts.append('from ' + fmt_ref(evt.base_ref_name))
elif not distinct_commits:
parts.append('at ' + fmt_hash(evt.after_sha))
if distinct_commits:
parts.append('+' + Tags.Bold(str(num_commits)))
parts.append('new commit' + ('s'
if num_commits > 1 else ''))
elif evt.deleted:
parts.append(Tags.Red('deleted ') + fmt_ref(evt.ref_name))
parts.append('at ' + fmt_hash(evt.before_sha))
elif evt.forced:
parts.append(Tags.Red('force-pushed ') + fmt_ref(evt.ref_name))
parts.append('from ' + fmt_hash(evt.before_sha) + ' to ' +
fmt_hash(evt.after_sha))
elif commits and not distinct_commits:
if evt.base_ref_name:
parts.append('merged ' + fmt_ref(evt.base_ref_name) + ' into '
+ fmt_ref(evt.ref_name))
else:
parts.append('fast-forwarded ' + fmt_ref(evt.ref_name))
parts.append('from ' + fmt_hash(evt.before_sha) + ' to ' +
fmt_hash(evt.after_sha))
else:
parts.append('pushed ' + Tags.Bold(str(num_commits)))
parts.append('new commit' + ('s' if num_commits > 1 else ''))
parts.append('to ' + fmt_ref(evt.ref_name))
self.bot.say(' '.join(str(p) for p in parts))
for commit in distinct_commits[:4]:
firstline = commit.message.split('\n')[0]
author = self.format_nickname(commit.author.name)
added = Tags.LtGreen(str(len(commit.added)))
modified = Tags.LtGreen(str(len(commit.modified)))
removed = Tags.Red(str(len(commit.removed)))
url = Tags.UnderlineBlue(utils.shorten_url(commit.url))
self.bot.say('%s by %s [%s|%s|%s] %s %s' %
(commit.hash[:6], author, added, modified, removed,
url, firstline))
if len(distinct_commits) > 4:
self.bot.say('... and %d more commits' %
(len(distinct_commits) - 4))
def handle_gh_pull_request_comment(self, evt):
self.bot.say('[%s] %s commented on #%s %s: %s' %
(Tags.UnderlinePink(evt.repo),
self.format_nickname(evt.author), evt.id, evt.hash[:6],
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def handle_gh_issue_comment(self, evt):
if evt.author == cfg.github.account.login:
return
self.bot.say('[%s] %s commented on #%s (%s): %s' % (
Tags.UnderlinePink(evt.repo), self.format_nickname(evt.author),
evt.id, evt.title, Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def handle_gh_pull_request_comment(self, evt):
self.bot.say('[%s] %s commented on #%s %s: %s' % (
Tags.UnderlinePink(evt.repo), self.format_nickname(evt.author),
evt.id, evt.hash[:6],
Tags.UnderlineBlue(utils.shorten_url(evt.url))))
def test_shorten(self):
""" Test the shorten function """
url = "http://example.com"
size = 5
self.assertEqual(len(shorten_url(url, size)), size)
def handle_gh_pull_request(self, evt):
self.bot.say('[%s] %s %s pull request #%d: %s (%s...%s): %s' % (
Tags.UnderlinePink(evt.repo), self.format_nickname(evt.author),
evt.action, evt.id, evt.title, Tags.Purple(evt.base_ref_name),
Tags.Purple(evt.head_ref_name),
Tags.UnderlineBlue(utils.shorten_url(evt.url))))