def test_owly_bad_key():
b = Shortener(shorteners.OWLY_SHORTENER)
with pytest.raises(TypeError):
b.short('http://www.test.com')
with pytest.raises(TypeError):
b.expand('http://www.test.com')
def test_owly_bad_key():
b = Shortener('OwlyShortener')
with pytest.raises(TypeError):
b.short('http://www.test.com')
with pytest.raises(TypeError):
b.expand('http://www.test.com')
def test_google_bad_params():
s = Shortener(Shorteners.GOOGLE)
with pytest.raises(TypeError):
s.short(expanded)
with pytest.raises(TypeError):
s.expand(expanded)
def test_bitly_bad_keys():
s = Shortener(Shorteners.BITLY)
with pytest.raises(TypeError):
s.short(expanded)
with pytest.raises(TypeError):
s.expand(shorten)
def test_qrcode():
s = Shortener('TinyurlShortener')
url = 'http://www.google.com'
mock_url = '{}?url={}'.format(s.api_url, url)
shorten = 'http://tinyurl.com/test'
responses.add(responses.GET, mock_url, body=shorten,
match_querystring=True)
s.short(url)
# flake8: noqa
assert s.qrcode() == 'http://chart.apis.google.com/chart?cht=qr&chl={0}&chs=120x120'.format(shorten)
def test_shortener_debug_enabled():
url = 'http://www.test.com'
small = 'http://small.com'
responses.add(responses.GET, url, body=small)
responses.add(responses.GET, small, body=url)
s = Shortener(debug=True)
s.short('http://www.test.com')
s.expand('http://small.com')
with pytest.raises(NotImplementedError):
s.total_clicks('http://small.com')
def test_is_valid_url():
bad = 'www.google.com'
good = 'http://www.google.com'
assert is_valid_url(good)
assert not is_valid_url(bad)
s = Shortener('TinyurlShortener')
with pytest.raises(ValueError):
url = 'http://12'
s.short(url)
with pytest.raises(ValueError):
url = 'www.11.xom'
s.expand(url)
def tweet_post(title, url, subid):
#post found scores to twitter
CONSUMERKEY = keys[0]
CONSUMERSECRET = keys[1]
ACCESSKEY = keys[2]
ACCESSSECRET = keys[3]
nonDuplicateFlag = True
auth = tweepy.OAuthHandler(CONSUMERKEY, CONSUMERSECRET)
auth.set_access_token(ACCESSKEY, ACCESSSECRET)
api = tweepy.API(auth)
#Clean link to prepare for tweeting
title = tweet_cleanup(title)
shortener = Shortener('TinyurlShortener')
tinyurl = shortener.short(url)
tweet = tweet_cleanup(title)
if subid in postedTweets:
nonDuplicateFlag = False
return
post = title + tinyurl
try:
api.update_status(post) #post the tweet
except:
print('Tweet not posted. Check Issue. Length= ', len(post))
return
postedTweets.append(subid)
update_db()
def item_create(form):
record = db(db.geo_item.f_name == request.vars.f_name).select().first()
if record.f_qrcode == None:
arg = str(record.id)
path = 'http://siri.pythonanywhere.com/byui_art/default/item_details?itemId=' + arg + '&qr=True'
tiny = ""
shortSuccess = False
try:
shortener = Shortener('Tinyurl')
tiny = "{}".format(shortener.short(path))
session.tinyCreateException = tiny
shortSuccess = True
except:
session.tinyCreateException = "There was a problem with the url shortener. Please try again."
if shortSuccess:
code = qrcode.make(tiny)
else:
code = qrcode.make(path)
qrName='geo_item.f_qrcode.%s.jpg' % (uuid.uuid4())
code.save(request.folder + 'uploads/qrcodes/' + qrName, 'JPEG')
record.update_record(f_qrcode=qrName)
qr_text(qrName, record.f_name, record.f_alt5, tiny)
try:
imgPath = request.folder + 'uploads/' + record.f_image
thumbName = 'geo_item.f_thumb.%s.%s.jpg' % (uuid.uuid4(), record.id)
thumb = Image.open(request.folder + 'uploads/' + record.f_image)
thumb.thumbnail((350, 10000), Image.ANTIALIAS)
thumb.save(request.folder + 'uploads/thumbs/' + thumbName, 'JPEG')
record.update_record(f_thumb=thumbName)
except:
session.itemCreateException = "there was a problem updating the image in item_create: " + record.f_name
return dict()
def coll_create(form):
record = db(db.geo_collection.f_name == request.vars.f_name).select().first()
arg = str(record.id)
path = 'http://siri.pythonanywhere.com/byui_art/default/collection_details?collectionId=' + arg + '&qr=True'
tiny = ""
#######################################################################
################## TESTING PYSHORTENERS : TINYURL #####################
#######################################################################
try:
shortener = Shortener('TinyurlShortener')
tiny = "{}".format(shortener.short(path))
except:
session.tinyCreateException = "There was a problem with the url shortener. Please try again."
#######################################################################
###################### END PYSHORTENERS : TINYURL #####################
#######################################################################
code = qrcode.make(tiny)
qrName='geo_collection.f_qrcode.%s.jpg' % (uuid.uuid4())
code.save(request.folder + 'uploads/qrcodes/' + qrName, 'JPEG')
qr_text(qrName, record.f_name, record.f_location, tiny)
record.update_record(f_qrcode=qrName)
return dict()
def url_shorten(service, url):
supported_services = ['Isgd']
url = url if url.startswith('http') else ('http://%s' % url)
if service not in supported_services:
print 'Service "{}" is not supported, supported services are: {}'\
.format(service, supported_services)
return
try:
shortener = Shortener(service)
print shortener.short(url)
except ValueError:
print 'Invalid url given'
except Exception as e:
print 'Unable to shorten the url'
def get_item_str(self, item):
global api_key
global cache
global shortener_service
if api_key:
shortener = Shortener(shortener_service, api_key=api_key)
else:
shortener = None
short_url = None
if cache is not None:
short_url = cache.get(item['link'])
if short_url is None:
# Retry 3 times for goo.gl API
for i in range(0,3):
try:
short_url = shortener.short(item['link'])
if short_url is not None:
logger.debug("Saving short url to cache")
cache.put(item['link'], short_url)
except Exception as e:
logger.debug("Shortener threw exception {}".format(e.message))
continue
break
else:
logger.debug("Short url cache hit")
if short_url is None:
url = item['link']
else:
url = short_url
return '[%s] %s <%s>' % (''.join([c for c in self.name][0:18]), item['title'], url)
def hello():
googl = Shortener('GoogleShortener')
return """
Hello World! Testing www.google.com
Shorten url:{} - Expanded:{}
""".format(googl.short('http://www.google.com'),
googl.expand('http://goo.gl/fbsS')),
def hello():
short = Shortener('Tinyurl')
print("""
Hello World! Testing TinyurlShortener with www.google.com URL
Shorten url: {}
Expanded: {}
""".format(short.short('http://www.google.com'),
short.expand('http://goo.gl/fbsS')))
def test_custom_shortener():
class MyShortenerWithBlackJackAndHookers(BaseShortener):
def short(self, url):
return url
s = Shortener(MyShortenerWithBlackJackAndHookers)
url = 'http://www.test.com'
assert s.short(url) == url
def tweet():
args = get_args()
creds = load_credentials()
shortener = Shortener('Google', api_key=creds[0])
tweet = Twitter(auth=OAuth(creds[1], creds[2], creds[3], creds[4]))
url = "http://127.0.0.1:" + str(args.port) + "/raw_data"
response = urllib.urlopen(url)
dump = json.loads(response.read())
new = copy.deepcopy(dump)
old = {
'pokemons': []
}
if os.path.isfile('data.json'):
with open('data.json') as data_file:
old = json.load(data_file)
# Deletes encounter id for next step
for e_new in new['pokemons']:
for e_old in old['pokemons']:
if e_new['encounter_id'] == e_old['encounter_id']:
del e_new['encounter_id']
break
# Existing encounter ids are rare pokemon
# This entire step is to parse the data for a tweet
for e_new in new['pokemons']:
if e_new['pokemon_id'] in rares:
if 'encounter_id' in e_new:
location = str(Geocoder.reverse_geocode(e_new['latitude'], e_new['longitude'])[0]).split(',')
destination = location[0]
if len(location) == 5:
destination += ", " + location[1]
time = datetime.datetime.fromtimestamp(e_new['disappear_time']/1000)
ampm = "AM"
hour = time.hour
gmap = 'https://www.google.com/maps/dir/Current+Location/' \
+ str(e_new['latitude']) + ',' + str(e_new['longitude'])
if hour > 12:
hour -= 12
ampm = "PM"
elif hour == 12:
ampm = "PM"
elif hour == 0:
hour = 12
tweeting = "{} at {} until {}:{}:{} {}. #PokemonGo {}".format( \
e_new['pokemon_name'], destination, \
hour, str(time.minute).zfill(2), str(time.second).zfill(2), ampm, \
shortener.short(gmap))
tweet.statuses.update(status=tweeting)
print tweeting
# Google api timeout
t.sleep(0.5)
with open('data.json', 'w') as outfile:
json.dump(dump, outfile)
def make_link(group_name, event_id, shorten=True):
url = "https://meetup.com/{group_name}/events/{event_id}".format(
group_name=group_name,
event_id=event_id)
if shorten:
shortener = Shortener('Tinyurl')
try:
url = shortener.short(url)
except ReadTimeout:
pass
return url
def make_short(url, service):
if service == "google":
shortener = Shortener("Google", api_key=api_key)
elif service == "bitly":
shortener = Shortener("Bitly", bitly_token=access_token)
elif service == "owly":
shortener = Shortener("Owly", api_key=api_key)
elif service == "tinyurl":
shortener = Shortener("Tinyurl")
new_url = shortener.short(url)
return new_url
def sensu_event(self, kwargs):
host_params = ['action', 'client', 'check', 'occurrences', 'timestamp']
params = {
param: kwargs.get(param, None)
for param in host_params
}
logging.info("Received request on host endpoint: " + str(params))
logging.debug("Unpacked parameters:" + str(kwargs))
hostname = params['client']['name']
check = params['check']['name']
output = self._truncate_string(params['check']['output'], length=250)
# Custom attributes
# broadcast is a custom check attribute to know where to send IRC notifications
try:
if 'broadcast' in params['check']:
broadcast = params['check']['broadcast']
elif 'custom' in params['check'] and 'broadcast' in params['check']['custom']:
broadcast = params['check']['custom']['broadcast']
else:
logging.info("This notification does not have a broadcast assigned, not doing anything with it.")
return False
except KeyError as e:
logging.info("KeyError when trying to set broadcast config: " + str(e))
return False
dashboard = self.bot_config.MONITORING_DASHBOARD
check_url = "{0}/#/client/uchiwa/{1}?check={2}".format(dashboard,
hostname,
check)
shortener = Shortener('Tinyurl')
check_url = shortener.short(check_url)
msg_type = {
'create': "NEW: {0} - {1} @ {2} |#| {3}",
'resolve': "RESOLVED: {0} - {1} @ {2} |#| {3}",
'flapping': "FLAPPING: {0} - {1} @ {2} |#| {3}",
'unknown': "UNKNOWN: {0} - {1} @ {2} |#| {3}"
}
if params['action'] in msg_type:
msg = msg_type[params['action']].format(hostname, check, check_url,
output)
else:
msg = msg_type['unknown'].format(hostname, check, check_url,
output)
self._monitoring_broadcast(msg, broadcast=broadcast)
def tinyurlShort(self):
# Determine if URL is absolute or relative
tof = bool(urllib.parse.urlparse(self).netloc)
if tof is True:
return(self)
else:
self = (urllib.parse.urljoin('http://www.autotrader.com/', self))
# Shorten URL or catch exceptions
try:
shortener = Shortener('TinyurlShortener', timeout=9000)
autotraderFile.write((shortener.short(self)) + '\n\n')
except Exception as err:
autotraderFile.write('ERROR: Check the log.' + '\n\n')
logging.error(str(err) + '\n')
def exhi_create(form):
record = db(db.geo_exhibit.f_name == request.vars.f_name).select().first()
arg = str(record.id)
path = 'http://siri.pythonanywhere.com/byui_art/default/exhibit_details?exhibitId=' + arg + '&qr=True'
tiny = ""
try:
shortener = Shortener('TinyurlShortener')
tiny = "{}".format(shortener.short(path))
except:
session.tinyCreateException = "There was a problem with the url shortener. Please try again."
code = qrcode.make(tiny)
qrName='geo_exhibit.f_qrcode.%s.jpg' % (uuid.uuid4())
code.save(request.folder + 'uploads/qrcodes/' + qrName, 'JPEG')
qr_text(qrName, record.f_name, '', tiny)
record.update_record(f_qrcode=qrName)
def generateText(match):
report = ""
report += "Update: "
report += getTeamAlliance(match)
if (didTeamWin(match) == 1):
report += (" won " + getMatchTitle(match) + " against ")
elif (didTeamWin(match) == -1):
report += (" lost " + getMatchTitle(match) + " to ")
elif (didTeamWin(match) == 0):
report += (" tied " + getMatchTitle(match) + " with ")
report += (getOpposingAlliance(match) + ", " + getScore(match))
report += ". Full Tournament Results: "
shortener = Shortener('GoogleShortener', api_key=shortenerAPIKey)
robotEventsURL = "http://www.robotevents.com/%s.html#tabs-results" % eventSKU
report += shortener.short(robotEventsURL)
return report
def get_send_url(self):
# If we don't have a URL in the first place, return empty strings
# Or return the short URL if it has already been made
if not self.story_url:
return (None, None)
elif self.story_short_url:
return (self.story_url, self.story_short_url)
elif self.section.shorten_links == False:
return(self.story_url, self.story_url)
# Set up for Bitly Goodness
try:
BITLY_ACCESS_TOKEN = MeowSetting.objects.get(
setting_key='bitly_access_token').setting_value
shortener = Shortener('Bitly', bitly_token=BITLY_ACCESS_TOKEN)
except:
print("[WARN] URL Shortener is not properly configured!")
return(self.story_url, self.story_url)
# api=bitly_api.Connection(access_token=BITLY_ACCESS_TOKEN)
# If Bitly fails, we'll just continue with our canonical URL
# try:
# short_url = api.shorten(self.story_url)['url']
# except bitly_api.BitlyError as e:
# self.log_error(e, self.section)
# short_url = None
# except:
# e = sys.exc_info()[0]
# short_url = None
# self.log_error(e, self.section)
try:
short_url = shortener.short(self.story_url)
except:
e = sys.exc_info()[0]
short_url = None
self.log_error(e, self.section)
if short_url:
self.story_short_url = short_url
self.save()
return (self.story_url, short_url)
else:
return (self.story_url, self.story_url)
def on_post(self, req, resp):
"""Handles incomming slack requests and returns lmgtfy link"""
text = req.get_param('text').decode('ascii')
token = req.get_param('token').decode('ascii')
user_id = req.get_param('user_id').decode('ascii')
channel_id = req.get_param('channel_id').decode('ascii')
response_url = req.get_param('response_url').decode('ascii')
url = "http://lmgtfy.com/?q=" + text.replace(" ","+")
shortener = Shortener('Tinyurl')
json_response = {
"token": token,
"response_type": "in_channel",
"channel": channel_id,
"text": "Here's an answer to your question: " + shortener.short(url),
"as_user": True,
"scope": 'chat:write:user'
}
resp.body = json.dumps(json_response)
def index(request):
form = UrlInputForm()
if request.method == 'POST':
form = UrlInputForm(request.POST)
if form.is_valid():
url = request.POST.get('url')
shortener = Shortener('TinyurlShortener')
short = {
'long_url': url[7:], # exclude 'http://' from url string
'short_url': shortener.short(url)[19:], # exclude 'http://tinyurl.com/
'user': random.choice(User.objects.all())
}
try:
shorted = Shorted.objects.get(long_url=short['long_url'])
except ObjectDoesNotExist:
shorted = Shorted.objects.create(**short)
return HttpResponseRedirect('/links/'+str(shorted.short_url))
return render(request, 'app/index.html', {'form': form})
class VkMusicSearcher:
def __init__(self, vk_api_token, google_api_token):
__session = vk.Session(access_token=vk_api_token)
__google_api_token = google_api_token
self.vk_api = vk.API(__session)
self.shortener = Shortener('Google', api_key=__google_api_token)
# result_count <= 9
self.result_count = 9
def print_menu(self, local_count):
print "\nMenu:"
print "-> Next page - [n]"
print "-> Get info about audio - [1-{0}]".format(local_count - 1)
print "-> Exit menu - [q]"
def search_music(self, query):
offset = 0
for counter in range(10):
query_result = self.vk_api.audio.search(q=query, count=self.result_count, offset=offset)
local_count = len(query_result)
query_result.pop(0)
music_counter = 1
for q in query_result:
print "{0}) {1}\t: {2}".format(music_counter, uformat(q["artist"]), uformat(q["title"]))
music_counter += 1
self.print_menu(local_count)
option = raw_input("\nInput option: ").lower()
if option == "n":
offset += local_count
continue
elif option == "q":
break
elif search(r"^[1-{0}]$".format(local_count - 1), option):
music_number = int(option)
music_info = query_result[music_number - 1]
print "----> {0} : {1} | {2}\n".format(uformat(music_info["artist"]), uformat(music_info["title"]),
self.shortener.short(music_info["url"]))
else:
continue
def main(argv, list_of_tags):
if len(argv) > 1:
url = argv
if len(argv) > 20:
shortener = Shortener('TinyurlShortener')
url = shortener.short(url.strip())
else:
print("ERROR")
url = "http://www.clarifai.com/img/metro-north.jpg"
command = 'curl -H "Authorization: Bearer b3A9PCVEzVkAijC1CC0qEUPNKcS9GE" --data-urlencode "url='
command = command + url + '" https://api.clarifai.com/v1/tag/'
command = command + " | python -mjson.tool > test.json"
os.system(command)
with open('test.json') as test:
data = json.load(test)
with open('tag.txt', 'w') as f:
pprint(data['results'][0]['result']['tag']['classes'], f)
with open('tag.txt', 'r') as f:
for x in f:
list_of_tags.append(x.split("'")[1])
def test_adfly_bad_params():
s = Shortener('AdflyShortener')
with pytest.raises(TypeError):
s.short(expanded)
def test_short_method_bad_url():
s = Shortener()
with pytest.raises(ValueError):
s.short('test.com')
def worker(url, host):
shortener = Shortener(host, timeout=10)
short_url = shortener.short(url)
return short_url
def shrink_url(url):
shortener = Shortener("Bitly", bitly_token=os.environ["BITLY_ACCESS_TOKEN"])
return shortener.short(url)
def test_adfly_bad_params():
s = Shortener(Shorteners.ADFLY)
with pytest.raises(TypeError):
s.short(expanded)
link = s.find(href=re.compile('download'))
get_link = link.get('href')
# shrinkParsedURL
access_token = 'my_bitly_token'
tinyurl_short = Shortener('Tinyurl')
bitly_short = Shortener('Bitly', bitly_token=access_token)
while not req_proxy.generate_proxied_request(get_link):
print('\nNext proxy for Parsed URL')
else:
print('\nConnected to Parsed URL!')
pass
shrink_url = bitly_short.short(get_link)
if shrink_url:
print('\nBitLy: ' + str(shrink_url))
else:
shrink_url = tinyurl_short.short(get_link)
print('\nTinyurl: ' + str(shrink_url))
# downloadShrinkURL
while not req_proxy.generate_proxied_request(shrink_url):
print('\nNext proxy for Shrink URL')
else:
print('\nConnected to Shrink URL!')
pass
print('\nDownloading: ' + query + ' via Short URL --> ' + shrink_url)
stats_filtered = stats_filtered + 1
# Twitter
tweet_text = post_title
tweet_size_allowed = tweet_size - tweet_link_size
# Mastodon
toot_text = post_title
# Shorten text if needed
if (len(tweet_text) >= (tweet_size_allowed)):
tweet_text = tweet_text[:tweet_size_allowed - 1]
# Add link
try:
bitly_article_url = shortener.short(web_page.url)
tweet_text = tweet_text + " " + bitly_article_url
toot_text = toot_text + " " + bitly_article_url
except:
tweet_text = tweet_text + " " + web_page.url
toot_text = toot_text + " " + web_page.url
# Add source if possible
if (len(tweet_text) + len(rss_twitter) < tweet_size_allowed):
tweet_text = tweet_text + " " + rss_twitter
toot_text = toot_text + " " + rss_twitter
if doToot and not filtered_post:
out_img = kr_utils.cleanImage_url(web_page, out_img)
if out_img:
def shorten_url(url):
# Goo.gl Shortener
api_key = "AIzaSyBRICfYzs7Q-5ojvnXks2dq213z_fPXqSQ"
shortener = Shortener('Google', api_key=api_key)
return shortener.short(url)
parameters = {
"earth_date": date,
"api_key": "CbSqjyfnKECX7WWkRUg5ux4hLwUkMbh0svWuxMfr"
}
response = requests.get(
"https://api.nasa.gov/mars-photos/api/v1/rovers/curiosity/photos",
params=parameters)
jresponse = response.json()
photos = jresponse["photos"]
for i in photos:
url = i["img_src"]
url = short.short(url)
camera = i["camera"]["full_name"]
roverid = i["rover"]["id"]
rovername = i["rover"]["name"]
landed = i["rover"]["landing_date"]
landed = englishdate(landed)
output = "The image can be found at {0} it was taken on the {1} camera of the rover id {2} '{3}' which landed on {4}".format(
url, camera, roverid, rovername, landed)
time.sleep(1)
print(output)
imagestore.write(str(output))
imagestore.write("\n")
def shorten_url(url):
shortener = Shortener('Google', api_key=GOOGLE_URL_SHORTENER_API_KEY)
short_url = shortener.short(url)
return short_url
class SCDLBot:
def __init__(self,
tg_bot_token,
botan_token=None,
google_shortener_api_key=None,
sc_auth_token=None,
store_chat_id=None,
no_flood_chat_ids=None,
alert_chat_ids=None,
dl_dir="/tmp/scdlbot",
dl_timeout=300,
max_convert_file_size=80000000,
chat_storage_file="/tmp/scdlbotdata",
app_url=None,
serve_audio=False):
self.SERVE_AUDIO = serve_audio
if self.SERVE_AUDIO:
self.MAX_TG_FILE_SIZE = 19000000
else:
self.MAX_TG_FILE_SIZE = 47000000
self.SITES = {
"sc": "soundcloud",
"scapi": "api.soundcloud",
"bc": "bandcamp",
"yt": "youtu",
}
self.APP_URL = app_url
self.DL_TIMEOUT = dl_timeout
self.MAX_CONVERT_FILE_SIZE = max_convert_file_size
self.HELP_TEXT = get_response_text('help.tg.md')
self.SETTINGS_TEXT = get_response_text('settings.tg.md')
self.DL_TIMEOUT_TEXT = get_response_text('dl_timeout.txt').format(
self.DL_TIMEOUT // 60)
self.WAIT_TEXT = get_response_text('wait.txt')
self.NO_AUDIO_TEXT = get_response_text('no_audio.txt')
self.NO_URLS_TEXT = get_response_text('no_urls.txt')
self.OLG_MSG_TEXT = get_response_text('old_msg.txt')
self.REGION_RESTRICTION_TEXT = get_response_text(
'region_restriction.txt')
self.DIRECT_RESTRICTION_TEXT = get_response_text(
'direct_restriction.txt')
self.LIVE_RESTRICTION_TEXT = get_response_text('live_restriction.txt')
# self.chat_storage = {}
self.chat_storage = shelve.open(chat_storage_file, writeback=True)
for chat_id in no_flood_chat_ids:
self.init_chat(
chat_id=chat_id,
chat_type=Chat.PRIVATE if chat_id > 0 else Chat.SUPERGROUP,
flood="no")
self.ALERT_CHAT_IDS = set(alert_chat_ids) if alert_chat_ids else set()
self.STORE_CHAT_ID = store_chat_id
self.DL_DIR = dl_dir
self.botan_token = botan_token if botan_token else None
self.shortener = Shortener('Google', api_key=google_shortener_api_key
) if google_shortener_api_key else None
config = configparser.ConfigParser()
config['scdl'] = {}
config['scdl']['path'] = self.DL_DIR
if sc_auth_token:
config['scdl']['auth_token'] = sc_auth_token
config_dir = os.path.join(os.path.expanduser('~'), '.config', 'scdl')
config_path = os.path.join(config_dir, 'scdl.cfg')
os.makedirs(config_dir, exist_ok=True)
with open(config_path, 'w') as config_file:
config.write(config_file)
self.updater = Updater(token=tg_bot_token)
dispatcher = self.updater.dispatcher
start_command_handler = CommandHandler('start',
self.help_command_callback)
dispatcher.add_handler(start_command_handler)
help_command_handler = CommandHandler('help',
self.help_command_callback)
dispatcher.add_handler(help_command_handler)
settings_command_handler = CommandHandler(
'settings', self.settings_command_callback)
dispatcher.add_handler(settings_command_handler)
dl_command_handler = CommandHandler('dl',
self.common_command_callback,
filters=~Filters.forwarded,
pass_args=True)
dispatcher.add_handler(dl_command_handler)
link_command_handler = CommandHandler('link',
self.common_command_callback,
filters=~Filters.forwarded,
pass_args=True)
dispatcher.add_handler(link_command_handler)
message_with_links_handler = MessageHandler(
Filters.text & (Filters.entity(MessageEntity.URL)
| Filters.entity(MessageEntity.TEXT_LINK)),
self.common_command_callback)
dispatcher.add_handler(message_with_links_handler)
button_query_handler = CallbackQueryHandler(self.button_query_callback)
dispatcher.add_handler(button_query_handler)
inline_query_handler = InlineQueryHandler(self.inline_query_callback)
dispatcher.add_handler(inline_query_handler)
unknown_handler = MessageHandler(Filters.command,
self.unknown_command_callback)
dispatcher.add_handler(unknown_handler)
dispatcher.add_error_handler(self.error_callback)
self.bot_username = self.updater.bot.get_me().username
self.RANT_TEXT_PRIVATE = "Read /help to learn how to use me"
self.RANT_TEXT_PUBLIC = "[Start me in PM to read help and learn how to use me](t.me/{}?start=1)".format(
self.bot_username)
def start(self,
use_webhook=False,
webhook_port=None,
cert_file=None,
cert_key_file=None,
webhook_host="0.0.0.0",
url_path="scdlbot"):
if use_webhook:
url_path = url_path.replace(":", "")
self.updater.start_webhook(
listen=webhook_host,
port=webhook_port,
url_path=url_path,
)
# cert=cert_file if cert_file else None,
# key=cert_key_file if cert_key_file else None,
# webhook_url=urljoin(app_url, url_path))
self.updater.bot.set_webhook(
url=urljoin(self.APP_URL, url_path),
certificate=open(cert_file, 'rb') if cert_file else None)
else:
self.updater.start_polling()
logger.warning("Bot started")
self.updater.idle()
def unknown_command_callback(self, bot, update):
pass
# bot.send_message(chat_id=update.message.chat_id, text="Unknown command")
def error_callback(self, bot, update, error):
try:
raise error
except Unauthorized:
# remove update.message.chat_id from conversation list
logger.debug('Update {} caused Unauthorized error: {}'.format(
update, error))
except BadRequest:
# handle malformed requests - read more below!
logger.debug('Update {} caused BadRequest error: {}'.format(
update, error))
except TimedOut:
# handle slow connection problems
logger.debug('Update {} caused TimedOut error: {}'.format(
update, error))
except NetworkError:
# handle other connection problems
logger.debug('Update {} caused NetworkError: {}'.format(
update, error))
except ChatMigrated as e:
# the chat_id of a group has changed, use e.new_chat_id instead
logger.debug('Update {} caused ChatMigrated error: {}'.format(
update, error))
except TelegramError:
# handle all other telegram related errors
logger.debug('Update {} caused TelegramError: {}'.format(
update, error))
def init_chat(self,
message=None,
chat_id=None,
chat_type=None,
flood="yes"):
if message:
chat_id = str(message.chat_id)
chat_type = message.chat.type
else:
chat_id = str(chat_id)
if chat_id not in self.chat_storage:
self.chat_storage[chat_id] = {}
if "settings" not in self.chat_storage[chat_id]:
self.chat_storage[chat_id]["settings"] = {}
if "mode" not in self.chat_storage[chat_id]["settings"]:
if chat_type == Chat.PRIVATE:
self.chat_storage[chat_id]["settings"]["mode"] = "dl"
else:
self.chat_storage[chat_id]["settings"]["mode"] = "ask"
if "flood" not in self.chat_storage[chat_id]["settings"]:
self.chat_storage[chat_id]["settings"]["flood"] = flood
if "rant_msg_ids" not in self.chat_storage[chat_id]["settings"]:
self.chat_storage[chat_id]["settings"]["rant_msg_ids"] = []
self.chat_storage.sync()
# logger.debug("Current chat_storage: %r", self.chat_storage)
def cleanup_chat(self, chat_id):
for msg_id in self.chat_storage[str(chat_id)]:
if msg_id != "settings":
timedelta = datetime.now() - self.chat_storage[str(
chat_id)][msg_id]["message"].date
if timedelta.days > 0:
self.chat_storage[str(chat_id)].pop(msg_id)
self.chat_storage.sync()
@run_async
def log_and_botan_track(self, event_name, message=None):
logger.info("Event: %s", event_name)
if message:
# TODO: add to local db
if self.botan_token:
return botan_track(self.botan_token, message, event_name)
else:
return False
def rant_and_cleanup(self,
bot,
chat_id,
rant_text,
reply_to_message_id=None):
rant_msg = bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=rant_text,
parse_mode='Markdown',
disable_web_page_preview=True)
flood = self.chat_storage[str(chat_id)]["settings"]["flood"]
if flood == "no":
for rant_msg_id in self.chat_storage[str(
chat_id)]["settings"]["rant_msg_ids"]:
try:
bot.delete_message(chat_id=chat_id, message_id=rant_msg_id)
except:
pass
self.chat_storage[str(
chat_id)]["settings"]["rant_msg_ids"].remove(rant_msg_id)
self.chat_storage[str(chat_id)]["settings"]["rant_msg_ids"].append(
rant_msg.message_id)
self.chat_storage.sync()
def help_command_callback(self, bot, update):
self.init_chat(update.message)
event_name = "help"
entities = update.message.parse_entities(
types=[MessageEntity.BOT_COMMAND])
for entity_value in entities.values():
event_name = entity_value.replace("/", "").replace(
"@{}".format(self.bot_username), "")
break
self.log_and_botan_track(event_name, update.message)
chat_id = update.message.chat_id
chat_type = update.message.chat.type
reply_to_message_id = update.message.message_id
flood = self.chat_storage[str(chat_id)]["settings"]["flood"]
if chat_type != Chat.PRIVATE and flood == "no":
self.rant_and_cleanup(bot,
chat_id,
self.RANT_TEXT_PUBLIC,
reply_to_message_id=reply_to_message_id)
else:
bot.send_message(chat_id=chat_id,
text=self.HELP_TEXT,
parse_mode='Markdown',
disable_web_page_preview=True)
def get_settings_inline_keyboard(self, chat_id):
mode = self.chat_storage[str(chat_id)]["settings"]["mode"]
flood = self.chat_storage[str(chat_id)]["settings"]["flood"]
emoji_yes = "✅"
emoji_no = "❌"
button_dl = InlineKeyboardButton(text=" ".join(
[emoji_yes if mode == "dl" else emoji_no, "Download"]),
callback_data=" ".join(
["settings", "dl"]))
button_link = InlineKeyboardButton(
text=" ".join([emoji_yes if mode == "link" else emoji_no,
"Links"]),
callback_data=" ".join(["settings", "link"]))
button_ask = InlineKeyboardButton(
text=" ".join([emoji_yes if mode == "ask" else emoji_no, "Ask"]),
callback_data=" ".join(["settings", "ask"]))
button_flood = InlineKeyboardButton(
text=" ".join([emoji_yes if flood == "yes" else emoji_no,
"Flood"]),
callback_data=" ".join(["settings", "flood"]))
button_close = InlineKeyboardButton(
text=" ".join([emoji_no, "Close settings"]),
callback_data=" ".join(["settings", "close"]))
inline_keyboard = InlineKeyboardMarkup(
[[button_dl, button_link, button_ask],
[button_flood, button_close]])
return inline_keyboard
def settings_command_callback(self, bot, update):
self.init_chat(update.message)
self.log_and_botan_track("settings")
chat_id = update.message.chat_id
bot.send_message(
chat_id=chat_id,
parse_mode='Markdown',
reply_markup=self.get_settings_inline_keyboard(chat_id),
text=self.SETTINGS_TEXT)
def common_command_callback(self, bot, update, args=None):
self.init_chat(update.message)
chat_id = update.message.chat_id
chat_type = update.message.chat.type
reply_to_message_id = update.message.message_id
entities = update.message.parse_entities(
types=[MessageEntity.BOT_COMMAND])
if not entities:
command_passed = False
# if no command then it is just a message and use default mode
mode = self.chat_storage[str(chat_id)]["settings"]["mode"]
else:
command_passed = True
# try to determine mode from command
mode = None
for entity_value in entities.values():
mode = entity_value.replace("/", "").replace(
"@{}".format(self.bot_username), "")
break
if not mode:
mode = "dl"
if command_passed and not args:
rant_text = self.RANT_TEXT_PRIVATE if chat_type == Chat.PRIVATE else self.RANT_TEXT_PUBLIC
rant_text += "\nYou can simply send message with links (to download) OR command as `/{} <links>`.".format(
mode)
self.rant_and_cleanup(bot,
chat_id,
rant_text,
reply_to_message_id=reply_to_message_id)
return
# apologize and send TYPING: always in PM and only when it's command in non-PM
apologize = chat_type == Chat.PRIVATE or command_passed
if apologize:
bot.send_chat_action(chat_id=chat_id, action=ChatAction.TYPING)
urls = self.prepare_urls(msg_or_text=update.message,
direct_urls=(mode == "link"))
logger.debug(urls)
if not urls:
if apologize:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.NO_URLS_TEXT,
parse_mode='Markdown')
else:
botan_event_name = ("{}_cmd".format(mode)) if command_passed else (
"{}_msg".format(mode))
self.log_and_botan_track(botan_event_name, update.message)
if mode == "dl":
wait_message = bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
parse_mode='Markdown',
text=get_italic(self.WAIT_TEXT))
for url in urls:
self.download_url_and_send(
bot,
url,
urls[url],
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
wait_message_id=wait_message.message_id)
elif mode == "link":
wait_message = bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
parse_mode='Markdown',
text=get_italic(self.WAIT_TEXT))
link_text = self.get_link_text(urls)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
parse_mode='Markdown',
disable_web_page_preview=True,
text=link_text if link_text else self.NO_URLS_TEXT)
bot.delete_message(chat_id=chat_id,
message_id=wait_message.message_id)
elif mode == "ask":
# ask: always in PM and only if good urls exist in non-PM
if chat_type == Chat.PRIVATE or "http" in " ".join(
urls.values()):
orig_msg_id = str(reply_to_message_id)
self.chat_storage[str(chat_id)][orig_msg_id] = {
"message": update.message,
"urls": urls
}
question = "🎶 links found, what to do?"
button_dl = InlineKeyboardButton(text="✅ Download",
callback_data=" ".join(
[orig_msg_id, "dl"]))
button_link = InlineKeyboardButton(
text="❇️ Links",
callback_data=" ".join([orig_msg_id, "link"]))
button_cancel = InlineKeyboardButton(
text="❎",
callback_data=" ".join([orig_msg_id, "nodl"]))
inline_keyboard = InlineKeyboardMarkup(
[[button_dl, button_link, button_cancel]])
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
reply_markup=inline_keyboard,
text=question)
self.cleanup_chat(chat_id)
def button_query_callback(self, bot, update):
btn_msg = update.callback_query.message
self.init_chat(btn_msg)
user_id = update.callback_query.from_user.id
btn_msg_id = btn_msg.message_id
chat = btn_msg.chat
chat_id = chat.id
chat_type = chat.type
orig_msg_id, action = update.callback_query.data.split()
if orig_msg_id == "settings":
if chat_type != Chat.PRIVATE:
chat_member_status = chat.get_member(user_id).status
if chat_member_status not in [
ChatMember.ADMINISTRATOR, ChatMember.CREATOR
] and user_id not in self.ALERT_CHAT_IDS:
self.log_and_botan_track("settings_fail")
update.callback_query.answer(text="You're not chat admin")
return
self.log_and_botan_track("settings_{}".format(action), btn_msg)
if action == "close":
bot.delete_message(chat_id, btn_msg_id)
else:
setting_changed = False
if action in ["dl", "link", "ask"]:
current_setting = self.chat_storage[str(
chat_id)]["settings"]["mode"]
if action != current_setting:
setting_changed = True
self.chat_storage[str(
chat_id)]["settings"]["mode"] = action
elif action in ["flood"]:
current_setting = self.chat_storage[str(
chat_id)]["settings"]["flood"]
setting_changed = True
self.chat_storage[str(chat_id)]["settings"][
action] = "no" if current_setting == "yes" else "yes"
if setting_changed:
self.chat_storage.sync()
update.callback_query.answer(text="Settings changed")
update.callback_query.edit_message_reply_markup(
parse_mode='Markdown',
reply_markup=self.get_settings_inline_keyboard(
chat_id))
else:
update.callback_query.answer(text="Settings not changed")
elif orig_msg_id in self.chat_storage[str(chat_id)]:
msg_from_storage = self.chat_storage[str(chat_id)].pop(orig_msg_id)
orig_msg = msg_from_storage["message"]
urls = msg_from_storage["urls"]
self.log_and_botan_track("{}_msg".format(action), orig_msg)
if action == "dl":
update.callback_query.answer(text=self.WAIT_TEXT)
wait_message = update.callback_query.edit_message_text(
parse_mode='Markdown', text=get_italic(self.WAIT_TEXT))
for url in urls:
self.download_url_and_send(
bot,
url,
urls[url],
chat_id=chat_id,
reply_to_message_id=orig_msg_id,
wait_message_id=wait_message.message_id)
elif action == "link":
update.callback_query.answer(text=self.WAIT_TEXT)
wait_message = update.callback_query.edit_message_text(
parse_mode='Markdown', text=get_italic(self.WAIT_TEXT))
urls = self.prepare_urls(urls.keys(), direct_urls=True)
link_text = self.get_link_text(urls)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=orig_msg_id,
parse_mode='Markdown',
disable_web_page_preview=True,
text=link_text if link_text else self.NO_URLS_TEXT)
bot.delete_message(chat_id=chat_id,
message_id=wait_message.message_id)
elif action == "nodl":
bot.delete_message(chat_id=chat_id, message_id=btn_msg_id)
else:
update.callback_query.answer(text=self.OLG_MSG_TEXT)
bot.delete_message(chat_id=chat_id, message_id=btn_msg_id)
def inline_query_callback(self, bot, update):
self.log_and_botan_track("link_inline")
inline_query_id = update.inline_query.id
text = update.inline_query.query
results = []
urls = self.prepare_urls(msg_or_text=text, direct_urls=True)
for url in urls:
for direct_url in urls[url].splitlines(
): # TODO: fix non-mp3 and allow only sc/bc
logger.debug(direct_url)
results.append(
InlineQueryResultAudio(id=str(uuid4()),
audio_url=direct_url,
title="FAST_INLINE_DOWNLOAD"))
try:
bot.answer_inline_query(inline_query_id, results)
except:
pass
def prepare_urls(self, msg_or_text, direct_urls=False):
if isinstance(msg_or_text, Message):
urls = []
url_entities = msg_or_text.parse_entities(
types=[MessageEntity.URL])
for entity in url_entities:
url_str = url_entities[entity]
logger.debug("Entity URL Parsed: %s", url_str)
if "://" not in url_str:
url_str = "http://{}".format(url_str)
urls.append(URL(url_str))
text_link_entities = msg_or_text.parse_entities(
types=[MessageEntity.TEXT_LINK])
for entity in text_link_entities:
url_str = entity.url
logger.debug("Entity Text Link Parsed: %s", url_str)
urls.append(URL(url_str))
else:
urls = find_all_links(msg_or_text, default_scheme="http")
urls_dict = {}
for url in urls:
url_text = url.to_text(True)
url_parts_num = len([part for part in url.path_parts if part])
try:
if (
# SoundCloud: tracks, sets and widget pages, no /you/ pages
(self.SITES["sc"] in url.host and
(2 <= url_parts_num <= 3
or self.SITES["scapi"] in url_text) and
(not "you" in url.path_parts)) or
# Bandcamp: tracks and albums
(self.SITES["bc"] in url.host and
(2 <= url_parts_num <= 2)) or
# YouTube: videos and playlists
(self.SITES["yt"] in url.host and
("youtu.be" in url.host or "watch" in url.path
or "playlist" in url.path))):
if direct_urls or self.SITES["yt"] in url.host:
urls_dict[url_text] = get_direct_urls(url_text)
else:
urls_dict[url_text] = "http"
elif not any(
(site in url.host for site in self.SITES.values())):
urls_dict[url_text] = get_direct_urls(url_text)
except ProcessExecutionError:
logger.debug("youtube-dl get url failed: %s", url_text)
except URLError as exc:
urls_dict[url_text] = exc.status
return urls_dict
def get_link_text(self, urls):
link_text = ""
for i, url in enumerate(urls):
link_text += "[Source Link #{}]({}) | `{}`\n".format(
str(i + 1), url,
URL(url).host)
direct_urls = urls[url].splitlines()
for j, direct_url in enumerate(direct_urls):
if "http" in direct_url:
content_type = ""
if "googlevideo" in direct_url:
if "audio" in direct_url:
content_type = "Audio"
else:
content_type = "Video"
if self.shortener:
try:
direct_url = self.shortener.short(direct_url)
# botan.shorten_url(original_url, botan_token, uid)
except:
pass
link_text += "• {} [Direct Link]({})\n".format(
content_type, direct_url)
return link_text
@run_async
def download_url_and_send(self,
bot,
url,
direct_urls,
chat_id,
reply_to_message_id=None,
wait_message_id=None):
bot.send_chat_action(chat_id=chat_id, action=ChatAction.RECORD_AUDIO)
download_dir = os.path.join(self.DL_DIR, str(uuid4()))
shutil.rmtree(download_dir, ignore_errors=True)
os.makedirs(download_dir)
status = 0
if direct_urls == "direct":
status = -3
elif direct_urls == "country":
status = -4
elif direct_urls == "live":
status = -5
else:
if (self.SITES["sc"] in url
and self.SITES["scapi"] not in url) or (self.SITES["bc"]
in url):
cmd_name = "scdl"
cmd_args = []
cmd = None
cmd_input = None
if self.SITES["sc"] in url and self.SITES["scapi"] not in url:
cmd_name = "scdl"
cmd_args = (
"-l",
url, # URL of track/playlist/user
"-c", # Continue if a music already exist
"--path",
download_dir, # Download the music to a custom path
"--onlymp3", # Download only the mp3 file even if the track is Downloadable
"--addtofile", # Add the artist name to the filename if it isn't in the filename already
"--addtimestamp", # Adds the timestamp of the creation of the track to the title (useful to sort chronologically)
"--no-playlist-folder",
# Download playlist tracks into directory, instead of making a playlist subfolder
)
cmd = scdl_bin
cmd_input = None
elif self.SITES["bc"] in url:
cmd_name = "bandcamp-dl"
cmd_args = (
"--base-dir",
download_dir, # Base location of which all files are downloaded
"--template",
"%{track} - %{artist} - %{title} [%{album}]", # Output filename template
"--overwrite", # Overwrite tracks that already exist
"--group", # Use album/track Label as iTunes grouping
"--embed-art", # Embed album art (if available)
"--no-slugify", # Disable slugification of track, album, and artist names
url, # URL of album/track
)
cmd = bandcamp_dl_bin
cmd_input = "yes"
logger.info("%s starts: %s", cmd_name, url)
cmd_proc = cmd[cmd_args].popen(stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
try:
cmd_stdout, cmd_stderr = cmd_proc.communicate(
input=cmd_input, timeout=self.DL_TIMEOUT)
cmd_retcode = cmd_proc.returncode
if cmd_retcode or "Error resolving url" in cmd_stderr:
raise ProcessExecutionError(cmd_args, cmd_retcode,
cmd_stdout, cmd_stderr)
logger.info("%s succeeded: %s", cmd_name, url)
status = 1
except TimeoutExpired:
cmd_proc.kill()
logger.info("%s took too much time and dropped: %s", url)
status = -1
except ProcessExecutionError:
logger.exception("%s failed: %s" % (cmd_name, url))
if status == 0:
cmd_name = "youtube-dl"
cmd = youtube_dl_func
# TODO: set different ydl_opts for different sites
ydl_opts = {
'format':
'bestaudio/best',
'outtmpl':
os.path.join(download_dir, '%(title)s.%(ext)s'),
# default: %(autonumber)s - %(title)s-%(id)s.%(ext)s
'postprocessors': [
{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '128',
},
# {'key': 'EmbedThumbnail',}, {'key': 'FFmpegMetadata',},
],
}
queue = Queue()
cmd_args = (
url,
ydl_opts,
queue,
)
logger.info("%s starts: %s", cmd_name, url)
cmd_proc = Process(target=cmd, args=cmd_args)
cmd_proc.start()
try:
cmd_retcode, cmd_stderr = queue.get(block=True,
timeout=self.DL_TIMEOUT)
cmd_stdout = ""
cmd_proc.join()
if cmd_retcode:
raise ProcessExecutionError(cmd_args, cmd_retcode,
cmd_stdout, cmd_stderr)
# raise cmd_status #TODO: pass and re-raise original Exception?
logger.info("%s succeeded: %s", cmd_name, url)
status = 1
except Empty:
cmd_proc.join(1)
if cmd_proc.is_alive():
cmd_proc.terminate()
logger.info("%s took too much time and dropped: %s", cmd_name,
url)
status = -1
except ProcessExecutionError:
logger.exception("%s failed: %s" % (cmd_name, url))
status = -2
gc.collect()
if status == -1:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.DL_TIMEOUT_TEXT,
parse_mode='Markdown')
elif status == -2:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.NO_AUDIO_TEXT,
parse_mode='Markdown')
elif status == -3:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.DIRECT_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == -4:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.REGION_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == -5:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.LIVE_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == 1:
file_list = []
for d, dirs, files in os.walk(download_dir):
for file in files:
file_list.append(os.path.join(d, file))
for file in sorted(file_list):
file_name = os.path.split(file)[-1]
file_parts = []
try:
file_parts = self.split_audio_file(file)
except FileNotSupportedError as exc:
if not (exc.file_format in ["m3u", "jpg", "jpeg", "png"]):
logger.warning("Unsupported file format: %s",
file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, downloaded file `{}` is in format I could not yet convert or send"
.format(file_name),
parse_mode='Markdown')
except FileTooLargeError as exc:
logger.info("Large file for convert: %s", file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, downloaded file `{}` is `{}` MB and it is larger than I could convert (`{} MB`)"
.format(file_name, exc.file_size // 1000000,
self.MAX_CONVERT_FILE_SIZE // 1000000),
parse_mode='Markdown')
except FileConvertedPartiallyError as exc:
file_parts = exc.file_parts
logger.exception("Pydub failed: %s" % file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text="*Sorry*, not enough memory to convert file `{}`.."
.format(file_name),
parse_mode='Markdown')
except FileNotConvertedError as exc:
logger.exception("Pydub failed: %s" % file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text="*Sorry*, not enough memory to convert file `{}`.."
.format(file_name),
parse_mode='Markdown')
try:
caption = "Downloaded from {} with @{}\n".format(
URL(url).host, self.bot_username)
if "qP303vxTLS8" in url:
caption += "\n" + random.choice([
"Скачал музла, машина эмпэтри дала!",
"У тебя талант, братан! Ка-какой? Качать онлайн!",
"Слушаю и не плачУ, то, что скачал вчера",
"Всё по чесноку, если скачал, отгружу музла!",
"Дёрнул за канат, и телега поймала трэкан!",
"Сегодня я качаю, и трэки не влазят мне в RAM!",
])
flood = self.chat_storage[str(
chat_id)]["settings"]["flood"]
sent_audio_ids = self.send_audio_file_parts(
bot, chat_id, file_parts,
reply_to_message_id if flood == "yes" else None,
caption if flood == "yes" else None)
except FileSentPartiallyError as exc:
sent_audio_ids = exc.sent_audio_ids
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, could not send file `{}` or some of it's parts.."
.format(file_name),
parse_mode='Markdown')
logger.warning("Sending some parts failed: %s" % file_name)
if not self.SERVE_AUDIO:
shutil.rmtree(download_dir, ignore_errors=True)
if wait_message_id: # TODO: delete only once
try:
bot.delete_message(chat_id=chat_id, message_id=wait_message_id)
except:
pass
# downloader = URLopener()
# file_name, headers = downloader.retrieve(url.to_text(full_quote=True))
# patoolib.extract_archive(file_name, outdir=DL_DIR)
# os.remove(file_name)
def split_audio_file(self, file=""):
file_root, file_ext = os.path.splitext(file)
file_format = file_ext.replace(".", "").lower()
if file_format not in ["mp3", "m4a", "mp4"]:
raise FileNotSupportedError(file_format)
file_size = os.path.getsize(file)
if file_size > self.MAX_CONVERT_FILE_SIZE:
raise FileTooLargeError(file_size)
file_parts = []
if file_size <= self.MAX_TG_FILE_SIZE:
if file_format == "mp3":
file_parts.append(file)
else:
logger.info("Converting: %s", file)
try:
sound = AudioSegment.from_file(file, file_format)
file_converted = file.replace(file_ext, ".mp3")
sound.export(file_converted, format="mp3")
del sound
gc.collect()
file_parts.append(file_converted)
except (OSError, MemoryError) as exc:
gc.collect()
raise FileNotConvertedError
else:
logger.info("Splitting: %s", file)
try:
id3 = mutagen.id3.ID3(file, translate=False)
except:
id3 = None
parts_number = file_size // self.MAX_TG_FILE_SIZE + 1
try:
sound = AudioSegment.from_file(file, file_format)
part_size = len(sound) / parts_number
for i in range(parts_number):
file_part = file.replace(
file_ext, ".part{}{}".format(str(i + 1), file_ext))
part = sound[part_size * i:part_size * (i + 1)]
part.export(file_part, format="mp3")
del part
gc.collect()
if id3:
try:
id3.save(file_part, v1=2, v2_version=4)
except:
pass
file_parts.append(file_part)
# https://github.com/jiaaro/pydub/issues/135
# https://github.com/jiaaro/pydub/issues/89#issuecomment-75245610
del sound
gc.collect()
except (OSError, MemoryError) as exc:
gc.collect()
raise FileConvertedPartiallyError(file_parts)
return file_parts
def send_audio_file_parts(self,
bot,
chat_id,
file_parts,
reply_to_message_id=None,
caption=None):
sent_audio_ids = []
for index, file in enumerate(file_parts):
path = pathlib.Path(file)
file_name = os.path.split(file)[-1]
# file_name = translit(file_name, 'ru', reversed=True)
logger.info("Sending: %s", file_name)
bot.send_chat_action(chat_id=chat_id,
action=ChatAction.UPLOAD_AUDIO)
caption_ = " ".join(
["Part", str(index + 1), "of",
str(len(file_parts))]) if len(file_parts) > 1 else ""
if caption:
caption_ = caption + caption_
for i in range(3):
try:
audio = str(
urljoin(self.APP_URL,
str(path.relative_to(self.DL_DIR))))
logger.debug(audio)
if not self.SERVE_AUDIO:
audio = open(file, 'rb')
audio_msg = bot.send_audio(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
audio=audio,
caption=caption_)
sent_audio_ids.append(audio_msg.audio.file_id)
logger.info("Sending succeeded: %s", file_name)
break
except TelegramError:
if i == 2:
logger.exception(
"Sending failed because of TelegramError: %s" %
file_name)
if len(sent_audio_ids) != len(file_parts):
raise FileSentPartiallyError(sent_audio_ids)
return sent_audio_ids
from pyshorteners import Shortener
url = 'http://www.google.com'
api_key = 'AIzaSyAU_zzFUPzICsSUuwpBHE-OUaqDwN-4JaQY'
shortener = Shortener('Google', api_key=api_key)
print("My short url is {}".format(shortener.short(url)))
def test_bad_key():
s = Shortener(Shorteners.AWSM)
with pytest.raises(TypeError):
s.short(expanded)
def test_short_method_bad_url():
s = Shortener()
with pytest.raises(ValueError):
s.short('test.com')
def shorten_url(self, url):
gat = os.environ['BITLY_GAT']
shortener = Shortener('Bitly', bitly_token=gat)
return format(shortener.short(url))
def _get_short_url_generic(url):
shortener = Shortener('Isgd')
return shortener.short(url)
class SCDLBot:
MAX_TG_FILE_SIZE = 45000000
SITES = {
"sc": "soundcloud",
"scapi": "api.soundcloud",
"bc": "bandcamp",
"yt": "youtu",
}
def __init__(self,
tg_bot_token,
botan_token=None,
google_shortener_api_key=None,
bin_path="",
sc_auth_token=None,
store_chat_id=None,
no_clutter_chat_ids=None,
alert_chat_ids=None,
dl_dir="/tmp/scdl",
dl_timeout=3600,
max_convert_file_size=500000000):
self.DL_TIMEOUT = dl_timeout
self.MAX_CONVERT_FILE_SIZE = max_convert_file_size
self.HELP_TEXT = self.get_response_text('help.tg.md')
self.DL_TIMEOUT_TEXT = self.get_response_text('dl_timeout.txt').format(
self.DL_TIMEOUT // 60)
self.WAIT_TEXT = self.get_response_text('wait.txt')
self.NO_AUDIO_TEXT = self.get_response_text('no_audio.txt')
self.NO_URLS_TEXT = self.get_response_text('no_urls.txt')
self.REGION_RESTRICTION_TEXT = self.get_response_text(
'region_restriction.txt')
self.DIRECT_RESTRICTION_TEXT = self.get_response_text(
'direct_restriction.txt')
self.LIVE_RESTRICTION_TEXT = self.get_response_text(
'live_restriction.txt')
self.NO_CLUTTER_CHAT_IDS = set(
no_clutter_chat_ids) if no_clutter_chat_ids else set()
self.ALERT_CHAT_IDS = set(alert_chat_ids) if alert_chat_ids else set()
self.STORE_CHAT_ID = store_chat_id
self.DL_DIR = dl_dir
self.scdl = local[os.path.join(bin_path, 'scdl')]
self.bandcamp_dl = local[os.path.join(bin_path, 'bandcamp-dl')]
self.youtube_dl = local[os.path.join(bin_path, 'youtube-dl')]
self.botan_token = botan_token if botan_token else None
self.shortener = Shortener('Google', api_key=google_shortener_api_key
) if google_shortener_api_key else None
self.msg_store = {}
self.rant_msg_ids = {}
config = configparser.ConfigParser()
config['scdl'] = {}
config['scdl']['path'] = self.DL_DIR
if sc_auth_token:
config['scdl']['auth_token'] = sc_auth_token
config_dir = os.path.join(os.path.expanduser('~'), '.config', 'scdl')
config_path = os.path.join(config_dir, 'scdl.cfg')
os.makedirs(config_dir, exist_ok=True)
with open(config_path, 'w') as config_file:
config.write(config_file)
self.updater = Updater(token=tg_bot_token)
dispatcher = self.updater.dispatcher
start_command_handler = CommandHandler('start',
self.start_command_callback)
dispatcher.add_handler(start_command_handler)
help_command_handler = CommandHandler('help',
self.help_command_callback)
dispatcher.add_handler(help_command_handler)
clutter_command_handler = CommandHandler('clutter',
self.clutter_command_callback)
dispatcher.add_handler(clutter_command_handler)
dl_command_handler = CommandHandler('dl',
self.dl_command_callback,
filters=~Filters.forwarded,
pass_args=True)
dispatcher.add_handler(dl_command_handler)
link_command_handler = CommandHandler('link',
self.link_command_callback,
filters=~Filters.forwarded,
pass_args=True)
dispatcher.add_handler(link_command_handler)
message_with_links_handler = MessageHandler(
Filters.text & (Filters.entity(MessageEntity.URL)
| Filters.entity(MessageEntity.TEXT_LINK)),
self.message_callback)
dispatcher.add_handler(message_with_links_handler)
message_callback_query_handler = CallbackQueryHandler(
self.message_callback_query_callback)
dispatcher.add_handler(message_callback_query_handler)
inline_query_handler = InlineQueryHandler(self.inline_query_callback)
dispatcher.add_handler(inline_query_handler)
unknown_handler = MessageHandler(Filters.command,
self.unknown_command_callback)
dispatcher.add_handler(unknown_handler)
dispatcher.add_error_handler(self.error_callback)
self.bot_username = self.updater.bot.get_me().username
self.RANT_TEXT_PRIVATE = "Read /help to learn how to use me"
self.RANT_TEXT_PUBLIC = "[Press here and start to read help in my PM to learn how to use me](t.me/" + self.bot_username + "?start=1)"
def start(self,
use_webhook=False,
app_url=None,
webhook_port=None,
cert_file=None,
webhook_host="0.0.0.0",
url_path="scdlbot"):
if use_webhook:
url_path = url_path.replace(":", "")
self.updater.start_webhook(listen=webhook_host,
port=webhook_port,
url_path=url_path)
self.updater.bot.set_webhook(
url=urljoin(app_url, url_path),
certificate=open(cert_file, 'rb') if cert_file else None)
else:
self.updater.start_polling()
# self.send_alert(self.updater.bot, "bot restarted")
self.updater.idle()
@staticmethod
def get_response_text(file_name):
# https://stackoverflow.com/a/20885799/2490759
path = '/'.join(('texts', file_name))
return pkg_resources.resource_string(__name__, path).decode("UTF-8")
@staticmethod
def md_italic(text):
return "".join(["_", text, "_"])
def log_and_botan_track(self, event_name, message=None):
logger.info("Event: %s", event_name)
if message:
if self.botan_token:
try:
# uid = message.chat_id
uid = message.from_user.id
except AttributeError:
logger.warning('No chat_id in message')
return False
data = json.loads(message.to_json())
return botan.track(self.botan_token, uid, data, event_name)
else:
return False
def send_alert(self, bot, text, url=""):
for alert_chat_id in self.ALERT_CHAT_IDS:
try:
bot.send_message(
chat_id=alert_chat_id,
text="BOT ADMIN ALERT\nURL or file failed:\n" + url +
"\n" + text)
except:
pass
def rant_and_cleanup(self,
bot,
chat_id,
rant_text,
reply_to_message_id=None):
rant_msg = bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=rant_text,
parse_mode='Markdown',
disable_web_page_preview=True)
if chat_id in self.NO_CLUTTER_CHAT_IDS:
if not chat_id in self.rant_msg_ids.keys():
self.rant_msg_ids[chat_id] = []
else:
for rant_msg_id in self.rant_msg_ids[chat_id]:
try:
bot.delete_message(chat_id=chat_id,
message_id=rant_msg_id)
except:
pass
self.rant_msg_ids[chat_id].remove(rant_msg_id)
self.rant_msg_ids[chat_id].append(rant_msg.message_id)
def unknown_command_callback(self, bot, update):
pass
# bot.send_message(chat_id=update.message.chat_id, text="Unknown command")
def error_callback(self, bot, update, error):
try:
raise error
except Unauthorized:
# remove update.message.chat_id from conversation list
logger.debug('Update {} caused Unauthorized error: {}'.format(
update, error))
except BadRequest:
# handle malformed requests - read more below!
logger.debug('Update {} caused BadRequest error: {}'.format(
update, error))
except TimedOut:
# handle slow connection problems
logger.debug('Update {} caused TimedOut error: {}'.format(
update, error))
except NetworkError:
# handle other connection problems
logger.debug('Update {} caused NetworkError: {}'.format(
update, error))
except ChatMigrated as e:
# the chat_id of a group has changed, use e.new_chat_id instead
logger.debug('Update {} caused ChatMigrated error: {}'.format(
update, error))
except TelegramError:
# handle all other telegram related errors
logger.debug('Update {} caused TelegramError: {}'.format(
update, error))
def start_command_callback(self, bot, update):
self.help_command_callback(bot, update, event_name="start")
def help_command_callback(self, bot, update, event_name="help"):
chat_id = update.message.chat_id
chat_type = update.message.chat.type
reply_to_message_id = update.message.message_id
if (chat_type != "private") and (chat_id in self.NO_CLUTTER_CHAT_IDS):
self.rant_and_cleanup(bot,
chat_id,
self.RANT_TEXT_PUBLIC,
reply_to_message_id=reply_to_message_id)
else:
bot.send_message(chat_id=chat_id,
text=self.HELP_TEXT,
parse_mode='Markdown',
disable_web_page_preview=True)
self.log_and_botan_track(event_name, update.message)
def clutter_command_callback(self, bot, update):
chat_id = update.message.chat_id
if chat_id in self.NO_CLUTTER_CHAT_IDS:
self.NO_CLUTTER_CHAT_IDS.remove(chat_id)
bot.send_message(
chat_id=chat_id,
text=
"Chat cluttering is now ON. I *will send audios as replies* to messages with links.",
parse_mode='Markdown',
disable_web_page_preview=True)
else:
self.NO_CLUTTER_CHAT_IDS.add(chat_id)
bot.send_message(
chat_id=chat_id,
text=
"Chat cluttering is now OFF. I *will not send audios as replies* to messages with links.",
parse_mode='Markdown',
disable_web_page_preview=True)
self.log_and_botan_track("clutter", update.message)
def inline_query_callback(self, bot, update):
inline_query_id = update.inline_query.id
text = update.inline_query.query
results = []
urls = self.prepare_urls(text=text, get_direct_urls=True)
if urls:
for url in urls:
# self.download_and_send(bot, url, self.STORE_CHAT_ID, inline_query_id=update.inline_query.id)
for direct_url in urls[url].splitlines(
): #TODO: fix non-mp3 and allow only sc/bc
logger.debug(direct_url)
results.append(
InlineQueryResultAudio(id=str(uuid4()),
audio_url=direct_url,
title="FAST_INLINE_DOWNLOAD"))
try:
bot.answer_inline_query(inline_query_id, results)
except:
pass
self.log_and_botan_track("link_inline")
def dl_command_callback(self, bot, update, args=None, event_name="dl"):
chat_id = update.message.chat_id
chat_type = update.message.chat.type
reply_to_message_id = update.message.message_id
apologize = not (event_name == "msg" and chat_type != "private")
if event_name != "msg" and not args:
rant_text = self.RANT_TEXT_PRIVATE if chat_type == "private" else self.RANT_TEXT_PUBLIC
rant_text += "\nYou can simply send message with links (to download) OR command as `/{} <links>`.".format(
event_name)
self.rant_and_cleanup(
bot,
chat_id,
rant_text,
reply_to_message_id=update.message.message_id)
return
if apologize:
bot.send_chat_action(chat_id=chat_id, action=ChatAction.TYPING)
urls = self.prepare_urls(
msg=update.message,
get_direct_urls=(event_name == "link")) # text=" ".join(args)
if not urls:
if apologize:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.NO_URLS_TEXT,
parse_mode='Markdown')
return
else:
logger.debug(urls)
if event_name == "dl":
wait_message = bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
parse_mode='Markdown',
text=self.md_italic(self.WAIT_TEXT))
self.log_and_botan_track("dl_cmd", update.message)
for url in urls:
self.download_url_and_send(
bot,
url,
urls[url],
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
wait_message_id=wait_message.message_id)
elif event_name == "link":
wait_message = bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
parse_mode='Markdown',
text=self.md_italic(self.WAIT_TEXT))
self.log_and_botan_track("link_cmd", update.message)
link_text = ""
for i, link in enumerate("\n".join(urls.values()).split()):
logger.debug(link)
if self.shortener:
try:
link = self.shortener.short(link)
except:
pass
link_text += "[Download Link #" + str(
i + 1) + "](" + link + ")\n"
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
parse_mode='Markdown',
text=link_text)
bot.delete_message(chat_id=chat_id,
message_id=wait_message.message_id)
elif event_name == "msg":
if chat_type == "private":
wait_message = bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
parse_mode='Markdown',
text=self.md_italic(self.WAIT_TEXT))
self.log_and_botan_track("dl_msg", update.message)
for url in urls:
self.download_url_and_send(
bot,
url,
urls[url],
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
wait_message_id=wait_message.message_id)
else:
if "http" in " ".join(urls.values()):
orig_msg_id = str(reply_to_message_id)
if not chat_id in self.msg_store.keys():
self.msg_store[chat_id] = {}
self.msg_store[chat_id][orig_msg_id] = {
"message": update.message,
"urls": urls
}
button_dl = InlineKeyboardButton(
text="✅ Yes",
callback_data=" ".join([orig_msg_id, "dl"]))
button_link = InlineKeyboardButton(
text="✅ Get links",
callback_data=" ".join([orig_msg_id, "link"]))
button_cancel = InlineKeyboardButton(
text="❎ No",
callback_data=" ".join([orig_msg_id, "nodl"]))
inline_keyboard = InlineKeyboardMarkup(
[[button_dl, button_cancel]])
question = "🎶 links found. Download it?"
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
reply_markup=inline_keyboard,
text=question)
self.log_and_botan_track("dl_msg_income")
def link_command_callback(self, bot, update, args=None):
self.dl_command_callback(bot, update, args, event_name="link")
def message_callback(self, bot, update):
self.dl_command_callback(bot, update, event_name="msg")
def message_callback_query_callback(self, bot, update):
chat_id = update.callback_query.message.chat_id
btn_msg_id = update.callback_query.message.message_id
orig_msg_id, action = update.callback_query.data.split()
if chat_id in self.msg_store:
if orig_msg_id in self.msg_store[chat_id]:
orig_msg = self.msg_store[chat_id][orig_msg_id]["message"]
urls = self.msg_store[chat_id][orig_msg_id]["urls"]
if action == "dl":
update.callback_query.answer(text=self.WAIT_TEXT)
edited_msg = update.callback_query.edit_message_text(
parse_mode='Markdown',
text=self.md_italic(self.WAIT_TEXT))
self.log_and_botan_track(("dl_msg"), orig_msg)
for url in urls:
self.download_url_and_send(
bot,
url,
urls[url],
chat_id=chat_id,
reply_to_message_id=orig_msg_id,
wait_message_id=edited_msg.message_id)
elif action == "nodl":
# update.callback_query.answer(text="Cancelled!", show_alert=True)
bot.delete_message(chat_id=chat_id, message_id=btn_msg_id)
self.log_and_botan_track(("nodl_msg"), orig_msg)
elif action == "link":
self.log_and_botan_track(("link_msg"), orig_msg)
self.msg_store[chat_id].pop(orig_msg_id)
for msg_id in self.msg_store[chat_id]:
timedelta = datetime.now(
) - self.msg_store[chat_id][msg_id]["message"].date
if timedelta.days > 0:
self.msg_store[chat_id].pop(msg_id)
return
update.callback_query.answer(
text="Sorry, very old message that I don't remember.")
bot.delete_message(chat_id=chat_id, message_id=btn_msg_id)
@staticmethod
def youtube_dl_download_url(url, ydl_opts, queue=None):
ydl = youtube_dl.YoutubeDL(ydl_opts)
try:
ydl.download([url])
except Exception as exc:
ydl_status = str(exc)
# ydl_status = exc #TODO
else:
ydl_status = 0
if queue:
queue.put(ydl_status)
def youtube_dl_get_direct_urls(self, url):
ret_code, std_out, std_err = self.youtube_dl["--get-url",
url].run(retcode=None)
# TODO: case when one page has multiple videos some available some not
if "returning it as such" in std_err:
return "direct"
elif "proxy server" in std_err:
return "proxy"
elif "yt_live_broadcast" in std_out:
return "live"
else:
return std_out
def prepare_urls(self, msg=None, text=None, get_direct_urls=False):
if text:
urls = find_all_links(text, default_scheme="http")
elif msg:
urls = []
for url_str in msg.parse_entities(types=["url"]).values():
if "://" not in url_str:
url_str = "http://" + url_str
urls.append(URL(url_str))
else:
logger.debug("Text or msg is required")
return
urls_dict = {}
for url in urls:
url_text = url.to_text(True)
url_parts_num = len([part for part in url.path_parts if part])
if (
# SoundCloud: tracks, sets and widget pages
(self.SITES["sc"] in url.host and
(2 <= url_parts_num <= 3 or self.SITES["scapi"] in url_text))
or
# Bandcamp: tracks and albums
(self.SITES["bc"] in url.host and (2 <= url_parts_num <= 2)) or
# YouTube: videos and playlists
(self.SITES["yt"] in url.host and
("youtu.be" in url.host or "watch" in url.path
or "playlist" in url.path))):
if get_direct_urls or self.SITES["yt"] in url.host:
direct_urls = self.youtube_dl_get_direct_urls(url_text)
if direct_urls:
urls_dict[url_text] = direct_urls
else:
urls_dict[url_text] = "http"
elif not any((site in url.host for site in self.SITES.values())):
direct_urls = self.youtube_dl_get_direct_urls(url_text)
if direct_urls:
urls_dict[url_text] = direct_urls
if not urls_dict:
logger.info("No supported URLs found")
return urls_dict
@run_async
def download_url_and_send(self,
bot,
url,
direct_urls,
chat_id,
reply_to_message_id=None,
wait_message_id=None,
inline_query_id=None):
bot.send_chat_action(chat_id=chat_id, action=ChatAction.RECORD_AUDIO)
download_dir = os.path.join(self.DL_DIR, str(uuid4()))
shutil.rmtree(download_dir, ignore_errors=True)
os.makedirs(download_dir)
scdl_cmd = self.scdl[
"-l", url, # URL of track/playlist/user
"-c", # Continue if a music already exist
"--path", download_dir, # Download the music to a custom path
"--onlymp3", # Download only the mp3 file even if the track is Downloadable
"--addtofile", # Add the artist name to the filename if it isn't in the filename already
]
bandcamp_dl_cmd = self.bandcamp_dl[
"--base-dir",
download_dir, # Base location of which all files are downloaded
"--template",
"%{track} - %{artist} - %{title} [%{album}]", # Output filename template
"--overwrite", # Overwrite tracks that already exist
"--group", # Use album/track Label as iTunes grouping
"--embed-art", # Embed album art (if available)
"--no-slugify", # Disable slugification of track, album, and artist names
url # URL of album/track
]
# TODO: different ydl_opts for different sites
ydl_opts = {
'format':
'bestaudio/best',
'outtmpl':
os.path.join(download_dir, '%(title)s.%(ext)s'
), # %(autonumber)s - %(title)s-%(id)s.%(ext)s
'postprocessors': [
{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '128',
},
# {
# 'key': 'EmbedThumbnail',
# },
# {
# 'key': 'FFmpegMetadata',
# },
],
}
status = 0
if direct_urls == "direct":
status = -3
elif direct_urls == "proxy":
status = -4
elif direct_urls == "live":
status = -5
else:
logger.info("Trying to download URL: %s", url)
if self.SITES["sc"] in url and self.SITES["scapi"] not in url:
logger.info("scdl starts...")
try:
cmd_popen = scdl_cmd.popen(stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
try:
std_out, std_err = cmd_popen.communicate(
timeout=self.DL_TIMEOUT)
if cmd_popen.returncode or "Error resolving url" in std_err:
text = "scdl process failed" + "\nstdout:\n" + std_out + "\nstderr:\n" + std_err
logger.error(text)
self.send_alert(bot, text, url)
else:
text = "scdl succeeded"
logger.info(text)
status = 1
except TimeoutExpired:
text = "Download took with scdl too long, dropped"
logger.warning(text)
cmd_popen.kill()
status = -1
except Exception as exc:
text = "scdl start failed"
logger.exception(text)
self.send_alert(bot, text + "\n" + str(exc), url)
elif self.SITES["bc"] in url:
logger.info("bandcamp-dl starts...")
try:
cmd_popen = bandcamp_dl_cmd.popen(stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
try:
std_out, std_err = cmd_popen.communicate(
input="yes", timeout=self.DL_TIMEOUT)
if cmd_popen.returncode:
text = "bandcamp-dl process failed" + "\nstdout:\n" + std_out + "\nstderr:\n" + std_err
logger.error(text)
self.send_alert(bot, text, url)
else:
text = "bandcamp-dl succeeded"
logger.info(text)
status = 1
except TimeoutExpired:
text = "bandcamp-dl took too much time and dropped"
logger.warning(text)
cmd_popen.kill()
status = -1
except Exception as exc:
text = "bandcamp-dl start failed"
logger.exception(text)
self.send_alert(bot, text + "\n" + str(exc), url)
if status == 0:
logger.info("youtube-dl starts...")
queue = multiprocessing.Queue()
ydl = multiprocessing.Process(target=self.youtube_dl_download_url,
args=(
url,
ydl_opts,
queue,
))
ydl.start()
try:
ydl_status = queue.get(block=True, timeout=self.DL_TIMEOUT)
ydl.join()
if ydl_status:
raise Exception(ydl_status)
# raise ydl_status
text = "youtube-dl succeeded"
logger.info(text)
status = 1
except Empty:
ydl.join(1)
if ydl.is_alive():
ydl.terminate()
text = "youtube-dl took too much time and dropped"
logger.warning(text)
status = -1
except Exception as exc:
text = "youtube-dl failed"
logger.exception(text)
self.send_alert(bot, text + "\n" + str(exc), url)
status = -2
gc.collect()
if status == -1:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.DL_TIMEOUT_TEXT,
parse_mode='Markdown')
elif status == -2:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.NO_AUDIO_TEXT,
parse_mode='Markdown')
elif status == -3:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.DIRECT_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == -4:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.REGION_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == -5:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.LIVE_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == 1:
if chat_id in self.NO_CLUTTER_CHAT_IDS:
reply_to_message_id_ = None
else:
reply_to_message_id_ = reply_to_message_id
file_list = []
for d, dirs, files in os.walk(download_dir):
for file in files:
file_list.append(os.path.join(d, file))
send_status_flag = True
for file in sorted(file_list):
send_status = self.split_and_send_audio_file(
bot, chat_id, reply_to_message_id_, file)
if send_status != "success":
send_status_flag = False
if not send_status_flag:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.NO_AUDIO_TEXT,
parse_mode='Markdown')
shutil.rmtree(download_dir, ignore_errors=True)
if wait_message_id: # TODO: delete only once or append errors
try:
bot.delete_message(chat_id=chat_id, message_id=wait_message_id)
except:
pass
# if inline_query_id:
# results = []
# for audio_id in sent_audio_ids:
# if audio_id:
# results.append(InlineQueryResultCachedAudio(id=str(uuid4()), audio_file_id=audio_id))
# bot.answer_inline_query(inline_query_id, results)
# downloader = URLopener()
# try:
# file_name, headers = downloader.retrieve(url.to_text(full_quote=True))
# patoolib.extract_archive(file_name, outdir=DL_DIR)
# os.remove(file_name)
# except Exception as exc:
# return str(exc)
# # return str(sys.exc_info()[0:1])
#
# return "success"
def split_and_send_audio_file(self,
bot,
chat_id,
reply_to_message_id=None,
file=""):
sent_audio_ids = []
file_root, file_ext = os.path.splitext(file)
file_name = os.path.split(file)[-1]
file_format = file_ext.replace(".", "")
if not (file_format == "mp3" or file_format == "m4a"
or file_format == "mp4"):
logger.warning("Unsupported file format: %s", file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, downloaded file `{}` is in format I could not yet send or convert"
.format(file_name),
parse_mode='Markdown')
return "format"
file_size = os.path.getsize(file)
if file_size > self.MAX_CONVERT_FILE_SIZE:
logger.warning("Large file for convert: %s", file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, downloaded file `{}` is larger than I could convert".
format(file_name),
parse_mode='Markdown')
return "size"
parts_number = 1
file_parts = []
if file_size <= self.MAX_TG_FILE_SIZE:
file_parts.append(file)
else:
logger.info("Splitting: %s", file_name)
try:
id3 = mutagen.id3.ID3(file, translate=False)
except:
id3 = None
parts_number = file_size // self.MAX_TG_FILE_SIZE + 1
try:
sound = AudioSegment.from_file(file, file_format)
part_size = len(sound) / parts_number
for i in range(parts_number):
file_part = file.replace(file_ext,
".part" + str(i + 1) + file_ext)
part = sound[part_size * i:part_size * (i + 1)]
part.export(file_part, format="mp3")
del part
if id3:
id3.save(file_part, v1=2, v2_version=4)
file_parts.append(file_part)
# https://github.com/jiaaro/pydub/issues/135
# https://github.com/jiaaro/pydub/issues/89#issuecomment-75245610
del sound
gc.collect()
except (OSError, MemoryError) as exc:
text = "Failed pydub convert"
logger.exception(text)
self.send_alert(bot, text + "\n" + str(exc), file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, not enough memory to convert file `{}`, you may try again later.."
.format(file_name),
parse_mode='Markdown')
gc.collect()
return "memory"
file_parts_copy = list(file_parts)
for index, file in enumerate(file_parts):
logger.info("Sending: %s", file_name)
bot.send_chat_action(chat_id=chat_id,
action=ChatAction.UPLOAD_AUDIO)
# file = translit(file, 'ru', reversed=True)
caption = ""
if file_size > self.MAX_TG_FILE_SIZE:
caption += "\n" + " ".join(
["Part", str(index + 1), "of",
str(parts_number)])
for i in range(3):
try:
if (i == 0) and (chat_id not in self.NO_CLUTTER_CHAT_IDS):
caption = "Downloaded with @{}".format(
self.bot_username) + caption
audio_msg = bot.send_audio(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
audio=open(file, 'rb'),
caption=caption)
sent_audio_ids.append(audio_msg.audio.file_id)
file_parts_copy.remove(file)
logger.info("Sending success: %s", file_name)
break
except TelegramError as exc:
logger.exception(
"Sending failed because of TelegramError: %s",
file_name)
self.send_alert(bot, "TelegramError:\n" + str(exc),
file_name)
if not file_parts_copy:
return "success"
else:
logger.warning("Sending total failed: %s", file_name)
return "send"
def test_bad_key():
s = Shortener(Shorteners.AWSM)
with pytest.raises(TypeError):
s.short(expanded)
class RSSReader(BotPlugin):
def configure(self, configuration):
if configuration is not None and configuration != {}:
config = dict(chain(CONFIG_TEMPLATE.items(),
configuration.items()))
else:
config = CONFIG_TEMPLATE
super(RSSReader, self).configure(config)
def get_configuration_template(self):
return CONFIG_TEMPLATE
def check_configuration(self, configuration):
# configuration now happens via other than the default means
pass
@botcmd(template='feeds')
def rssreader_feeds(self, *args):
"""Show all feeds that are checked for updates by RSSReader."""
return {'feeds': self.config['FEEDS']}
@botcmd(template='subscriptions')
def rssreader_subscriptions(self, *args):
"""Show channels which will receive new content from specific feeds."""
subs = self.config['SUBSCRIPTIONS']
joined_subs = {k: self.list_format(v) for k, v in subs.items()}
return {'subscriptions': joined_subs}
# An alias for rssreader_subscriptions
@botcmd(template='subscriptions')
def rssreader_subs(self, *args):
"""Show channels which will receive new content from specific feeds."""
return self.rssreader_subscriptions(args)
@botcmd(admin_only=True, split_args_with=None)
def rssreader_add(self, msg, args):
"""Add URL(s) of feed(s) to be checked for updates."""
for feed in args:
hash = self.hash_feed(feed)
self.config['FEEDS'][hash] = feed
self.config['SUBSCRIPTIONS'][hash] = []
self.save_config()
return 'Added [{}] {}'.format(hash, self.list_format(args))
@botcmd(admin_only=True, split_args_with=None)
def rssreader_rm(self, msg, args):
"""Remove feed(s) from the update loop."""
removed = []
for feed in args:
if feed not in self.config['FEEDS']:
yield 'Feed ID {} was not found'.format(feed)
continue
del self.config['FEEDS'][feed]
del self.config['SUBSCRIPTIONS'][feed]
removed.append(feed)
self.save_config()
if len(removed) > 0:
yield 'Removed {}'.format(self.list_format(removed))
@botcmd(admin_only=True, split_args_with=None)
def rssreader_subscribe(self, msg, args):
"""Make a feed send updates to given channel(s)."""
feed = args[0]
if feed not in self.config['FEEDS']:
msg_404 = 'Feed with ID {} cannot be found. Please add it first'
return msg_404.format(feed)
channels = args[1:]
for channel in channels:
self.config['SUBSCRIPTIONS'][feed].append(channel)
self.save_config()
return 'Subscribed {} to feed {}'.format(self.list_format(channels),
feed)
@botcmd(admin_only=True, split_args_with=None)
def rssreader_unsubscribe(self, msg, args):
"""Unsubscribe channels/rooms from a specific feed."""
feed = args[0]
if feed not in self.config['FEEDS']:
msg_404 = 'Feed with ID {} cannot be found. Please add it first'
return msg_404.format(feed)
chans = args[1:]
subs = self.config['SUBSCRIPTIONS'][feed]
removed = []
for channel in chans:
try:
subs.remove(channel)
removed.append(channel)
except ValueError:
yield 'Channel {} is not subscribed to feed {}'.format(
channel, feed)
self.config['SUBSCRIPTIONS'][feed] = subs
self.save_config()
if len(removed) > 0:
removed_formatted = self.list_format(removed)
yield 'Unsubscribed {} from feed {}'.format(
removed_formatted, feed)
def activate(self):
self.shortener = Shortener('Isgd')
self.start_poller(self.config['UPDATE_INTERVAL'], self.check_feeds)
super(RSSReader, self).activate()
# Make sure the hash which holds information on feeds exists when the
# checker is ran
try:
if type(self['feeds']) is not dict:
self['feeds'] = {}
except KeyError:
self['feeds'] = {}
def hash_feed(self, feed_url, size=6):
"""Creates a hash ID for a feed URL."""
return hashlib.sha224(feed_url.encode('utf-8')).hexdigest()[:size]
def hash_entry(self, entry):
"""Creates a hash out of the feedparser's Entry. Uses just the title
and the link as that is what we care about in most cases."""
s = "{}{}".format(entry.title, entry.link).encode('utf-8')
return hashlib.sha224(s).hexdigest()
def check_feeds(self):
""""Periodically checks for new entries in given (configured) feeds."""
saved_feeds = self['feeds']
for id, feed in self.config['FEEDS'].items():
if feed not in saved_feeds:
saved_feeds[feed] = []
d = feedparser.parse(feed)
past_entries = saved_feeds[feed]
i = 1
# Take the oldest entries first.
for entry in reversed(d.entries):
hash = self.hash_entry(entry)
if hash in past_entries:
continue
if i > self.config['MAX_STORIES']:
break
self.sender(d, entry, id)
i += 1
past_entries.insert(0, hash)
saved_feeds[feed] = past_entries[:self.config['ENTRY_CACHE_SIZE']]
self['feeds'] = saved_feeds
return ''
def sender(self, d, entry, feed_id):
"""A helper function that takes care of sending the entry that we
regard as 'new' to proper places. Moreover, it takes care of formatting
the raw entry into textual representation and shortening the entry
link if it is too long."""
link = entry.link
if len(link) > self.config['MAX_LINK_LENGTH']:
link = self.shortener.short(link)
s = self.config['MSG_FORMAT'].format(d.feed.title, entry.title, link)
for channel in self.config['SUBSCRIPTIONS'][feed_id]:
identifier = self.build_identifier(channel)
self.send(identifier, s)
def list_format(self, list):
n = len(list)
if n > 1:
return ('{}, ' * (len(list) - 2) + '{} and {}').format(*list)
elif n > 0:
return list[0]
else:
return ''
def save_config(self):
"""Save edited configuration to Errbot's internal structures."""
return self._bot.plugin_manager.set_plugin_configuration(
'RSSReader', self.config)
}
class_name = 'lay-overflow-hidden word-break--word mt-5'
class_price = 'pro-price variant-BC con-emphasize font-primary--bold mr-5'
frm = '*****@*****.**' # sender's mail
to = os.environ.get('EMAIL_ADDRESS') # receiver's mail
password = os.environ.get('EMAIL_PASSWORD')
with open('url.txt', 'r') as f:
URL = f.read()
# check if the URL leads to MALL.CZ
if 'https://www.mall.cz/' in URL:
shortener = Shortener('Tinyurl')
URL = shortener.short(URL) # shorts the URL -> www.tinyurl.com/link
else:
print('URL address in file "url.txt" isn\'t link to website MALL.CZ. ')
print('If you want run the code properly, ')
input('replace the link with URL, that is leading to it. ')
exit()
class File:
def read(self):
with open('price.txt', 'r+') as f: # opens the text file
latest_price = f.read()
latest_price = int(latest_price)
return latest_price
def change(self):
def short(self, url):
shorten = Shortener('Bitly', bitly_token='dd800abec74d5b12906b754c630cdf1451aea9e0')
return shorten.short(url)
def urlShortner(search_args):
shortener = Shortener('Tinyurl', timeout=9000)
short_url = shortener.short('http://' + search_args)
return (discord.Embed(title=short_url, type='rich'))
def maybe_market_to_twitter(bounty, event_name):
"""Tweet the specified Bounty event.
Args:
bounty (dashboard.models.Bounty): The Bounty to be marketed.
event_name (str): The name of the event.
Returns:
bool: Whether or not the twitter notification was sent successfully.
"""
if not settings.TWITTER_CONSUMER_KEY or (event_name not in ['new_bounty', 'remarket_bounty']) or (
bounty.get_natural_value() < 0.0001) or (bounty.network != settings.ENABLE_NOTIFICATIONS_ON_NETWORK):
return False
return False # per 2018/01/22 convo with vivek / kevin, these tweets have low engagement
# we are going to test manually promoting these tweets for a week and come back to revisit this
api = twitter.Api(
consumer_key=settings.TWITTER_CONSUMER_KEY,
consumer_secret=settings.TWITTER_CONSUMER_SECRET,
access_token_key=settings.TWITTER_ACCESS_TOKEN,
access_token_secret=settings.TWITTER_ACCESS_SECRET,
)
tweet_txts = [
"Earn {} {} {} now by completing this task: \n\n{}",
"Oppy to earn {} {} {} for completing this task: \n\n{}",
"Is today the day you (a) boost your OSS rep (b) make some extra cash? 🤔 {} {} {} \n\n{}",
]
if event_name == 'remarket_bounty':
tweet_txts = tweet_txts + [
"Gitcoin open task of the day is worth {} {} {} ⚡️ \n\n{}",
"Task of the day 💰 {} {} {} ⚡️ \n\n{}",
]
if event_name == 'new_bounty':
tweet_txts = tweet_txts + [
"Extra! Extra 🗞🗞 New Funded Issue, Read all about it 👇 {} {} {} \n\n{}",
"Hot off the blockchain! 🔥🔥🔥 There's a new task worth {} {} {} \n\n{}",
"💰 New Task Alert.. 💰 Earn {} {} {} for working on this 👇 \n\n{}",
]
random.shuffle(tweet_txts)
tweet_txt = tweet_txts[0]
shortener = Shortener('Tinyurl')
new_tweet = tweet_txt.format(
round(bounty.get_natural_value(), 4),
bounty.token_name,
f"({bounty.value_in_usdt} USD @ ${convert_token_to_usdt(bounty.token_name)}/{bounty.token_name})" if bounty.value_in_usdt else "",
shortener.short(bounty.get_absolute_url())
)
new_tweet = new_tweet + " " + github_org_to_twitter_tags(bounty.org_name) # twitter tags
if bounty.keywords: # hashtags
for keyword in bounty.keywords.split(','):
_new_tweet = new_tweet + " #" + str(keyword).lower().strip()
if len(_new_tweet) < 140:
new_tweet = _new_tweet
try:
api.PostUpdate(new_tweet)
except Exception as e:
print(e)
return False
return True
def maybe_market_to_twitter(bounty, event_name):
"""Tweet the specified Bounty event.
Args:
bounty (dashboard.models.Bounty): The Bounty to be marketed.
event_name (str): The name of the event.
Returns:
bool: Whether or not the twitter notification was sent successfully.
"""
if not settings.TWITTER_CONSUMER_KEY:
return False
if bounty.get_natural_value() < 0.0001:
return False
if bounty.network != settings.ENABLE_NOTIFICATIONS_ON_NETWORK:
return False
api = twitter.Api(
consumer_key=settings.TWITTER_CONSUMER_KEY,
consumer_secret=settings.TWITTER_CONSUMER_SECRET,
access_token_key=settings.TWITTER_ACCESS_TOKEN,
access_token_secret=settings.TWITTER_ACCESS_SECRET,
)
tweet_txts = [
"Earn {} {} {} now by completing this task: \n\n{}",
"Oppy to earn {} {} {} for completing this task: \n\n{}",
"Is today the day you (a) boost your OSS rep (b) make some extra cash? 🤔 {} {} {} \n\n{}",
]
if event_name == 'remarket_bounty':
tweet_txts = tweet_txts + [
"Gitcoin open task of the day is worth {} {} {} ⚡️ \n\n{}",
"Task of the day 💰 {} {} {} ⚡️ \n\n{}",
]
elif event_name == 'new_bounty':
tweet_txts = tweet_txts + [
"Extra! Extra 🗞🗞 New Funded Issue, Read all about it 👇 {} {} {} \n\n{}",
"Hot off the blockchain! 🔥🔥🔥 There's a new task worth {} {} {} \n\n{}",
"💰 New Task Alert.. 💰 Earn {} {} {} for working on this 👇 \n\n{}",
]
elif event_name == 'increase_payout':
tweet_txts = ['Increased Payout on {} {} {}\n{}']
elif event_name == 'start_work':
tweet_txts = ['Work started on {} {} {}\n{}']
elif event_name == 'stop_work':
tweet_txts = ['Work stopped on {} {} {}\n{}']
elif event_name == 'work_done':
tweet_txts = ['Work done on {} {} {}\n{}']
elif event_name == 'work_submitted':
tweet_txts = ['Work submitted on {} {} {}\n{}']
elif event_name == 'killed_bounty':
tweet_txts = ['Bounty killed on {} {} {}\n{}']
random.shuffle(tweet_txts)
tweet_txt = tweet_txts[0]
url = bounty.get_absolute_url()
is_short = False
for shortener in ['Tinyurl', 'Adfly', 'Isgd', 'QrCx']:
try:
if not is_short:
shortener = Shortener(shortener)
response = shortener.short(url)
if response != 'Error' and 'http' in response:
url = response
is_short = True
except Exception:
pass
new_tweet = tweet_txt.format(
round(bounty.get_natural_value(), 4), bounty.token_name,
f"({bounty.value_in_usdt_now} USD @ ${round(convert_token_to_usdt(bounty.token_name),2)}/{bounty.token_name})"
if bounty.value_in_usdt_now else "", url)
new_tweet = new_tweet + " " + github_org_to_twitter_tags(
bounty.org_name) # twitter tags
if bounty.keywords: # hashtags
for keyword in bounty.keywords.split(','):
_new_tweet = new_tweet + " #" + str(keyword).lower().strip()
if len(_new_tweet) < 140:
new_tweet = _new_tweet
try:
api.PostUpdate(new_tweet)
except Exception as e:
print(e)
return False
return True
def handle(msg):
chat_id = msg['chat']['id'] # Stores chat id for referencing message
try:
command = msg['text'] # Filters text from the message sent
except KeyError as k:
try:
file_id = (msg['photo'][2])['file_id']
file_path = bot.getFile(file_id)
f_path = "https://api.telegram.org/file/bot221225786:AAElg0gODaJi7-xy0AM68eKH5moyuXZOzh0/" + file_path[
'file_path']
#file=bot.download_file(f_path)
bot.sendPhoto(chat_id, file_id)
return
except e:
print(e)
command = "Not Applicable"
bot.sendMessage(
chat_id, "Please try again with a command or an image /help")
return
print('Got command: %s' % command)
# Here Starts the command interpretation
if command == '/roll':
bot.sendMessage(chat_id, random.randint(1, 6))
elif command == '/time':
bot.sendMessage(chat_id, str(datetime.datetime.now()))
elif command == '/hi':
bot.sendMessage(chat_id, "Hello")
elif command == '/weather':
temperature = "Temperature > " + str(
w.get_temperature(unit='celsius')['temp']
) + " degree celcius\nMaximum Temperature > " + str(
w.get_temperature(unit='celsius')['temp_max']
) + " degree celcius\nMinimum Temperature > " + str(
w.get_temperature(unit='celsius')['temp_min']) + " degree celcius"
wind = "Speed > " + str(w.get_wind()['speed']) + "\nDegrees > " + str(
w.get_wind()['deg']) + " degrees clockwise from North direction"
pressure = "Sea Level > " + str(
w.get_pressure()['sea_level']) + "\nPressure > " + str(
w.get_pressure()['press'])
bot.sendMessage(
chat_id, "New Delhi,India\n( " + w.get_detailed_status() +
" )\n\nTemperature Details :\n" + temperature +
"\n\nWind Speed Details :\n" + wind + "\n\nCloud Coverage : \n" +
str(w.get_clouds()) + "%" + "\n\nHumidity : \n" +
str(w.get_humidity()) + "%" + "\n\nPressure Details :\n" +
pressure +
"\n\nData fetched by openweathermap API.All copyrights reserved")
elif '/wiki' in command:
try:
ny = wikipedia.summary(command[5:len(str(command))], sentences=7)
bot.sendMessage(chat_id, ny)
except wikipedia.exceptions.DisambiguationError as e:
stri = "This may refer to :\n\n"
for i, topic in enumerate(e.options):
stri = stri + str(i) + " " + topic + "\n"
stri = stri + "\nPlease choose anyone from above options"
bot.sendMessage(chat_id, stri)
except wikipedia.exceptions.PageError as e:
bot.sendMessage(chat_id, "No partial/full match found for this")
elif command == '/help':
bot.sendMessage(
chat_id, """List of supported commands is\n
/hi - Greet Your Device\n
/roll - Rolls a dice\n
/weather - Tells detailed current weather report of Raspberry Pi's location\n
/time - Tells current date and time\n
/wiki <Topic Name> - Does a topic search on wikipedia and gives a summary of the topic.Try long tapping /wiki in autofill\n
/torrent <magnet link/torrent url/infohash> - Adds and downloads torrent to your raspberry pi remotely\n
/torrent_status - Give the detailed status of your torrent(s) you have added/downloaded\n
/url <URL> - Shorten the given URL using Google API(goo.gl).\n
/url_exp <Shortened URL> - Expands the given shortened url made using Google API\n
/speedtest - Does a detailed network speed test using ookla's speedtest API\n
/yt <Youtube video link> - Creates the shortened download link for given youtube video\n
/news <Topic> - Displays top 10 latest headlines fetched by Google News API about given toipc using Beautiful soup py Library.
\n\nSee your autofill for quick selection of command or tap '/' icon on right side of your chat textbox.\n
For Commands with parameters,you can long tap the autosuggestion for quick typing and type your parameter followed by a space."""
)
elif '/torrent ' in command:
os.system("deluge-console add Desktop " + command[8:len(str(command))])
bot.sendMessage(chat_id, "Torrent Successfully added")
elif command == '/torrent_status':
p = os.popen("deluge-console info")
q = p.read()
try:
bot.sendMessage(chat_id, str(q))
except telepot.exception.TelegramError as e:
bot.sendMessage(chat_id,
"No added torrents found for remote download")
p.close()
elif '/url ' in command:
url = str(command[5:len(command)])
if validators.url(url):
shortener = Shortener('Google', api_key=api_key)
bot.sendMessage(chat_id,
"Shortened URL is\n" + str(shortener.short(url)))
else:
bot.sendMessage(chat_id, "Please enter a valid url")
elif '/url_exp ' in command:
url = str(command[9:len(command)])
shortener = Shortener('Google', api_key=api_key)
bot.sendMessage(chat_id, "Expanded URL is\n" + shortener.expand(url))
elif command == '/speedtest':
bot.sendMessage(
chat_id,
"""Wait for a while until we check and measure speed of system's network.
If result does'nt come in 30 seconds,Try again.Little patience is appreciated..."""
)
try:
p = str(subprocess.check_output(["speedtest-cli"]))
q = p[2:len(p) - 1]
r = q.replace("\\r", "")
s = r.split("\\n")
bot.sendMessage(chat_id, '\n'.join(s))
except:
bot.sendMessage(
chat_id,
"Something went wrong,Please try again\n Or\nTry some other commands /help"
)
elif '/news ' in command:
headers = {
'user-agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
}
tempurl = "https://www.google.com/search?q=%s&num=10&start=10&tbm=nws#q=%s&tbas=0&tbs=sbd:1&tbm=nws&gl=d"
news_topic = command[6:]
url = tempurl % (news_topic, news_topic)
print(url)
ahrefs = []
titles = []
req = requests.get(url, headers=headers)
soup = bs4.BeautifulSoup(req.text, "html.parser")
#you don't even have to process the div container
#just go strait to <a> and using indexing get "href"
#headlines
ahref = [a["href"] for a in soup.find_all("a", class_="_HId")]
#"buckets"
ahref += [a["href"] for a in soup.find_all("a", class_="_sQb")]
ahrefs.append(ahref)
#or get_text() will return the array inside the hyperlink
#the title you want
title = [a.get_text() for a in soup.find_all("a", class_="_HId")]
title += [a.get_text() for a in soup.find_all("a", class_="_sQb")]
titles.append(title)
#print(ahrefs)
titles = str(titles)
titles = titles.strip("[[]]")
titles = titles.replace('"', '\'')
titles = " " + titles
tit = titles.split(',')
ans = ""
k = 0
for i in tit:
if str(i)[0] == " " and str(i)[1] == "'":
ans = ans + "\n" + str(k + 1) + ". " + str(i)
k = k + 1
else:
ans = ans + "\n" + str(i)
bot.sendMessage(
chat_id, "Top " + str(k) +
" latest news headlines for the given topic are :\n\n" + ans)
elif '/yt ' in command:
bot.sendMessage(
chat_id,
"Wait until we create the download link,Sitback and relax..")
url = command[4:len(command)]
ydl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s%(ext)s'})
with ydl:
result = ydl.extract_info(url,
download=False) # We just need the info
if 'entries' in result:
# Can be a playlist or a list of videos
video = result['entries'][0]
else:
# Just a video
video = result
video_url = video['url']
p = shortener.short(video_url)
bot.sendMessage(chat_id,
"Download link for given youtube video is:\n" + p)
elif '/cal ' in command:
ans = eval(str(command[5:len(command)]))
bot.sendMessage(chat_id, "Answer is:\n" + ans)
else:
bot.sendMessage(
chat_id,
"Type /help for list of supported commands till now,There are many more to come!!"
)
