def get_status_code(url):
"""
Gets the current status code of a full url or short url. Short urls will
be prepended with https.
"""
if valid_url(url):
return get(url).status_code
elif valid_url("https://" + url):
return get("https://" + url).status_code
else:
raise InvalidURL(f"{url} isn't a valid URL!")
def get_url_file_size(url: str) -> int:
"""
Takes a URL specified resource, and gets its size (in bytes)
:param url: resource whose size is being queried
:return size:
"""
size: int = 0
if url:
try:
assert (valid_url(url))
# fetching the header information
info = requests.head(url)
content_length = info.headers['Content-Length']
size: int = int(content_length)
return size
except ValidationFailure:
logger.error(f"get_url_file_size(url: '{str(url)}') is invalid?")
return -2
except KeyError:
logger.error(
f"get_url_file_size(url: '{str(url)}') doesn't have a 'Content-Length' value in its header?"
)
return -3
except Exception as exc:
logger.error(f"get_url_file_size(url:'{str(url)}'): {str(exc)}")
# TODO: invalidate the size invariant to propagate a call error
# for now return -1 to encode the error state
return -1
return size
def songLink(songRow):
try:
link = songRow.find_all('td')[0].select("a[data-sample]")[0].attrs['data-sample'].strip()
if valid_url(link):
return link
else:
return None
except:
return None
def __call__(self, *args, **kwargs):
super().__call__(*args, **kwargs)
if valid_url(self._query):
exit_code, error_message, text = self.parse_response()
if exit_code == 1:
self.send_telegram_message('`{}`'.format(text))
else:
self.send_telegram_message(
'*❗️ Error Message*: {e!s}'.format(e=error_message))
else:
self.send_telegram_message('No valid url provided.')
def validateDomain(url):
"""check if the url is valid"""
try:
if not valid_url(url):
raise ArgumentTypeError('\n{}[x] Invalid url.{}\n'.format(BR, S))
elif not valid_domain(getNetloc(url)):
raise ArgumentTypeError('\n{}[x] Invalid domain.{}\n'.format(
BR, S))
else:
return url
except Exception, e:
print e
sys.exit(0)
def upload_from_link(
bucket,
object_key,
source,
client=None, # not used here. EC2 level aws cli used instead
callback=None):
"""
Transfers a file resource to S3 from a URL location. Note that this
method is totally agnostic as to specific (KGX) file format and content.
:param bucket: in S3
:param object_key: of target S3 object
:param source: url of resource to be uploaded to S3
:param callback: e.g. progress monitor
:param client: for S3 - ignored (aws CLI used instead)
"""
# make sure we're getting a valid url
assert (valid_url(source))
try:
s3_object_target = f"s3://{bucket}/{object_key}"
cmd = f"curl -L {source}| aws s3 cp - {s3_object_target}"
with Popen(cmd,
bufsize=1,
universal_newlines=True,
stderr=PIPE,
shell=True).stderr as proc_stderr:
previous: int = 0
callback(0)
for line in proc_stderr:
# The 'line' is the full curl progress meter
field = line.split()
if not field:
continue
current: int = _cbr(field[3])
if current < 0:
continue
if previous < current:
callback(current - previous)
previous = current
except RuntimeWarning:
logger.warning("URL transfer cancelled by exception?")
async def add_song(self,
name,
no_message=False,
maxlen=10,
priority=False,
channel=None,
**metadata):
on_error = functools.partial(self.failed_info, channel=channel)
try:
self.adding_songs = True
if valid_url(name):
info = await self.downloader.extract_info(self.bot.loop,
url=name,
on_error=on_error,
download=False)
else:
info = await self._search(name, on_error=on_error)
if info is None:
if not no_message:
return await channel.send(
'No songs found or a problem with YoutbeDL that I cannot fix :('
)
return
if 'entries' in info:
entries = info['entries']
size = len(entries)
if size > maxlen >= 0: # Max playlist size
await channel.send(
f'Playlist is too big. Max size is {maxlen}')
return
if entries[0]['ie_key'].lower() != 'youtube':
await channel.send(
'Only youtube playlists are currently supported')
return
url = 'https://www.youtube.com/watch?v=%s'
title = info['title']
if priority:
await channel.send(
'Playlists queued with playnow will be reversed except for the first song',
delete_after=60)
message = await channel.send(f'Processing {size} songs')
t = time.time()
songs = deque()
first = True
progress = 0
async def progress_info():
nonlocal message
while progress <= size:
try:
await asyncio.sleep(3)
t2 = time.time() - t
eta = progress / t2
if eta == 0:
eta = 'Undefined'
else:
eta = seconds2str(max(size / eta - t2, 0))
s = 'Loading playlist. Progress {}/{}\nETA {}'.format(
progress, size, eta)
message = await message.edit(s)
except asyncio.CancelledError:
await self.bot.delete_message(message)
except:
return
await message.delete()
task = self.bot.loop.create_task(progress_info())
async def _on_error(e):
try:
if not no_message:
await channel.send(
'Failed to process {}\n{}'.format(
entry.get('title', entry.get['id']), e))
except discord.HTTPException:
pass
return False
for entry in entries:
progress += 1
info = await self.downloader.extract_info(
self.bot.loop,
url=url % entry['id'],
download=False,
on_error=_on_error)
if info is False:
continue
if info is None:
try:
if not no_message:
await channel.send(
'Failed to process {}'.format(
entry.get('title', entry['id'])))
except discord.HTTPException:
pass
continue
song = Song(playlist=self,
config=self.bot.config,
**metadata)
song.info_from_dict(**info)
if not priority:
await self._append_song(song)
else:
if first:
await self._append_song(song, priority=priority)
first = False
else:
songs.append(song)
task.cancel()
if songs:
await self._append_song(songs.popleft(), priority=priority)
songs.reverse()
for song in songs:
self.playlist.appendleft(song)
if not no_message:
if priority:
msg = 'Enqueued playlist %s to the top' % title
else:
msg = 'Enqueued playlist %s' % title
return await channel.send(msg, delete_after=60)
else:
await self._add_from_info(priority=priority,
channel=channel,
no_message=no_message,
metadata=metadata,
**info)
finally:
self.adding_songs = False
def __set__(self, instance, value):
if (value is not None) and (not valid_url(value)):
raise ValueError(f"{self.name} has an invalid url format of {value}")
super().__set__(instance, value)
