def get_status_code(url): """ Gets the current status code of a full url or short url. Short urls will be prepended with https. """ if valid_url(url): return get(url).status_code elif valid_url("https://" + url): return get("https://" + url).status_code else: raise InvalidURL(f"{url} isn't a valid URL!")
def get_url_file_size(url: str) -> int: """ Takes a URL specified resource, and gets its size (in bytes) :param url: resource whose size is being queried :return size: """ size: int = 0 if url: try: assert (valid_url(url)) # fetching the header information info = requests.head(url) content_length = info.headers['Content-Length'] size: int = int(content_length) return size except ValidationFailure: logger.error(f"get_url_file_size(url: '{str(url)}') is invalid?") return -2 except KeyError: logger.error( f"get_url_file_size(url: '{str(url)}') doesn't have a 'Content-Length' value in its header?" ) return -3 except Exception as exc: logger.error(f"get_url_file_size(url:'{str(url)}'): {str(exc)}") # TODO: invalidate the size invariant to propagate a call error # for now return -1 to encode the error state return -1 return size
def songLink(songRow): try: link = songRow.find_all('td')[0].select("a[data-sample]")[0].attrs['data-sample'].strip() if valid_url(link): return link else: return None except: return None
def __call__(self, *args, **kwargs): super().__call__(*args, **kwargs) if valid_url(self._query): exit_code, error_message, text = self.parse_response() if exit_code == 1: self.send_telegram_message('`{}`'.format(text)) else: self.send_telegram_message( '*❗️ Error Message*: {e!s}'.format(e=error_message)) else: self.send_telegram_message('No valid url provided.')
def validateDomain(url): """check if the url is valid""" try: if not valid_url(url): raise ArgumentTypeError('\n{}[x] Invalid url.{}\n'.format(BR, S)) elif not valid_domain(getNetloc(url)): raise ArgumentTypeError('\n{}[x] Invalid domain.{}\n'.format( BR, S)) else: return url except Exception, e: print e sys.exit(0)
def upload_from_link( bucket, object_key, source, client=None, # not used here. EC2 level aws cli used instead callback=None): """ Transfers a file resource to S3 from a URL location. Note that this method is totally agnostic as to specific (KGX) file format and content. :param bucket: in S3 :param object_key: of target S3 object :param source: url of resource to be uploaded to S3 :param callback: e.g. progress monitor :param client: for S3 - ignored (aws CLI used instead) """ # make sure we're getting a valid url assert (valid_url(source)) try: s3_object_target = f"s3://{bucket}/{object_key}" cmd = f"curl -L {source}| aws s3 cp - {s3_object_target}" with Popen(cmd, bufsize=1, universal_newlines=True, stderr=PIPE, shell=True).stderr as proc_stderr: previous: int = 0 callback(0) for line in proc_stderr: # The 'line' is the full curl progress meter field = line.split() if not field: continue current: int = _cbr(field[3]) if current < 0: continue if previous < current: callback(current - previous) previous = current except RuntimeWarning: logger.warning("URL transfer cancelled by exception?")
async def add_song(self, name, no_message=False, maxlen=10, priority=False, channel=None, **metadata): on_error = functools.partial(self.failed_info, channel=channel) try: self.adding_songs = True if valid_url(name): info = await self.downloader.extract_info(self.bot.loop, url=name, on_error=on_error, download=False) else: info = await self._search(name, on_error=on_error) if info is None: if not no_message: return await channel.send( 'No songs found or a problem with YoutbeDL that I cannot fix :(' ) return if 'entries' in info: entries = info['entries'] size = len(entries) if size > maxlen >= 0: # Max playlist size await channel.send( f'Playlist is too big. Max size is {maxlen}') return if entries[0]['ie_key'].lower() != 'youtube': await channel.send( 'Only youtube playlists are currently supported') return url = 'https://www.youtube.com/watch?v=%s' title = info['title'] if priority: await channel.send( 'Playlists queued with playnow will be reversed except for the first song', delete_after=60) message = await channel.send(f'Processing {size} songs') t = time.time() songs = deque() first = True progress = 0 async def progress_info(): nonlocal message while progress <= size: try: await asyncio.sleep(3) t2 = time.time() - t eta = progress / t2 if eta == 0: eta = 'Undefined' else: eta = seconds2str(max(size / eta - t2, 0)) s = 'Loading playlist. Progress {}/{}\nETA {}'.format( progress, size, eta) message = await message.edit(s) except asyncio.CancelledError: await self.bot.delete_message(message) except: return await message.delete() task = self.bot.loop.create_task(progress_info()) async def _on_error(e): try: if not no_message: await channel.send( 'Failed to process {}\n{}'.format( entry.get('title', entry.get['id']), e)) except discord.HTTPException: pass return False for entry in entries: progress += 1 info = await self.downloader.extract_info( self.bot.loop, url=url % entry['id'], download=False, on_error=_on_error) if info is False: continue if info is None: try: if not no_message: await channel.send( 'Failed to process {}'.format( entry.get('title', entry['id']))) except discord.HTTPException: pass continue song = Song(playlist=self, config=self.bot.config, **metadata) song.info_from_dict(**info) if not priority: await self._append_song(song) else: if first: await self._append_song(song, priority=priority) first = False else: songs.append(song) task.cancel() if songs: await self._append_song(songs.popleft(), priority=priority) songs.reverse() for song in songs: self.playlist.appendleft(song) if not no_message: if priority: msg = 'Enqueued playlist %s to the top' % title else: msg = 'Enqueued playlist %s' % title return await channel.send(msg, delete_after=60) else: await self._add_from_info(priority=priority, channel=channel, no_message=no_message, metadata=metadata, **info) finally: self.adding_songs = False
def __set__(self, instance, value): if (value is not None) and (not valid_url(value)): raise ValueError(f"{self.name} has an invalid url format of {value}") super().__set__(instance, value)