def test_sanitize_file_name(self): """ Check if `sanitize_file_name` function correctly modifies a given file name. The function is expected to lowercase the file name, remove parentheses and square brackets, and replace spaces with underscores. """ file_name = sanitize_file_name('Test (a) [b].txt') self.assertEquals(file_name, 'test_a_b.txt')
def clean_data(self): """ Perform custom clean-up steps on `data` field of the UploadForm. When calling the `is_valid()` method on a form, Django performs a series of default steps for validating and cleaning user submitted data. This method adds custom logic to the clean-up process for the `data` field: The name of the user submitted file is altered to make sure that it conforms to a format that won't confuse any subsequent subprocess calls. """ data = self.cleaned_data['data'] data.name = sanitize_file_name(data.name) return data
def generate_image(text, preset, save_video=False): i = 0 output_dir = sanitize_folder_name('{}_{}'.format(text.replace(' ', '_'), i)) while os.path.exists(output_dir) and os.path.isdir(output_dir): i += 1 output_dir = sanitize_folder_name('{}_{}'.format( text.replace(' ', '_'), i)) os.mkdir(output_dir) os.chdir(output_dir) model = Imagine(text=text, num_layers=preset['num_layers'], save_every=preset['save_every'], image_width=preset['image_width'], lr=preset['lr'], iterations=preset['iterations'], save_progress=True, save_video=save_video, save_gif=False) # Write text used to generate the piece with open('name.txt', 'w+') as f: f.write(text) # Overwrite the output path of images model.textpath = sanitize_file_name(text) for epoch in trange(preset['epochs'], desc='epoch'): for i in trange(preset['iterations'], desc='iteration'): model.train_step(epoch, i) del model gc.collect() torch.cuda.empty_cache() if save_video: gen_video('./', del_imgs=True) os.chdir('../')
def getFilesList(self): """Get the list of files from a room and prepare the information of each file """ wait = ui.WebDriverWait(self.driver, 3) # Wait for the list of files and get them try: files = wait.until( lambda driver: driver.find_elements_by_css_selector( "#file_list .filelist_file")) except TimeoutException: self.logger.error("Couldn't find the list of files, aborting...") return (Result.ERROR, None) # Get all files information files_list_output = [] for file_elem in files: file_left_part = file_elem.find_element_by_class_name( "file_left_part") file_right_part = file_elem.find_element_by_class_name( "file_right_part") url = file_left_part.get_attribute("href") file_tag = file_left_part.find_element_by_class_name( "file_tag").get_attribute("innerHTML") file_size_expiration = file_right_part.get_attribute("innerHTML") size_expiration_pattern = re.compile(r"^(.*?)<.*>(.*)<\/span>") size_expiration_info = size_expiration_pattern.findall( file_size_expiration) file_size = size_expiration_info[0][0] file_expiration = size_expiration_info[0][1] file_id, real_file_name = get_file_id_and_name(url) file_name_without_extension, extension = get_file_extension( real_file_name) files_list_output.append({ "id": file_id, "url": url, "name": sanitize_file_name(file_name_without_extension), "extension": extension, "tag": file_tag, "size": humanfriendly.parse_size(file_size), "expiration": file_expiration }) if config.download_oldest_first: files_list_output = files_list_output[::-1] return (Result.SUCCESS, files_list_output)
def download(self, query, user_message=lambda text: True): url = None match = self.mp3_dns_regex.search(query) if match: url = match.group(0) match = self.mp3_ip4_regex.search(query) if match: url = match.group(0) if url is None: raise UnappropriateArgument() self.logger.debug("Sending HEAD to url: " + url) media_dir = self.config.get("downloader", "media_dir", fallback="media") file_dir = os.path.join(os.getcwd(), media_dir) file_name = sanitize_file_name( parse.unquote(url).split("/")[-1] + ".mp3") file_path = os.path.join(file_dir, file_name) if os.path.exists(file_path) and os.path.getsize(file_path) > 0: title, artist, duration = get_mp3_info(file_path) title = remove_links(title) artist = remove_links(artist) return file_path, title, artist, duration user_message("Скачиваем...") self.logger.debug("Querying URL") try: response_head = requests.head(url, allow_redirects=True) except requests.exceptions.ConnectionError as e: raise UrlOrNetworkProblem(e) if response_head.status_code != 200: raise BadReturnStatus(response_head.status_code) try: file_size = int(response_head.headers['content-length']) except KeyError: self.logger.error("No content-length header. Headers: %s", str(response_head.headers)) raise MediaSizeUnspecified() if file_size > 1000000 * self.config.getint( "downloader", "max_file_size", fallback=self._default_max_size): raise MediaIsTooBig() self.get_file( url=url, file_path=file_path, file_size=file_size, percent_callback=lambda p: user_message("Скачиваем [%d%%]...\n" % int(p)), ) title, artist, duration = get_mp3_info(file_path) title = remove_links(title) artist = remove_links(artist) if duration > self.config.getint("downloader", "max_duration", fallback=self._default_max_duration): os.unlink(file_path) raise MediaIsTooLong() self.touch_without_creation(file_path) return file_path, title, artist, duration
def download(self, query, user_message=lambda text: True): result_id = query["id"] self.logger.debug("Downloading result #" + str(result_id)) base_uri = self.config.get("downloader_html", "base_uri") download_xpath = self.config.get("downloader_html", "download_page_xpath") media_dir = self.config.get("downloader", "media_dir", fallback="media") try: song = self.songs_cache[result_id] except KeyError: self.logger.error("No search cache entry for id " + result_id) raise DownloaderException( "Внутренняя ошибка (запрошенная песня отсутствует в кэше поиска)" ) if song["duration"] > self.config.getint( "downloader", "max_duration", fallback=self._default_max_duration): raise MediaIsTooLong(song["duration"]) headers = self.get_headers() search_request = requests.get((base_uri + song["link"]), headers=headers) if search_request.status_code != 200: raise BadReturnStatus(search_request.status_code) tree = lxml.html.fromstring(search_request.text) right_part: str = tree.xpath(download_xpath)[0] # if right_part.startswith("//"): # right_part = right_part[1:] download_uri = base_uri + right_part file_name = sanitize_file_name("html-" + str(result_id) + '.mp3') file_path = os.path.join(os.getcwd(), media_dir, file_name) if self.is_in_cache(file_path): self.logger.info("File %s already in cache" % result_id) return file_path, song["title"], song["artist"], song["duration"] if not os.path.exists(os.path.join(os.getcwd(), media_dir)): os.makedirs(os.path.join(os.getcwd(), media_dir)) self.logger.debug("Media dir have been created: %s" % os.path.join(os.getcwd(), media_dir)) self.logger.info("Downloading song #" + result_id) user_message("Скачиваем...\n%s — %s" % (song["artist"], song["title"])) file_size = None if not self.skip_head: response_head = requests.head( download_uri, headers=self.get_headers(), allow_redirects=True, stream=True, ) if response_head.status_code != 200: raise BadReturnStatus(response_head.status_code) try: file_size = int(response_head.headers['content-length']) except KeyError as e: self.logger.error( "No header \"content-length\". More information below\n" + str(e)) raise ApiError if file_size > 1000000 * self.config.getint( "downloader", "max_file_size", fallback=self._default_max_size): raise MediaIsTooBig(file_size) sleep(1) self.get_file( url=download_uri, file_path=file_path, file_size=file_size, percent_callback=lambda p: user_message("Скачиваем [%d%%]...\n%s — %s" % (int(p), song["artist"], song["title"])), headers=self.get_headers(), ) self.logger.debug("Download completed #" + str(result_id)) self.touch_without_creation(file_path) self.logger.debug("File stored in path: " + file_path) return file_path, song["title"], song["artist"], song["duration"]
def download(self, query, user_message=lambda text: True): match = self.yt_regex.search(query) if match: url = match.group(0) else: raise UnappropriateArgument() self.logger.info("Getting url: " + url) user_message("Загружаем информацию о видео...") media_dir = self.config.get("downloader", "media_dir", fallback="media") try: video = YouTube(url, on_progress_callback=self.video_download_progress) stream = video.streams.filter(only_audio=True).first() except Exception: traceback.print_exc() raise ApiError() video_id = video.video_id video_details = video.player_config_args.get('player_response', {}).get('videoDetails', {}) if video_id is None: raise UrlProblem() try: video_title = html.unescape(video.title) self.logger.debug("Video title [using primary method]: " + video_title) except KeyError: video_title = html.unescape(video_details.get('title', 'Unknown YT video')) self.logger.debug("Video title [using fallback method]: " + video_title) video_title = remove_links(video_title) try: file_size = int(stream.filesize) except HTTPError as e: traceback.print_exc() raise BadReturnStatus(e.code) if file_size > 1000000 * self.config.getint("downloader", "max_file_size", fallback=self._default_max_size): raise MediaIsTooBig() file_dir = media_dir file_name = sanitize_file_name("youtube-" + str(video_id)) seconds = video.length if seconds > self.config.getint("downloader", "max_duration", fallback=self._default_max_duration): raise MediaIsTooLong() self.download_status[str(video_id)] = { "start_time": time.time(), "last_update": time.time(), "file_size": file_size, "stream": stream, "title": video_title, "user_message": user_message, } file_path = os.path.join(file_dir, file_name) + ".mp4" if self.is_in_cache(file_path): self.logger.debug("Loading from cache: " + file_path) return file_path, video_title, "", seconds if not os.path.exists(file_dir): os.makedirs(file_dir) self.logger.debug("Media dir have been created: " + file_dir) self.logger.info("Downloading audio from video: " + video_id) user_message("Скачиваем...\n%s" % video_title) try: stream.download(output_path=file_dir, filename=file_name) except HTTPError as e: traceback.print_exc() raise BadReturnStatus(e.code) self.touch_without_creation(file_path) self.logger.debug("File stored in path: " + file_path) return file_path, video_title, "", seconds