def test_sanitize_file_name(self):
"""
Check if `sanitize_file_name` function correctly modifies a
given file name.
The function is expected to lowercase the file name, remove
parentheses and square brackets, and replace spaces with
underscores.
"""
file_name = sanitize_file_name('Test (a) [b].txt')
self.assertEquals(file_name, 'test_a_b.txt')
def clean_data(self):
"""
Perform custom clean-up steps on `data` field of the
UploadForm.
When calling the `is_valid()` method on a form, Django
performs a series of default steps for validating and cleaning
user submitted data. This method adds custom logic to the
clean-up process for the `data` field: The name of the user
submitted file is altered to make sure that it conforms to a
format that won't confuse any subsequent subprocess calls.
"""
data = self.cleaned_data['data']
data.name = sanitize_file_name(data.name)
return data
def generate_image(text, preset, save_video=False):
i = 0
output_dir = sanitize_folder_name('{}_{}'.format(text.replace(' ', '_'),
i))
while os.path.exists(output_dir) and os.path.isdir(output_dir):
i += 1
output_dir = sanitize_folder_name('{}_{}'.format(
text.replace(' ', '_'), i))
os.mkdir(output_dir)
os.chdir(output_dir)
model = Imagine(text=text,
num_layers=preset['num_layers'],
save_every=preset['save_every'],
image_width=preset['image_width'],
lr=preset['lr'],
iterations=preset['iterations'],
save_progress=True,
save_video=save_video,
save_gif=False)
# Write text used to generate the piece
with open('name.txt', 'w+') as f:
f.write(text)
# Overwrite the output path of images
model.textpath = sanitize_file_name(text)
for epoch in trange(preset['epochs'], desc='epoch'):
for i in trange(preset['iterations'], desc='iteration'):
model.train_step(epoch, i)
del model
gc.collect()
torch.cuda.empty_cache()
if save_video:
gen_video('./', del_imgs=True)
os.chdir('../')
def getFilesList(self):
"""Get the list of files from a room and prepare the information
of each file
"""
wait = ui.WebDriverWait(self.driver, 3)
# Wait for the list of files and get them
try:
files = wait.until(
lambda driver: driver.find_elements_by_css_selector(
"#file_list .filelist_file"))
except TimeoutException:
self.logger.error("Couldn't find the list of files, aborting...")
return (Result.ERROR, None)
# Get all files information
files_list_output = []
for file_elem in files:
file_left_part = file_elem.find_element_by_class_name(
"file_left_part")
file_right_part = file_elem.find_element_by_class_name(
"file_right_part")
url = file_left_part.get_attribute("href")
file_tag = file_left_part.find_element_by_class_name(
"file_tag").get_attribute("innerHTML")
file_size_expiration = file_right_part.get_attribute("innerHTML")
size_expiration_pattern = re.compile(r"^(.*?)<.*>(.*)<\/span>")
size_expiration_info = size_expiration_pattern.findall(
file_size_expiration)
file_size = size_expiration_info[0][0]
file_expiration = size_expiration_info[0][1]
file_id, real_file_name = get_file_id_and_name(url)
file_name_without_extension, extension = get_file_extension(
real_file_name)
files_list_output.append({
"id":
file_id,
"url":
url,
"name":
sanitize_file_name(file_name_without_extension),
"extension":
extension,
"tag":
file_tag,
"size":
humanfriendly.parse_size(file_size),
"expiration":
file_expiration
})
if config.download_oldest_first:
files_list_output = files_list_output[::-1]
return (Result.SUCCESS, files_list_output)
def download(self, query, user_message=lambda text: True):
url = None
match = self.mp3_dns_regex.search(query)
if match:
url = match.group(0)
match = self.mp3_ip4_regex.search(query)
if match:
url = match.group(0)
if url is None:
raise UnappropriateArgument()
self.logger.debug("Sending HEAD to url: " + url)
media_dir = self.config.get("downloader",
"media_dir",
fallback="media")
file_dir = os.path.join(os.getcwd(), media_dir)
file_name = sanitize_file_name(
parse.unquote(url).split("/")[-1] + ".mp3")
file_path = os.path.join(file_dir, file_name)
if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
title, artist, duration = get_mp3_info(file_path)
title = remove_links(title)
artist = remove_links(artist)
return file_path, title, artist, duration
user_message("Скачиваем...")
self.logger.debug("Querying URL")
try:
response_head = requests.head(url, allow_redirects=True)
except requests.exceptions.ConnectionError as e:
raise UrlOrNetworkProblem(e)
if response_head.status_code != 200:
raise BadReturnStatus(response_head.status_code)
try:
file_size = int(response_head.headers['content-length'])
except KeyError:
self.logger.error("No content-length header. Headers: %s",
str(response_head.headers))
raise MediaSizeUnspecified()
if file_size > 1000000 * self.config.getint(
"downloader", "max_file_size",
fallback=self._default_max_size):
raise MediaIsTooBig()
self.get_file(
url=url,
file_path=file_path,
file_size=file_size,
percent_callback=lambda p: user_message("Скачиваем [%d%%]...\n" %
int(p)),
)
title, artist, duration = get_mp3_info(file_path)
title = remove_links(title)
artist = remove_links(artist)
if duration > self.config.getint("downloader",
"max_duration",
fallback=self._default_max_duration):
os.unlink(file_path)
raise MediaIsTooLong()
self.touch_without_creation(file_path)
return file_path, title, artist, duration
def download(self, query, user_message=lambda text: True):
result_id = query["id"]
self.logger.debug("Downloading result #" + str(result_id))
base_uri = self.config.get("downloader_html", "base_uri")
download_xpath = self.config.get("downloader_html",
"download_page_xpath")
media_dir = self.config.get("downloader",
"media_dir",
fallback="media")
try:
song = self.songs_cache[result_id]
except KeyError:
self.logger.error("No search cache entry for id " + result_id)
raise DownloaderException(
"Внутренняя ошибка (запрошенная песня отсутствует в кэше поиска)"
)
if song["duration"] > self.config.getint(
"downloader", "max_duration",
fallback=self._default_max_duration):
raise MediaIsTooLong(song["duration"])
headers = self.get_headers()
search_request = requests.get((base_uri + song["link"]),
headers=headers)
if search_request.status_code != 200:
raise BadReturnStatus(search_request.status_code)
tree = lxml.html.fromstring(search_request.text)
right_part: str = tree.xpath(download_xpath)[0]
# if right_part.startswith("//"):
# right_part = right_part[1:]
download_uri = base_uri + right_part
file_name = sanitize_file_name("html-" + str(result_id) + '.mp3')
file_path = os.path.join(os.getcwd(), media_dir, file_name)
if self.is_in_cache(file_path):
self.logger.info("File %s already in cache" % result_id)
return file_path, song["title"], song["artist"], song["duration"]
if not os.path.exists(os.path.join(os.getcwd(), media_dir)):
os.makedirs(os.path.join(os.getcwd(), media_dir))
self.logger.debug("Media dir have been created: %s" %
os.path.join(os.getcwd(), media_dir))
self.logger.info("Downloading song #" + result_id)
user_message("Скачиваем...\n%s — %s" % (song["artist"], song["title"]))
file_size = None
if not self.skip_head:
response_head = requests.head(
download_uri,
headers=self.get_headers(),
allow_redirects=True,
stream=True,
)
if response_head.status_code != 200:
raise BadReturnStatus(response_head.status_code)
try:
file_size = int(response_head.headers['content-length'])
except KeyError as e:
self.logger.error(
"No header \"content-length\". More information below\n" +
str(e))
raise ApiError
if file_size > 1000000 * self.config.getint(
"downloader", "max_file_size",
fallback=self._default_max_size):
raise MediaIsTooBig(file_size)
sleep(1)
self.get_file(
url=download_uri,
file_path=file_path,
file_size=file_size,
percent_callback=lambda p:
user_message("Скачиваем [%d%%]...\n%s — %s" %
(int(p), song["artist"], song["title"])),
headers=self.get_headers(),
)
self.logger.debug("Download completed #" + str(result_id))
self.touch_without_creation(file_path)
self.logger.debug("File stored in path: " + file_path)
return file_path, song["title"], song["artist"], song["duration"]
def download(self, query, user_message=lambda text: True):
match = self.yt_regex.search(query)
if match:
url = match.group(0)
else:
raise UnappropriateArgument()
self.logger.info("Getting url: " + url)
user_message("Загружаем информацию о видео...")
media_dir = self.config.get("downloader", "media_dir", fallback="media")
try:
video = YouTube(url, on_progress_callback=self.video_download_progress)
stream = video.streams.filter(only_audio=True).first()
except Exception:
traceback.print_exc()
raise ApiError()
video_id = video.video_id
video_details = video.player_config_args.get('player_response', {}).get('videoDetails', {})
if video_id is None:
raise UrlProblem()
try:
video_title = html.unescape(video.title)
self.logger.debug("Video title [using primary method]: " + video_title)
except KeyError:
video_title = html.unescape(video_details.get('title', 'Unknown YT video'))
self.logger.debug("Video title [using fallback method]: " + video_title)
video_title = remove_links(video_title)
try:
file_size = int(stream.filesize)
except HTTPError as e:
traceback.print_exc()
raise BadReturnStatus(e.code)
if file_size > 1000000 * self.config.getint("downloader", "max_file_size", fallback=self._default_max_size):
raise MediaIsTooBig()
file_dir = media_dir
file_name = sanitize_file_name("youtube-" + str(video_id))
seconds = video.length
if seconds > self.config.getint("downloader", "max_duration", fallback=self._default_max_duration):
raise MediaIsTooLong()
self.download_status[str(video_id)] = {
"start_time": time.time(),
"last_update": time.time(),
"file_size": file_size,
"stream": stream,
"title": video_title,
"user_message": user_message,
}
file_path = os.path.join(file_dir, file_name) + ".mp4"
if self.is_in_cache(file_path):
self.logger.debug("Loading from cache: " + file_path)
return file_path, video_title, "", seconds
if not os.path.exists(file_dir):
os.makedirs(file_dir)
self.logger.debug("Media dir have been created: " + file_dir)
self.logger.info("Downloading audio from video: " + video_id)
user_message("Скачиваем...\n%s" % video_title)
try:
stream.download(output_path=file_dir, filename=file_name)
except HTTPError as e:
traceback.print_exc()
raise BadReturnStatus(e.code)
self.touch_without_creation(file_path)
self.logger.debug("File stored in path: " + file_path)
return file_path, video_title, "", seconds
