async def parse_single_file(session,
queue,
base_path,
poly_id,
poly_type="s",
name=None,
password=None):
if poly_type != "s":
raise NotSingleFile()
auth = BasicAuth(login=poly_id,
password="******" if password is None else password)
async with session.request("PROPFIND",
url=WEBDAV_PUBLIC_URL,
data=PROPFIND_DATA,
headers=BASIC_HEADER,
auth=auth) as response:
xml = await response.text()
tree = ET.fromstring(xml)
if len(tree) != 1:
raise NotSingleFile()
response = tree[0]
status = go_down_tree(response, "d:propstat", "d:status", to_text=True)
if status != "HTTP/1.1 200 OK":
return
url = f"{INDEX_URL}s/{poly_id}/download"
orig_filename = await cache.check_filename(session,
url,
session_kwargs={"auth": auth})
if orig_filename is None:
raise NotSingleFile()
extension = get_extension(orig_filename)
if name:
filename = add_extension(name, extension)
else:
filename = orig_filename
await queue.put({
"url": url,
"path": safe_path_join(base_path, filename),
"session_kwargs": {
"auth": auth
},
})
def upload_avatar(name=None):
"""
"""
extension = get_extension(name)
up = UploadAvatar()
path_image = up.ajax_upload(UP_AVATARS_TMP_FOLDER, extension)
if up.allowed_file() == False:
success = False
message = g.users_msg('error_upload_1')
else:
up.thumb(AVATAR_IMAGE_SIZE['large'], os.path.join(UP_AVATARS_TMP_FOLDER, path_image))
if path_image:
success = True
message = path_image
if success:
return dict(success=True, message=message)
return dict(success=False, errors=[{ "message": message }])
def upload_avatar(name=None):
"""
"""
extension = get_extension(name)
up = UploadAvatar()
path_image = up.ajax_upload(UP_AVATARS_TMP_FOLDER, extension)
if up.allowed_file() == False:
success = False
message = g.users_msg('error_upload_1')
else:
up.thumb(AVATAR_IMAGE_SIZE['large'],
os.path.join(UP_AVATARS_TMP_FOLDER, path_image))
if path_image:
success = True
message = path_image
if success:
return dict(success=True, message=message)
return dict(success=False, errors=[{"message": message}])
def _test(self, path, from_widget):
if not os.path.isabs(path):
raise ValueError("Not an absolute path")
if not os.path.exists(path):
raise ValueError("Path does not exist or is not valid")
if self.only_folder and not os.path.isdir(path):
raise ValueError("Must be a folder")
if self.file_extensions is not None:
if not os.path.isfile(path):
raise ValueError("Path must be a file")
if "." not in path:
raise ValueError("File must have an extension")
if utils.get_extension(path) not in self.file_extensions:
raise ValueError(
f"File extension must be one of these: ({', '.join(self.file_extensions)})"
)
def allowed_file(self):
""" """
self.extension = get_extension(self.filename)
return '.' in self.filename and self.extension in ALLOWED_EXTENSIONS
def allowed_file(self):
""" """
self.extension = get_extension(self.filename)
return '.' in self.filename and self.extension in ALLOWED_EXTENSIONS
async def download_if_not_exist(session,
path,
url,
site_settings,
cancellable_pool,
with_extension=True,
session_kwargs=None,
allowed_extensions=None,
forbidden_extensions=None,
checksum=None,
signal_handler=None,
unique_key=None):
if session_kwargs is None:
session_kwargs = {}
if allowed_extensions is None:
allowed_extensions = []
if forbidden_extensions is None:
forbidden_extensions = []
allowed_extensions += site_settings.allowed_extensions
forbidden_extensions += site_settings.forbidden_extensions
if isinstance(url, str):
url = URL(url)
domain = url.host
timeout = aiohttp.ClientTimeout(total=0)
if os.path.isabs(path):
raise ValueError("Absolutes paths are not allowed")
absolute_path = os.path.join(site_settings.base_path, path)
if not with_extension:
guess_extension = await cache.check_extension(
session, str(url), session_kwargs=session_kwargs)
if guess_extension is None:
logger.warning(f"Could not retrieve the extension for {url}")
return
absolute_path += "." + guess_extension
force = False
if checksum is not None:
force = not cache.is_checksum_same(absolute_path, checksum)
elif site_settings.force_download and domain not in FORCE_DOWNLOAD_BLACKLIST:
force = True
if os.path.exists(absolute_path) and not force:
return
if os.path.exists(absolute_path):
headers = session_kwargs.get("headers", {})
etag = cache.get_etag(absolute_path)
if etag is not None:
headers["If-None-Match"] = etag
if headers:
session_kwargs["headers"] = headers
if os.path.exists(absolute_path):
action = ACTION_REPLACE
else:
action = ACTION_NEW
file_name = os.path.basename(absolute_path)
file_extension = get_extension(file_name)
if is_extension_forbidden(extension=file_extension,
forbidden_extensions=forbidden_extensions,
allowed_extensions=allowed_extensions):
return
async with session.get(url, timeout=timeout, **session_kwargs) as response:
response.raise_for_status()
response_headers = response.headers
if response.status == 304:
logger.debug(f"File '{absolute_path}' not modified")
cache.save_checksum(absolute_path, checksum)
return
if file_extension.lower() in MOVIE_EXTENSIONS:
logger.info(f"Starting to download {file_name}")
pathlib.Path(os.path.dirname(absolute_path)).mkdir(parents=True,
exist_ok=True)
if action == ACTION_REPLACE and site_settings.keep_replaced_files:
dir_path = os.path.dirname(absolute_path)
pure_name, extension = split_name_extension(file_name)
old_file_name = f"{pure_name}-old.{extension}"
old_absolute_path = os.path.join(dir_path, old_file_name)
os.replace(absolute_path, old_absolute_path)
try:
with open(absolute_path, 'wb') as f:
while True:
chunk = await response.content.read(8192)
if not chunk:
break
f.write(chunk)
except BaseException as e:
os.remove(absolute_path)
logger.debug(f"Removed file {absolute_path}")
raise e
if site_settings.highlight_difference and \
action == ACTION_REPLACE and \
site_settings.keep_replaced_files and \
file_extension.lower() == "pdf":
logger.debug(f"Adding highlights to {absolute_path}")
temp_file_name = f"{pure_name}-temp.{extension}"
temp_absolute_path = os.path.join(dir_path, temp_file_name)
future = cancellable_pool.apply(
functools.partial(pdf_highlighter.add_differ_highlight,
new_path=absolute_path,
old_path=old_absolute_path,
out_path=temp_absolute_path))
try:
await future
os.replace(temp_absolute_path, old_absolute_path)
except asyncio.CancelledError as e:
os.replace(old_absolute_path, absolute_path)
logger.debug(f"Reverted old file {absolute_path}")
raise e
except Exception as e:
logger.warning(
f"Could not add pdf highlight to {absolute_path}. {type(e).__name__}: {e}"
)
signal_handler.got_warning(
unique_key,
f"Could not add pdf highlight to {absolute_path}. {type(e).__name__}: {e}"
)
finally:
if os.path.exists(temp_absolute_path):
logger.debug(f"Removed temp file {temp_absolute_path}")
os.remove(temp_absolute_path)
if "ETag" in response_headers:
cache.save_etag(absolute_path, response.headers["ETag"])
elif domain not in FORCE_DOWNLOAD_BLACKLIST:
logger.warning(
f"url: {url} had not an etag and is not in the blacklist")
cache.save_checksum(absolute_path, checksum)
if action == ACTION_REPLACE:
if site_settings.keep_replaced_files and os.path.exists(
old_absolute_path):
signal_handler.replaced_file(unique_key, absolute_path,
old_absolute_path)
else:
signal_handler.replaced_file(unique_key, absolute_path)
method_msg = "Replaced"
elif action == ACTION_NEW:
signal_handler.added_new_file(unique_key, absolute_path)
method_msg = "Added new"
else:
method_msg = "Unexpected action"
start = {
"name": f"{method_msg} file: '{{}}'",
"var": file_name,
"priority": 100,
"cut": "back",
}
end = {
"name": " in '{}'",
"var": os.path.dirname(absolute_path),
"priority": -100,
"cut": "front",
}
logger.info(fit_sections_to_console(start, end, margin=1))
async def download(session,
queue,
base_path,
url,
password=None,
file_name=None):
domain = re.match(r"https?://([^.]*\.?)zoom.us", url).group(1)
agent_header = {
"referer":
f"https://{domain}zoom.us/",
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/74.0.3729.169 "
"Safari/537.36")
}
async with session.get(url, headers=agent_header) as response:
html = await response.text()
if password is not None:
meet_id_regex = re.compile("<input[^>]*")
for inp in meet_id_regex.findall(html):
input_split = inp.split()
if input_split[2] == 'id="meetId"':
meet_id = input_split[3][7:-1]
break
data = {
"id": meet_id,
"passwd": password,
"action": "viewdetailpage",
"recaptcha": ""
}
check_url = f"https://{domain}zoom.us/rec/validate_meet_passwd"
async with session.post(check_url, data=data,
headers=agent_header) as response:
pass
async with session.get(url, headers=agent_header) as response:
html = await response.text()
metadata = _get_page_meta(html, ("viewMp4Url", "topic"))
if metadata is None:
logger.warning(f"Zoom url: {url} has no video")
return None
vid_url = metadata.get("viewMp4Url", None)
if vid_url is None:
raise LoginError("Could not Login")
extension = get_extension(vid_url.split("?")[0].split("/")[-1])
name = file_name or metadata.get("topic")
# We need to disable the decoding of the url, because zoom is not RFC-compliant (btw f**k zoom).
await queue.put({
"url":
URL(vid_url, encoded=True),
"path":
safe_path_join(base_path, add_extension(name, extension)),
"session_kwargs":
dict(headers=agent_header)
})
