async def parse_single_file(session, queue, base_path, poly_id, poly_type="s", name=None, password=None): if poly_type != "s": raise NotSingleFile() auth = BasicAuth(login=poly_id, password="******" if password is None else password) async with session.request("PROPFIND", url=WEBDAV_PUBLIC_URL, data=PROPFIND_DATA, headers=BASIC_HEADER, auth=auth) as response: xml = await response.text() tree = ET.fromstring(xml) if len(tree) != 1: raise NotSingleFile() response = tree[0] status = go_down_tree(response, "d:propstat", "d:status", to_text=True) if status != "HTTP/1.1 200 OK": return url = f"{INDEX_URL}s/{poly_id}/download" orig_filename = await cache.check_filename(session, url, session_kwargs={"auth": auth}) if orig_filename is None: raise NotSingleFile() extension = get_extension(orig_filename) if name: filename = add_extension(name, extension) else: filename = orig_filename await queue.put({ "url": url, "path": safe_path_join(base_path, filename), "session_kwargs": { "auth": auth }, })
def upload_avatar(name=None): """ """ extension = get_extension(name) up = UploadAvatar() path_image = up.ajax_upload(UP_AVATARS_TMP_FOLDER, extension) if up.allowed_file() == False: success = False message = g.users_msg('error_upload_1') else: up.thumb(AVATAR_IMAGE_SIZE['large'], os.path.join(UP_AVATARS_TMP_FOLDER, path_image)) if path_image: success = True message = path_image if success: return dict(success=True, message=message) return dict(success=False, errors=[{ "message": message }])
def upload_avatar(name=None): """ """ extension = get_extension(name) up = UploadAvatar() path_image = up.ajax_upload(UP_AVATARS_TMP_FOLDER, extension) if up.allowed_file() == False: success = False message = g.users_msg('error_upload_1') else: up.thumb(AVATAR_IMAGE_SIZE['large'], os.path.join(UP_AVATARS_TMP_FOLDER, path_image)) if path_image: success = True message = path_image if success: return dict(success=True, message=message) return dict(success=False, errors=[{"message": message}])
def _test(self, path, from_widget): if not os.path.isabs(path): raise ValueError("Not an absolute path") if not os.path.exists(path): raise ValueError("Path does not exist or is not valid") if self.only_folder and not os.path.isdir(path): raise ValueError("Must be a folder") if self.file_extensions is not None: if not os.path.isfile(path): raise ValueError("Path must be a file") if "." not in path: raise ValueError("File must have an extension") if utils.get_extension(path) not in self.file_extensions: raise ValueError( f"File extension must be one of these: ({', '.join(self.file_extensions)})" )
def allowed_file(self): """ """ self.extension = get_extension(self.filename) return '.' in self.filename and self.extension in ALLOWED_EXTENSIONS
async def download_if_not_exist(session, path, url, site_settings, cancellable_pool, with_extension=True, session_kwargs=None, allowed_extensions=None, forbidden_extensions=None, checksum=None, signal_handler=None, unique_key=None): if session_kwargs is None: session_kwargs = {} if allowed_extensions is None: allowed_extensions = [] if forbidden_extensions is None: forbidden_extensions = [] allowed_extensions += site_settings.allowed_extensions forbidden_extensions += site_settings.forbidden_extensions if isinstance(url, str): url = URL(url) domain = url.host timeout = aiohttp.ClientTimeout(total=0) if os.path.isabs(path): raise ValueError("Absolutes paths are not allowed") absolute_path = os.path.join(site_settings.base_path, path) if not with_extension: guess_extension = await cache.check_extension( session, str(url), session_kwargs=session_kwargs) if guess_extension is None: logger.warning(f"Could not retrieve the extension for {url}") return absolute_path += "." + guess_extension force = False if checksum is not None: force = not cache.is_checksum_same(absolute_path, checksum) elif site_settings.force_download and domain not in FORCE_DOWNLOAD_BLACKLIST: force = True if os.path.exists(absolute_path) and not force: return if os.path.exists(absolute_path): headers = session_kwargs.get("headers", {}) etag = cache.get_etag(absolute_path) if etag is not None: headers["If-None-Match"] = etag if headers: session_kwargs["headers"] = headers if os.path.exists(absolute_path): action = ACTION_REPLACE else: action = ACTION_NEW file_name = os.path.basename(absolute_path) file_extension = get_extension(file_name) if is_extension_forbidden(extension=file_extension, forbidden_extensions=forbidden_extensions, allowed_extensions=allowed_extensions): return async with session.get(url, timeout=timeout, **session_kwargs) as response: response.raise_for_status() response_headers = response.headers if response.status == 304: logger.debug(f"File '{absolute_path}' not modified") cache.save_checksum(absolute_path, checksum) return if file_extension.lower() in MOVIE_EXTENSIONS: logger.info(f"Starting to download {file_name}") pathlib.Path(os.path.dirname(absolute_path)).mkdir(parents=True, exist_ok=True) if action == ACTION_REPLACE and site_settings.keep_replaced_files: dir_path = os.path.dirname(absolute_path) pure_name, extension = split_name_extension(file_name) old_file_name = f"{pure_name}-old.{extension}" old_absolute_path = os.path.join(dir_path, old_file_name) os.replace(absolute_path, old_absolute_path) try: with open(absolute_path, 'wb') as f: while True: chunk = await response.content.read(8192) if not chunk: break f.write(chunk) except BaseException as e: os.remove(absolute_path) logger.debug(f"Removed file {absolute_path}") raise e if site_settings.highlight_difference and \ action == ACTION_REPLACE and \ site_settings.keep_replaced_files and \ file_extension.lower() == "pdf": logger.debug(f"Adding highlights to {absolute_path}") temp_file_name = f"{pure_name}-temp.{extension}" temp_absolute_path = os.path.join(dir_path, temp_file_name) future = cancellable_pool.apply( functools.partial(pdf_highlighter.add_differ_highlight, new_path=absolute_path, old_path=old_absolute_path, out_path=temp_absolute_path)) try: await future os.replace(temp_absolute_path, old_absolute_path) except asyncio.CancelledError as e: os.replace(old_absolute_path, absolute_path) logger.debug(f"Reverted old file {absolute_path}") raise e except Exception as e: logger.warning( f"Could not add pdf highlight to {absolute_path}. {type(e).__name__}: {e}" ) signal_handler.got_warning( unique_key, f"Could not add pdf highlight to {absolute_path}. {type(e).__name__}: {e}" ) finally: if os.path.exists(temp_absolute_path): logger.debug(f"Removed temp file {temp_absolute_path}") os.remove(temp_absolute_path) if "ETag" in response_headers: cache.save_etag(absolute_path, response.headers["ETag"]) elif domain not in FORCE_DOWNLOAD_BLACKLIST: logger.warning( f"url: {url} had not an etag and is not in the blacklist") cache.save_checksum(absolute_path, checksum) if action == ACTION_REPLACE: if site_settings.keep_replaced_files and os.path.exists( old_absolute_path): signal_handler.replaced_file(unique_key, absolute_path, old_absolute_path) else: signal_handler.replaced_file(unique_key, absolute_path) method_msg = "Replaced" elif action == ACTION_NEW: signal_handler.added_new_file(unique_key, absolute_path) method_msg = "Added new" else: method_msg = "Unexpected action" start = { "name": f"{method_msg} file: '{{}}'", "var": file_name, "priority": 100, "cut": "back", } end = { "name": " in '{}'", "var": os.path.dirname(absolute_path), "priority": -100, "cut": "front", } logger.info(fit_sections_to_console(start, end, margin=1))
async def download(session, queue, base_path, url, password=None, file_name=None): domain = re.match(r"https?://([^.]*\.?)zoom.us", url).group(1) agent_header = { "referer": f"https://{domain}zoom.us/", "User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/74.0.3729.169 " "Safari/537.36") } async with session.get(url, headers=agent_header) as response: html = await response.text() if password is not None: meet_id_regex = re.compile("<input[^>]*") for inp in meet_id_regex.findall(html): input_split = inp.split() if input_split[2] == 'id="meetId"': meet_id = input_split[3][7:-1] break data = { "id": meet_id, "passwd": password, "action": "viewdetailpage", "recaptcha": "" } check_url = f"https://{domain}zoom.us/rec/validate_meet_passwd" async with session.post(check_url, data=data, headers=agent_header) as response: pass async with session.get(url, headers=agent_header) as response: html = await response.text() metadata = _get_page_meta(html, ("viewMp4Url", "topic")) if metadata is None: logger.warning(f"Zoom url: {url} has no video") return None vid_url = metadata.get("viewMp4Url", None) if vid_url is None: raise LoginError("Could not Login") extension = get_extension(vid_url.split("?")[0].split("/")[-1]) name = file_name or metadata.get("topic") # We need to disable the decoding of the url, because zoom is not RFC-compliant (btw f**k zoom). await queue.put({ "url": URL(vid_url, encoded=True), "path": safe_path_join(base_path, add_extension(name, extension)), "session_kwargs": dict(headers=agent_header) })