示例#1
0
def get_urls(url):
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    first_iframe_url = soup.find("iframe").get("src")
    f_url = parse_url(first_iframe_url)
    m3u8_page_url = "%s://%s%s" % (p_url.scheme, BASE_STREAM, f_url.path)
    headers.update({"Referer": BASE_STREAM_REF})
    html = http_get(m3u8_page_url, headers=headers)
    urls = generic_m3u8_searcher.search(html.text)
    return urls
示例#2
0
    def __init__(self,
                 url='',
                 username='',
                 password='',
                 head=None,
                 password_stores=None):
        parsed = common.parse_url(url, username, password)
        # --username and --password override URL credentials
        self.username = parsed[0]
        self.password = parsed[1]
        self.svn_url = parsed[2]

        self.init_ra_and_client()

        self.svn_url = self.remote.get_url()
        self.uuid = self.remote.get_uuid()
        self.root = self.remote.get_repos_root()
        assert self.svn_url.startswith(self.root)

        # *will* have a leading '/', would not if we used get_repos_root2
        self.subdir = self.svn_url[len(self.root):]
        if not self.subdir or self.subdir[-1] != '/':
            self.subdir += '/'
        # the RA interface always yields quoted paths, but the editor interface
        # expects unquoted paths
        self.subdir = urllib.unquote(self.subdir)
        self.hasdiff3 = True
        self.autoprops_config = common.AutoPropsConfig()
示例#3
0
    def __init__(self,
                 url='',
                 username='',
                 password='',
                 head=None,
                 password_stores=None):
        parsed = common.parse_url(url, username, password)
        # --username and --password override URL credentials
        self.username = parsed[0]
        self.password = parsed[1]
        self.svn_url = core.svn_path_canonicalize(parsed[2])
        self.auth_baton_pool = core.Pool()
        self.auth_baton = _create_auth_baton(self.auth_baton_pool,
                                             password_stores)
        # self.init_ra_and_client() assumes that a pool already exists
        self.pool = core.Pool()

        self.init_ra_and_client()
        self.uuid = ra.get_uuid(self.ra, self.pool)
        self.svn_url = ra.get_session_url(self.ra, self.pool)
        self.root = ra.get_repos_root(self.ra, self.pool)
        assert self.svn_url.startswith(self.root)
        # *will* have a leading '/', would not if we used get_repos_root2
        self.subdir = self.svn_url[len(self.root):]
        if not self.subdir or self.subdir[-1] != '/':
            self.subdir += '/'
        # the RA interface always yields quoted paths, but the editor interface
        # expects unquoted paths
        self.subdir = urllib.unquote(self.subdir)
        self.hasdiff3 = True
        self.autoprops_config = common.AutoPropsConfig()
示例#4
0
def get_urls(url):
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    b64_str = re.search(r"window\.atob\('(.*)'\)", html.text).group(1)
    de_str = base64.b64decode(b64_str).decode("utf-8")
    return [de_str]
示例#5
0
    def __init__(self, url='', username='', password='', head=None,
                 password_stores=None, meta=None):
        parsed = common.parse_url(url, username, password)
        # --username and --password override URL credentials
        self.username = parsed[0]
        self.password = parsed[1]
        self.svn_url = core.svn_path_canonicalize(parsed[2])
        self.auth_baton_pool = core.Pool()
        self.auth_baton = _create_auth_baton(self.auth_baton_pool, password_stores)
        # self.init_ra_and_client() assumes that a pool already exists
        self.pool = core.Pool()

        self.init_ra_and_client()
        self.uuid = ra.get_uuid(self.ra, self.pool)
        self.svn_url = ra.get_session_url(self.ra, self.pool)
        self.root = ra.get_repos_root(self.ra, self.pool)
        assert self.svn_url.startswith(self.root)
        # *will* have a leading '/', would not if we used get_repos_root2
        self.subdir = self.svn_url[len(self.root):]
        if not self.subdir or self.subdir[-1] != '/':
            self.subdir += '/'
        # the RA interface always yields quoted paths, but the editor interface
        # expects unquoted paths
        self.subdir = urllib.unquote(self.subdir)
        self.hasdiff3 = True
        self.autoprops_config = common.AutoPropsConfig()

        # store the svn meta object for use with branch skipping
        self.meta = meta
示例#6
0
    def __init__(self, url='', username='', password='', head=None,
                 password_stores=None, meta=None):
        parsed = common.parse_url(url, username, password)
        # --username and --password override URL credentials
        self.username = parsed[0]
        self.password = parsed[1]
        self.svn_url = parsed[2]

        self.init_ra_and_client()

        self.svn_url = self.remote.get_url()
        self.uuid = self.remote.get_uuid()
        self.root = self.remote.get_repos_root()
        assert self.svn_url.startswith(self.root)

        # *will* have a leading '/', would not if we used get_repos_root2
        self.subdir = self.svn_url[len(self.root):]
        if not self.subdir or self.subdir[-1] != '/':
            self.subdir += '/'
        # the RA interface always yields quoted paths, but the editor interface
        # expects unquoted paths
        self.subdir = urllib.unquote(self.subdir)
        self.hasdiff3 = True
        self.autoprops_config = common.AutoPropsConfig()

        # store the svn meta object for use with branch skipping
        self.meta = meta
示例#7
0
def get_urls(url):
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    iframe = soup.find("iframe")
    iframe_url = "%s://%s/%s" % (p_url.scheme, p_url.netloc, iframe.get("src"))
    return generic_m3u8_searcher.get_urls(iframe_url)
示例#8
0
def get_urls(url):
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    iframe = soup.find("iframe")
    iframe_url = iframe.get("src")
    html = http_get(iframe_url, headers=headers)
    m3u8 = re.search(r"source: \'(.*)\'", html.text).group(1)
    return [m3u8]
示例#9
0
文件: node.py 项目: szoth/semosancus
 def bind_socket(self):
     socket = context.socket(zmq.PULL)
     address, port = parse_url(self.address)
     if '.onion' in address:
         bindAddress = 'tcp://127.0.0.1'
     else:
         bindAddress = 'tcp://%s' % address
     logger.debug("%s:%d" % (bindAddress, port))
     socket.bind("%s:%d" % (bindAddress, port))
     return socket
示例#10
0
def get_urls(url):
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    f_iframe_url = soup.find("iframe").get("src")
    headers.update({"Referer": url})
    html = http_get(f_iframe_url, headers=headers)
    rSI = algo(html.text)
    return [rSI]
示例#11
0
def get_urls(url):
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    iframe = soup.find("iframe")
    html = http_get(iframe.get("src"), headers=headers)
    b64_str = re.search(r"window\.atob\('(.*)'\)", html.text).group(1)
    de_str = base64.b64decode(b64_str).decode("utf-8")
    return [de_str]
示例#12
0
def get_urls(url):
    headers = header_random_agent()
    cookies = {}
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    cookies.update(html.cookies)
    soup = BeautifulSoup(html.text, 'html.parser')
    f_iframe_1_url = soup.find("iframe").get("src")
    headers.update({"Referer": url})
    html = http_get(f_iframe_1_url, headers=headers, cookies=cookies)
    return [wstreamto(html.text)]
示例#13
0
文件: node.py 项目: szoth/semosancus
 def __init__(self, address):
     self.address = "tcp://%s:%d" % parse_url(address)
     if not os.path.isdir('nodes'):
         os.mkdir('nodes')
     directory = node_dir(self.address)
     if not os.path.isdir(directory):
         os.mkdir(directory)
     self.directory = directory
     open(os.path.join(directory, 'address'), 'w').write(self.address)
     self.peers = shelve.open(os.path.join(directory, 'peers.shelf'))
     self.routing = shelve.open(os.path.join(directory, 'routing.shelf'))
     self.agree = shelve.open(os.path.join(directory, 'agree.shelf'))
     self.postponed = {}
示例#14
0
def get_urls(url):
    headers = header_random_agent()
    parsed_url = parse_url(url)
    html = http_get(url, headers=headers)
    urls = search(html.text)
    formatted = []
    for u in urls:
        if u.startswith("//"):
            formatted.append("%s:%s" % (parsed_url.scheme, u))
        else:
            formatted.append(u)
    no_duplicates = list(dict.fromkeys(formatted))
    return no_duplicates
示例#15
0
def get_urls(url):
    urls = []
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    f_iframe_1_url = soup.find("iframe").get("src")
    headers.update({"Referer": url})
    html = http_get(f_iframe_1_url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    try:
        source1 = re.search(r"file\": \"(.*?)\"", html.text).group(1)
        urls.append(source1)
    except:
        pass
    return urls
示例#16
0
def get_urls(url):
    p_url = parse_url(url)
    headers = header_random_agent()
    html = http_get(url, headers=headers)
    soup = BeautifulSoup(html.text, "html.parser")
    angel = re.search(r"angelthump.com/(.*?)/embed", html.text)
    headers.update({
        "Referer": url,
        "Origin": url,
    })
    if angel:
        angel_id = angel.group(1)
        return [M3U8_URL % angel_id]
    else:
        xyz = soup.find(allowfullscreen="true")
        xyz_url = "%s:%s" % (p_url.scheme, xyz.get("src"))
        html = http_get(xyz_url, headers=headers)
        return xyzembed(html.text)
示例#17
0
def get_urls(url):
    headers = header_random_agent()
    cookies = {}
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    cookies.update(html.cookies)
    soup = BeautifulSoup(html.text, 'html.parser')
    f_iframe_1_url = soup.find("iframe").get("src")
    headers.update({"Referer": url})
    html = http_get(f_iframe_1_url, headers=headers, cookies=cookies)
    cookies.update(html.cookies)
    soup = BeautifulSoup(html.text, 'html.parser')
    f_iframe_2_url = soup.find("iframe").get("src")
    html = http_get(f_iframe_2_url, headers=headers)
    cookies.update(html.cookies)
    soup = BeautifulSoup(html.text, 'html.parser')
    f_iframe_3_url = soup.find("iframe").get("src")
    headers.update({"Referer": f_iframe_3_url})
    html = http_get(f_iframe_3_url, headers=headers)
    b64_str = re.search(r"window\.atob\(\"(.*)\"\)", html.text).group(1)
    de_str = base64.b64decode(b64_str).decode("utf-8") 
    return [de_str]
示例#18
0
def get_urls(url):
    headers = header_random_agent()
    p_url = parse_url(url)
    html = http_get(url, headers=headers)
    vidgstream = re.search(r'var vidgstream = \"(.*)\"', html.text).group(1)
    params = {
        "idgstream": vidgstream,
        "serverid": "",
    }
    headers.update({
        "Referer": url,
        "Origin": url,
        "Accept-Encoding": "compress"
    })
    print(HLS_URL, params, headers)
    resp = http_get(HLS_URL, params=params, headers=headers)
    print(resp.text)
    json = resp.json()
    rawUrl = json["rawUrl"]
    if rawUrl == 'null':
        return []
    return [rawUrl]
示例#19
0
def can_handle(url):
    p_url = parse_url(url)
    return p_url.netloc == BASE
示例#20
0
def can_handle(url):
    p_url = parse_url(url)
    return p_url.netloc.strip("www.") == BASE