Exemplo n.º 1
0
def request(flow: mitmproxy.http.HTTPFlow) -> None:
    scf_servers = choice(scf_servers)
    r = flow.request
    data = {
        "url": r.pretty_url,
        "method": r.method,
        "headers": dict(r.headers),
        "cookies": dict(r.cookies),
        "params": dict(r.query),
        "data": b64encode(r.raw_content).decode("ascii"),
    }

    flow.request = flow.request.make(
        "POST",
        url=scf_servers,
        content=json.dumps(data),
        headers={
            "Accept":
            "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "Accept-Encoding": "gzip, deflate, compress",
            "Accept-Language": "en-us;q=0.8",
            "Cache-Control": "max-age=0",
            "Connection": "close",
            "user-agent":
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36",
            "host": urlparse(scf_servers).netloc,
            "SCF-Token": SCF_TOKRN,
        })
Exemplo n.º 2
0
	def register_node(self, address):
		"""
		Add a new node to the list of nodes

		:param address: <str> Address a node. Eg. 'http://192.168.0.5:50000'
		:return: None
		"""

		parsed_url = urlparse(address)
		self.nodes.add(parsed_url.netloc)
Exemplo n.º 3
0
def download(url, timeout=180):
    parsed_url = parse.urlparse(url)
    file_name = path.basename(parsed_url.path)

    sleep_time = random.choice(RANDOM_SLEEP_TIMES)

    time.sleep(sleep_time)

    r = requests.get(url, timeout=timeout)

    return Image(file_name=file_name, file_content=r.content)
Exemplo n.º 4
0
    def _add_to_con(self, con):
        """
            Add connection information to connection data
        """
        con["display"] = True
        if self.simplify:
            url = urlparse.urlparse(con["source"]).netloc
            if url:
                con["source"] = url

            url = urlparse.urlparse(con["destination"]).netloc
            if url:
                con["destination"] = url

        self._add_weak_loc(con["source"])
        self._add_weak_loc(con["destination"])

        for a in self.data["connections"]:
            d = DictDiffer(a, con)
            if not d.anychange():
                return

        self.data["connections"].append(con)
Exemplo n.º 5
0
    def test_add_entry(self):
        response = self.client.post("/entry/add", data={
            "title": "Test Entry",
            "content": "Test content"
        })

        self.assertEqual(response.status_code, 302)
        self.assertEqual(urlparse(response.location).path, "/")
        entries = session.query(Entry).all()
        self.assertEqual(len(entries), 1)

        entry = entries[0]
        self.assertEqual(entry.title, "Test Entry")
        self.assertEqual(entry.content, "Test content")
        self.assertEqual(entry.author, self.user)
Exemplo n.º 6
0
    def _add_to_loc(self, loc):
        """
            Add location information to location data
        """
        loc["display"] = True
        if self.simplify:
            url = urlparse.urlparse(loc["url"]).netloc
            if url:
                loc["url"] = url

        for a in self.data["locations"]:
            d = DictDiffer(a, loc)
            if not d.anychange():
                return

        self.data["locations"].append(loc)
 def add_node(self,address):
     parsed_url=urlparse(address) 
     self.nodes.add(parsed_url_netloc)
Exemplo n.º 8
0
def parse_youtube_url(url):
    url = clean_url(url)
    u = normalize_url(url,
                      strip_lang_subdomains=True,
                      strip_trailing_slash=True)
    parsed = urlparse(url)
    # URL pattern youtu.be/VIDEO_ID
    if parsed.netloc == 'youtu.be':
        if "/" not in u:
            return "home", None
        url_id = u.split("/")[1]
        url_id = u.split("?")[0]
        url_id = u.split("%")[0]
        return "video", url_id
    # URL pattern youtube.googleapis.com/v/VIDEO_ID
    if parsed.netloc == 'youtube.googleapis.com':
        if "/v/" in u:
            url_id = u.split("/")[2]
        else:
            raise (Exception("Wrong url format %s" % u))
        return "video", url_id
    if parsed.netloc in [
            'img.youtube.com', 'gaming.youtube.com', 'music.youtube.com',
            'studio.youtube.com'
    ]:
        return "irrelevant", None
    if parsed.netloc.endswith('youtube.com'):
        if u in ["youtube.com"] and not parsed.fragment:
            return "home", None
        stem0 = parsed.path.split("/")[1]
        stem1 = parsed.path.split("/")[2] if "/" in parsed.path.lstrip(
            "/") else None
        queryargs = parsed.query.split("&")
        if stem0 in [
                "t", "yt", "results", "playlist", "artist", "channels",
                "audiolibrary", "feed", "intl", "musicpremium", "premium",
                "show", "watch_videos", "comment", "creators",
                "profile_redirector", "static", "view_play_list", "index"
        ]:
            return "irrelevant", None
        # URL pattern youtube.com/channel/CHANNEL_ID
        if stem0 == "channel":
            return "channel", stem1
        # URL pattern youtube.com/user/USER_ID
        if stem0 in ["user", "c"]:
            return "user", stem1
        # URL pattern youtube.com/profile_videos?user=USER_ID
        if stem0 == "attribution_link":
            uarg = [arg for arg in queryargs if arg.startswith("u=")]
            if len(uarg):
                return parse_youtube_url("http://youtube.com" +
                                         unquote(uarg[0].split("=")[1]))
        if stem0 in ["profile_videos", "subscription_center"]:
            uarg = [
                arg for arg in queryargs
                if arg.startswith("user="******"add_user="******"user", uarg[0].split("=")[1]
        # URL pattern youtube.com/v/VIDEO_ID
        if stem0 in ["v", "embed", "video"]:
            return "video", stem1
        # URL pattern youtube.com/watch?v=VIDEO_ID
        if stem0 in [
                "watch", "redirect", "comment_servlet", "all_comments",
                "watch_popup"
        ]:
            varg = [arg for arg in queryargs if arg.startswith("v=")]
            if len(varg):
                return "video", varg[0].split("=")[1]
            return "video", None
        if stem0 in ["edit", "swf"]:
            varg = [arg for arg in queryargs if arg.startswith("video_id=")]
            if len(varg):
                return "video", varg[0].split("=")[1]
            return "video", None
        # URL pattern youtube.com/#%2Fwatch%3Fv%3DVIDEO_ID
        if "v%3D" in parsed.query:
            fquery = unquote(parsed.query)
            queryargs = fquery.split("?")[1].split("&")
            varg = [arg for arg in queryargs if arg.startswith("v=")]
            if len(varg):
                return "video", varg[0].split("=")[1]
        if "v%3D" in parsed.fragment:
            fquery = unquote(parsed.fragment)
            queryargs = fquery.split("?")[1].split("&")
            varg = [arg for arg in queryargs if arg.startswith("v=")]
            if len(varg):
                return "video", varg[0].split("=")[1]
        if "continue=" in parsed.query:
            urlarg = [arg for arg in queryargs
                      if arg.startswith("continue=")][0].split("=")[1]
            return parse_youtube_url(unquote(urlarg))
        if not stem1 and (not parsed.query or parsed.query
                          in ["sub_confirmation=1"]) and not parsed.fragment:
            return "user", stem0
    return "error", None