コード例 #1
0
ファイル: Sqlcore.py プロジェクト: yishanyu/spider
    def load(self, data):
        total = 0
        jump = 0
        down = 0
        fail = 0
        for i in data:
            total += 1
            if self.found(i):
                jump += 1
                if self._cfgs["vmode"] == "full":
                    self.output(
                        str(total) + ".跳过:" + urllib.parse.urlparse(i).path)

                #continue
            else:
                if self._cfgs["vmode"] == "full":
                    self.output(
                        str(total) + ".[正在下载]:" +
                        urllib.parse.urlparse(i).path)

                try:
                    ua = Util.randUA()
                    req = urllib.request.Request(i, headers={"User-Agent": ua})
                    response = urllib.request.urlopen(req)
                except urllib.error.URLError as reason:
                    fail += 1
                    self.output("URLError:" + str(reason))
                except urllib.error.HTTPError as reason:
                    fail += 1
                    self.output("HTTPError:" + str(reason))
                else:
                    down += 1
                    img = response.read()
                    with open(self.fullpath(i), "wb") as f:
                        f.write(img)

        self.output("实时总数%s,跳过%s,下载%s,失败%s" % (total, jump, down, fail))
        return True
コード例 #2
0
#!/usr/bin/python3
#-*-coding:utf-8-*-


from tools import Util


'''
入口功能test
'''
#u = Util.Util()
#ua = u.getUA()
#print(ua)

print(Util.randUA())

if __name__ == "__main__":
    print("welcome")