def downloadEp(ep): src = BeautifulSoup( http.request("GET", "http://anime.megamanonline.org" + ep.p.a["href"]).data, "html.parser") vimeourl = src.find("iframe")["src"] index = vimeourl.rindex("/") + 1 idvid = vimeourl[index:index + 8] vimeourl = "https://player.vimeo.com/video/" + idvid res = "" try: res = json.loads( http.request("POST", vimeourl + "/check-password?referrer=http://www.google.com", fields={ "password": "******" }).data.decode("utf-8")) except: print("Unable to download : " + ep.p.text.strip()) sys.stdout.flush() return print("Downloading : " + ep.p.text.strip()) sys.stdout.flush() download(res["request"]["files"]["progressive"][0]["url"], res["video"]["title"] + ".mp4", trackers=(hasher, progress)) return
def fetch(self): p = self.check_cache() if p is not None: return p release_list = yarg.get(self.name).release(self.version) preferred_release = self.pick_best_wheel(release_list) if preferred_release is None: raise NoWheelError('No compatible wheels found for {0.name} {0.version}'.format(self)) download_to = get_cache_dir() / 'pypi' / self.name / self.version try: download_to.mkdir(parents=True) except OSError as e: # Py2 compatible equivalent of FileExistsError if e.errno != errno.EEXIST: raise target = download_to / preferred_release.filename from . import __version__ hasher = HashTracker(hashlib.md5()) headers = {'user-agent': 'pynsist/'+__version__} logger.info('Downloading wheel: %s', preferred_release.url) download(preferred_release.url, str(target), headers=headers, trackers=(hasher,)) if hasher.hashobj.hexdigest() != preferred_release.md5_digest: target.unlink() raise ValueError('Downloaded wheel corrupted: {}'.format(preferred_release.url)) return target
def fetch(self): p = self.check_cache() if p is not None: return p release_list = yarg.get(self.name).release(self.version) preferred_release = self.pick_best_wheel(release_list) if preferred_release is None: raise NoWheelError( 'No compatible wheels found for {0.name} {0.version}'.format( self)) download_to = get_cache_dir() / 'pypi' / self.name / self.version try: download_to.mkdir(parents=True) except OSError as e: # Py2 compatible equivalent of FileExistsError if e.errno != errno.EEXIST: raise target = download_to / preferred_release.filename from . import __version__ hasher = HashTracker(hashlib.md5()) headers = {'user-agent': 'pynsist/' + __version__} logger.info('Downloading wheel: %s', preferred_release.url) download(preferred_release.url, str(target), headers=headers, trackers=(hasher, )) if hasher.hashobj.hexdigest() != preferred_release.md5_digest: target.unlink() raise ValueError('Downloaded wheel corrupted: {}'.format( preferred_release.url)) return target
def importLospec(url): for extension in EXTENSIONS: url = url.replace(extension, "") url_gpl = url + ".gpl" filename = os.path.basename(url_gpl) metadata = MetaData(url) PALETTES_PATH = "" if metadata.author != "": PALETTES_PATH = EXTENSIONS_PATH.joinpath(metadata.author) else: PALETTES_PATH = EXTENSIONS_PATH.joinpath("lospec-palettes") PALETTES_PATH.mkdir(parents=True, exist_ok=True) JSON_FILE_PATH = PALETTES_PATH.joinpath("package.json") jsonmetadata = "" if not JSON_FILE_PATH.exists(): src = pkg_resources.resource_string(__name__, "template.json").decode("utf-8") jsonmetadata = json.loads(src) if metadata.author != "": jsonmetadata["name"] = metadata.author jsonmetadata["displayName"] = metadata.author jsonmetadata[ "description"] = "Palettes created by " + metadata.author + " on Lospec" jsonmetadata["author"]["name"] = metadata.author jsonmetadata["author"]["url"] = metadata.authorUrl else: f = open(JSON_FILE_PATH, mode="r") jsonmetadata = json.load(f) f.close() download(url_gpl, PALETTES_PATH.joinpath(filename), trackers=(hasher, progress)) name = metadata.paletteName jsonmetadata["contributes"]["palettes"].append({ "id": name, "path": filename }) f = open(JSON_FILE_PATH, "w") json.dump(jsonmetadata, f, indent=4, sort_keys=True) f.close()
def fetchZip(url, folderPath): print('fetching', url, '-->', tempDir) zipName = folderPath + '.zip' try: os.rmdir(folderPath) except: pass try: os.remove(folderPath + '.zip') except: pass download(url, zipName) untar(zipName)
def get_from_pypi(self): """Download a compatible wheel from PyPI. Downloads to the cache directory and returns the destination as a Path. Raises NoWheelError if no compatible wheel is found. """ try: pypi_pkg = yarg.get(self.name) except yarg.HTTPError as e: if e.status_code == 404: raise NoWheelError("No package named {} found on PyPI".format( self.name)) raise release_list = pypi_pkg.release(self.version) if release_list is None: raise NoWheelError( "No release {0.version} for package {0.name}".format(self)) preferred_release = self.pick_best_wheel(release_list) if preferred_release is None: raise NoWheelError( 'No compatible wheels found for {0.name} {0.version}'.format( self)) download_to = get_cache_dir() / 'pypi' / self.name / self.version try: download_to.mkdir(parents=True) except OSError: # Ignore OSError only if the directory exists if not download_to.is_dir(): raise target = download_to / preferred_release.filename from . import __version__ hasher = HashTracker(hashlib.md5()) headers = {'user-agent': 'pynsist/' + __version__} logger.info('Downloading wheel: %s', preferred_release.url) download(preferred_release.url, str(target), headers=headers, trackers=(hasher, )) if hasher.hashobj.hexdigest() != preferred_release.md5_digest: target.unlink() raise ValueError('Downloaded wheel corrupted: {}'.format( preferred_release.url)) return target
def process_wheel(filename, url, size, md5, sha256, tmpdir): """ Process an individual wheel. The wheel is downloaded from ``url`` to the directory ``tmpdir``, analyzed with `inspect_wheel()`, and then deleted. The wheel's size and digests are also checked against ``size``, ``md5``, and ``sha256`` (provided by PyPI) to verify download integrity. :return: the results of the call to `inspect_wheel()` """ fpath = join(tmpdir, filename) log.info("Downloading %s from %s ...", filename, url) # Write "user-agent" in lowercase so it overrides requests_download's # header correctly: download(url, fpath, headers={"user-agent": USER_AGENT}) log.info("Inspecting %s ...", filename) try: about = inspect_wheel(fpath) finally: os.remove(fpath) if about["file"]["size"] != size: log.error( "Wheel %s: size mismatch: PyPI reports %d, got %d", size, about["file"]["size"], ) raise ValueError( f'Size mismatch: PyPI reports {size}, got {about["file"]["size"]}') for alg, expected in [("md5", md5), ("sha256", sha256)]: if expected is not None and expected != about["file"]["digests"][alg]: log.error( "Wheel %s: %s hash mismatch: PyPI reports %s, got %s", alg, expected, about["file"]["digests"][alg], ) raise ValueError(f"{alg} hash mismatch: PyPI reports {expected}," f' got {about["file"]["digests"][alg]}') log.info("Finished inspecting %s", filename) return about
path = title + '/' + file_name if (os.path.exists(path)): print('Ep ' + file_name + ' giá presente') c += 1 else: link = link.replace('http:/', 'http://') link = link.replace('http:///', 'http://') link = link.replace('https:///', 'http://') link = link.replace('https:/', 'http://') hasher = HashTracker(hashlib.sha256()) progress = ProgressTracker(DataTransferBar()) print('Download di ') print(colored(file_name, 'green')) download(link, path, trackers=(hasher, progress)) manc = num - i manc = str(manc) print('Download finito con successo!! ne mancano ' + manc) i += 1 c += 1 if i < num: print('Si passa al prossimo \n') else: print('Fine!') print('\n\n\nIl programma si sta chiudendo, arrivederci') time.sleep(10) exit()
def download_unpack(url): with tempfile.TemporaryDirectory() as td: path = os.path.join(td, urlparse(url).path.split('/')[-1]) download(url, path) unpacked = unpack(path) return unpacked
def StartMain(scrapingurl,mail,passwod,mail1,passwod1): cookiesList = [{ '_ga': 'GA1.2.1700526595.1562353772', 'DG_IID': '1F0A32E9-312D-3A37-8C76-FD423D7DD7CB', 'DG_UID': 'E6B29B47-0D07-39EA-A8F3-1AB8491A4715', 'DG_ZID': '37A2504E-56CB-300B-B808-EB880195D761', 'DG_ZUID': '2B4EFECC-4336-3C79-BCC7-A7C324BC319E', 'DG_HID': '64CD7855-FFF4-3BFA-8377-3EE62110EDEB', 'DG_SID': '112.72.98.74:OmLiYC7xWc8CPD6H/+kymRTCrOSypamOScLOHFnVals', 'S': '0heemi1em5jbqojqub42ixu3', 'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1RvQYgw@@ZNh', '_gid': 'GA1.2.2050803316.1563099665', '_gat': '1', },{ '_ga': 'GA1.2.1700526595.1562353772', 'DG_IID': '1F0A32E9-312D-3A37-8C76-FD423D7DD7CB', 'DG_UID': 'E6B29B47-0D07-39EA-A8F3-1AB8491A4715', 'DG_ZID': '27F459B7-78F4-3C41-829C-3D4510B8B951', 'DG_ZUID': '5854C0B3-CA7E-3481-980A-E0D020504B2D', 'DG_HID': '1130CA69-827F-3D85-937F-42EC972B6C64', 'DG_SID': '112.72.98.74:OmLiYC7xWc8CPD6H/+kymRTCrOSypamOScLOHFnVals', 'S': 'gfo3tjlodpjpe4ughyk32rqo', 'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1RPQYgw@@SRx', '_gid': 'GA1.2.1078663764.1562455209', },{ 'S': '4b4lnmpmj4hvbzxt5cqshfn0', 'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1T/QYgw@@1bn', 'DG_IID': '7E32FE40-F925-36CB-A97F-3585432EACC1', 'DG_UID': 'A3E172C9-2C5A-340B-8A0B-C508599D120A', 'DG_ZID': 'F469F273-DE68-3459-9B68-2E21D8CFF660', 'DG_ZUID': '3AE599EF-6EDF-3CA9-9EA9-810D8021112A', 'DG_HID': '46968B3C-425D-3F2C-A43F-B27DEC81C6DD', 'DG_SID': '171.236.93.85:Z+uesN5AOoUjpB5Ki4IKyjqkKSw8zy0i5snXaiRSh30', '_ga': 'GA1.2.1757745723.1562341012', '_gid': 'GA1.2.1190884154.1562341012', },{ '_ga': 'GA1.2.1700526595.1562353772', 'DG_IID': '457A4A4E-4102-3439-9F48-FD3A8823AF10', 'DG_UID': '440AF410-834A-3F82-B1B5-FBADE7ACA144', 'DG_ZID': '27F459B7-78F4-3C41-829C-3D4510B8B951', 'DG_ZUID': '5854C0B3-CA7E-3481-980A-E0D020504B2D', 'DG_HID': '1130CA69-827F-3D85-937F-42EC972B6C64', 'DG_SID': '112.72.98.74:OmLiYC7xWc8CPD6H/+kymRTCrOSypamOScLOHFnVals', 'S': 'c4rs2tee41nm2vr0nev3rwde', 'A': 'D1F7D6A15C8DE6302987AA17C881E14F7C7C326616633E001FE691317EE4C43B49DC50D4E69F27EDEE6D1BC32883FB6EDD57D2CFDC7FAD3868E645EC975A5BC933CD7727D2D4D88079F1E3FACCACFC14CB511604D2AFB9AF2EA7DB8914EA5CB611CCC9FBEAD13ECA1C7CB8386CFDEB28D6F4C5A1D0740CF517D4F36FA53B0CD632A4C1852D9AB845F01E15561921AE6618425E50B51CAE7CD37A0CEF36203778', '_gid': 'GA1.2.1807101646.1562724451', 'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1R/QYgw@@Y+Q', }] headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:67.0) Gecko/20100101 Firefox/67.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', } params = ( ('Key', 'ce8c842e8e1846a487d9112b1a5a04e8'), ('App', 'TREB'), ) total=muc=moi=cu=0 duongdan="./" if not os.path.exists(duongdan+"images"): os.makedirs(duongdan+"images") #os.system("chmod -R 777 "+duongdan+"images") if not os.path.exists(duongdan+"data"): os.makedirs(duongdan+"data") #os.system("chmod -R 777 "+duongdan+"data") cookies = random.choice(cookiesList) response = requests.get(scrapingurl, headers=headers, cookies=cookies) strhtml = response.text html = lxml.html.fromstring(strhtml) Data = html.xpath('//div[@class="formitem form viewform"]') if len(Data) != 0: for row in Data: total = total+1 item = {} dl = row.xpath('.//div[@class="formitem formgroup horizontal"][1]/div[1]/span[1]/span/text()') MLS = str(dl[1]) item['MLS'] = MLS dulieu = row.xpath('.//span[@class="formitem formfield"]//span/text()') for i in range(0,5,1): item['Address_'+str(i)]=dulieu[i] dt=row.xpath('.//span[@class="formitem formfield"]') for idx,rs in enumerate(dt): if(rs.xpath('./label')): if(rs.xpath('./span/text()')): item[str(rs.xpath('./label/text()')[0])] = rs.xpath('./span/text()')[0] else: item[str(rs.xpath('./label/text()')[0])] = "" else: if (rs.xpath('./span/text()')): item["label" + str(idx)] = rs.xpath('./span/text()')[0] else: item["label" + str(idx)] = "" filename = "data/"+MLS+".json" if os.path.exists(filename): cu = cu + 1 with open(filename) as f: item = json.load(f) WebsiteLoading(item,mail,passwod) KijiOpenWebsite(item,mail1,passwod1) else: moi = moi + 1 with open(filename, 'w') as outfile: json.dump(item, outfile) print("Save json file: " + MLS) Anh = row.xpath('.//div[@class="imageset_container "]//img/@data-multi-photos') anh = json.loads(Anh[0]) dlanh = anh['multi-photos'] i=1 if not os.path.exists("images/" + MLS): os.makedirs(duongdan + "/images/" + MLS) # os.system("chmod -R 777 images/"+MLS) print("Images download for.... :" + MLS) for dla in dlanh: imgf=MLS+"_"+str(i)+".jpg" imgurl=str(dla['url']).replace('&size=250','') if not os.path.exists("images/"+MLS+"/"+imgf): download(imgurl,"images/"+MLS+"/"+imgf) i=i+1 WebsiteLoading(item,mail, passwod) KijiOpenWebsite(item,mail1,passwod1) print("Uploading End: " + item["MLS"]) print("Uploading End") messagebox.showinfo("Success", "Uploading End") else: print("Wrong Url") messagebox.showwarning("Warning","Incorrect Url~") textBox.delete('1.0', END)