def downloadEp(ep):
src = BeautifulSoup(
http.request("GET",
"http://anime.megamanonline.org" + ep.p.a["href"]).data,
"html.parser")
vimeourl = src.find("iframe")["src"]
index = vimeourl.rindex("/") + 1
idvid = vimeourl[index:index + 8]
vimeourl = "https://player.vimeo.com/video/" + idvid
res = ""
try:
res = json.loads(
http.request("POST",
vimeourl +
"/check-password?referrer=http://www.google.com",
fields={
"password": "******"
}).data.decode("utf-8"))
except:
print("Unable to download : " + ep.p.text.strip())
sys.stdout.flush()
return
print("Downloading : " + ep.p.text.strip())
sys.stdout.flush()
download(res["request"]["files"]["progressive"][0]["url"],
res["video"]["title"] + ".mp4",
trackers=(hasher, progress))
return
def fetch(self):
p = self.check_cache()
if p is not None:
return p
release_list = yarg.get(self.name).release(self.version)
preferred_release = self.pick_best_wheel(release_list)
if preferred_release is None:
raise NoWheelError('No compatible wheels found for {0.name} {0.version}'.format(self))
download_to = get_cache_dir() / 'pypi' / self.name / self.version
try:
download_to.mkdir(parents=True)
except OSError as e:
# Py2 compatible equivalent of FileExistsError
if e.errno != errno.EEXIST:
raise
target = download_to / preferred_release.filename
from . import __version__
hasher = HashTracker(hashlib.md5())
headers = {'user-agent': 'pynsist/'+__version__}
logger.info('Downloading wheel: %s', preferred_release.url)
download(preferred_release.url, str(target), headers=headers,
trackers=(hasher,))
if hasher.hashobj.hexdigest() != preferred_release.md5_digest:
target.unlink()
raise ValueError('Downloaded wheel corrupted: {}'.format(preferred_release.url))
return target
def fetch(self):
p = self.check_cache()
if p is not None:
return p
release_list = yarg.get(self.name).release(self.version)
preferred_release = self.pick_best_wheel(release_list)
if preferred_release is None:
raise NoWheelError(
'No compatible wheels found for {0.name} {0.version}'.format(
self))
download_to = get_cache_dir() / 'pypi' / self.name / self.version
try:
download_to.mkdir(parents=True)
except OSError as e:
# Py2 compatible equivalent of FileExistsError
if e.errno != errno.EEXIST:
raise
target = download_to / preferred_release.filename
from . import __version__
hasher = HashTracker(hashlib.md5())
headers = {'user-agent': 'pynsist/' + __version__}
logger.info('Downloading wheel: %s', preferred_release.url)
download(preferred_release.url,
str(target),
headers=headers,
trackers=(hasher, ))
if hasher.hashobj.hexdigest() != preferred_release.md5_digest:
target.unlink()
raise ValueError('Downloaded wheel corrupted: {}'.format(
preferred_release.url))
return target
def importLospec(url):
for extension in EXTENSIONS:
url = url.replace(extension, "")
url_gpl = url + ".gpl"
filename = os.path.basename(url_gpl)
metadata = MetaData(url)
PALETTES_PATH = ""
if metadata.author != "":
PALETTES_PATH = EXTENSIONS_PATH.joinpath(metadata.author)
else:
PALETTES_PATH = EXTENSIONS_PATH.joinpath("lospec-palettes")
PALETTES_PATH.mkdir(parents=True, exist_ok=True)
JSON_FILE_PATH = PALETTES_PATH.joinpath("package.json")
jsonmetadata = ""
if not JSON_FILE_PATH.exists():
src = pkg_resources.resource_string(__name__,
"template.json").decode("utf-8")
jsonmetadata = json.loads(src)
if metadata.author != "":
jsonmetadata["name"] = metadata.author
jsonmetadata["displayName"] = metadata.author
jsonmetadata[
"description"] = "Palettes created by " + metadata.author + " on Lospec"
jsonmetadata["author"]["name"] = metadata.author
jsonmetadata["author"]["url"] = metadata.authorUrl
else:
f = open(JSON_FILE_PATH, mode="r")
jsonmetadata = json.load(f)
f.close()
download(url_gpl,
PALETTES_PATH.joinpath(filename),
trackers=(hasher, progress))
name = metadata.paletteName
jsonmetadata["contributes"]["palettes"].append({
"id": name,
"path": filename
})
f = open(JSON_FILE_PATH, "w")
json.dump(jsonmetadata, f, indent=4, sort_keys=True)
f.close()
def fetchZip(url, folderPath):
print('fetching', url, '-->', tempDir)
zipName = folderPath + '.zip'
try:
os.rmdir(folderPath)
except:
pass
try:
os.remove(folderPath + '.zip')
except:
pass
download(url, zipName)
untar(zipName)
def get_from_pypi(self):
"""Download a compatible wheel from PyPI.
Downloads to the cache directory and returns the destination as a Path.
Raises NoWheelError if no compatible wheel is found.
"""
try:
pypi_pkg = yarg.get(self.name)
except yarg.HTTPError as e:
if e.status_code == 404:
raise NoWheelError("No package named {} found on PyPI".format(
self.name))
raise
release_list = pypi_pkg.release(self.version)
if release_list is None:
raise NoWheelError(
"No release {0.version} for package {0.name}".format(self))
preferred_release = self.pick_best_wheel(release_list)
if preferred_release is None:
raise NoWheelError(
'No compatible wheels found for {0.name} {0.version}'.format(
self))
download_to = get_cache_dir() / 'pypi' / self.name / self.version
try:
download_to.mkdir(parents=True)
except OSError:
# Ignore OSError only if the directory exists
if not download_to.is_dir():
raise
target = download_to / preferred_release.filename
from . import __version__
hasher = HashTracker(hashlib.md5())
headers = {'user-agent': 'pynsist/' + __version__}
logger.info('Downloading wheel: %s', preferred_release.url)
download(preferred_release.url,
str(target),
headers=headers,
trackers=(hasher, ))
if hasher.hashobj.hexdigest() != preferred_release.md5_digest:
target.unlink()
raise ValueError('Downloaded wheel corrupted: {}'.format(
preferred_release.url))
return target
def process_wheel(filename, url, size, md5, sha256, tmpdir):
"""
Process an individual wheel. The wheel is downloaded from ``url`` to the
directory ``tmpdir``, analyzed with `inspect_wheel()`, and then deleted.
The wheel's size and digests are also checked against ``size``, ``md5``,
and ``sha256`` (provided by PyPI) to verify download integrity.
:return: the results of the call to `inspect_wheel()`
"""
fpath = join(tmpdir, filename)
log.info("Downloading %s from %s ...", filename, url)
# Write "user-agent" in lowercase so it overrides requests_download's
# header correctly:
download(url, fpath, headers={"user-agent": USER_AGENT})
log.info("Inspecting %s ...", filename)
try:
about = inspect_wheel(fpath)
finally:
os.remove(fpath)
if about["file"]["size"] != size:
log.error(
"Wheel %s: size mismatch: PyPI reports %d, got %d",
size,
about["file"]["size"],
)
raise ValueError(
f'Size mismatch: PyPI reports {size}, got {about["file"]["size"]}')
for alg, expected in [("md5", md5), ("sha256", sha256)]:
if expected is not None and expected != about["file"]["digests"][alg]:
log.error(
"Wheel %s: %s hash mismatch: PyPI reports %s, got %s",
alg,
expected,
about["file"]["digests"][alg],
)
raise ValueError(f"{alg} hash mismatch: PyPI reports {expected},"
f' got {about["file"]["digests"][alg]}')
log.info("Finished inspecting %s", filename)
return about
path = title + '/' + file_name
if (os.path.exists(path)):
print('Ep ' + file_name + ' giá presente')
c += 1
else:
link = link.replace('http:/', 'http://')
link = link.replace('http:///', 'http://')
link = link.replace('https:///', 'http://')
link = link.replace('https:/', 'http://')
hasher = HashTracker(hashlib.sha256())
progress = ProgressTracker(DataTransferBar())
print('Download di ')
print(colored(file_name, 'green'))
download(link, path, trackers=(hasher, progress))
manc = num - i
manc = str(manc)
print('Download finito con successo!! ne mancano ' + manc)
i += 1
c += 1
if i < num:
print('Si passa al prossimo \n')
else:
print('Fine!')
print('\n\n\nIl programma si sta chiudendo, arrivederci')
time.sleep(10)
exit()
def download_unpack(url):
with tempfile.TemporaryDirectory() as td:
path = os.path.join(td, urlparse(url).path.split('/')[-1])
download(url, path)
unpacked = unpack(path)
return unpacked
def StartMain(scrapingurl,mail,passwod,mail1,passwod1):
cookiesList = [{
'_ga': 'GA1.2.1700526595.1562353772',
'DG_IID': '1F0A32E9-312D-3A37-8C76-FD423D7DD7CB',
'DG_UID': 'E6B29B47-0D07-39EA-A8F3-1AB8491A4715',
'DG_ZID': '37A2504E-56CB-300B-B808-EB880195D761',
'DG_ZUID': '2B4EFECC-4336-3C79-BCC7-A7C324BC319E',
'DG_HID': '64CD7855-FFF4-3BFA-8377-3EE62110EDEB',
'DG_SID': '112.72.98.74:OmLiYC7xWc8CPD6H/+kymRTCrOSypamOScLOHFnVals',
'S': '0heemi1em5jbqojqub42ixu3',
'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1RvQYgw@@ZNh',
'_gid': 'GA1.2.2050803316.1563099665',
'_gat': '1',
},{
'_ga': 'GA1.2.1700526595.1562353772',
'DG_IID': '1F0A32E9-312D-3A37-8C76-FD423D7DD7CB',
'DG_UID': 'E6B29B47-0D07-39EA-A8F3-1AB8491A4715',
'DG_ZID': '27F459B7-78F4-3C41-829C-3D4510B8B951',
'DG_ZUID': '5854C0B3-CA7E-3481-980A-E0D020504B2D',
'DG_HID': '1130CA69-827F-3D85-937F-42EC972B6C64',
'DG_SID': '112.72.98.74:OmLiYC7xWc8CPD6H/+kymRTCrOSypamOScLOHFnVals',
'S': 'gfo3tjlodpjpe4ughyk32rqo',
'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1RPQYgw@@SRx',
'_gid': 'GA1.2.1078663764.1562455209',
},{
'S': '4b4lnmpmj4hvbzxt5cqshfn0',
'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1T/QYgw@@1bn',
'DG_IID': '7E32FE40-F925-36CB-A97F-3585432EACC1',
'DG_UID': 'A3E172C9-2C5A-340B-8A0B-C508599D120A',
'DG_ZID': 'F469F273-DE68-3459-9B68-2E21D8CFF660',
'DG_ZUID': '3AE599EF-6EDF-3CA9-9EA9-810D8021112A',
'DG_HID': '46968B3C-425D-3F2C-A43F-B27DEC81C6DD',
'DG_SID': '171.236.93.85:Z+uesN5AOoUjpB5Ki4IKyjqkKSw8zy0i5snXaiRSh30',
'_ga': 'GA1.2.1757745723.1562341012',
'_gid': 'GA1.2.1190884154.1562341012',
},{
'_ga': 'GA1.2.1700526595.1562353772',
'DG_IID': '457A4A4E-4102-3439-9F48-FD3A8823AF10',
'DG_UID': '440AF410-834A-3F82-B1B5-FBADE7ACA144',
'DG_ZID': '27F459B7-78F4-3C41-829C-3D4510B8B951',
'DG_ZUID': '5854C0B3-CA7E-3481-980A-E0D020504B2D',
'DG_HID': '1130CA69-827F-3D85-937F-42EC972B6C64',
'DG_SID': '112.72.98.74:OmLiYC7xWc8CPD6H/+kymRTCrOSypamOScLOHFnVals',
'S': 'c4rs2tee41nm2vr0nev3rwde',
'A': 'D1F7D6A15C8DE6302987AA17C881E14F7C7C326616633E001FE691317EE4C43B49DC50D4E69F27EDEE6D1BC32883FB6EDD57D2CFDC7FAD3868E645EC975A5BC933CD7727D2D4D88079F1E3FACCACFC14CB511604D2AFB9AF2EA7DB8914EA5CB611CCC9FBEAD13ECA1C7CB8386CFDEB28D6F4C5A1D0740CF517D4F36FA53B0CD632A4C1852D9AB845F01E15561921AE6618425E50B51CAE7CD37A0CEF36203778',
'_gid': 'GA1.2.1807101646.1562724451',
'Qr1B4j3mrGOWWlAIrB7u5C7QWf8G4A@@': 'v1R/QYgw@@Y+Q',
}]
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:67.0) Gecko/20100101 Firefox/67.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
}
params = (
('Key', 'ce8c842e8e1846a487d9112b1a5a04e8'),
('App', 'TREB'),
)
total=muc=moi=cu=0
duongdan="./"
if not os.path.exists(duongdan+"images"):
os.makedirs(duongdan+"images")
#os.system("chmod -R 777 "+duongdan+"images")
if not os.path.exists(duongdan+"data"):
os.makedirs(duongdan+"data")
#os.system("chmod -R 777 "+duongdan+"data")
cookies = random.choice(cookiesList)
response = requests.get(scrapingurl, headers=headers, cookies=cookies)
strhtml = response.text
html = lxml.html.fromstring(strhtml)
Data = html.xpath('//div[@class="formitem form viewform"]')
if len(Data) != 0:
for row in Data:
total = total+1
item = {}
dl = row.xpath('.//div[@class="formitem formgroup horizontal"][1]/div[1]/span[1]/span/text()')
MLS = str(dl[1])
item['MLS'] = MLS
dulieu = row.xpath('.//span[@class="formitem formfield"]//span/text()')
for i in range(0,5,1):
item['Address_'+str(i)]=dulieu[i]
dt=row.xpath('.//span[@class="formitem formfield"]')
for idx,rs in enumerate(dt):
if(rs.xpath('./label')):
if(rs.xpath('./span/text()')):
item[str(rs.xpath('./label/text()')[0])] = rs.xpath('./span/text()')[0]
else:
item[str(rs.xpath('./label/text()')[0])] = ""
else:
if (rs.xpath('./span/text()')):
item["label" + str(idx)] = rs.xpath('./span/text()')[0]
else:
item["label" + str(idx)] = ""
filename = "data/"+MLS+".json"
if os.path.exists(filename):
cu = cu + 1
with open(filename) as f:
item = json.load(f)
WebsiteLoading(item,mail,passwod)
KijiOpenWebsite(item,mail1,passwod1)
else:
moi = moi + 1
with open(filename, 'w') as outfile:
json.dump(item, outfile)
print("Save json file: " + MLS)
Anh = row.xpath('.//div[@class="imageset_container "]//img/@data-multi-photos')
anh = json.loads(Anh[0])
dlanh = anh['multi-photos']
i=1
if not os.path.exists("images/" + MLS):
os.makedirs(duongdan + "/images/" + MLS)
# os.system("chmod -R 777 images/"+MLS)
print("Images download for.... :" + MLS)
for dla in dlanh:
imgf=MLS+"_"+str(i)+".jpg"
imgurl=str(dla['url']).replace('&size=250','')
if not os.path.exists("images/"+MLS+"/"+imgf):
download(imgurl,"images/"+MLS+"/"+imgf)
i=i+1
WebsiteLoading(item,mail, passwod)
KijiOpenWebsite(item,mail1,passwod1)
print("Uploading End: " + item["MLS"])
print("Uploading End")
messagebox.showinfo("Success", "Uploading End")
else:
print("Wrong Url")
messagebox.showwarning("Warning","Incorrect Url~")
textBox.delete('1.0', END)
