def __init__(self, zip_file, tmp_dir):
"""
Initialize LocalZipExtractor.
:param zip_file: a path object or a string to a zip that contains a full recovery ROM.
:param tmp_dir: output directory to place the extracted zip in.
"""
self.zip_url = zip_file if "http" in zip_file or "ota.d.miui.com" in zip_file else ""
self.zip_file_path = Path(zip_file) if not self.zip_url and isinstance(zip_file, str) else ""
self.files = []
self._extractor = RemoteZip(self.zip_url) if self.zip_url else ZipFile(self.zip_file_path)
self.handler = AndroidOneZip(self.zip_file_path, tmp_dir, self._extractor) \
if "payload.bin" in str(self._extractor.namelist()) \
else StandardZip(self.zip_file_path, tmp_dir, self._extractor)
async def _(event):
""" remote link to unzip archive """
await event.edit("`Processing...`")
textx = await event.get_reply_message()
message = event.pattern_match.group(1)
if message:
pass
elif textx:
message = textx.text
else:
await event.edit("`Usage: .runzip <url>`")
return
reply = ''
links = re.findall(r'\.zip', textx)
if not links:
reply = "`No .zip extension link found!`"
await event.edit(reply)
else:
with RemoteZip(links) as zip:
zip.extractall(filedir)
x = zip.extractall(filedir)
await borg.send_file(
event.chat_id,
x,
caption="unzipped",
# progress_callback=lambda d, t: asyncio.get_event_loop().create_task(
# progress(d, t, event, c_time, "trying to upload")
# )
)
await event.edit(reply)
def fetch_1033_sepbb(device_identifier, version, is_verbose):
cellular = is_cellular(device_identifier)
ipswme_device_info = requests.get(
f'https://api.ipsw.me/v4/device/{device_identifier}?type=ipsw')
device_info = ipswme_device_info.json()
for x in range(0, len(device_info['firmwares'])):
if device_info['firmwares'][x]['version'] == '10.3.3':
ipsw_download = device_info['firmwares'][x]['url']
utils.log(
f'[VERBOSE] Found 10.3.3 IPSW to extract SEP from\n[VERBOSE] URL: {ipsw_download}',
is_verbose)
break
with RemoteZip(ipsw_download) as ipsw:
os.makedirs('work/tmp/1033_SEPBB')
ipsw.extract(
f'Firmware/all_flash/sep-firmware.{device_info["boardconfig"][:3]}.RELEASE.im4p',
'work/tmp/1033_SEPBB')
if cellular:
ipsw.extract('Firmware/Mav7Mav8-7.60.00.Release.bbfw',
'work/tmp/1033_SEPBB')
os.rename(
f'work/tmp/1033_SEPBB/Firmware/all_flash/sep-firmware.{device_info["boardconfig"][:3]}.RELEASE.im4p',
'work/tmp/1033_SEPBB/sep-firmware.im4p')
if cellular:
os.rename(
f'work/tmp/1033_SEPBB/Firmware/Mav7Mav8-7.60.00.Release.bbfw',
'work/tmp/1033_SEPBB/baseband.bbfw')
def grab_latest_llb_iboot(device_identifier, firm_bundle, firm_bundle_number):
ipswme_device_info = requests.get(
f'https://api.ipsw.me/v4/device/{device_identifier}?type=ipsw')
device_info = ipswme_device_info.json()
with open(f'{firm_bundle}/Info.json') as f:
data = json.load(f)
if firm_bundle_number != 1337:
hardware_model = data['devices'][firm_bundle_number]['boardconfig']
else:
hardware_model = data['boardconfig']
for x in range(0, len(device_info['firmwares'])):
if device_info['firmwares'][x]['signed'] == True:
ipsw_download = device_info['firmwares'][x]['url']
with RemoteZip(ipsw_download) as ipsw:
os.makedirs('work/tmp')
os.chdir('work/tmp/')
ipsw.extract(f'Firmware/all_flash/LLB.{hardware_model}.RELEASE.im4p')
ipsw.extract(f'Firmware/all_flash/iBoot.{hardware_model}.RELEASE.im4p')
os.chdir('../../')
shutil.copy(
f'work/tmp/Firmware/all_flash/LLB.{hardware_model}.RELEASE.im4p',
'work/ipsw/Firmware/all_flash/')
shutil.copy(
f'work/tmp/Firmware/all_flash/iBoot.{hardware_model}.RELEASE.im4p',
'work/ipsw/Firmware/all_flash/')
shutil.rmtree('work/tmp')
def download_file(url, file, real_filename):
with RemoteZip(url) as zip:
zip.extract(file)
os.rename(file, real_filename)
if os.path.isdir(file.split('/')[0]):
shutil.rmtree(file.split('/')[0])
def grab_file(url: str, filename: str) -> str:
"""Partialzip file from remote server."""
with RemoteZip(url) as zipfile:
filenames = zipfile.namelist()
for fname in filenames:
zinfo = zipfile.getinfo(fname)
if filename in zinfo.filename and ".plist" not in zinfo.filename:
filename = zinfo.filename.split("/")[-1]
print("[i] downloading %s" % filename)
extract_and_clean(zipfile, zinfo.filename, filename)
return filename
return filename
def downloadBuildManifest(self):
shit = APIParser(self.device, self.version)
buildid = shit.iOSToBuildid()
shit.linksForDevice('ipsw')
with open(f'{self.device}.json', 'r') as file:
data = json.load(file)
i = 0
buildidFromJsonFile = data['firmwares'][i]['buildid']
while buildidFromJsonFile != buildid:
i += 1
buildidFromJsonFile = data['firmwares'][i]['buildid']
url = data['firmwares'][i]['url']
manifest = 'BuildManifest.plist'
# Start the process of reading and extracting a file from a url
#print(f'Downloading manifest for {self.version}, {buildid}')
zip = RemoteZip(url)
zip.extract(manifest)
# This can be done better
os.rename(manifest, f'BuildManifest_{self.device}_{self.version}_{buildid}.plist')
#print('Done downloading!')
zip.close()
file.close()
def grab_latest_llb_iboot(device_identifier, version, firm_bundle,
firm_bundle_number, is_verbose):
is_ipsw_apfs = is_apfs(version, is_verbose)
with open(f'{firm_bundle}/Info.json') as f:
data = json.load(f)
if firm_bundle_number != 1337:
hardware_model = data['devices'][firm_bundle_number]['boardconfig']
else:
hardware_model = data['boardconfig']
ipswme_device_info = requests.get(
f'https://api.ipsw.me/v4/device/{device_identifier}?type=ipsw')
device_info = ipswme_device_info.json()
for x in range(0, len(device_info['firmwares'])):
if version.startswith('10.'):
if device_info['firmwares'][x]['version'] == '10.3.3':
ipsw_download = device_info['firmwares'][x]['url']
utils.log(
f'[VERBOSE] Found 10.3.3 IPSW to extract LLB and iBoot from\n[VERBOSE] URL: {ipsw_download}',
is_verbose)
elif device_info['firmwares'][x]['signed'] == True:
ipsw_download = device_info['firmwares'][x]['url']
utils.log(
f'[VERBOSE] Found signed IPSW to extract LLB and iBoot from\n[VERBOSE] URL: {ipsw_download}',
is_verbose)
with RemoteZip(ipsw_download) as ipsw:
os.makedirs('work/tmp/LLB_iBoot')
os.chdir('work/tmp/LLB_iBoot')
ipsw.extract(f'Firmware/all_flash/LLB.{hardware_model}.RELEASE.im4p')
ipsw.extract(f'Firmware/all_flash/iBoot.{hardware_model}.RELEASE.im4p')
os.chdir('../../../')
if is_ipsw_apfs:
ipsw_dir = 'work/ipsw/Firmware/all_flash/'
llb_file = f'LLB.{hardware_model}.RELEASE.im4p'
iboot_file = f'iBoot.{hardware_model}.RELEASE.im4p'
elif hardware_model == 'iphone6':
full_boardconfig = device_info['boardconfig']
semi_boardconfig = full_boardconfig[:3]
ipsw_dir = f'work/ipsw/Firmware/all_flash/all_flash.{full_boardconfig}.production/'
llb_file = f'LLB.{semi_boardconfig}.RELEASE.im4p'
iboot_file = f'iBoot.{semi_boardconfig}.RELEASE.im4p'
os.rename(
f'work/tmp/LLB_iBoot/Firmware/all_flash/LLB.{hardware_model}.RELEASE.im4p',
f'{ipsw_dir + llb_file}')
os.rename(
f'work/tmp/LLB_iBoot/Firmware/all_flash/iBoot.{hardware_model}.RELEASE.im4p',
f'{ipsw_dir + iboot_file}')
shutil.rmtree('work/tmp/LLB_iBoot/')
def list_files(url):
files_to_process = []
with RemoteZip(url) as zip:
for zip_info in zip.infolist():
path = zip_info.filename
if 'all_flash' in path or 'dfu' in path:
if path.endswith('im4p'):
files_to_process.append(path)
if 'kernelcache' in path:
files_to_process.append(path)
files_to_process.sort()
return files_to_process
def readFromRemoteArchive(self, path: str, save: bool, out=None) -> bytes:
url = self.getArchiveURL()
with RemoteZip(url) as f:
stuff = f.namelist()
for i in range(len(stuff)):
if path in stuff[i]:
path = stuff[i]
break
data = f.read(path)
if save:
name = os.path.basename(path)
if out:
if not os.path.exists(out):
os.makedirs(out)
try:
os.path.exists(out)
except IOError:
raise
name = os.path.join(out, name)
if os.path.isfile(name):
raise FileExistsError
if not os.path.exists(name):
with open(name, 'wb') as p:
p.write(data)
else:
return data
def downloadFileFromArchive(self, path, output):
buildid = self.iOSToBuildid()
self.linksForDevice('ipsw')
with open(f'{self.device}.json', 'r') as file:
data = json.load(file)
i = 0
buildidFromJsonFile = data['firmwares'][i]['buildid']
while buildidFromJsonFile != buildid:
i += 1
buildidFromJsonFile = data['firmwares'][i]['buildid']
url = data['firmwares'][i]['url']
filename = splitToFileName(url)
zip = RemoteZip(url)
print(f"Extracting: {path}, from {filename}")
zip.extract(path)
zip.close()
if output:
shutil.move(path, output)
file.close()
def enable(
source: str,
group: FontGroup = FontGroup.REGULAR,
force: bool = False,
font_folder=Path.getcwd() / "fonts",
):
"""
All-in-one function to deploy free Font Awesome fonts.
It fetches the font file with the version from the given Font Awesome kit script file,
converts the css and registers it as primary font.
:param source: the script for your font kit from https://fontawesome.com/kits
OR a Font Awesome version
OR `~kivysome.LATEST`
:param group: a valid ::class:`~kivysome.FontGroup`
:param force: to ignore the existing files and refetch them by force
:param font_folder: to folder to put the font files, "fonts" is used as default
"""
font_folder = Path(font_folder)
font_folder.mkdir_p()
if semver.VersionInfo.isvalid(source) or source == LATEST:
version = source
else:
pool_manager = urllib3.PoolManager()
try:
result = pool_manager.request("GET", source)
except MaxRetryError as e:
raise ValueError("the given link could not be accessed") from e
if not result.status == 200:
raise ValueError("the given link did not return a correct status code")
content = result.data.decode("utf-8")
if '"license":"free"' not in content:
raise ValueError(
"the given link is not referencing a free version of a Font Awesome kit"
)
version_match = _VERSION_MATCHER.match(content)
if version_match is None:
raise ValueError("no version could be parsed from the given link")
version = version_match.group(1)
if version == LATEST:
repo = "FortAwesome/Font-Awesome"
version = lastversion.latest(repo, "version")
elif not semver.VersionInfo.isvalid(version):
raise ValueError(f"matched version {version} is no valid semantic version")
download_link = f"https://use.fontawesome.com/releases/v{version}/fontawesome-free-{version}-web.zip"
ttf_local = font_folder / f"fa-{group.value}-{version}.ttf"
css_local = ttf_local.with_suffix(".css")
fontd_local = ttf_local.with_suffix(".fontd")
if force or not ttf_local.isfile() or not css_local.isfile():
try:
with RemoteZip(download_link) as remote_zip:
ttf_file: str = ""
css_file: str = ""
for file_name in remote_zip.namelist():
if group.value in file_name and file_name.endswith(".ttf"):
ttf_file = file_name
elif file_name.endswith("all.css"):
css_file = file_name
if not ttf_file or not css_file:
raise ValueError(
f"the required font files could not be found in the remote zip file {download_link}"
)
if ttf_local.exists():
if not remote_zip.getinfo(ttf_file).file_size == ttf_local.size:
ttf_local.remove()
if css_local.exists():
if not remote_zip.getinfo(css_file).file_size == css_local.size:
css_local.remove()
if not ttf_local.exists():
target_path = remote_zip.extract(ttf_file, font_folder)
Path(target_path).move(ttf_local)
if not css_local.exists():
target_path = remote_zip.extract(css_file, font_folder)
Path(target_path).move(css_local)
for tmp_folder in font_folder.dirs(f"*{version}*"):
if len(list(tmp_folder.walkfiles())) == 0:
tmp_folder.rmtree()
except RemoteIOError as e:
raise ValueError(
f"the referenced Font Awesome version {version} could not be downloaded from GitHub"
) from e
if force or not fontd_local.isfile():
create_fontdict_file(css_local, fontd_local)
register(f"fontawesome-{group}", ttf_local, fontd_local)
async def _(event):
if event.fwd_from:
return
textx = await event.get_reply_message()
message = event.pattern_match.group(1)
event.pattern_match.group(1)
if message:
pass
elif textx:
message = textx.text
else:
await event.edit("`Usage: .direct <url>`")
return
if not os.path.isdir(filedir):
os.makedirs(filedir)
reply = ''
links = message
if not links:
reply = "`No links found!`"
await event.edit(reply)
if links:
with RemoteZip(message) as zip:
zip.extractall(filedir)
x = zip.extractall(filedir)
filename = sorted(get_lst_of_files(x, []))
for single_file in filename:
if os.path.exists(single_file):
# https://stackoverflow.com/a/678242/4723940
caption_rts = os.path.basename(single_file)
force_document = True
supports_streaming = False
document_attributes = []
if single_file.endswith(
(".mp4", ".mp3", ".flac", ".webm")):
metadata = extractMetadata(createParser(single_file))
duration = 0
width = 0
height = 0
if metadata.has("duration"):
duration = metadata.get('duration').seconds
if os.path.exists(thumb_image_path):
metadata = extractMetadata(
createParser(thumb_image_path))
if metadata.has("width"):
width = metadata.get("width")
if metadata.has("height"):
height = metadata.get("height")
document_attributes = [
DocumentAttributeVideo(duration=duration,
w=width,
h=height,
round_message=False,
supports_streaming=True)
]
try:
await borg.send_file(
event.chat_id,
single_file,
caption=f"remote unzip `{caption_rts}`",
force_document=force_document,
supports_streaming=supports_streaming,
allow_cache=False,
reply_to=event.message.id,
attributes=document_attributes,
# progress_callback=lambda d, t: asyncio.get_event_loop().create_task(
# progress(d, t, event, c_time, "trying to upload")
# )
)
except Exception as e:
await borg.send_message(event.chat_id,
"{} caused `{}`".format(
caption_rts, str(e)),
reply_to=event.message.id)
# some media were having some issues
continue
os.remove(single_file)
#!/usr/bin/env python from remotezip import RemoteZip url = "http://ak.englishtown.com/Juno/FierceRabbit/FullContentPackages/Revision1/0A/1/courseware-full-0A-381-1.zip" remoteZip = RemoteZip(url) remoteZip.getTableOfContents() print remoteZip.tableOfContents # sample extraction uncompressedFile = remoteZip.extractFile(remoteZip.tableOfContents[0]['filename'])
results = re.findall(regex, email_content)[0]
url = ''.join(results)
fd_exports.append(url)
download_dir = 'input/exports'
login_url = 'https://www.flowdock.com/login'
fd_username = config['flowdock_user']
fd_password = config['flowdock_password']
# To download the zip files we need to authenticate
browser = mechanicalsoup.StatefulBrowser()
browser.open(login_url)
login_form = browser.select_form()
browser['user_session[email]'] = fd_username
browser['user_session[password]'] = fd_password
login_response = browser.submit_selected()
request_headers = login_response.request.headers
flow_name_regex = r"/([a-zA-Z-]+)-2020-"
for fd_export in fd_exports:
print(fd_export)
flow_name = re.findall(flow_name_regex, fd_export)[0]
zip = RemoteZip(fd_export, headers=request_headers)
messages_file = zip.getinfo('messages.json')
output_dir = '%s/%s/' % (download_dir, flow_name)
zip.extract(messages_file, output_dir)
class ZipExtractor:
"""ZipExtractor provides methods for dealing with local and remote zip files."""
zip_file_path: Union[Path, str]
zip_file: str
zip_url: str
handler: Union[StandardZip, AndroidOneZip]
def __init__(self, zip_file, tmp_dir):
"""
Initialize LocalZipExtractor.
:param zip_file: a path object or a string to a zip that contains a full recovery ROM.
:param tmp_dir: output directory to place the extracted zip in.
"""
self.zip_url = zip_file if "http" in zip_file or "ota.d.miui.com" in zip_file else ""
self.zip_file_path = Path(zip_file) if not self.zip_url and isinstance(zip_file, str) else ""
self.files = []
self._extractor = RemoteZip(self.zip_url) if self.zip_url else ZipFile(self.zip_file_path)
self.handler = AndroidOneZip(self.zip_file_path, tmp_dir, self._extractor) \
if "payload.bin" in str(self._extractor.namelist()) \
else StandardZip(self.zip_file_path, tmp_dir, self._extractor)
def exists(self) -> bool:
"""
Check if the local zip file exists.
:return: True if zip file exists, False otherwise.
"""
return self.zip_file_path.exists() if self.zip_file_path else head(self.zip_url).ok
def get_files_list(self):
"""
Get contents of the local zip file as list and store it in files attribute.
:return:
"""
self.files = self._extractor.namelist()
def get_file_name(self) -> str:
"""
Get input zip file name and return it as a string.
:return: a string of the input zip file name.
"""
return self.zip_file_path.name if self.zip_file_path else self.zip_url.split('/')[-1]
def prepare(self):
if isinstance(self.handler, AndroidOneZip):
self.files = self.handler.prepare()
def extract(self, files_to_extract: List[str]):
"""
Extract a list of files from the zip file
:param files_to_extract: a list of files to extract
:return:
"""
self.handler.extract(files_to_extract)
def close(self):
"""
Close the zip file.
:return:
"""
self._extractor.close()
