def unmount_everything():
partitions = Partitions()
partitions.unmount(paths.dev)
partitions.unmount(paths.proc)
partitions.unmount(paths.sys)
partitions.unmount(paths.boot)
partitions.unmount(paths.efi_mnt)
partitions.unmount(paths.memtest_mnt)
sh.zfs.umount('-a')
# zfs doesn't seem to want to give up this mount with the -a option above
zfs_mounts = sh.zfs.mount()
if str(paths.zroot) in zfs_mounts:
sh.zfs.umount(paths.zroot)
sh.rmdir(paths.zroot)
def move_private_key(dirname, private_key_name):
''' Move the private key to the dirname.
By default openssl puts private keys in a 'private' subdir. Some
apps need them in a different dir, such as the dir where the pub
keys are.
'''
private_dir = os.path.join(dirname, 'private')
private_key_path = os.path.join(private_dir, private_key_name)
new_private_key_path = os.path.join(dirname, private_key_name)
with locked():
sh.mv(private_key_path, new_private_key_path)
log('moved {} to {}'.format(private_key_path, new_private_key_path))
# remove openssl's 'private' dir when empty
if not os.listdir(private_dir):
sh.rmdir(private_dir)
log('removed {}'.format(os.path.join(private_dir, private_key_name)))
def autounmount(image, mount_point, user=False):
path_exist_or_exit(mount_point)
if image['type'] == 'MBR':
try_unmount(mount_point, user)
try_unmount(LOOP_DIR, user=True)
for idx, part in enumerate(image['partitionTable']):
target_folder = os.path.join(mount_point, 'p' + str(idx + 1))
try_unmount(target_folder, user)
if user and os.path.exists(target_folder):
sh.rmdir(target_folder, _fg=True, _ok_code=range(255))
if not user and os.path.exists(target_folder):
sh.sudo.rmdir(target_folder, _fg=True, _ok_code=range(255))
elif image['type'] == 'CPIO':
if not user:
os.system(
'cd {} && sudo find . | sudo cpio -H newc --quiet -o > "{}"'.
format(mount_point, image['path']))
safely_clean_dir(mount_point, user)
else:
try_unmount(mount_point, user)
if __name__ == '__main__':
argv_len = len(sys.argv)
ts_list = []
mp4_file = None
if argv_len == 3:
m3u8_file, mp4_file = sys.argv[1:]
print m3u8_file, mp4_file
with open(m3u8_file) as m3u8_f:
content = m3u8_f.readlines()
for line in content:
if line.endswith(".ts\n"):
ts_list.append(line.replace("\n", ""))
ts_path = os.path.join(cwd, "ts_files")
sh.mkdir("-p", ts_path)
ts_file_list = download_ts(ts_path, ts_list)
print ts_file_list
mp4_path = os.path.join(cwd, mp4_file)
with open(mp4_path, 'wa+') as f:
for ts_file in ts_file_list:
with open(ts_file) as ts:
f.write(ts.read())
sh.rmdir("-rf", ts_path)
for line in sh.mount(_iter=True):
for mount in mounted:
if mount in line and "ro" in line:
ro.append(mount)
# Unmount all missing drives
for mount in mounted:
if mount not in mounts or mount in ro:
print(f"Unmounting {mount}")
try:
sh.umount(mount)
except Exception as e:
pass
try:
sh.rmdir(mount)
except Exception as e:
pass
# Mount all present drives
for drive in drives:
mount = mountpoint(drive)
# Abort if folder already exists
if path.exists(mount):
print(f"{mount} already exists!")
continue
sh.mkdir("-p", mount)
sh.chmod("go-w", mount)
print(f"Mounting {drive} to {mount}")
try:
sh.mount("-o", "errors=continue", drive, mount)
def extractSeq(fastqDir, outDir, lmdbPath, threads, splitInput, cutoff):
try:
os.mkdir(outDir)
except:
logger.warning(f"{outDir} existed!!")
if not splitInput:
allR1Path = glob.glob(f"{fastqDir}*R1*")
allR2Path = [x.replace("R1", "R2") for x in allR1Path]
else:
fastqTemp = outDir + "tempSplited/"
try:
sh.mkdir(fastqTemp)
except:
logger.warning(f"{fastqTemp} existed!!")
allR1Path = glob.glob(f"{fastqDir}*_R1*")
allR2Path = [x.replace("R1", "R2") for x in allR1Path]
allSplitedPath = [
fastqTemp + re.search(r"[\w\W]+?(?=_R1)",
x.split("/")[-1])[0] + "/" for x in allR1Path
]
if allR1Path[0].endswith(".gz"):
formatGz = True
else:
formatGz = False
splitedNum = threads // len(allSplitedPath)
if splitedNum <= 1:
allR1Path = glob.glob(f"{fastqDir}*R1*")
allR2Path = [x.replace("R1", "R2") for x in allR1Path]
if allR1Path[0].endswith(".gz"):
logger.error("format gz, please uncompress it.")
1 / 0
else:
mPResults = []
with multiP(threads // 2) as mP:
for singleR1Path, singleR2Path, singleSplitedPath in zip(
allR1Path, allR2Path, allSplitedPath):
mPResults.append(
mP.submit(
sh.seqkit,
"split2",
"-f",
"-1",
singleR1Path,
"-2",
singleR2Path,
p=splitedNum,
O=singleSplitedPath,
j=2,
))
tempAllSplitedR1Path = glob.glob(f"{fastqTemp}*/*R1*")
tempAllSplitedR2Path = [
x.replace("R1", "R2") for x in tempAllSplitedR1Path
]
sampleId = set([
re.search(r"(?<=tempSplited/)[\w\W]+?(?=_L)", x)[0]
for x in tempAllSplitedR1Path
])
if len(sampleId) != 1:
allSample = ", ".join(sampleId)
logger.warning(f"MORE THAN ONE INPUT SAMPLES: {allSample}")
sampleId = sampleId.pop()
logger.warning(f"The prefix will change to {sampleId}")
else:
sampleId = sampleId.pop()
i = 0
formatGzUseThreadContents = []
for tempSingleSplitedR1Path, tempSingleSplitedR2Path in zip(
tempAllSplitedR1Path, tempAllSplitedR2Path):
i += 1
if formatGz:
sh.mv(
tempSingleSplitedR1Path,
f"{fastqTemp}{sampleId}_L{i:03}_R1_001.fastq.gz",
)
sh.mv(
tempSingleSplitedR2Path,
f"{fastqTemp}{sampleId}_L{i:03}_R2_001.fastq.gz",
)
formatGzUseThreadContents.append(
sh.gzip(
"-d",
f"{fastqTemp}{sampleId}_L{i:03}_R1_001.fastq.gz",
_bg=True,
))
formatGzUseThreadContents.append(
sh.gzip(
"-d",
f"{fastqTemp}{sampleId}_L{i:03}_R2_001.fastq.gz",
_bg=True,
))
else:
sh.mv(
tempSingleSplitedR1Path,
f"{fastqTemp}{sampleId}_L{i:03}_R1_001.fastq",
)
sh.mv(
tempSingleSplitedR2Path,
f"{fastqTemp}{sampleId}_L{i:03}_R2_001.fastq",
)
if formatGz:
[x.wait() for x in formatGzUseThreadContents]
for singleTempDir in glob.glob(f"{fastqTemp}*/"):
sh.rmdir(singleTempDir)
allR1Path = glob.glob(f"{fastqTemp}*R1*")
allR2Path = [x.replace("R1", "R2") for x in allR1Path]
allSubProcess = []
with multiP(threads) as mP:
for singleR1Path, singleR2Path in zip(allR1Path, allR2Path):
allSubProcess.append(
mP.submit(
processOneFastq,
singleR1Path,
singleR2Path,
lmdbPath,
outDir,
cutoff,
))
[x.result() for x in allSubProcess]
if not splitInput:
pass
else:
sh.rm("-rf", fastqTemp)
def _teardown_rsync(self):
if self.image_mount_point:
sh.sudo.umount(self.image_mount_point)
sh.rmdir(self.image_mount_point)
self.image_mount_point = None
def main():
global MY_TMP, MY_USB, MY_PROP, MY_OUT, DIR_TMP, MNT_TMP, APK_TMP, ZIP_TMP, ODEX_TMP, TAR_TMP, MSC_TMP, IMAGE, VENDOR, KEEPSTUFF, VENDORMODE, SUB_DIR, MY_FULL_DIR, MY_DIR
args = parse_arguments()
# if no args
print('Args.filepath: ' + args.filepath)
print('Args.vendor: ' + args.vendor)
if (args.filepath != None and args.vendor != None):
print('setting variables')
IMAGE = args.filepath
VENDOR = args.vendor
KEEPSTUFF = (
args.keepstuff
) # keep all the decompiled/unpackaged stuff for later analysis
VENDORMODE = (args.vendormode
) # should be provided as 0 unless alternate mode
JOB_ID = str(args.index)
MY_DIR = MY_DIR + VENDOR
MY_FULL_DIR = MY_FULL_DIR + VENDOR
print('home value: ' + HOME)
DIR_TMP = HOME + '/atsh_tmp' + JOB_ID
print('dir_tmp:' + DIR_TMP)
MNT_TMP = HOME + '/atsh_tmp' + JOB_ID + '/mnt'
APK_TMP = HOME + '/atsh_apk' + JOB_ID
ZIP_TMP = HOME + '/atsh_zip' + JOB_ID
ODEX_TMP = HOME + '/atsh_odex' + JOB_ID
TAR_TMP = HOME + '/atsh_tar' + JOB_ID
MSC_TMP = HOME + '/atsh_msc' + JOB_ID
else:
print('filepath and/or vendor are empty')
temp_str = 'error'
print()
print('---------------------------------------------------')
print('Welcome to Connor Short\'s Android extraction tool!')
print('----------------------------------------------------------')
print()
print('**********************************************************')
print()
print(
'This tool was created in cohesion with FICS. The tool is based of a previous iteration'
)
print(
'of android extraction where AT commands were pulled from Android image files.'
)
print()
print(
'It also relies heavily on code from the previous python iteration of Android Extract'
)
print('developed by Sam Simon')
print()
print('For more information on the previous tool, please visit:')
print('www.atcommands.org')
print()
print()
print('**********************************************************')
print()
print()
fo2 = open('2', 'wt')
if (USINGDEPPATH == 1) and (DEPPATH == ''):
print('ERROR: variable DEPPATH not initialized on line 64', file=fo2)
print(
' : if not using DEPPATH and manually updated all dependency locations on lines 67-85',
file=fo2)
print(' : set USINGDEPPATH=0 to disable this check.', file=fo2)
print('', file=fo2)
print(
'For additional guidance and a full list of dependencies, please refer to the provided README.',
file=fo2)
exit(1)
# print usage if not enough arguments provided
if (args.filepath is None or args.vendor is None or args.index is None
or args.vendormode is None):
print_how_to()
print()
print()
exit(0)
elif (args.vendormode == 0):
print('WARN : VENDoRMODE has been set to 0!')
print(
'WARN : some images may require alternative steps for extraction, in which case you should supply',
file=fo2)
print('an additional argument (1). currently applies to:', file=fo2)
print(
'password protected Samsung (.zip) image files from firmwarefile.com',
file=fo2)
print('Continuing after defaulting to 0!', file=fo2)
print()
VENDORMODE = 0
print('ALERT: Now initiating extraction process')
#os.mkdir(TOP_DIR)
os.makedirs(TOP_DIR, exist_ok=True)
#os.mkdir(MY_DIR)
os.makedirs(MY_DIR, exist_ok=True)
cp(IMAGE, MY_DIR)
os.chdir(MY_DIR)
VENDOR = subprocess.run(['basename', VENDOR, '-expanded'],
universal_newlines=True,
stdout=subprocess.PIPE).stdout.rstrip('\n')
print('The current vendor: ' + VENDOR)
IMAGE = subprocess.run(['basename', IMAGE],
universal_newlines=True,
stdout=subprocess.PIPE).stdout.rstrip('\n')
print('ALERT: Cleaning up temporary files from prior run (if any).')
clean_up()
#Create logs for vendor formats that cannot be handled
if (VENDOR == "samsung" and not os.path.isfile(MY_PROP)):
open(TIZ_LOG, 'w+')
# Assume name.suffix format
if (VENDOR == "asus"):
DIR_PRE = subprocess.run(
["echo", IMAGE, "|", "cut", "-d", "?", "-f", "1"],
universal_newlines=True,
stdout=subprocess.PIPE,
shell=True).stdout.rstrip("\n")
SUB_EXT = DIR_PRE[-4:]
SUB_DIR = DIR_PRE[:-4]
else:
print('Image: ' + IMAGE)
#DIR_PRE= subprocess.run(["basename", IMAGE], universal_newlines=True, stdout=subprocess.PIPE, shell=True).stdout.rstrip("\n")
DIR_PRE = getBasename(IMAGE)
SUB_EXT = DIR_PRE[-4:]
SUB_DIR = DIR_PRE[:-4]
print('Output will be available in: ' + SUB_DIR)
os.makedirs(SUB_DIR, exist_ok=True)
mv(IMAGE, SUB_DIR)
os.chdir(SUB_DIR)
print('Unzipping the image file...')
if (VENDOR == 'aosp'):
main_unzip_result = at_unzip(IMAGE, None)
elif (VENDOR == 'samsung'):
os.makedirs(SUB_SUB_TMP)
DECSUFFIX = IMAGE[-4:]
if (DECSUFFIX == '.zip'):
#Deal with vendormode versions later
main_unzip_result = at_unzip(IMAGE, SUB_SUB_TMP)
else:
print('The archive format is not currently supported')
os.chdir(SUB_SUB_TMP)
fileList = getFiles()
if (len(fileList) == 1 and os.path.isdir(fileList[0])):
cp('-r', fileList[0] + '/' + '*', '.')
shutil.rmtree(fileList[0])
os.chdir('..')
if (main_unzip_result == False):
print(
'Sorry, there is currently no support for decompressing this image!'
)
exit(0)
os.remove(IMAGE)
# NOTE: assume there is only 1 dir after unzipping
print('current directory ' + os.getcwd())
#print('ls results ' + subprocess.Popen(['ls','|','head','-1'], stdout=subprocess.PIPE))
SUB_SUB_DIR = subprocess.run(
['ls', '|', 'head', '-1'],
universal_newlines=True,
stdout=subprocess.PIPE,
shell=True).stdout.rstrip('\n').partition('\n')[0]
#MY_TMP = MY_TMP
if (not os.path.isfile(MY_TMP)):
open(MY_TMP, 'w+')
MY_TMP = os.getcwd() + '/' + MY_TMP
if (not os.path.isfile(MY_USB)):
open(MY_USB, 'w+')
MY_USB = os.getcwd() + '/' + MY_USB
if (not os.path.isfile(MY_PROP)):
open(MY_PROP, 'w+')
MY_PROP = os.getcwd() + '/' + MY_PROP
MY_OUT = os.getcwd() + '/' + MY_OUT
if (os.path.isdir(SUB_SUB_DIR)):
os.chdir(SUB_SUB_DIR)
else:
print('ERROR: More than 1 sub directory found!')
exit(0)
if (VENDOR == 'aosp'):
print('Beginning aosp extraction process')
extract_aosp()
if (VENDOR == 'samsung'):
print('Beginning samsung extraction process')
extract_samsung()
print('Summarizing the findings...')
if (KEEPSTUFF == 0):
rmdir(SUB_SUB_DIR)
cat(MY_TMP, _out=open(MY_OUT, 'w+'))