def update_from_cli(**kwargs):
tests_root = kwargs["tests_root"]
path = kwargs["path"]
assert tests_root is not None
m = None
logger = get_logger()
if not kwargs.get("rebuild", False):
try:
m = manifest.load(tests_root, path)
except manifest.ManifestVersionMismatch:
logger.info("Manifest version changed, rebuilding")
m = None
else:
logger.info("Updating manifest")
if m is None:
m = manifest.Manifest(None)
update(tests_root,
kwargs["url_base"],
m,
ignore_local=kwargs.get("ignore_local", False))
manifest.write(m, path)
def __zipwrite(self, outputfp):
"""
Write the document to an open file pointer
This is where the real work is done
@param outputfp instance of zipfile.ZipFile
"""
assert (isinstance(outputfp, zipfile.ZipFile))
self._z = outputfp
self._now = time.localtime()[:6]
self.manifest = manifest.Manifest()
# Write mimetype
zi = zipfile.ZipInfo('mimetype', self._now)
zi.compress_type = zipfile.ZIP_STORED
zi.external_attr = UNIXPERMS
self._z.writestr(zi, self.mimetype.encode("utf-8"))
self._saveXmlObjects(self, u"")
# Write pictures
self._savePictures(self, u"")
# Write the thumbnail
if self.thumbnail is not None:
self.manifest.addElement(
manifest.FileEntry(fullpath=u"Thumbnails/", mediatype=u''))
self.manifest.addElement(
manifest.FileEntry(fullpath=u"Thumbnails/thumbnail.png",
mediatype=u''))
zi = zipfile.ZipInfo(u"Thumbnails/thumbnail.png", self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
zi.external_attr = UNIXPERMS
self._z.writestr(zi, self.thumbnail)
# Write any extra files
for op in self._extra:
if op.filename == u"META-INF/documentsignatures.xml":
continue # Don't save signatures
self.manifest.addElement(
manifest.FileEntry(fullpath=op.filename,
mediatype=op.mediatype))
if sys.version_info[0] == 3:
zi = zipfile.ZipInfo(op.filename, self._now)
else:
zi = zipfile.ZipInfo(op.filename.encode('utf-8'), self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
zi.external_attr = UNIXPERMS
if op.content is not None:
self._z.writestr(zi, op.content)
# Write manifest
zi = zipfile.ZipInfo(u"META-INF/manifest.xml", self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
zi.external_attr = UNIXPERMS
self._z.writestr(zi, self.__manifestxml())
del self._z
del self._now
del self.manifest
def _zipwrite(self, outputfp):
""" Write the document to an open file pointer """
self._z = outputfp
self._now = time.localtime()[:6]
self.manifest = manifest.Manifest()
# Write mimetype
zi = zipfile.ZipInfo('mimetype', self._now)
zi.compress_type = zipfile.ZIP_STORED
zi.external_attr = UNIXPERMS
self._z.writestr(zi, self.mimetype)
self._saveXmlObjects(self, "")
# Write pictures
self._savePictures(self, "")
# Write the thumbnail
if self.thumbnail is not None:
self.manifest.addElement(
manifest.FileEntry(fullpath="Thumbnails/", mediatype=''))
self.manifest.addElement(
manifest.FileEntry(fullpath="Thumbnails/thumbnail.png",
mediatype=''))
zi = zipfile.ZipInfo("Thumbnails/thumbnail.png", self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
zi.external_attr = UNIXPERMS
self._z.writestr(zi, self.thumbnail)
# Write any extra files
for op in self._extra:
if op.filename == "META-INF/documentsignatures.xml":
continue # Don't save signatures
self.manifest.addElement(
manifest.FileEntry(fullpath=op.filename,
mediatype=op.mediatype))
zi = zipfile.ZipInfo(op.filename.encode('utf-8'), self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
zi.external_attr = UNIXPERMS
if op.content is not None:
self._z.writestr(zi, op.content)
# Write manifest
zi = zipfile.ZipInfo("META-INF/manifest.xml", self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
zi.external_attr = UNIXPERMS
self._z.writestr(zi, self.manifestxml())
del self._z
del self._now
del self.manifest
def save(self, outputfile, addsuffix=False):
""" Save the document under the filename """
self.manifest = manifest.Manifest()
mimetype = self.mimetype
suffix = odmimetypes.get(mimetype, '.xxx')
if outputfile == '-':
self._z = zipfile.ZipFile(sys.stdout, "w")
else:
if addsuffix:
outputfile = outputfile + suffix
self._z = zipfile.ZipFile(outputfile, "w")
self._now = time.localtime()[:6]
# Write mimetype
zi = zipfile.ZipInfo('mimetype', self._now)
zi.compress_type = zipfile.ZIP_STORED
self._z.writestr(zi, self.mimetype)
self._saveXmlObjects(self, "")
# Write pictures
self._savePictures(self, "")
# Write the thumbnail
if self.thumbnail is not None:
self.manifest.addElement(
manifest.FileEntry(fullpath="Thumbnails/", mediatype=''))
self.manifest.addElement(
manifest.FileEntry(fullpath="Thumbnails/thumbnail.png",
mediatype=''))
zi = zipfile.ZipInfo("Thumbnails/thumbnail.png", self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
self._z.writestr(zi, self.thumbnail)
# Write manifest
zi = zipfile.ZipInfo("META-INF/manifest.xml", self._now)
zi.compress_type = zipfile.ZIP_DEFLATED
self._z.writestr(zi, self.manifestxml())
self._z.close()
del self._z
del self._now
del self.manifest
def update_manifest(self,
manifest_path,
tests_path,
url_base="/",
recreate=False):
self.logger.info("Updating test manifest %s" % manifest_path)
json_data = None
if not recreate:
try:
with open(manifest_path) as f:
json_data = json.load(f)
except IOError:
#If the existing file doesn't exist just create one from scratch
pass
if not json_data:
manifest_file = manifest.Manifest(None, url_base)
else:
manifest_file = manifest.Manifest.from_json(json_data)
manifest.update(tests_path, url_base, manifest_file)
manifest.write(manifest_file, manifest_path)
def update_from_cli(**kwargs):
tests_root = kwargs["tests_root"]
path = kwargs["path"]
assert tests_root is not None
m = None
if kwargs["download"]:
download_from_github(path, tests_root)
if not kwargs.get("rebuild", False):
try:
m = manifest.load(tests_root, path)
except manifest.ManifestVersionMismatch:
logger.info("Manifest version changed, rebuilding")
m = None
if m is None:
m = manifest.Manifest(kwargs["url_base"])
changed = update(tests_root, m, working_copy=kwargs["work"])
if changed:
manifest.write(m, path)
def open(self, path):
"""
open an existing image package at the targeted path
"""
logger = logging.getLogger(sys._getframe().f_code.co_name)
self.path = validate_path(path, 'directory')
self.id = os.path.basename(self.path)
# verify original and master and metadata and checksums
# TBD
# open manifest and metadata
self.manifest = manifest.Manifest(
os.path.join(self.path, 'manifest-sha1.txt'))
try:
self.metadata = metadata.Metadata(
os.path.join(self.path, 'meta.xml'))
except IOError:
logger.warning(
"no meta.xml file was found in the package for this image ({0})"
.format(self.id))
raise
# see if there is an original file yet
filenames = self.manifest.get_all().keys()
for filename in filenames:
if 'original.' in filename:
front, extension = os.path.splitext(filename)
if 'sha1' not in extension:
self.original = filename
try:
o = self.original
except AttributeError:
logger.error(
"no original image file was found in the package for this image ({0})"
.format(self.id))
raise
self.make_overview()
def create(self, path, id, original_path):
"""
create a new image package at the targeted path
"""
real_path = validate_path(path, 'directory')
os.makedirs(
os.path.join(real_path, id, 'temp')
) # don't we need to destroy this when done? what is it for?
self.path = os.path.join(real_path, id)
self.id = id
self.manifest = manifest.Manifest(os.path.join(self.path,
'manifest-sha1.txt'),
create=True)
self.__import_original__(original_path)
self.master = self.__generate_master__()
self.original = os.path.basename(original_path)
self.make_derivatives()
self.metadata = metadata.Metadata(os.path.join(self.path, 'meta.xml'),
create=True,
exiftool_json=os.path.join(
self.path, 'original-exif.json'))
self.make_overview()
self.__append_event__(
'created package at {path}'.format(path=self.path))
def coverage_for_apk(apk_name, coverage_type):
ella_run_dir = utils.apk_name_to_ellaout_dir(apk_name)
cov_dat = get_latest_coverage_dat_file(ella_run_dir)
headers = utils.parse_covdat_headers(cov_dat)
if headers[
"recorder"] == "com.apposcopy.ella.runtime.MethodSequenceRecorder":
if coverage_type == "method":
cov = coverage.MethodCoverageGraph()
elif coverage_type == "activity":
am = manifest.Manifest(
os.path.join(ella_run_dir, "apktool-out",
"AndroidManifest.xml"))
cov = coverage.ActivityCoverageGraph(am)
else:
raise Exception("Not implemented: " + coverage_type +
" coverage, for recorder " + headers["recorder"])
elif headers[
"recorder"] == "com.apposcopy.ella.runtime.MethodCoverageRecorder":
if coverage_type == "method":
cov = coverage.MethodCoverageSet()
elif coverage_type == "activity":
am = manifest.Manifest(
os.path.join(ella_run_dir, "apktool-out",
"AndroidManifest.xml"))
cov = coverage.ActivityCoverageSet(am)
else:
raise Exception("Not implemented: " + coverage_type +
" coverage, for recorder " + headers["recorder"])
else:
raise Exception("Unknown recorder type: " + headers["recorder"])
covids = []
with open(os.path.join(ella_run_dir, "covids"), "r") as covidsf:
for line in covidsf:
method_sig = line.strip()
covids.append(method_sig)
cov.announce_method(method_sig)
if headers[
"recorder"] == "com.apposcopy.ella.runtime.MethodSequenceRecorder":
start_time = -1
with open(cov_dat, "r") as coveragef:
for (time, method_id) in read_coverage(coveragef):
if start_time == -1:
start_time = time
cov.setStartTime(start_time)
assert len(covids) > method_id
method_sig = covids[method_id]
cov.notify_covered_method(method_sig, time)
elif headers[
"recorder"] == "com.apposcopy.ella.runtime.MethodCoverageRecorder":
with open(cov_dat, "r") as coveragef:
for line in coveragef:
line = line.strip()
if ":" in line or line == "###": continue # Ignore header data
method_id = int(line)
assert len(covids) > method_id
method_sig = covids[method_id]
cov.notify_covered_method(method_sig)
else:
assert False # Unreachable
if coverage_type == "method":
print "Method coverage:", cov.get_current_coverage()
elif coverage_type == "activity":
print "Activity coverage:", cov.get_current_coverage()
def create_manifest(self):
logger.info("Creating test manifest")
manifest.setup_git(self.tests_root)
manifest_file = manifest.Manifest(None)
manifest.update(manifest_file)
manifest.write(manifest_file, self.manifest_path)
def __init__(self):
self.manifest = _manifest.Manifest("./", {})
def create_manifest(self, manifest_path, tests_path, url_base="/"):
self.logger.info("Creating test manifest %s" % manifest_path)
manifest_file = manifest.Manifest(None, url_base)
manifest.update(tests_path, url_base, manifest_file)
manifest.write(manifest_file, manifest_path)
def load_next_manifest(self):
#
# Do we have any nore images to process?
#
print "ManifestProcessor:load_next_manifest(): Called"
#
# Unmount previous mount point
#
if self.mount_point:
try:
subprocess.check_call(["/bin/umount", self.mount_point ])
except subprocess.CalledProcessError:
print "Failed to unmount {}: {}".format(self.mount_point,
subprocess.CalledProcessError.returncode)
self.mount_point = None
self.current_manifest = None
if len(self.image_queue) == 0:
print "ManifestProcessor:load_next_manifest(): Image queue is empty"
return False
print "ManifestProcessor:load_next_manifest(): #{}".format(len(self.image_queue))
image_path = self.image_queue.pop()
print "Will process update image: {}".format(image_path)
# Mount the file system
self.mount_point = "/tmp/swlm/{}".format(os.getpid())
print "Will create mount point: {}".format(self.mount_point)
try:
os.makedirs(self.mount_point)
except os.OSError as e:
print "Failed to create {}: {}".format(self.mount_point, e)
pass
try:
subprocess.check_call(["/bin/mount", image_path, self.mount_point ])
except subprocess.CalledProcessError:
print "Failed to mount {} on {}: {}".format(image_path,
self.mount_point,
subprocess.CalledProcessError.returncode)
return False
# Create the new manifest object
try:
self.current_manifest = manifest.Manifest([], [], [], self)
except Exception as e:
print "Manifest exception: {}".format(e)
# Specify manifest file to load
manifest_file= "{}/update_manifest.json".format(self.mount_point)
if not self.current_manifest.load_from_file(manifest_file):
print "Failed to load manifest {}".format(manifest_file)
self.current_manifest = None
# Unmount file system
try:
subprocess.check_call(["/bin/umount", self.mount_point ])
except subprocess.CalledProcessError:
print "Failed to unmount {}: {}".format(self.mount_point,
subprocess.CalledProcessError.returncode)
self.mount_point = None
return False
return True
def load_next_manifest(self):
"""Load next manifest from image
Mount the squashfs image and load the manifest in it for processing.
@return True if successful, False otherwise
"""
#
# Do we have any nore images to process?
#
logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Called.')
#
# Unmount previous mount point
#
if self.mount_point:
try:
command = list(settings.SQUASHFS_UNMOUNT_CMD)
command.append(self.mount_point)
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed to unmount %s: %s.',
self.mount_point, e)
self.mount_point = None
self.current_manifest = None
if len(self.image_queue) == 0:
logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Image queue is empty.')
return False
logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Manifests in queue: %s.', len(self.image_queue))
image_path = self.image_queue.pop()
logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Processing update image: %s.', image_path)
# Mount the file system
if not settings.SQUASHFS_MOUNT_POINT.endswith('/'):
self.mount_point = settings.SQUASHFS_MOUNT_POINT + '/' + str(os.getpid())
else:
self.mount_point = settings.SQUASHFS_MOUNT_POINT + str(os.getpid())
logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Creating mount point: %s.', self.mount_point)
try:
os.makedirs(self.mount_point)
except OSError as e:
logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed creating mount point %s: %s.', self.mount_point, e)
pass
try:
command = settings.SQUASHFS_MOUNT_CMD.format(
mount_point=self.mount_point,
image_path=image_path).split()
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed mounting %s on %s: %s.',
image_path, self.mount_point, e)
return False
# Create the new manifest object
# Specify manifest file to load
self.manifest_file= "{}/update_manifest.json".format(self.mount_point)
try:
self.current_manifest = manifest.Manifest(self.mount_point, self.manifest_file, self.dbstore)
except Exception as e:
logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed loading manifest: %s.', e)
traceback.print_exc()
if not self.current_manifest:
self.current_manifest = None
# Unmount file system
try:
command = list(settings.SQUASHFS_UNMOUNT_CMD)
command.append(self.mount_point)
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed unmounting %s: %s.',
self.mount_point, e)
self.mount_point = None
return False
return True
