def update_from_cli(**kwargs): tests_root = kwargs["tests_root"] path = kwargs["path"] assert tests_root is not None m = None logger = get_logger() if not kwargs.get("rebuild", False): try: m = manifest.load(tests_root, path) except manifest.ManifestVersionMismatch: logger.info("Manifest version changed, rebuilding") m = None else: logger.info("Updating manifest") if m is None: m = manifest.Manifest(None) update(tests_root, kwargs["url_base"], m, ignore_local=kwargs.get("ignore_local", False)) manifest.write(m, path)
def __zipwrite(self, outputfp): """ Write the document to an open file pointer This is where the real work is done @param outputfp instance of zipfile.ZipFile """ assert (isinstance(outputfp, zipfile.ZipFile)) self._z = outputfp self._now = time.localtime()[:6] self.manifest = manifest.Manifest() # Write mimetype zi = zipfile.ZipInfo('mimetype', self._now) zi.compress_type = zipfile.ZIP_STORED zi.external_attr = UNIXPERMS self._z.writestr(zi, self.mimetype.encode("utf-8")) self._saveXmlObjects(self, u"") # Write pictures self._savePictures(self, u"") # Write the thumbnail if self.thumbnail is not None: self.manifest.addElement( manifest.FileEntry(fullpath=u"Thumbnails/", mediatype=u'')) self.manifest.addElement( manifest.FileEntry(fullpath=u"Thumbnails/thumbnail.png", mediatype=u'')) zi = zipfile.ZipInfo(u"Thumbnails/thumbnail.png", self._now) zi.compress_type = zipfile.ZIP_DEFLATED zi.external_attr = UNIXPERMS self._z.writestr(zi, self.thumbnail) # Write any extra files for op in self._extra: if op.filename == u"META-INF/documentsignatures.xml": continue # Don't save signatures self.manifest.addElement( manifest.FileEntry(fullpath=op.filename, mediatype=op.mediatype)) if sys.version_info[0] == 3: zi = zipfile.ZipInfo(op.filename, self._now) else: zi = zipfile.ZipInfo(op.filename.encode('utf-8'), self._now) zi.compress_type = zipfile.ZIP_DEFLATED zi.external_attr = UNIXPERMS if op.content is not None: self._z.writestr(zi, op.content) # Write manifest zi = zipfile.ZipInfo(u"META-INF/manifest.xml", self._now) zi.compress_type = zipfile.ZIP_DEFLATED zi.external_attr = UNIXPERMS self._z.writestr(zi, self.__manifestxml()) del self._z del self._now del self.manifest
def _zipwrite(self, outputfp): """ Write the document to an open file pointer """ self._z = outputfp self._now = time.localtime()[:6] self.manifest = manifest.Manifest() # Write mimetype zi = zipfile.ZipInfo('mimetype', self._now) zi.compress_type = zipfile.ZIP_STORED zi.external_attr = UNIXPERMS self._z.writestr(zi, self.mimetype) self._saveXmlObjects(self, "") # Write pictures self._savePictures(self, "") # Write the thumbnail if self.thumbnail is not None: self.manifest.addElement( manifest.FileEntry(fullpath="Thumbnails/", mediatype='')) self.manifest.addElement( manifest.FileEntry(fullpath="Thumbnails/thumbnail.png", mediatype='')) zi = zipfile.ZipInfo("Thumbnails/thumbnail.png", self._now) zi.compress_type = zipfile.ZIP_DEFLATED zi.external_attr = UNIXPERMS self._z.writestr(zi, self.thumbnail) # Write any extra files for op in self._extra: if op.filename == "META-INF/documentsignatures.xml": continue # Don't save signatures self.manifest.addElement( manifest.FileEntry(fullpath=op.filename, mediatype=op.mediatype)) zi = zipfile.ZipInfo(op.filename.encode('utf-8'), self._now) zi.compress_type = zipfile.ZIP_DEFLATED zi.external_attr = UNIXPERMS if op.content is not None: self._z.writestr(zi, op.content) # Write manifest zi = zipfile.ZipInfo("META-INF/manifest.xml", self._now) zi.compress_type = zipfile.ZIP_DEFLATED zi.external_attr = UNIXPERMS self._z.writestr(zi, self.manifestxml()) del self._z del self._now del self.manifest
def save(self, outputfile, addsuffix=False): """ Save the document under the filename """ self.manifest = manifest.Manifest() mimetype = self.mimetype suffix = odmimetypes.get(mimetype, '.xxx') if outputfile == '-': self._z = zipfile.ZipFile(sys.stdout, "w") else: if addsuffix: outputfile = outputfile + suffix self._z = zipfile.ZipFile(outputfile, "w") self._now = time.localtime()[:6] # Write mimetype zi = zipfile.ZipInfo('mimetype', self._now) zi.compress_type = zipfile.ZIP_STORED self._z.writestr(zi, self.mimetype) self._saveXmlObjects(self, "") # Write pictures self._savePictures(self, "") # Write the thumbnail if self.thumbnail is not None: self.manifest.addElement( manifest.FileEntry(fullpath="Thumbnails/", mediatype='')) self.manifest.addElement( manifest.FileEntry(fullpath="Thumbnails/thumbnail.png", mediatype='')) zi = zipfile.ZipInfo("Thumbnails/thumbnail.png", self._now) zi.compress_type = zipfile.ZIP_DEFLATED self._z.writestr(zi, self.thumbnail) # Write manifest zi = zipfile.ZipInfo("META-INF/manifest.xml", self._now) zi.compress_type = zipfile.ZIP_DEFLATED self._z.writestr(zi, self.manifestxml()) self._z.close() del self._z del self._now del self.manifest
def update_manifest(self, manifest_path, tests_path, url_base="/", recreate=False): self.logger.info("Updating test manifest %s" % manifest_path) json_data = None if not recreate: try: with open(manifest_path) as f: json_data = json.load(f) except IOError: #If the existing file doesn't exist just create one from scratch pass if not json_data: manifest_file = manifest.Manifest(None, url_base) else: manifest_file = manifest.Manifest.from_json(json_data) manifest.update(tests_path, url_base, manifest_file) manifest.write(manifest_file, manifest_path)
def update_from_cli(**kwargs): tests_root = kwargs["tests_root"] path = kwargs["path"] assert tests_root is not None m = None if kwargs["download"]: download_from_github(path, tests_root) if not kwargs.get("rebuild", False): try: m = manifest.load(tests_root, path) except manifest.ManifestVersionMismatch: logger.info("Manifest version changed, rebuilding") m = None if m is None: m = manifest.Manifest(kwargs["url_base"]) changed = update(tests_root, m, working_copy=kwargs["work"]) if changed: manifest.write(m, path)
def open(self, path): """ open an existing image package at the targeted path """ logger = logging.getLogger(sys._getframe().f_code.co_name) self.path = validate_path(path, 'directory') self.id = os.path.basename(self.path) # verify original and master and metadata and checksums # TBD # open manifest and metadata self.manifest = manifest.Manifest( os.path.join(self.path, 'manifest-sha1.txt')) try: self.metadata = metadata.Metadata( os.path.join(self.path, 'meta.xml')) except IOError: logger.warning( "no meta.xml file was found in the package for this image ({0})" .format(self.id)) raise # see if there is an original file yet filenames = self.manifest.get_all().keys() for filename in filenames: if 'original.' in filename: front, extension = os.path.splitext(filename) if 'sha1' not in extension: self.original = filename try: o = self.original except AttributeError: logger.error( "no original image file was found in the package for this image ({0})" .format(self.id)) raise self.make_overview()
def create(self, path, id, original_path): """ create a new image package at the targeted path """ real_path = validate_path(path, 'directory') os.makedirs( os.path.join(real_path, id, 'temp') ) # don't we need to destroy this when done? what is it for? self.path = os.path.join(real_path, id) self.id = id self.manifest = manifest.Manifest(os.path.join(self.path, 'manifest-sha1.txt'), create=True) self.__import_original__(original_path) self.master = self.__generate_master__() self.original = os.path.basename(original_path) self.make_derivatives() self.metadata = metadata.Metadata(os.path.join(self.path, 'meta.xml'), create=True, exiftool_json=os.path.join( self.path, 'original-exif.json')) self.make_overview() self.__append_event__( 'created package at {path}'.format(path=self.path))
def coverage_for_apk(apk_name, coverage_type): ella_run_dir = utils.apk_name_to_ellaout_dir(apk_name) cov_dat = get_latest_coverage_dat_file(ella_run_dir) headers = utils.parse_covdat_headers(cov_dat) if headers[ "recorder"] == "com.apposcopy.ella.runtime.MethodSequenceRecorder": if coverage_type == "method": cov = coverage.MethodCoverageGraph() elif coverage_type == "activity": am = manifest.Manifest( os.path.join(ella_run_dir, "apktool-out", "AndroidManifest.xml")) cov = coverage.ActivityCoverageGraph(am) else: raise Exception("Not implemented: " + coverage_type + " coverage, for recorder " + headers["recorder"]) elif headers[ "recorder"] == "com.apposcopy.ella.runtime.MethodCoverageRecorder": if coverage_type == "method": cov = coverage.MethodCoverageSet() elif coverage_type == "activity": am = manifest.Manifest( os.path.join(ella_run_dir, "apktool-out", "AndroidManifest.xml")) cov = coverage.ActivityCoverageSet(am) else: raise Exception("Not implemented: " + coverage_type + " coverage, for recorder " + headers["recorder"]) else: raise Exception("Unknown recorder type: " + headers["recorder"]) covids = [] with open(os.path.join(ella_run_dir, "covids"), "r") as covidsf: for line in covidsf: method_sig = line.strip() covids.append(method_sig) cov.announce_method(method_sig) if headers[ "recorder"] == "com.apposcopy.ella.runtime.MethodSequenceRecorder": start_time = -1 with open(cov_dat, "r") as coveragef: for (time, method_id) in read_coverage(coveragef): if start_time == -1: start_time = time cov.setStartTime(start_time) assert len(covids) > method_id method_sig = covids[method_id] cov.notify_covered_method(method_sig, time) elif headers[ "recorder"] == "com.apposcopy.ella.runtime.MethodCoverageRecorder": with open(cov_dat, "r") as coveragef: for line in coveragef: line = line.strip() if ":" in line or line == "###": continue # Ignore header data method_id = int(line) assert len(covids) > method_id method_sig = covids[method_id] cov.notify_covered_method(method_sig) else: assert False # Unreachable if coverage_type == "method": print "Method coverage:", cov.get_current_coverage() elif coverage_type == "activity": print "Activity coverage:", cov.get_current_coverage()
def create_manifest(self): logger.info("Creating test manifest") manifest.setup_git(self.tests_root) manifest_file = manifest.Manifest(None) manifest.update(manifest_file) manifest.write(manifest_file, self.manifest_path)
def __init__(self): self.manifest = _manifest.Manifest("./", {})
def create_manifest(self, manifest_path, tests_path, url_base="/"): self.logger.info("Creating test manifest %s" % manifest_path) manifest_file = manifest.Manifest(None, url_base) manifest.update(tests_path, url_base, manifest_file) manifest.write(manifest_file, manifest_path)
def load_next_manifest(self): # # Do we have any nore images to process? # print "ManifestProcessor:load_next_manifest(): Called" # # Unmount previous mount point # if self.mount_point: try: subprocess.check_call(["/bin/umount", self.mount_point ]) except subprocess.CalledProcessError: print "Failed to unmount {}: {}".format(self.mount_point, subprocess.CalledProcessError.returncode) self.mount_point = None self.current_manifest = None if len(self.image_queue) == 0: print "ManifestProcessor:load_next_manifest(): Image queue is empty" return False print "ManifestProcessor:load_next_manifest(): #{}".format(len(self.image_queue)) image_path = self.image_queue.pop() print "Will process update image: {}".format(image_path) # Mount the file system self.mount_point = "/tmp/swlm/{}".format(os.getpid()) print "Will create mount point: {}".format(self.mount_point) try: os.makedirs(self.mount_point) except os.OSError as e: print "Failed to create {}: {}".format(self.mount_point, e) pass try: subprocess.check_call(["/bin/mount", image_path, self.mount_point ]) except subprocess.CalledProcessError: print "Failed to mount {} on {}: {}".format(image_path, self.mount_point, subprocess.CalledProcessError.returncode) return False # Create the new manifest object try: self.current_manifest = manifest.Manifest([], [], [], self) except Exception as e: print "Manifest exception: {}".format(e) # Specify manifest file to load manifest_file= "{}/update_manifest.json".format(self.mount_point) if not self.current_manifest.load_from_file(manifest_file): print "Failed to load manifest {}".format(manifest_file) self.current_manifest = None # Unmount file system try: subprocess.check_call(["/bin/umount", self.mount_point ]) except subprocess.CalledProcessError: print "Failed to unmount {}: {}".format(self.mount_point, subprocess.CalledProcessError.returncode) self.mount_point = None return False return True
def load_next_manifest(self): """Load next manifest from image Mount the squashfs image and load the manifest in it for processing. @return True if successful, False otherwise """ # # Do we have any nore images to process? # logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Called.') # # Unmount previous mount point # if self.mount_point: try: command = list(settings.SQUASHFS_UNMOUNT_CMD) command.append(self.mount_point) subprocess.check_call(command) except subprocess.CalledProcessError as e: logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed to unmount %s: %s.', self.mount_point, e) self.mount_point = None self.current_manifest = None if len(self.image_queue) == 0: logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Image queue is empty.') return False logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Manifests in queue: %s.', len(self.image_queue)) image_path = self.image_queue.pop() logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Processing update image: %s.', image_path) # Mount the file system if not settings.SQUASHFS_MOUNT_POINT.endswith('/'): self.mount_point = settings.SQUASHFS_MOUNT_POINT + '/' + str(os.getpid()) else: self.mount_point = settings.SQUASHFS_MOUNT_POINT + str(os.getpid()) logger.debug('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Creating mount point: %s.', self.mount_point) try: os.makedirs(self.mount_point) except OSError as e: logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed creating mount point %s: %s.', self.mount_point, e) pass try: command = settings.SQUASHFS_MOUNT_CMD.format( mount_point=self.mount_point, image_path=image_path).split() subprocess.check_call(command) except subprocess.CalledProcessError as e: logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed mounting %s on %s: %s.', image_path, self.mount_point, e) return False # Create the new manifest object # Specify manifest file to load self.manifest_file= "{}/update_manifest.json".format(self.mount_point) try: self.current_manifest = manifest.Manifest(self.mount_point, self.manifest_file, self.dbstore) except Exception as e: logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed loading manifest: %s.', e) traceback.print_exc() if not self.current_manifest: self.current_manifest = None # Unmount file system try: command = list(settings.SQUASHFS_UNMOUNT_CMD) command.append(self.mount_point) subprocess.check_call(command) except subprocess.CalledProcessError as e: logger.error('SoftwareLoadingManager.ManifestProcessor.load_next_manifest(): Failed unmounting %s: %s.', self.mount_point, e) self.mount_point = None return False return True