def remove_addon(self, addon_id): """Remove the add-on as specified by the id :param addon_id: id of the add-on to be removed """ path = self.get_addon_path(addon_id) mozfile.remove(path)
def download(self, url, target_folder=None): """ Downloads an add-on from the specified URL to the target folder :param url: URL of the add-on (XPI file) :param target_folder: Folder to store the XPI file in """ response = urllib2.urlopen(url) fd, path = tempfile.mkstemp(suffix='.xpi') os.write(fd, response.read()) os.close(fd) if not self.is_addon(path): mozfile.remove(path) raise AddonFormatError('Not a valid add-on: %s' % url) # Give the downloaded file a better name by using the add-on id details = self.addon_details(path) new_path = path.replace('.xpi', '_%s.xpi' % details.get('id')) # Move the add-on to the target folder if requested if target_folder: new_path = os.path.join(target_folder, os.path.basename(new_path)) os.rename(path, new_path) return new_path
def check_for_crashes(self, test_name=None): test_name = test_name or self.last_test dump_dir = self.device.pull_minidumps() crashed = BaseRunner.check_for_crashes( self, dump_directory=dump_dir, test_name=test_name) mozfile.remove(dump_dir) return crashed
def set_default_settings(self): filename = 'settings.json' defaults = DEFAULT_SETTINGS.copy() defaults.update(self.testvars.get('settings', {})) defaults = self.modify_settings(defaults) if self.locale != 'undefined': defaults['language.current'] = self.locale if self.device.is_desktop_b2g: directory = self.marionette.instance.profile_path path = os.path.join(directory, filename) else: directory = '/system/b2g/defaults' path = posixpath.join(directory, filename) settings = json.loads(self.device.file_manager.pull_file(path)) for name, value in defaults.items(): self.logger.debug('Setting %s to %s' % (name, value)) settings[name] = value td = tempfile.mkdtemp() try: tf = os.path.join(td, filename) with open(tf, 'w') as f: json.dump(settings, f) if not self.device.is_desktop_b2g: self.device.manager.remount() self.device.file_manager.push_file(tf, directory) finally: mozfile.remove(td)
def get_gaia_info(self, app_zip): tempdir = tempfile.mkdtemp() try: gaia_commit = os.path.join(tempdir, 'gaia_commit.txt') try: zip_file = zipfile.ZipFile(app_zip.name) with open(gaia_commit, 'w') as f: f.write(zip_file.read('resources/gaia_commit.txt')) except zipfile.BadZipfile: self._logger.info('Unable to unzip application.zip, falling ' 'back to system unzip') from subprocess import call call(['unzip', '-j', app_zip.name, 'resources/gaia_commit.txt', '-d', tempdir]) with open(gaia_commit) as f: changeset, date = f.read().splitlines() self._info['gaia_changeset'] = re.match( '^\w{40}$', changeset) and changeset or None self._info['gaia_date'] = date except KeyError: self._logger.warning( 'Unable to find resources/gaia_commit.txt in ' 'application.zip') finally: mozfile.remove(tempdir)
def process_test_job(data): global logger logger = logger or structuredlog.get_default_logger() build_name = "{}-{} {}".format(data['platform'], data['buildtype'], data['test']) logger.debug("now processing a '{}' job".format(build_name)) log_url = None for name, url in data['blobber_files'].iteritems(): if name in settings['structured_log_names']: log_url = url break log_path = _download_log(log_url) try: backend = settings['datastore'] db_args = config.database store = get_storage_backend(backend, **db_args) # TODO commit metadata about the test run handler = StoreResultsHandler(store) with open(log_path, 'r') as log: iterator = reader.read(log) reader.handle_log(iterator, handler) finally: mozfile.remove(log_path)
def vendor(self, ignore_modified=False): self.populate_logger() self.log_manager.enable_unstructured() if not ignore_modified: self.check_modified_files() cargo = self.get_cargo_path() if not self.check_cargo_version(cargo): self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)') return else: self.log(logging.DEBUG, 'cargo_version', {}, 'cargo is new enough') have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines()) if not have_vendor: self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor') self.run_process(args=[cargo, 'install', 'cargo-vendor']) else: self.log(logging.DEBUG, 'cargo_vendor', {}, 'cargo-vendor already intalled') vendor_dir = mozpath.join(self.topsrcdir, 'third_party/rust') self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % vendor_dir) mozfile.remove(vendor_dir) # Once we require a new enough cargo to switch to workspaces, we can # just do this once on the workspace root crate. for crate_root in ('toolkit/library/rust/', 'toolkit/library/gtest/rust'): path = mozpath.join(self.topsrcdir, crate_root) self._run_command_in_srcdir(args=[cargo, 'generate-lockfile', '--manifest-path', mozpath.join(path, 'Cargo.toml')]) self._run_command_in_srcdir(args=[cargo, 'vendor', '--sync', mozpath.join(path, 'Cargo.lock'), vendor_dir]) #TODO: print stats on size of files added/removed, warn or error # when adding very large files (bug 1306078) self.repository.add_remove_files(vendor_dir)
def python_test(self, *args, **kwargs): try: tempdir = os.environ[b'PYTHON_TEST_TMP'] = str(tempfile.mkdtemp(suffix='-python-test')) return self.run_python_tests(*args, **kwargs) finally: import mozfile mozfile.remove(tempdir)
def computeSNRAndDelay(self): snr_delay = "-1.000,-1" if not os.path.exists(_MEDIA_TOOLS_): return False, "SNR Tool not found" cmd = [_MEDIA_TOOLS_, '-c', 'snr', '-r', _INPUT_FILE_, '-t', _RECORDED_NO_SILENCE_] cmd = [str(s) for s in cmd] output = subprocess.check_output(cmd) # SNR_Delay=1.063,5 result = re.search('SNR_DELAY=(\d+\.\d+),(\d+)', output) # delete the recorded file with no silence mozfile.remove(_RECORDED_NO_SILENCE_) if result: snr_delay = str(result.group(1)) + ',' + str(result.group(2)) return True, snr_delay else: """ We return status as True since SNR computation went through successfully but scores computation failed due to severly degraded audio quality. """ return True, snr_delay
def cleanup_pending_crash_reports(): """ Delete any pending crash reports. The presence of pending crash reports may be reported by the browser, affecting test results; it is best to ensure that these are removed before starting any browser tests. Firefox stores pending crash reports in "<UAppData>/Crash Reports". If the browser is not running, it cannot provide <UAppData>, so this code tries to anticipate its value. See dom/system/OSFileConstants.cpp for platform variations of <UAppData>. """ if mozinfo.isWin: location = os.path.expanduser("~\\AppData\\Roaming\\Mozilla\\Firefox\\Crash Reports") elif mozinfo.isMac: location = os.path.expanduser("~/Library/Application Support/firefox/Crash Reports") else: location = os.path.expanduser("~/.mozilla/firefox/Crash Reports") logger = get_logger() if os.path.exists(location): try: mozfile.remove(location) logger.info("Removed pending crash reports at '%s'" % location) except: pass
def bootstrap(self): try: name = self.boot.instance.__class__.__name__ name = name[:name.index('Bootstrapper')].lower() if hasattr(self, 'pre_bootstrap_%s' % name): getattr(self, 'pre_bootstrap_%s' % name)() else: self.boot.finished = PLATFORM_NOT_IMPLEMENTED + SEE_MDN_DOCS self.boot.instance.packages.extend(self.extra_packages) if name == 'osx': # for osx, use the shell script provided in the documentation self.boot.instance.ensure_xcode() self._download('https://raw.github.com/mozilla-b2g/B2G/master/scripts/bootstrap-mac.sh') print('Running the mac bootstrap script...') ret = subprocess.call(['bash', 'bootstrap-mac.sh']) os.remove('bootstrap-mac.sh') print(self.boot.finished) return ret self.boot.bootstrap() if hasattr(self, 'post_bootstrap_%s' % name): getattr(self, 'post_bootstrap_%s' % name)() finally: if self.mozboot_dir: import mozfile mozfile.remove(self.mozboot_dir)
def setupProfile(self, prefs=None): """Sets up the user profile on the device. :param prefs: String of user_prefs to add to the profile. Defaults to a standard b2g testing profile. """ # currently we have no custom prefs to set (when bug 800138 is fixed, # we will probably want to enable marionette on an external ip by # default) if not prefs: prefs = "" #remove previous user.js if there is one if not self.profileDir: self.profileDir = tempfile.mkdtemp() our_userJS = os.path.join(self.profileDir, "user.js") mozfile.remove(our_userJS) #copy profile try: self.getFile(self.userJS, our_userJS) except subprocess.CalledProcessError: pass #if we successfully copied the profile, make a backup of the file if os.path.exists(our_userJS): self.shellCheckOutput(['dd', 'if=%s' % self.userJS, 'of=%s.orig' % self.userJS]) with open(our_userJS, 'a') as user_file: user_file.write("%s" % prefs) self.pushFile(our_userJS, self.userJS) self.restartB2G() self.setupMarionette()
def getDirectory(self, remoteDir, localDir, checkDir=True): localDir = os.path.normpath(localDir) remoteDir = os.path.normpath(remoteDir) copyRequired = False originalLocal = localDir if self._adb_version >= '1.0.36' and \ os.path.isdir(localDir) and self.dirExists(remoteDir): # See do_sync_pull in # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp # Work around change in behavior in adb 1.0.36 where if # the local destination directory exists, adb pull will # copy the source directory *into* the destination # directory otherwise it will copy the source directory # *onto* the destination directory. # # If the destination directory does exist, pull to its # parent directory. If the source and destination leaf # directory names are different, pull the source directory # into a temporary directory and then copy the temporary # directory onto the destination. localName = os.path.basename(localDir) remoteName = os.path.basename(remoteDir) if localName != remoteName: copyRequired = True tempParent = tempfile.mkdtemp() localDir = os.path.join(tempParent, remoteName) else: localDir = '/'.join(localDir.rstrip('/').split('/')[:-1]) self._runCmd(["pull", remoteDir, localDir]).wait() if copyRequired: dir_util.copy_tree(localDir, originalLocal) mozfile.remove(tempParent)
def _download(self, url, dest, finished_callback, chunk_size, session): # save the file under a temporary name # this allow to not use a broken file in case things went really bad # while downloading the file (ie the python interpreter is killed # abruptly) temp_dest = dest + '.tmp' bytes_so_far = 0 try: with closing(session.get(url, stream=True)) as response: total_size = int(response.headers['Content-length'].strip()) self._update_progress(bytes_so_far, total_size) with open(temp_dest, 'wb') as f: for chunk in response.iter_content(chunk_size): if self.is_canceled(): break if chunk: f.write(chunk) bytes_so_far += len(chunk) self._update_progress(bytes_so_far, total_size) except: self.__error = sys.exc_info() try: if self.is_canceled() or self.__error: mozfile.remove(temp_dest) else: # if all goes well, then rename the file to the real dest mozfile.remove(dest) # just in case it already existed os.rename(temp_dest, dest) finally: if finished_callback: finished_callback(self)
def pushDir(self, localDir, remoteDir, retryLimit=None): # adb "push" accepts a directory as an argument, but if the directory # contains symbolic links, the links are pushed, rather than the linked # files; we either zip/unzip or re-copy the directory into a temporary # one to get around this limitation retryLimit = retryLimit or self.retryLimit if not self.dirExists(remoteDir): self.mkDirs(remoteDir+"/x") if self._useZip: try: localZip = tempfile.mktemp() + ".zip" remoteZip = remoteDir + "/adbdmtmp.zip" subprocess.Popen(["zip", "-r", localZip, '.'], cwd=localDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False) mozfile.remove(localZip) data = self._runCmd(["shell", "unzip", "-o", remoteZip, "-d", remoteDir]).stdout.read() self._checkCmd(["shell", "rm", remoteZip], retryLimit=retryLimit) if re.search("unzip: exiting", data) or re.search("Operation not permitted", data): raise Exception("unzip failed, or permissions error") except: self._logger.info("zip/unzip failure: falling back to normal push") self._useZip = False self.pushDir(localDir, remoteDir, retryLimit=retryLimit) else: tmpDir = tempfile.mkdtemp() # copytree's target dir must not already exist, so create a subdir tmpDirTarget = os.path.join(tmpDir, "tmp") shutil.copytree(localDir, tmpDirTarget) self._checkCmd(["push", tmpDirTarget, remoteDir], retryLimit=retryLimit) mozfile.remove(tmpDir)
def _prepare_tests(self, revision, tests_url): use_cache = settings['MAX_TESTS_CACHE_SIZE'] > 0 if use_cache and revision in tests_cache: # the tests bundle is possibly being downloaded by another thread, # wait a bit before downloading ourselves. timeout = 300 # 5 minutes start = datetime.datetime.now() while datetime.datetime.now() - start < datetime.timedelta(seconds=timeout): if tests_cache[revision] != None: logger.debug("using pre-downloaded tests bundle for revision '{}'".format(revision)) # another thread has already downloaded the bundle for this revision, woohoo! return tests_cache[revision] time.sleep(1) logger.debug("downloading tests bundle for revision '{}'".format(revision)) if use_cache: # let other threads know we are already downloading this rev tests_cache[revision] = None if len(tests_cache) >= settings['MAX_TESTS_CACHE_SIZE']: # clean up the oldest revision, it most likely isn't needed anymore mozfile.remove(tests_cache.popitem(last=False)[1]) # FIFO tf = mozfile.NamedTemporaryFile(suffix='.zip') with open(tf.name, 'wb') as f: f.write(self._download(tests_url)) tests_path = tempfile.mkdtemp() mozfile.extract(tf.name, tests_path) if use_cache: tests_cache[revision] = tests_path return tests_path
def clean(self): """ Clean up temp folders created with the instance creation. """ mozfile.remove(self.option('dir')) for symbol_path in self.symbol_paths.values(): mozfile.remove(symbol_path)
def fetch_and_unpack(self, revision, target): '''Fetch and unpack upstream source''' url = self.upstream_snapshot(revision) self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}') prefix = 'aom-' + revision filename = prefix + '.tar.gz' with open(filename, 'wb') as f: req = requests.get(url, stream=True) for data in req.iter_content(4096): f.write(data) tar = tarfile.open(filename) bad_paths = filter(lambda name: name.startswith('/') or '..' in name, tar.getnames()) if any(bad_paths): raise Exception("Tar archive contains non-local paths," "e.g. '%s'" % bad_paths[0]) self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target) mozfile.remove(target) self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.') tar.extractall(target) # Github puts everything properly down a directory; move it up. if all(map(lambda name: name.startswith(prefix), tar.getnames())): tardir = mozpath.join(target, prefix) os.system('mv %s/* %s/.* %s' % (tardir, tardir, target)) os.rmdir(tardir) # Remove the tarball. mozfile.remove(filename)
def __init__(self, userName, password, serverUri, tbVersion, binary, cachePath="profileCache", reset=False, *args, **kwargs): self.profileName = "%s-tb%d-%s" % (userName, tbVersion, time.strftime("%Y-%m-%d", time.localtime())) profilePath = os.path.join(cachePath, self.profileName) if reset: print "Reseting profile in",profilePath mozfile.remove(profilePath) super(ObmProfile, self).__init__(profile=profilePath, *args, **kwargs) self.userName = userName self.password = password self.serverUri = serverUri self.tbVersion = tbVersion # Thunderbird 3 doesn't have 64-bit NSS libraries on mac, use the old # signons file for this version if self.tbVersion > 3: self.signons = SignonsSQLFile(profilePath, os.path.dirname(binary)) else: self.signons = Signons3File(os.path.join(profilePath, "signons3.txt")) self.overrides = CertOverrideFile(os.path.join(profilePath,"cert_override.txt")) self.initProfile() self.flush()
def _run_tests(tags): application_folder = None try: # Backup current tags test_tags = self.test_tags application_folder = self.duplicate_application(source_folder) self.bin = mozinstall.get_binary(application_folder, 'Firefox') self.test_tags = tags super(UpdateTestRunner, self).run_tests(tests) except Exception: self.exc_info = sys.exc_info() self.logger.error('Failure during execution of the update test.', exc_info=self.exc_info) finally: self.test_tags = test_tags self.logger.info('Removing copy of the application at "%s"' % application_folder) try: mozfile.remove(application_folder) except IOError as e: self.logger.error('Cannot remove copy of application: "%s"' % str(e))
def __iter__(self): for path, extra in self.dump_files: rv = self._process_dump_file(path, extra) yield rv if self.remove_symbols: mozfile.remove(self.symbols_path)
def clean(self): """Clean up addons in the profile.""" # Remove all add-ons installed for addon in self._addons: # TODO (bug 934642) # Once we have a proper handling of add-ons we should kill the id # from self._addons once the add-on is removed. For now lets forget # about the exception try: self.remove_addon(addon) except IOError: pass # Remove all downloaded add-ons for addon in self.downloaded_addons: mozfile.remove(addon) # restore backups if self.backup_dir and os.path.isdir(self.backup_dir): extensions_path = os.path.join(self.profile, 'extensions', 'staged') for backup in os.listdir(self.backup_dir): backup_path = os.path.join(self.backup_dir, backup) shutil.move(backup_path, extensions_path) if not os.listdir(self.backup_dir): mozfile.remove(self.backup_dir) # reset instance variables to defaults self._internal_init()
def run_tests(self): """ Start the execution of the tests. """ manifest = manifestparser.TestManifest( manifests=[os.path.join(self.repository.path, self.manifest_path)], strict=False) tests = manifest.active_tests(**mozinfo.info) # instantiate handlers logger = mozmill.logger.LoggerListener(log_file=self.options.logfile, console_level=self.debug and 'DEBUG' or 'INFO', file_level=self.debug and 'DEBUG' or 'INFO', debug=self.debug) handlers = [logger] if self.options.report_url: self.report = reports.DashboardReport(self.options.report_url, self) handlers.append(self.report) if self.options.junit_file: filename = files.get_unique_filename(self.options.junit_file, self.testrun_index) self.junit_report = reports.JUnitReport(filename, self) handlers.append(self.junit_report) # instantiate MozMill profile_path = os.path.join(self.workspace, 'profile') self.mozlogger.info('Creating profile: %s' % profile_path) profile_args = dict(profile=profile_path, addons=self.addon_list, preferences=self.preferences, ) mozmill_args = dict(app=self.options.application, binary=self._application, handlers=handlers, profile_args=profile_args, ) if self.timeout: mozmill_args['jsbridge_timeout'] = self.timeout self._mozmill = mozmill.MozMill.create(**mozmill_args) self.graphics = None for listener in self.listeners: self._mozmill.add_listener(listener[0], eventType=listener[1]) self._mozmill.persisted.update(self.persisted) try: self._mozmill.run(tests, self.options.restart) finally: self.results = self._mozmill.finish() self.mozlogger.info('Removing profile: %s' % profile_path) mozfile.remove(profile_path) # Whenever a test fails it has to be marked, so we quit with the correct exit code self.last_failed_tests = self.last_failed_tests or self.results.fails self.testrun_index += 1
def __init__(self, sources=None, host=None, device_serial=None, adb_host=None, adb_port=None, **kwargs): B2GVersion.__init__(self, sources, **kwargs) try: import mozdevice except ImportError: self._logger.critical("mozdevice is required to get the version" " of a remote device") raise dm = mozdevice.DeviceManagerADB(deviceSerial=device_serial, serverHost=adb_host, serverPort=adb_port) if not sources: path = 'system/sources.xml' if dm.fileExists(path): sources = StringIO(dm.pullFile(path)) else: self._logger.info('Unable to find %s' % path) tempdir = tempfile.mkdtemp() for ini in ('application', 'platform'): with open(os.path.join(tempdir, '%s.ini' % ini), 'w') as f: f.write(dm.pullFile('/system/b2g/%s.ini' % ini)) f.flush() self.get_gecko_info(tempdir) mozfile.remove(tempdir) for path in ['/system/b2g', '/data/local']: path += '/webapps/settings.gaiamobile.org/application.zip' if dm.fileExists(path): with tempfile.NamedTemporaryFile() as f: dm.getFile(path, f.name) self.get_gaia_info(f) break else: self._logger.warning('Error pulling gaia file') build_props = dm.pullFile('/system/build.prop') desired_props = { 'ro.build.version.incremental': 'device_firmware_version_incremental', 'ro.build.version.release': 'device_firmware_version_release', 'ro.build.date.utc': 'device_firmware_date', 'ro.product.device': 'device_id'} for line in build_props.split('\n'): if not line.strip().startswith('#') and '=' in line: key, value = [s.strip() for s in line.split('=', 1)] if key in desired_props.keys(): self._info[desired_props[key]] = value if self._info.get('device_id', '').lower() == 'flame': for prop in ['ro.boot.bootloader', 't2m.sw.version']: value = dm.shellCheckOutput(['getprop', prop]) if value: self._info['device_firmware_version_base'] = value break
def remove_downloaded_addons(self): for path in self.downloaded_addons: try: # Remove downloaded add-on self.mozlogger.info('Removing downloaded add-on: %s' % path) mozfile.remove(path) except: self.mozlogger.exception('Failed to remove downloaded add-on: %s' % path)
def check_for_crashes(self, browser_config, profile_dir, test_name): # check for minidumps minidumpdir = os.path.join(profile_dir, "minidumps") found = mozcrash.check_for_crashes(minidumpdir, browser_config["symbols_path"], test_name=test_name) mozfile.remove(minidumpdir) if found: raise TalosCrash("Found crashes after test run, terminating test")
def test_remove_path_that_does_not_exists(self): not_existing_path = os.path.join(self.tempdir, 'I_do_not_not_exists') try: mozfile.remove(not_existing_path) except OSError as exc: if exc.errno == errno.ENOENT: self.fail("removing non existing path must not raise error") raise
def remove_downloaded_update(self): """Remove an already downloaded update from the update staging directory. Hereby not only remove the update subdir but everything below 'updates'. """ path = os.path.dirname(self.software_update.staging_directory) self.logger.info('Clean-up update staging directory: {}'.format(path)) mozfile.remove(path)
def remove_downloaded_addons(self): for path in self.downloaded_addons: try: # Remove downloaded add-on print "*** Removing downloaded add-on '%s'." % path mozfile.remove(path) except: print "*** Failed to remove downloaded add-on '%s'." % path
def test_remove_readonly_file(self): """Test removing read-only files""" filepath = os.path.join(self.tempdir, *stubs.files[1]) mark_readonly(filepath) # However, mozfile should change write permission and then remove file. mozfile.remove(filepath) self.assertFalse(os.path.exists(filepath))
def check_for_crashes(self, browser_config, minidump_dir, test_name): # check for minidumps found = mozcrash.check_for_crashes(minidump_dir, browser_config['symbols_path'], test_name=test_name) mozfile.remove(minidump_dir) if found: raise TalosCrash("Found crashes after test run, terminating test")
def setup_benchmarks(self): """Make sure benchmarks are linked to the proper location in the objdir. Benchmarks can either live in-tree or in an external repository. In the latter case also clone/update the repository if necessary. """ print("Updating external benchmarks from {}".format( BENCHMARK_REPOSITORY)) # Set up the external repo external_repo_path = os.path.join(get_state_dir()[0], 'performance-tests') try: subprocess.check_output(['git', '--version']) except Exception as ex: print("Git is not available! Please install git and " "ensure it is included in the terminal path") raise ex if not os.path.isdir(external_repo_path): subprocess.check_call( ['git', 'clone', BENCHMARK_REPOSITORY, external_repo_path]) else: subprocess.check_call(['git', 'checkout', 'master'], cwd=external_repo_path) subprocess.check_call(['git', 'pull'], cwd=external_repo_path) subprocess.check_call(['git', 'checkout', BENCHMARK_REVISION], cwd=external_repo_path) # Link or copy benchmarks to the objdir benchmark_paths = ( os.path.join(external_repo_path, 'benchmarks'), os.path.join(self.topsrcdir, 'third_party', 'webkit', 'PerformanceTests'), ) benchmark_dest = os.path.join(self.topobjdir, 'testing', 'raptor', 'benchmarks') if not os.path.isdir(benchmark_dest): os.makedirs(benchmark_dest) for benchmark_path in benchmark_paths: for name in os.listdir(benchmark_path): path = os.path.join(benchmark_path, name) dest = os.path.join(benchmark_dest, name) if not os.path.isdir(path) or name.startswith('.'): continue if hasattr(os, 'symlink'): if not os.path.exists(dest): os.symlink(path, dest) else: # Clobber the benchmark in case a recent update removed any files. mozfile.remove(dest) shutil.copytree(path, dest)
def duplicate_application(self, application_folder): """Creates a copy of the specified binary.""" target_folder = os.path.join(self.workspace, 'binary.backup') self.logger.info('Creating a copy of the application at "%s".' % target_folder) mozfile.remove(target_folder) shutil.copytree(application_folder, target_folder) return target_folder
def _install(self, dest): self.tempdir = safe_mkdtemp() try: self.binary = mozinstall.get_binary( mozinstall.install(src=dest, dest=self.tempdir), self.app_name ) except Exception: remove(self.tempdir) raise
def check_for_crashes(self, dump_save_path=None, test_name=None): test_name = test_name or self.last_test dump_dir = self.device.pull_minidumps() crashed = BaseRunner.check_for_crashes(self, dump_directory=dump_dir, dump_save_path=dump_save_path, test_name=test_name) mozfile.remove(dump_dir) return crashed
def clean(self): try: mozfile.remove(self._tmp_dir) except Exception as e: LOG.info("Exception while removing profile directory: %s" % self._tmp_dir) LOG.info(e) if self.gecko_profile: self.gecko_profile.clean()
def setup_benchmarks(self): """Make sure benchmarks are linked to the proper location in the objdir. Benchmarks can either live in-tree or in an external repository. In the latter case also clone/update the repository if necessary. """ external_repo_path = os.path.join(get_state_dir(), "performance-tests") print("Updating external benchmarks from {}".format( BENCHMARK_REPOSITORY)) try: subprocess.check_output(["git", "--version"]) except Exception as ex: print("Git is not available! Please install git and " "ensure it is included in the terminal path") raise ex if not os.path.isdir(external_repo_path): print("Cloning the benchmarks to {}".format(external_repo_path)) subprocess.check_call( ["git", "clone", BENCHMARK_REPOSITORY, external_repo_path]) else: subprocess.check_call(["git", "checkout", "master"], cwd=external_repo_path) subprocess.check_call(["git", "pull"], cwd=external_repo_path) subprocess.check_call(["git", "checkout", BENCHMARK_REVISION], cwd=external_repo_path) # Link or copy benchmarks to the objdir benchmark_paths = ( os.path.join(external_repo_path, "benchmarks"), os.path.join(self.topsrcdir, "third_party", "webkit", "PerformanceTests"), ) benchmark_dest = os.path.join(self.topobjdir, "testing", "raptor", "benchmarks") if not os.path.isdir(benchmark_dest): os.makedirs(benchmark_dest) for benchmark_path in benchmark_paths: for name in os.listdir(benchmark_path): path = os.path.join(benchmark_path, name) dest = os.path.join(benchmark_dest, name) if not os.path.isdir(path) or name.startswith("."): continue if hasattr(os, "symlink") and os.name != "nt": if not os.path.exists(dest): os.symlink(path, dest) else: # Clobber the benchmark in case a recent update removed any files. mozfile.remove(dest) shutil.copytree(path, dest)
def test_restore_false(profile): # make a clone of this profile with restore=False clone = Profile.clone(profile.profile, restore=False) try: clone.cleanup() # clone should still be around on the filesystem assert os.path.exists(clone.profile) finally: mozfile.remove(clone.profile)
def test_remove_readonly_tree(self): """Test removing a read-only directory""" dirpath = os.path.join(self.tempdir, "nested_tree") mark_readonly(dirpath) # However, mozfile should change write permissions and remove dir. mozfile.remove(dirpath) self.assertFalse(os.path.exists(dirpath))
def test_remove_closed_file(self): """Test removing a closed file""" # Open a file in the generated stub filepath = os.path.join(self.tempdir, *stubs.files[1]) with open(filepath, 'w') as f: f.write('foo-bar') # Folder should be deleted on all platforms mozfile.remove(self.tempdir) self.assertFalse(os.path.exists(self.tempdir))
def duplicate_application(self, application_folder): """Creates a copy of the specified binary.""" target_folder = tempfile.mkdtemp('.binary-update-tests') self.logger.info('Creating a copy of the application at "%s".' % target_folder) mozfile.remove(target_folder) shutil.copytree(application_folder, target_folder) return target_folder
def __init__(self, sources=None, dm_type='adb', host=None, device_serial=None, **kwargs): B2GVersion.__init__(self, sources, **kwargs) if dm_type == 'adb': dm = mozdevice.DeviceManagerADB(deviceSerial=device_serial) elif dm_type == 'sut': if not host: raise Exception('A host for SUT must be supplied.') dm = mozdevice.DeviceManagerSUT(host=host) else: raise Exception('Unknown device manager type: %s' % dm_type) if not sources: path = 'system/sources.xml' if dm.fileExists(path): sources = StringIO(dm.pullFile(path)) else: self.info('Unable to find %s' % path) tempdir = tempfile.mkdtemp() for ini in ('application', 'platform'): with open(os.path.join(tempdir, '%s.ini' % ini), 'w') as f: f.write(dm.pullFile('/system/b2g/%s.ini' % ini)) f.flush() self.get_gecko_info(tempdir) mozfile.remove(tempdir) for path in ['/system/b2g', '/data/local']: path += '/webapps/settings.gaiamobile.org/application.zip' if dm.fileExists(path): with tempfile.NamedTemporaryFile() as f: dm.getFile(path, f.name) self.get_gaia_info(f) break else: self.warn('Error pulling gaia file') build_props = dm.pullFile('/system/build.prop') desired_props = { 'ro.build.version.incremental': 'device_firmware_version_incremental', 'ro.build.version.release': 'device_firmware_version_release', 'ro.build.date.utc': 'device_firmware_date', 'ro.product.device': 'device_id' } for line in build_props.split('\n'): if not line.strip().startswith('#') and '=' in line: key, value = [s.strip() for s in line.split('=', 1)] if key in desired_props.keys(): self._info[desired_props[key]] = value
def install(src, dest): """Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of the installation folder. :param src: Path to the install file :param dest: Path to install to (to ensure we do not overwrite any existent files the folder should not exist yet) """ src = os.path.realpath(src) dest = os.path.realpath(dest) if not is_installer(src): raise InvalidSource(src + ' is not valid installer file.') did_we_create = False if not os.path.exists(dest): did_we_create = True os.makedirs(dest) trbk = None try: install_dir = None if src.lower().endswith('.dmg'): install_dir = _install_dmg(src, dest) elif src.lower().endswith('.exe'): install_dir = _install_exe(src, dest) elif zipfile.is_zipfile(src) or tarfile.is_tarfile(src): install_dir = mozfile.extract(src, dest)[0] return install_dir except: cls, exc, trbk = sys.exc_info() if did_we_create: try: # try to uninstall this properly uninstall(dest) except: # uninstall may fail, let's just try to clean the folder # in this case try: mozfile.remove(dest) except: pass if issubclass(cls, Exception): error = InstallError('Failed to install "%s (%s)"' % (src, str(exc))) raise InstallError, error, trbk # any other kind of exception like KeyboardInterrupt is just re-raised. raise cls, exc, trbk finally: # trbk won't get GC'ed due to circular reference # http://docs.python.org/library/sys.html#sys.exc_info del trbk
def test_removing_open_file_with_retry(self): """Test removing a file in use with retry""" filepath = os.path.join(self.tempdir, *stubs.files[1]) with wait_file_opened_in_thread(filepath, 0.2): # on windows first attempt will fail, # and it will be retried until the thread leave the handle mozfile.remove(filepath) # Check deletion was successful self.assertFalse(os.path.exists(filepath))
def test_removing_already_deleted_file_with_retry(self): """Test removing a meanwhile removed file with retry""" filepath = os.path.join(self.tempdir, *stubs.files[1]) with wait_file_opened_in_thread(filepath, 0.2, True): # on windows first attempt will fail, and before # the retry the opened file will be deleted in the thread mozfile.remove(filepath) # Check deletion was successful self.assertFalse(os.path.exists(filepath))
def prepare_application(self, binary): TestRun.prepare_application(self, binary) # If a fallback update has to be performed, create a second copy # of the application to avoid running the installer twice if not self.options.no_fallback: self._backup_folder = os.path.join(self.workspace, 'binary_backup') self.mozlogger.info('Creating backup of binary: %s' % self._backup_folder) mozfile.remove(self._backup_folder) shutil.copytree(self._folder, self._backup_folder)
def clean(self): """Remove all webapps that were installed and restore profile to previous state""" if self._installed_apps: mozfile.remove(self.webapps_dir) if os.path.isdir(self.backup_dir): shutil.copytree(self.backup_dir, self.webapps_dir) mozfile.remove(self.backup_dir) self._apps = [] self._installed_apps = []
def python_test(self, *args, **kwargs): try: tempdir = str(tempfile.mkdtemp(suffix='-python-test')) if six.PY2: os.environ[b'PYTHON_TEST_TMP'] = tempdir else: os.environ['PYTHON_TEST_TMP'] = tempdir return self.run_python_tests(*args, **kwargs) finally: import mozfile mozfile.remove(tempdir)
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None): # adb "push" accepts a directory as an argument, but if the directory # contains symbolic links, the links are pushed, rather than the linked # files; we either zip/unzip or re-copy the directory into a temporary # one to get around this limitation retryLimit = retryLimit or self.retryLimit if self._useZip: self.removeDir(remoteDir) self.mkDirs(remoteDir + "/x") try: localZip = tempfile.mktemp() + ".zip" remoteZip = remoteDir + "/adbdmtmp.zip" proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir, processOutputLine=self._log) proc.run() proc.wait() self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False) mozfile.remove(localZip) data = self._runCmd( ["shell", "unzip", "-o", remoteZip, "-d", remoteDir]).output[0] self._checkCmd(["shell", "rm", remoteZip], retryLimit=retryLimit, timeout=self.short_timeout) if re.search("unzip: exiting", data) or re.search( "Operation not permitted", data): raise Exception("unzip failed, or permissions error") except: self._logger.warning(traceback.format_exc()) self._logger.warning( "zip/unzip failure: falling back to normal push") self._useZip = False self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout) else: # If the remote directory exists, newer implementations of # "adb push" will create a sub-directory, while older versions # will not! Bug 1285040 self.mkDirs(remoteDir + "/x") self.removeDir(remoteDir) tmpDir = tempfile.mkdtemp() # copytree's target dir must not already exist, so create a subdir tmpDirTarget = os.path.join(tmpDir, "tmp") shutil.copytree(localDir, tmpDirTarget) self._checkCmd(["push", tmpDirTarget, remoteDir], retryLimit=retryLimit, timeout=timeout) mozfile.remove(tmpDir)
def test_remove_symlink_for_system_path(self): """Test removing a symlink which points to a system folder""" symlink_path = os.path.join(self.tempdir, 'symlink') os.symlink(os.path.dirname(self.tempdir), symlink_path) self.assertTrue(os.path.islink(symlink_path)) # The folder with the contained symlink will be deleted but not the # original linked file mozfile.remove(symlink_path) self.assertFalse(os.path.exists(symlink_path))
def _install(self, dest): self.tempdir = safe_mkdtemp() try: with zipfile.ZipFile(dest, "r") as z: z.extractall(self.tempdir) self.binary = os.path.join(self.tempdir, "js" if mozinfo.os != "win" else "js.exe") # set the file executable os.chmod(self.binary, os.stat(self.binary).st_mode | stat.S_IEXEC) except Exception: remove(self.tempdir) raise
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None): # adb "push" accepts a directory as an argument, but if the directory # contains symbolic links, the links are pushed, rather than the linked # files; we either zip/unzip or re-copy the directory into a temporary # one to get around this limitation retryLimit = retryLimit or self.retryLimit if self._useZip: self.removeDir(remoteDir) self.mkDirs(remoteDir + "/x") try: localZip = tempfile.mktemp() + ".zip" remoteZip = remoteDir + "/adbdmtmp.zip" proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir, processOutputLine=self._log) proc.run() proc.wait() self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False) mozfile.remove(localZip) data = self._runCmd(["shell", "unzip", "-o", remoteZip, "-d", remoteDir]).output[0] self._checkCmd(["shell", "rm", remoteZip], retryLimit=retryLimit, timeout=self.short_timeout) if re.search("unzip: exiting", data) or re.search("Operation not permitted", data): raise Exception("unzip failed, or permissions error") except: self._logger.warning(traceback.format_exc()) self._logger.warning("zip/unzip failure: falling back to normal push") self._useZip = False self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout) else: localDir = os.path.normpath(localDir) remoteDir = os.path.normpath(remoteDir) tempParent = tempfile.mkdtemp() remoteName = os.path.basename(remoteDir) newLocal = os.path.join(tempParent, remoteName) dir_util.copy_tree(localDir, newLocal) # See do_sync_push in # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp # Work around change in behavior in adb 1.0.36 where if # the remote destination directory exists, adb push will # copy the source directory *into* the destination # directory otherwise it will copy the source directory # *onto* the destination directory. if self._adb_version >= '1.0.36': remoteDir = '/'.join(remoteDir.rstrip('/').split('/')[:-1]) try: if self._checkCmd(["push", newLocal, remoteDir], retryLimit=retryLimit, timeout=timeout): raise DMError("failed to push %s (copy of %s) to %s" % (newLocal, localDir, remoteDir)) except: raise finally: mozfile.remove(tempParent)
def test_remove_symlink(self): """Test removing a symlink""" file_path = os.path.join(self.tempdir, *stubs.files[1]) symlink_path = os.path.join(self.tempdir, 'symlink') os.symlink(file_path, symlink_path) self.assertTrue(os.path.islink(symlink_path)) # The linked folder and files should not be deleted mozfile.remove(symlink_path) self.assertFalse(os.path.exists(symlink_path)) self.assertTrue(os.path.exists(file_path))
def python_test(self, command_context, *args, **kwargs): try: tempdir = str(tempfile.mkdtemp(suffix="-python-test")) if six.PY2: os.environ[b"PYTHON_TEST_TMP"] = tempdir else: os.environ["PYTHON_TEST_TMP"] = tempdir return self.run_python_tests(*args, **kwargs) finally: import mozfile mozfile.remove(tempdir)
def __init__(self, upload_dir, raptor_config, test_config): self.upload_dir = upload_dir self.raptor_config = raptor_config self.test_config = test_config self.cleanup = True # Create a temporary directory into which the tests can put # their profiles. These files will be assembled into one big # zip file later on, which is put into the MOZ_UPLOAD_DIR. self.gecko_profile_dir = tempfile.mkdtemp() # Each test INI can specify gecko_profile_interval and entries but they # can be overrided by user input. gecko_profile_interval = raptor_config.get( "gecko_profile_interval", None) or test_config.get( "gecko_profile_interval", 1) gecko_profile_entries = raptor_config.get( "gecko_profile_entries", None) or test_config.get( "gecko_profile_entries", 1000000) # We need symbols_path; if it wasn't passed in on cmdline, set it # use objdir/dist/crashreporter-symbols for symbolsPath if none provided if (not self.raptor_config["symbols_path"] and self.raptor_config["run_local"] and "MOZ_DEVELOPER_OBJ_DIR" in os.environ): self.raptor_config["symbols_path"] = os.path.join( os.environ["MOZ_DEVELOPER_OBJ_DIR"], "dist", "crashreporter-symbols") # turn on crash reporter if we have symbols os.environ["MOZ_CRASHREPORTER_NO_REPORT"] = "1" if self.raptor_config["symbols_path"]: os.environ["MOZ_CRASHREPORTER"] = "1" else: os.environ["MOZ_CRASHREPORTER_DISABLE"] = "1" # Make sure no archive already exists in the location where # we plan to output our profiler archive self.profile_arcname = os.path.join( self.upload_dir, "profile_{0}.zip".format(test_config["name"])) LOG.info("Clearing archive {0}".format(self.profile_arcname)) mozfile.remove(self.profile_arcname) self.symbol_paths = { "FIREFOX": tempfile.mkdtemp(), "WINDOWS": tempfile.mkdtemp(), } LOG.info("Activating gecko profiling, temp profile dir:" " {0}, interval: {1}, entries: {2}".format( self.gecko_profile_dir, gecko_profile_interval, gecko_profile_entries))
def test_symlinked_binary(self): self._write_ini_files() # create a symlink of the binary in another directory and check # version against this symlink tempdir = tempfile.mkdtemp() try: browser_link = os.path.join(tempdir, os.path.basename(self.binary)) os.symlink(self.binary, browser_link) self._check_version(get_version(browser_link)) finally: mozfile.remove(tempdir)
def clone(cls, path_from, path_to=None, **kwargs): """Instantiate a temporary profile via cloning - path: path of the basis to clone - kwargs: arguments to the profile constructor """ if not path_to: tempdir = tempfile.mkdtemp() # need an unused temp dir name mozfile.remove(tempdir) # copytree requires that dest does not exist path_to = tempdir copytree(path_from, path_to) c = cls(path_to, **kwargs) c.create_new = True # deletes a cloned profile when restore is True return c
def test_removing_open_file_with_retry(self): """Test removing a file in use with retry""" filepath = os.path.join(self.tempdir, *stubs.files[1]) thread = FileOpenCloseThread(filepath, 1) thread.start() # Wait a bit so we can be sure the file has been opened time.sleep(.5) mozfile.remove(filepath) thread.join() # Check deletion was successful self.assertFalse(os.path.exists(filepath))
def _getRemoteHash(self, remoteFile): """ Return the md5 sum of a file on the device """ localFile = tempfile.mkstemp()[1] localFile = self._runPull(remoteFile, localFile) if localFile is None: return None md5 = self._getLocalHash(localFile) mozfile.remove(localFile) return md5