def install_java(): """Install java just like we do for Hadoop Base. This is the same method used to install java in HadoopBase: https://github.com/juju-solutions/jujubigdata/blob/master/jujubigdata/handlers.py#L134 This allows us to run Pig in local mode (which requires Java) without any Hadoop. If Hadoop comes along later, we'll already have java installed in a way that is compatible with the plugin. NOTE: this will go away if/when we support the java interface. """ env = utils.read_etc_env() java_installer = Path(jujuresources.resource_path('java-installer')) java_installer.chmod(0o755) output = check_output([java_installer], env=env).decode('utf8') lines = output.strip().splitlines() if len(lines) != 2: raise ValueError('Unexpected output from java-installer: %s' % output) java_home, java_version = lines if '_' in java_version: java_major, java_release = java_version.split("_") else: java_major, java_release = java_version, '' unitdata.kv().set('java.home', java_home) unitdata.kv().set('java.version', java_major) unitdata.kv().set('java.version.release', java_release)
def mkdtemp_rename(destination, chmod=None, **kwargs): """A wrapper for tempfile.mkdtemp that always cleans up. This wrapper sets defaults based on the class values.""" log = kwargs.pop('log', None) dest = Path(destination).normpath() kwargs.setdefault('dir', dest.parent) tmppath = Path(tempfile.mkdtemp(**kwargs)) try: yield tmppath try: tmppath.chmod(0o0755) tmppath.rename(dest) except OSError as e: if e.errno == errno.ENOENT: # the tmppath didn't exist?! log.exception("Shouldn't be here %r", tmppath) raise elif e.errno == errno.ENOTEMPTY: log.debug("Target already existed") else: raise finally: tmppath.rmtree_p()
def install_java(self): """ Run the java-installer resource to install Java and determine the JAVA_HOME and Java version. The java-installer must be idempotent and its only output (on stdout) should be two lines: the JAVA_HOME path, and the Java version, respectively. If there is an error installing Java, the installer should exit with a non-zero exit code. """ env = utils.read_etc_env() java_installer = Path(jujuresources.resource_path('java-installer')) java_installer.chmod(0o755) output = check_output([java_installer], env=env).decode('utf8') lines = output.strip().splitlines() if len(lines) != 2: raise ValueError('Unexpected output from java-installer: %s' % output) java_home, java_version = lines if '_' in java_version: java_major, java_release = java_version.split("_") else: java_major, java_release = java_version, '' unitdata.kv().set('java.home', java_home) unitdata.kv().set('java.version', java_major) unitdata.kv().set('java.version.release', java_release)
def install_consul(version, destination_directory='/usr/local/bin'): ''' Install the configured version of consul for the architecture. ''' if version: architecture = consul_arch() consul_file_name = 'consul_{0}_{1}.zip'.format(version, architecture) # Find the sha256sum for the specific file name. sha256sum = find_sha256sum(version, consul_file_name) print('Expecting {0} for {1}'.format(sha256sum, consul_file_name)) url = '{0}/{1}/{2}'.format(URL_PREFIX, version, consul_file_name) print('Fetching {0}'.format(url)) installer = archiveurl.ArchiveUrlFetchHandler() # Download and unzip the archive into the final destination directory. installer.install(url, dest=destination_directory, checksum=sha256sum, hash_type='sha256') consul = Path(destination_directory + '/consul') # Make the consul binary executable. consul.chmod(0o555)
def ensure_built(self): if self.should_rebuild(): if self.buildpath.exists(): self.buildpath.rmtree() self.buildpath.mkdir() download_urls = self._download_urls[self.version] if self.os_name == "linux": chrome_download_url = download_urls["linux_chrome"] chromedriver_download_url = download_urls["linux_chromedriver"] elif self.os_name == "mac": chrome_download_url = download_urls["mac_chrome"] chromedriver_download_url = download_urls["mac_chromedriver"] download_to = self.tmp / "chrome-{}.zip".format(self.version) utils.download_file(download_to, chrome_download_url) if self.os_name == "mac": # patool has a bug on mac that breaks chromium Command("unzip", download_to).in_dir(self.buildpath).run() else: utils.extract_archive(download_to, self.buildpath) download_to.remove() chrome_bin = Path(self.chrome_bin) chrome_bin.chmod(chrome_bin.stat().st_mode | stat.S_IEXEC) # Install chromedriver download_to = self.tmp / "chromedriver-{}.zip".format(self.version) utils.download_file(download_to, chromedriver_download_url) utils.extract_archive(download_to, self.buildpath) download_to.remove() chromedriver_bin = Path(self.chromedriver_bin) chromedriver_bin.chmod(chromedriver_bin.stat().st_mode | stat.S_IEXEC) self.verify() self.refingerprint()
class Job(models.Model): STATUS_CHOICES = ( ('L.STR', 'Starting the job'), ('L.RUN', 'Running'), ('L.FIN', 'Running'), ('L.CPL', 'Job completed'), ('L.ERR', 'Error'), ) # All other statuses mean the job is still running STATUS_FINISHED = ('L.CPL', 'L.ERR') job_id = models.AutoField(primary_key=True) job_name = models.CharField(max_length=100, blank=True) created = models.DateTimeField(auto_now_add=True) warning = models.CharField(max_length=1000, blank=True) error = models.CharField(max_length=1000, blank=True) deleted = models.BooleanField(default=False) restarted = models.IntegerField(default=0) touched = models.DateTimeField(auto_now=True) user = models.ForeignKey(User, on_delete=models.CASCADE) ip = models.GenericIPAddressField() celery_id = models.CharField(max_length=200, blank=True) status = models.CharField(max_length=5, choices=STATUS_CHOICES, default='L.STR') details_json = models.CharField(max_length=1000, blank=True) _temp_dir = '' def __str__(self): attr_list = [self.job_id, self.user, self.status] return '-'.join(map(str, attr_list)) def __getattr__(self, attr): json_str = super(Job, self).__getattribute__("details_json") if json_str: json_dict = json.loads(json_str) if attr in json_dict: return json_dict[attr] return super(Job, self).__getattribute__(attr) def get_dir(self): return env.JOBS_DIR.joinpath(str(self.job_id)) def get_output_dir(self): return self.get_dir().joinpath('output') def get_user_dir(self): return self.get_dir().joinpath('user') def get_user_json(self): return self.get_user_dir().joinpath('user.json') def get_log_file(self): return self.get_dir().joinpath('log.txt') def get_result_file(self): return self.get_output_dir().joinpath('result.txt') def get_result(self): with open(self.get_result_file(), 'r') as f: r = f.read().strip() return r def local_error(self, error): self.status = 'L.ERR' self.error = error self.save() def reset(self): """ Call this when a job is (re)started. Resets error and warning and increments 'restarted' field """ self.restarted += 1 self.error = '' self.warning = '' self.save() def start(self): if self.celery_id != '': result = submit_job.AsyncResult(self.celery_id) if not result.ready(): return [ f'Job {self.job_id} is already running (state: {result.state})' ] result = submit_job.apply_async((self.job_id, )) self.celery_id = result.task_id self.restarted += 1 self.error = '' self.warning = '' self.save() return [] def cancel(self): if self.celery_id != '': result = submit_job.AsyncResult(self.celery_id) if result.ready(): return [f'Job has already finished (state: {result.state})'] result.revoke(terminate=True) return [f'Job cancelled (state: {result.state})'] else: return ['Task id is empty'] def _make_temp_dir(self): self._temp_dir = Path(tempfile.mkdtemp(dir=env.TMP_DIR)) self._temp_dir.chmod(0o755) def _create_job_dir(self): job_dir = self.get_dir() if job_dir.isdir(): job_dir.rmtree(job_dir) self._temp_dir.move(job_dir) def check_user_input(self, form, files): error_list = [] self._make_temp_dir() user_dir = self._temp_dir.joinpath('user') user_dir.mkdir_p() # dump user files in user/ for name, file in files.items(): fpath = user_dir.joinpath(file.name) upload_file(file, fpath) # dump user data in user/user.json with open(user_dir.joinpath('user.json'), 'w') as f: data = { k: (v if not isinstance(v, File) else v.name) for k, v in form.cleaned_data.items() } json.dump(data, f, indent=4) return error_list def make_dir_and_start(self): self._create_job_dir() clean_json = self.get_user_json() with open(clean_json, 'r') as f: details_json = json.load(f) self.details_json = json.dumps(details_json) self.save() self.start()
def add_execute_permissions(path, mode=stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH): """Add Execute Permissions to a Path.""" path = Path(path) path.chmod(path.stat().st_mode | mode)
def test_chmod_str(self, tmpdir): tmpfile = Path(tmpdir) / 'file' tmpfile.touch() tmpfile.chmod('o-r') is_windows = platform.system() == 'Windows' assert is_windows or not (tmpfile.stat().st_mode & stat.S_IROTH)