def install_java():
"""Install java just like we do for Hadoop Base.
This is the same method used to install java in HadoopBase:
https://github.com/juju-solutions/jujubigdata/blob/master/jujubigdata/handlers.py#L134
This allows us to run Pig in local mode (which requires Java) without
any Hadoop. If Hadoop comes along later, we'll already have java installed
in a way that is compatible with the plugin.
NOTE: this will go away if/when we support the java interface.
"""
env = utils.read_etc_env()
java_installer = Path(jujuresources.resource_path('java-installer'))
java_installer.chmod(0o755)
output = check_output([java_installer], env=env).decode('utf8')
lines = output.strip().splitlines()
if len(lines) != 2:
raise ValueError('Unexpected output from java-installer: %s' % output)
java_home, java_version = lines
if '_' in java_version:
java_major, java_release = java_version.split("_")
else:
java_major, java_release = java_version, ''
unitdata.kv().set('java.home', java_home)
unitdata.kv().set('java.version', java_major)
unitdata.kv().set('java.version.release', java_release)
def mkdtemp_rename(destination, chmod=None, **kwargs):
"""A wrapper for tempfile.mkdtemp that always cleans up.
This wrapper sets defaults based on the class values."""
log = kwargs.pop('log', None)
dest = Path(destination).normpath()
kwargs.setdefault('dir', dest.parent)
tmppath = Path(tempfile.mkdtemp(**kwargs))
try:
yield tmppath
try:
tmppath.chmod(0o0755)
tmppath.rename(dest)
except OSError as e:
if e.errno == errno.ENOENT:
# the tmppath didn't exist?!
log.exception("Shouldn't be here %r", tmppath)
raise
elif e.errno == errno.ENOTEMPTY:
log.debug("Target already existed")
else:
raise
finally:
tmppath.rmtree_p()
def install_java(self):
"""
Run the java-installer resource to install Java and determine
the JAVA_HOME and Java version.
The java-installer must be idempotent and its only output (on stdout)
should be two lines: the JAVA_HOME path, and the Java version, respectively.
If there is an error installing Java, the installer should exit
with a non-zero exit code.
"""
env = utils.read_etc_env()
java_installer = Path(jujuresources.resource_path('java-installer'))
java_installer.chmod(0o755)
output = check_output([java_installer], env=env).decode('utf8')
lines = output.strip().splitlines()
if len(lines) != 2:
raise ValueError('Unexpected output from java-installer: %s' % output)
java_home, java_version = lines
if '_' in java_version:
java_major, java_release = java_version.split("_")
else:
java_major, java_release = java_version, ''
unitdata.kv().set('java.home', java_home)
unitdata.kv().set('java.version', java_major)
unitdata.kv().set('java.version.release', java_release)
def install_java(self):
"""
Run the java-installer resource to install Java and determine
the JAVA_HOME and Java version.
The java-installer must be idempotent and its only output (on stdout)
should be two lines: the JAVA_HOME path, and the Java version, respectively.
If there is an error installing Java, the installer should exit
with a non-zero exit code.
"""
env = utils.read_etc_env()
java_installer = Path(jujuresources.resource_path('java-installer'))
java_installer.chmod(0o755)
output = check_output([java_installer], env=env).decode('utf8')
lines = output.strip().splitlines()
if len(lines) != 2:
raise ValueError('Unexpected output from java-installer: %s' %
output)
java_home, java_version = lines
if '_' in java_version:
java_major, java_release = java_version.split("_")
else:
java_major, java_release = java_version, ''
unitdata.kv().set('java.home', java_home)
unitdata.kv().set('java.version', java_major)
unitdata.kv().set('java.version.release', java_release)
def install_java():
"""Install java just like we do for Hadoop Base.
This is the same method used to install java in HadoopBase:
https://github.com/juju-solutions/jujubigdata/blob/master/jujubigdata/handlers.py#L134
This allows us to run Pig in local mode (which requires Java) without
any Hadoop. If Hadoop comes along later, we'll already have java installed
in a way that is compatible with the plugin.
NOTE: this will go away if/when we support the java interface.
"""
env = utils.read_etc_env()
java_installer = Path(jujuresources.resource_path('java-installer'))
java_installer.chmod(0o755)
output = check_output([java_installer], env=env).decode('utf8')
lines = output.strip().splitlines()
if len(lines) != 2:
raise ValueError('Unexpected output from java-installer: %s' % output)
java_home, java_version = lines
if '_' in java_version:
java_major, java_release = java_version.split("_")
else:
java_major, java_release = java_version, ''
unitdata.kv().set('java.home', java_home)
unitdata.kv().set('java.version', java_major)
unitdata.kv().set('java.version.release', java_release)
def install_consul(version, destination_directory='/usr/local/bin'):
''' Install the configured version of consul for the architecture. '''
if version:
architecture = consul_arch()
consul_file_name = 'consul_{0}_{1}.zip'.format(version, architecture)
# Find the sha256sum for the specific file name.
sha256sum = find_sha256sum(version, consul_file_name)
print('Expecting {0} for {1}'.format(sha256sum, consul_file_name))
url = '{0}/{1}/{2}'.format(URL_PREFIX, version, consul_file_name)
print('Fetching {0}'.format(url))
installer = archiveurl.ArchiveUrlFetchHandler()
# Download and unzip the archive into the final destination directory.
installer.install(url, dest=destination_directory, checksum=sha256sum,
hash_type='sha256')
consul = Path(destination_directory + '/consul')
# Make the consul binary executable.
consul.chmod(0o555)
def install_consul(version, destination_directory='/usr/local/bin'):
''' Install the configured version of consul for the architecture. '''
if version:
architecture = consul_arch()
consul_file_name = 'consul_{0}_{1}.zip'.format(version, architecture)
# Find the sha256sum for the specific file name.
sha256sum = find_sha256sum(version, consul_file_name)
print('Expecting {0} for {1}'.format(sha256sum, consul_file_name))
url = '{0}/{1}/{2}'.format(URL_PREFIX, version, consul_file_name)
print('Fetching {0}'.format(url))
installer = archiveurl.ArchiveUrlFetchHandler()
# Download and unzip the archive into the final destination directory.
installer.install(url,
dest=destination_directory,
checksum=sha256sum,
hash_type='sha256')
consul = Path(destination_directory + '/consul')
# Make the consul binary executable.
consul.chmod(0o555)
def ensure_built(self):
if self.should_rebuild():
if self.buildpath.exists():
self.buildpath.rmtree()
self.buildpath.mkdir()
download_urls = self._download_urls[self.version]
if self.os_name == "linux":
chrome_download_url = download_urls["linux_chrome"]
chromedriver_download_url = download_urls["linux_chromedriver"]
elif self.os_name == "mac":
chrome_download_url = download_urls["mac_chrome"]
chromedriver_download_url = download_urls["mac_chromedriver"]
download_to = self.tmp / "chrome-{}.zip".format(self.version)
utils.download_file(download_to, chrome_download_url)
if self.os_name == "mac":
# patool has a bug on mac that breaks chromium
Command("unzip", download_to).in_dir(self.buildpath).run()
else:
utils.extract_archive(download_to, self.buildpath)
download_to.remove()
chrome_bin = Path(self.chrome_bin)
chrome_bin.chmod(chrome_bin.stat().st_mode | stat.S_IEXEC)
# Install chromedriver
download_to = self.tmp / "chromedriver-{}.zip".format(self.version)
utils.download_file(download_to, chromedriver_download_url)
utils.extract_archive(download_to, self.buildpath)
download_to.remove()
chromedriver_bin = Path(self.chromedriver_bin)
chromedriver_bin.chmod(chromedriver_bin.stat().st_mode
| stat.S_IEXEC)
self.verify()
self.refingerprint()
class Job(models.Model):
STATUS_CHOICES = (
('L.STR', 'Starting the job'),
('L.RUN', 'Running'),
('L.FIN', 'Running'),
('L.CPL', 'Job completed'),
('L.ERR', 'Error'),
)
# All other statuses mean the job is still running
STATUS_FINISHED = ('L.CPL', 'L.ERR')
job_id = models.AutoField(primary_key=True)
job_name = models.CharField(max_length=100, blank=True)
created = models.DateTimeField(auto_now_add=True)
warning = models.CharField(max_length=1000, blank=True)
error = models.CharField(max_length=1000, blank=True)
deleted = models.BooleanField(default=False)
restarted = models.IntegerField(default=0)
touched = models.DateTimeField(auto_now=True)
user = models.ForeignKey(User, on_delete=models.CASCADE)
ip = models.GenericIPAddressField()
celery_id = models.CharField(max_length=200, blank=True)
status = models.CharField(max_length=5,
choices=STATUS_CHOICES,
default='L.STR')
details_json = models.CharField(max_length=1000, blank=True)
_temp_dir = ''
def __str__(self):
attr_list = [self.job_id, self.user, self.status]
return '-'.join(map(str, attr_list))
def __getattr__(self, attr):
json_str = super(Job, self).__getattribute__("details_json")
if json_str:
json_dict = json.loads(json_str)
if attr in json_dict:
return json_dict[attr]
return super(Job, self).__getattribute__(attr)
def get_dir(self):
return env.JOBS_DIR.joinpath(str(self.job_id))
def get_output_dir(self):
return self.get_dir().joinpath('output')
def get_user_dir(self):
return self.get_dir().joinpath('user')
def get_user_json(self):
return self.get_user_dir().joinpath('user.json')
def get_log_file(self):
return self.get_dir().joinpath('log.txt')
def get_result_file(self):
return self.get_output_dir().joinpath('result.txt')
def get_result(self):
with open(self.get_result_file(), 'r') as f:
r = f.read().strip()
return r
def local_error(self, error):
self.status = 'L.ERR'
self.error = error
self.save()
def reset(self):
"""
Call this when a job is (re)started. Resets error and warning and increments 'restarted' field
"""
self.restarted += 1
self.error = ''
self.warning = ''
self.save()
def start(self):
if self.celery_id != '':
result = submit_job.AsyncResult(self.celery_id)
if not result.ready():
return [
f'Job {self.job_id} is already running (state: {result.state})'
]
result = submit_job.apply_async((self.job_id, ))
self.celery_id = result.task_id
self.restarted += 1
self.error = ''
self.warning = ''
self.save()
return []
def cancel(self):
if self.celery_id != '':
result = submit_job.AsyncResult(self.celery_id)
if result.ready():
return [f'Job has already finished (state: {result.state})']
result.revoke(terminate=True)
return [f'Job cancelled (state: {result.state})']
else:
return ['Task id is empty']
def _make_temp_dir(self):
self._temp_dir = Path(tempfile.mkdtemp(dir=env.TMP_DIR))
self._temp_dir.chmod(0o755)
def _create_job_dir(self):
job_dir = self.get_dir()
if job_dir.isdir():
job_dir.rmtree(job_dir)
self._temp_dir.move(job_dir)
def check_user_input(self, form, files):
error_list = []
self._make_temp_dir()
user_dir = self._temp_dir.joinpath('user')
user_dir.mkdir_p()
# dump user files in user/
for name, file in files.items():
fpath = user_dir.joinpath(file.name)
upload_file(file, fpath)
# dump user data in user/user.json
with open(user_dir.joinpath('user.json'), 'w') as f:
data = {
k: (v if not isinstance(v, File) else v.name)
for k, v in form.cleaned_data.items()
}
json.dump(data, f, indent=4)
return error_list
def make_dir_and_start(self):
self._create_job_dir()
clean_json = self.get_user_json()
with open(clean_json, 'r') as f:
details_json = json.load(f)
self.details_json = json.dumps(details_json)
self.save()
self.start()
def add_execute_permissions(path,
mode=stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH):
"""Add Execute Permissions to a Path."""
path = Path(path)
path.chmod(path.stat().st_mode | mode)
def test_chmod_str(self, tmpdir):
tmpfile = Path(tmpdir) / 'file'
tmpfile.touch()
tmpfile.chmod('o-r')
is_windows = platform.system() == 'Windows'
assert is_windows or not (tmpfile.stat().st_mode & stat.S_IROTH)
