def execute4(cmd, print_error=True, **kwargs):
logger = ElasticDevLogger("execute4")
output_to_json = kwargs.get("output_to_json", True)
#cmd = 'cd {}; (./run_order 2>&1 ; echo $? > {}) | tee -a {}; exit `cat {}`'.format(link,exit_file,logfile,exit_file)
exit_file = "/tmp/{}".format(id_generator(10,
chars=string.ascii_lowercase))
logfile = "/tmp/{}".format(id_generator(10, chars=string.ascii_lowercase))
cmd = '({} 2>&1 ; echo $? > {}) | tee -a {}; exit `cat {}`'.format(
cmd, exit_file, logfile, exit_file)
exitcode = os.system(cmd)
status = None
if exitcode == 0: status = True
output = open(logfile, "r").readlines()
if output_to_json and not isinstance(output, dict):
try:
output = json.loads(output)
except:
logger.warn("Could not convert output to json")
results = {"output": output, "status": status}
exit_error = kwargs.get("exit_error")
if exit_error and not status:
print output
exit(exitcode)
return results
def __init__(self, **kwargs):
self.classname = 'EC2_connections'
self.logger = ElasticDevLogger(self.classname)
self.logger.debug("Instantiating %s" % self.classname)
self.aws_default_region = os.environ["AWS_DEFAULT_REGION"]
self.aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
self.aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
def execute3(cmd, print_error=True, **kwargs):
logger = ElasticDevLogger("execute3")
logger.debug("Running command %s from directory %s" % (cmd, os.getcwd()))
output_queue = kwargs.get("output_queue")
env_vars = kwargs.get("env_vars")
output_to_json = kwargs.get("output_to_json", True)
if env_vars:
env_vars = env_vars.get()
for ek, ev in env_vars.iteritems():
if ev is None:
ev = "None"
elif not isinstance(ev, str) and not isinstance(ev, unicode):
ev = str(ev)
logger.debug(
"Setting environment variable {} to {}, type {}".format(
ek, ev, type(ev)))
os.environ[ek] = ev
exit_error = kwargs.get("exit_error")
process = Popen(cmd, shell=True, bufsize=0, stdout=PIPE, stderr=STDOUT)
output = process.communicate()[0]
if process.returncode != 0:
logger.aggmsg('exit code {}'.format(process.returncode, new=True))
logger.aggmsg(output, prt=True, cmethod="error")
results = {"status": False}
results["failed_message"] = output
results["output"] = output
results["exitcode"] = process.returncode
if output_queue: output_queue.put(results)
if exit_error: exit(process.returncode)
return results
if output_to_json and not isinstance(output, dict):
try:
output = json.loads(output)
except:
logger.warn("Could not convert output to json")
results = {"status": True}
results["output"] = output
if output_queue:
logger.debug("Attempting to place results in the output_queue")
try:
output_queue.put(results)
except:
logger.error("Could not append the results to the output_queue")
return results
def __init__(self, **kwargs):
ResourceCmdHelper.__init__(self)
self.classname = 'AwsCli'
self.logger = ElasticDevLogger(self.classname)
self.logger.debug("Instantiating %s" % self.classname)
self.file_config = None
self.file_config_loc = None
self.tempdir = None
self.resource_tags_keys = [
"tags", "name", "schedule_id", "job_instance_id", "job_id"
]
def __init__(self, **kwargs):
'''
# Testingyoyo - delete below
###################################################33
# old format:
###################################################33
# run_dir - current run directory - e.g. /tmp/ondisktmp/abc123/
# working_subdir - e.g. var/tmp/ansible
# app_dir - app directory is run_dir + working directory - e.g. /tmp/ondisktmp/abc123/var/tmp/ansible
###################################################33
# new format:
###################################################33
# run_dir -> exec_base_dir - e.g. /tmp/ondisktmp/abc123
# app_dir -> exec_dir - e.g. /tmp/ondisktmp/abc123/var/tmp/ansible
# working_subdir -> app_dir - e.g. var/tmp/ansible
# share_dir - share directory with docker or execution container - e.g. /var/tmp/share
# run_share_dir - share directory with stateful_id - e.g. /var/tmp/share/ABC123
'''
self.classname = 'ResourceCmdHelper'
self.logger = ElasticDevLogger(self.classname)
self.logger.debug("Instantiating %s" % self.classname)
self.cwd = os.getcwd()
self.exec_base_dir = os.environ.get("EXEC_BASE_DIR")
if not self.exec_base_dir: self.exec_base_dir = os.getcwd()
# must exists as environmental variables
self.must_exists = kwargs.get("must_exists", [])
self._set_stateful_params(**kwargs)
self._set_app_params(**kwargs)
self._set_exec_dir(**kwargs)
# by default, we set template_dir relative to the app_dir
# this can be over written by the inheriting class
self.template_dir = None
self.resources_dir = None
if hasattr(self, "exec_dir") and self.exec_dir:
self.template_dir = "{}/_ed_templates".format(self.exec_dir)
# ref 34532045732
self.resources_dir = os.path.join(self.exec_dir, "ed_resources")
self._set_docker_settings(**kwargs)
self._set_destroy_env_vars(**kwargs)
self._set_os_env_prefix(**kwargs)
self._get_docker_env_filepath()
self.output = []
def get_hash(data):
'''determines the hash of a data object'''
logger = ElasticDevLogger("get_hash")
try:
calculated_hash = hashlib.md5(data).hexdigest()
except:
logger.debug("Falling back to shellout md5sum for hash")
calculated_hash = shellout_hash(data)
if not calculated_hash:
logger.error("Could not calculate hash for %s" % data)
return False
return calculated_hash
def __init__(self, **kwargs):
ResourceCmdHelper.__init__(self)
self.classname = 'GcloudCli'
self.logger = ElasticDevLogger(self.classname)
self.logger.debug("Instantiating %s" % self.classname)
self.file_config = None
self.file_config_loc = None
self.tempdir = None
self.resource_tags_keys = [
"tags", "name", "schedule_id", "job_instance_id", "job_id"
]
self.share_dir = os.environ.get("SHARE_DIR", "/var/tmp/share")
self.stateful_dir = os.path.join(self.share_dir, id_generator(8))
self.docker_image = "google/cloud-sdk"
self.output = []
def __init__(self, **kwargs):
self.tmpdir = kwargs.get("tmpdir")
if not self.tmpdir: self.tmpdir = "/tmp"
self.subdir = kwargs.get("subdir", "ondisktmp")
if self.subdir:
self.basedir = "{}/{}".format(self.tmpdir, self.subdir)
else:
self.basedir = self.tmpdir
self.classname = "OnDiskTmpDir"
mkdir("/tmp/ondisktmpdir/log")
self.logger = ElasticDevLogger(self.classname)
if kwargs.get("init", True): self.set_dir(**kwargs)
def convert_str2json(_object, exit_error=None):
if isinstance(_object, dict): return _object
if isinstance(_object, list): return _object
logger = ElasticDevLogger("convert_str2json")
try:
_object = json.loads(_object)
#logger.debug("Success: Converting str to a json")
return _object
except:
pass
#logger.debug("Cannot convert str to a json. Will try to eval")
try:
_object = eval(_object)
#logger.debug("Success: Evaluating str to a json")
return _object
except:
#logger.debug("Cannot eval str to a json.")
if exit_error: exit(13)
return False
return _object