def _start_from_sources(self):
service_data = self.context.service_data(self.service_name)
assets_path = self.context.application.workspace + service_data["location"]
cmd_with_params = self.service_data["sources"]["cmd"]
force_pushdir(assets_path)
run_from_file = open("RUNNING_FROM", 'w')
run_from_file.write(self.run_from)
run_from_file.close()
makedirs_if_not_exists("logs")
seconds_remaining = SmPythonServiceStarter.PROCESS_STARTUP_TIMEOUT_SECONDS
with open("logs/stdout.txt", "wb") as out, open("logs/stderr.txt", "wb") as err:
subprocess.Popen(cmd_with_params, shell=False, env=os.environ.copy(), stdout=out, stderr=err, close_fds=True)
while seconds_remaining > 0 and not len(SmProcess.processes_matching("grunt")) > 0:
time.sleep(1)
seconds_remaining -= 1
if seconds_remaining < 10 or seconds_remaining % 5 == 0:
self.log("Waiting for Assets service to start: %s second%s before timeout" % (
seconds_remaining, "s" if seconds_remaining > 1 else ""))
if len(SmProcess.processes_matching("grunt")) == 1:
process = SmProcess.processes_matching("grunt")
for i, v in enumerate(process):
return v.pid
def _get_process_ids_for_processes_matching(regex):
command = "ps -eo pid,args | grep '%s' |grep -v 'grep %s' | awk '{print $1}'" % (
regex, regex)
ps_command = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
stdout, stderr = ps_command.communicate()
return map(int, stdout.split("\n")[:-1])
def clone_repo_if_required_raw(service_name, repo, path, context):
path_exists = True
if not os.path.exists(path):
path_exists = False
context.log("Source code for %s is missing, cloning..." % service_name)
os.chdir(context.application.workspace)
command = 'git clone %s' % repo
ps_command = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE)
ps_command.communicate()
if ps_command.returncode is not 0:
print b.fail + "ERROR: Unable to clone repo for '" + service_name + "'" + b.endc
if os.path.exists(path):
path_exists = True
print service_name + " - cloned"
else:
# TODO Should this just throw an exception?
context.log("Could not go to dir: " + path +
" do you have the project: " + service_name +
" checked out?")
elif os.path.exists(path + "/.git"):
#TODO: make non os dependent
print service_name + " - source exists"
else:
print b.warning + "WARNING: Nothing was cloned for '" + service_name + "' folder '" + path + "' already exists and it does not contain a repo" + b.endc
return path_exists
def _start_from_binary(self):
assets_target_path = self.context.get_microservice_target_path(
self.service_name)
assets_path = self.context.application.workspace + self.service_data[
"location"]
force_chdir(assets_path)
remove_if_exists("RUNNING_FROM")
force_chdir(assets_target_path)
if not self.context.offline:
nexus = SmNexus(self.context, self.service_name)
versions = nexus.get_all_versions(self.version, self.run_from)
for version in versions:
nexus.download_jar_if_necessary(self.run_from, version)
self._unzip_assets(versions)
cmd_with_params = self.service_data["binary"]["cmd"]
makedirs_if_not_exists("logs")
with open("logs/stdout.txt",
"wb") as out, open("logs/stderr.txt", "wb") as err:
return subprocess.Popen(cmd_with_params[0].split(),
shell=False,
env=os.environ.copy(),
stdout=out,
stderr=err,
close_fds=True).pid
def _is_pid_in_list(pid, command):
ps_command = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
ps_output = ps_command.stdout.read()
ps_pid_str = ps_output.strip()
for process in ps_pid_str.split("\n"):
if process and int(process) == pid:
return True
return False
def _is_command_running(command):
ps_command = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
ps_output = ps_command.stdout.read()
if u"\n" in ps_output:
raise MultipleProcessesRunningException(u"%s returned more than one process" % command)
ps_pid_str = ps_output.strip()
if ps_pid_str and int(ps_pid_str) == pid:
return True
return False
def processes_matching(regex):
command = "ps -eo ppid,pid,etime,rss,args | egrep -e '%s' | grep -vi 'grep -e'" % regex
ps_command = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
stdout, stderr = ps_command.communicate()
ps_output = stdout.split("\n")[:-1]
def process_line_to_object(process_line):
values = process_line.strip().split()
return SmProcess(int(values[0]), int(values[1]), values[2], values[3], values[4:])
ret = map(process_line_to_object, ps_output)
return ret
def all_processes(cls):
command = "ps -eo ppid,pid,etime,rss,args"
ps_command = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE)
stdout, stderr = ps_command.communicate()
ps_output = stdout.split("\n")[:-1]
def process_line_to_object(process_line):
values = process_line.strip().split()
return SmProcess(int(values[0]), int(values[1]), values[2],
values[3], values[4:])
return map(process_line_to_object, ps_output[1:])
def start(self):
try:
location = "."
if "location" in self.service_data:
location = self.service_data["location"]
microservice_path = os.path.join(
self.context.application.workspace, location)
os.chdir(microservice_path)
return subprocess.Popen(" ".join(self.get_start_command()),
cwd=microservice_path,
env=os.environ.copy(),
shell=True).pid
except Exception, e:
self.log("Could not start service due to exception: " + str(e))
def _get_service_name_for_pid(pid):
command = "ps -p %s -o command,args" % pid
ps_command = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
stdout, stderr = ps_command.communicate()
ps_output = stdout.split()
p = re.compile('-Dservice\.manager\.serviceName=([A-Z_]+)')
for i in ps_output:
t = p.match(i)
if t:
return t.group(1)
return None
def stop(self):
ps_command = "ps axo pid,command | grep '%s' | grep -v 'grep' | awk '{print $1}'" % SmPythonService.get_pattern(self)
ps = subprocess.Popen(ps_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pid_values = map(int, ps.stdout.read().split("\n")[:-1])
if len(pid_values) == 0:
return
self.log("Killing service '%s' (pid = %s)..." % (self.service_data["name"], str(pid_values)))
for pid_int in pid_values:
os.kill(pid_int, signal.SIGINT)
self.log("PID %d killed" % pid_int)
def waitForCondition(self, f, expected, time_out_secs=default_time_out):
dead_line = time.time() + time_out_secs
value = None
while (time.time() < dead_line):
value = f()
if value == expected: return
time.sleep(0.1)
command = "ps -eo ppid,pid,etime,rss,args"
ps_command = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE)
stdout, stderr = ps_command.communicate()
print(stdout)
self.assertEquals(value, expected)
def _get_running_process_args(context, service_name):
service = context.get_service(service_name)
pattern = service.get_pattern()
if service.is_started_on_default_port():
command = "ps -eo args | egrep '%s' | egrep -v 'egrep %s' | awk '{{print } }'" % (
pattern, pattern)
ps_command = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE)
ps_output = ps_command.stdout.read()
if ps_output:
return ps_output.split("\n")[:-1][0]
return ""
def stop(self, wait=False):
ps_command = "ps axo pid,command | grep '%s' | grep -v 'grep' | awk '{print $1}'" % SmPythonService.get_pattern(
self)
ps = subprocess.Popen(ps_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pid_values = map(int, ps.stdout.read().split("\n")[:-1])
if len(pid_values) == 0:
return
self.log("Stopping '%s'..." % (self.service_data["name"]), True)
for pid_int in pid_values:
kill_pid(self.context, pid_int, wait=wait)
self.log("PID %d killed" % pid_int, True)
def _start_from_binary(self):
assets_target_path = self.context.get_microservice_target_path(
self.service_name)
assets_path = self.context.application.workspace + self.service_data[
"location"]
force_chdir(assets_path)
remove_if_exists("RUNNING_FROM")
force_chdir(assets_target_path)
if not self.context.offline:
artifactory = SmArtifactory(self.context, self.service_name)
if self.version:
versions = [self.version]
elif self.context.assets_versions:
versions = self.context.assets_versions
self.log(
"Starting assets versions: %s" % (", ".join(versions)),
True)
else:
versions = artifactory.find_all_versions(self.run_from)
self.log(
"Starting assets versions: %s" % (", ".join(versions)),
True)
for version in versions:
self.context.log("\nStarting assets version: %s" % version)
artifactory.download_jar_if_necessary(self.run_from, version)
self._unzip_assets(versions)
cmd_with_params = self.service_data["binary"]["cmd"]
makedirs_if_not_exists("logs")
with open("logs/stdout.txt",
"wb") as out, open("logs/stderr.txt", "wb") as err:
return subprocess.Popen(cmd_with_params[0].split(),
shell=False,
env=os.environ.copy(),
stdout=out,
stderr=err,
close_fds=True).pid
def pull_rebase_repo(context, name, project_info):
if "sources" in project_info and "repo" in project_info["sources"]:
print "pulling '" + name + "' from repo '" + project_info["sources"][
"repo"] + "'"
path = context.application.workspace + project_info["location"]
if not os.path.exists(path):
print b.fail + "Nothing was pulled, it appears you have not cloned this repo yet: '" + name + "'" + b.endc + "\n"
else:
os.chdir(path)
command = 'git pull --rebase'
print "running: '" + command + "' from: '" + os.getcwd() + "'"
ps_command = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE)
stdout, stderr = ps_command.communicate()
if ps_command.returncode is not 0:
print b.fail + "Nothing was pulled, see output for: '" + name + "'" + b.endc
else:
print b.okgreen + "Pulled: '" + name + "'" + b.endc
print stdout
