def put_data(data, dest, container, append=False, user=None, perms=None):
if append and not path_exists(dest, container):
docker_exec(container, 'touch {}'.format(dest), user=user)
cmd = ''.join(('echo "', data, '" >>' if append else '" >', dest))
docker_exec(container, "/bin/sh -c '{}'".format(cmd), user=user, raises=True)
if perms:
utils.command('docker exec {} chmod {} {}'.format(container, perms, dest))
def put_file(source, dest, container, user=None, perms=None):
docker_cmd = 'docker cp {} {}:{}'.format(source, container, dest)
utils.command(docker_cmd, raises=True)
if user:
path_set_user(dest, user, container)
if perms:
set_permissions(dest, perms, container)
def collect_all_logs(self):
"""
Collect NNPI SysLog
"""
if not check_card_connection():
return False
# Collect from host
self.realtime_logs.add(utils.command(utils.get_command("dmesg"), log="host_dmesg"))
self.realtime_logs.add(utils.command(utils.get_command("cat /var/log/messages"), log="host_messages"))
# Collect from card
num_of_device = get_num_of_devs()
for dev in range(0, num_of_device):
with open(os.path.join(self.output_path, "sph.log_{}".format(str(dev))), 'a') as sphlog_file:
_, stdout = commands.getstatusoutput(utils.get_command("/opt/intel_nnpi/bin/nnpi_ctl log -history {} -devices {}".format(self.logh, dev)))
sphlog_file.write(stdout)
if len(stdout) > 0:
sphlog_file.write("\n")
with open(os.path.join(self.output_path, "messages_{}.log".format(str(dev))), 'a') as messages_file:
_, stdout = commands.getstatusoutput(utils.get_command("/opt/intel_nnpi/bin/nnpi_ctl log -sys_history {} -devices {}".format(self.logh, dev)))
messages_file.write(stdout)
messages_file.write("\n")
self.realtime_logs.add(utils.command(utils.get_command("/opt/intel_nnpi/bin/nnpi_ctl log -crash -devices {}".format(dev)), log="crash_{}.log".format(str(dev))))
for log in self.realtime_logs:
log.run(self.output_path)
self.logh = self.logh + 1
logger.info("log history: {}".format(self.logh))
def merge_yaml(self):
"""Function to generate merged yaml config based on the specs provided """
spec_file = OpenShiftComponent.retrieve_config_file(self, self.subdir, self.patch_config)
if not spec_file:
logging.info(f'{self.patch_config} file must be provided in the /projects/buildscript/scripts/configuration/{self.subdir} folder')
return 1
with open(spec_file) as spec_fp:
spec = spec_fp.read()
try:
spec_yaml = yaml.safe_load(spec)
except yaml.constructor.ConstructorError:
logging.error(f"Could not load {spec_file}. Please make sure that patch config file is in a valid yaml format.")
return 1
if not isinstance(spec_yaml, dict) or 'baseline-config' not in spec_yaml.keys():
logging.error(f'{spec_file} must specify a "baseline-config: <file>" directive')
return 1
if 'merge-rule' not in spec_yaml.keys():
logging.error(f'{spec_file} must specify "merge-rule: append|overwrite|merge" directive')
return 1
baseline_config = spec_yaml.pop('baseline-config')
# override baseline_config if user has explicitly provided one
if self.base_config:
self.base_config = os.path.join(os.path.dirname(spec_file), self.base_config)
baseline_file = OpenShiftComponent.retrieve_config_file(self, self.subdir, baseline_config, config_file=self.base_config)
if baseline_file is None:
logging.error(f'Unable to find a configuration file for {baseline_config}!')
return 1
logging.info(f'Using {baseline_file} as the base configuration')
merge_rule = spec_yaml.pop('merge-rule')
# create temp file for the patch-config
patch_file = f'{baseline_config}-patch'
with open(patch_file, 'w') as patch_fp:
yaml.safe_dump(spec_yaml, patch_fp,
encoding=('utf-8'), default_flow_style=False, allow_unicode=True)
if merge_rule == APPEND:
config, stderr, _ = utils.command(f'yq merge --append {baseline_file} {patch_file}')
elif merge_rule == OVERWRITE:
config, stderr, _ = utils.command(f'yq merge --overwrite {baseline_file} {patch_file}')
else:
config, stderr, _ = utils.command(f'yq merge {baseline_file} {patch_file}')
logging.error(stderr)
os.remove(patch_file)
with open(f'{self.patch_config}_merged', 'w') as merged_fp:
merged_fp.write(config)
logging.info(f'{self.patch_config}_merged file has been created with merged contents')
return 0
def mcq(self, f1, f2):
cmd = 'java -cp %s/mcq.ws.client-0.0.1-SNAPSHOT-jar-with-dependencies.jar pl.poznan.put.mcq.ws.client.Global -m %s -t %s >mcq.log' % (
BIN_DIR, f1, f2)
utils.command(cmd)
try:
v = float(open('mcq.log').read().strip())
except Exception:
v = 0
return v
def handle_clone():
'''
克隆所有项目到本地
'''
project_list = get_projects()
for project in project_list:
url = project['url']
folder = create_project_path(project['id'])
utils.command(
"git clone " + url + " " + folder
)
def isManaged(self):
"""Check if product is managed with bundleman."""
ret, output = command('svn pl %s | grep "bundleman"' % self.fpath,
do_raise=False)
if ret:
return False
return True
def has_subtitles(mkv_file):
"""
does the mkv file have subtitles?
"""
info = command(["mkvinfo", mkv_file])
sub_lines = []
for line_no, line in enumerate(info):
if "Track type" in line and "subtitles" in line:
sub_lines.append(line_no)
if not sub_lines:
return False
track_nos = []
for sub_line in sub_lines:
for line in info[sub_line:0:-1]:
if "Track number" in line:
track_nos.append(int(line.split(":").pop().strip()))
break
#extract the subtitle to a file
sub_sizes = []
for track_no in track_nos:
sub_out = "/tmp/subs.srt"
run(["mkvextract", "tracks", mkv_file, "%s:%s" % (track_no, sub_out)])
sub_sizes.append(path(sub_out).size)
for sub_size in sub_sizes:
#subtitle is too short, it's just the one liner
if sub_size < 500:
#remove the subtitle since they're meaningless
remove_subtitles(mkv_file)
return False
return True
def get_version(rel_url, ext_path, refs):
if 'tag' in refs:
return
if git_externals[rel_url]["vcs"] == "svn":
revision = command('svnversion', '-c').strip()
match = re_from_svnversion.search(revision)
if match:
revision = "svn:r" + match.group(2) # 565:56555 -> svn:r56555
else:
message = git("log", "--format=%b", "--grep", "git-svn-id:",
"-1")
match = re_from_git_svn_id.search(message)
if match:
revision = "svn:r" + match.group(1)
else:
error(
"Unsupported external format, should be svn or git-svn repo"
)
else:
branch_name = current_branch()
remote_name = git("config", "branch.%s.remote" % branch_name)
revision = git("log", "%s/%s" % (remote_name, branch_name), "-1",
"--format=%H")
info("Freeze {0} at {1}".format(rel_url, revision))
git_externals[rel_url]["ref"] = revision
def addChangelog(self, tag1, tag2):
"""Add a global CHANGELOG.txt file to the tag2 url."""
logger.info('Adding a global CHANGELOG.txt')
try:
changelog = self.getChangelog(tag1, tag2)
except ValueError:
return -1
f = NamedTemporaryFile('w+', prefix='bm-')
f.write(changelog)
f.flush()
tag_url = computeBundleTagUrl(self.bundle_url, tag2)
command('svn import %s %s/CHANGELOG.txt '
'-m"bundelman add CHANGELOG.txt to %s"' %
(f.name, tag_url, tag_url))
f.close()
return 0
def docker_network(name, cmd='create', raises=True):
allowed = ('create', 'remove')
if cmd not in allowed:
raise RuntimeError("Network command must be in {}, found {}".format(allowed, cmd))
ret = utils.command('docker network {} {}'.format(cmd, name))
if ret and raises:
raise RuntimeError("Could not {} network {}".format(cmd, name))
def get_version(rel_url, ext_path, refs):
if 'tag' in refs:
return
bare_svn = False
if git_externals[rel_url]["vcs"] == "svn":
revision = command('svnversion', '-c').strip()
match = re_from_svnversion.search(revision)
if match:
revision = "svn:r" + match.group(2) # 565:56555 -> svn:r56555
bare_svn = True
else:
message = git("log", "--format=%b", "--grep", "git-svn-id:", "-1")
match = re_from_git_svn_id.search(message)
if match:
revision = "svn:r" + match.group(1)
else:
here = os.path.relpath(os.getcwd(), repo_root)
error("Unsupported external format, svn or git-svn repo expected:\n\t{}".format(here))
else:
branch_name = current_branch()
remote_name = git("config", "branch.%s.remote" % branch_name)
revision = git("log", "%s/%s" % (remote_name, branch_name), "-1", "--format=%H")
info("Freeze {0} at {1}".format(rel_url, revision))
if messages and not bare_svn:
old = resolve_revision(git_externals[rel_url]["ref"])
new = resolve_revision(revision)
git("log", "--format=- %h %s", "{}..{}".format(old, new), capture=False)
git_externals[rel_url]["ref"] = revision
def isManaged(self):
"""Check if product is managed with bundleman."""
ret, output = command('svn pl %s | grep "bundleman"' % self.fpath,
do_raise=False)
if ret:
return False
return True
def main():
banner()
logger.info("Initializing...")
logger.info("Starting base docker")
container = create_container()
if not container:
return
host_port = docker_host_port(
CURRENT_CONTAINER)[HONEYPOT_DOCKER_SERVICE_PORT]
logger.info("Base docker has started")
logger.info("Creating initial iptables rules...")
local_ip = get_local_ip(INTERFACE)
out, ok = command("iptables -t nat -A PREROUTING -p tcp "
f"-d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
f"-j DNAT --to { local_ip }:{ host_port }")
if not ok:
return
out, ok = command(f"iptables -A INPUT -p tcp -i { INTERFACE } "
"--dport 22 -m state --state NEW,ESTABLISHED "
"-j ACCEPT")
if not ok:
return
out, ok = command(f"iptables -A OUTPUT -p tcp -o { INTERFACE } "
"--sport 22 -m state --state ESTABLISHED "
"-j ACCEPT")
if not ok:
return
out, ok = command(
f"iptables -A OUTPUT -p tcp --tcp-flags SYN,ACK SYN,ACK",
["-j", "LOG", "--log-prefix", "Connection established: "])
if not ok:
return
logger.info("Rules created. Honeydock is ready to go. :)")
handler = EventHandler(KERN_LOG_PATH)
watch_manager = WatchManager()
watch_manager.add_watch(handler.file_path, IN_MODIFY)
notifier = Notifier(watch_manager, handler)
notifier.loop()
def minify(fn):
filetypes = ['css', 'js', 'html']
_, ending = fn.rsplit('.', 1)
assert ending in filetypes
minifier = os.path.join(os.path.dirname(__file__), 'tools', 'minify_%s_bin' % ending)
print 'Minifying %s' % fn
stdout, stderr = command('%s %s %s' % (minifier, fn, fn))
def docker_run(image, container, host=None, parameters=None):
cmd = 'docker run -d '
cmd += '--name {} '.format(container)
cmd += '-h {} '.format(host or container)
if parameters:
cmd += parameters + ' '
cmd += image
print(utils.yellow(cmd))
return not utils.command(cmd)
def optimize_pngs(fns):
'''Takes a root directory path and a list of *.png filenames and replaces each file with
an optimzed version.'''
cwd = os.path.dirname(__file__)
optimize_bin = os.path.join(cwd, 'tools', 'optimize_image_bin')
for fn in fns:
print 'Optimizing %s' % fn
stdout, stderr = command('%s %s %s' % (optimize_bin, fn, fn))
def run_command(rel_url, ext_path, targets):
try:
info("External {}".format(get_repo_name(rel_url)))
output = decode_utf8(command(*subcommand))
info("Ok: CWD: {}, cmd: {}".format(os.getcwd(), subcommand))
echo(output)
except CommandError as err:
info("Command error {} CWD: {}, cmd: {}".format(err, os.getcwd(), subcommand))
error(str(err), exitcode=err.errcode)
def run_command(rel_url, ext_path, targets):
try:
info("External {}".format(get_repo_name(rel_url)))
output = decode_utf8(command(*subcommand))
info("Ok: CWD: {}, cmd: {}".format(os.getcwd(), subcommand))
echo(output)
except CommandError as err:
info("Command error {} CWD: {}, cmd: {}".format(
err, os.getcwd(), subcommand))
error(str(err), exitcode=err.errcode)
async def status(self, chan, src, msg):
"""
:name: sysinfo
:hook: cmd
:help: sysinfo - retrieve info for the tilde.team server
:args:
:aliases:
"""
res = utils.command(["bin/sysinfo"], "")
await self.msg(modname, chan, [f"~team status: {res}"])
def getSvnUrl(self):
"""Extract the svn url from the product path."""
status, output = command("svn info " + self.fpath + " | grep '^URL'",
do_raise=False)
if not status and len(output) == 1:
url_line = output[0].split(':', 1)
if url_line[0] == 'URL':
return url_line[1].strip()
msg = 'Invalid product path %s, or svn url.' % self.fpath
logger.error(msg)
raise ValueError(msg)
def docker_exec(container, cmd, user=None, stdout_only=True, return_code_only=False, raises=False):
docker_cmd = 'docker exec -i {} {} {}'.format('-u {}'.format(user) if user else '', container, cmd)
if return_code_only:
return utils.command(docker_cmd)
dock = utils.Command(docker_cmd)
if raises and dock.returncode:
raise RuntimeError(
"Error while executing <{}>: [{}]".format(docker_cmd, dock.stderr.strip() or dock.returncode))
if stdout_only:
return dock.stdout.strip()
return dock
def docker_rm(container_list: List[str]) -> None:
"""Remove all docker containers
:param container_list: docker container id list
"""
cmd = "docker rm"
out, ok = command(cmd, container_list)
if ok:
print("Containers removed!\n", out)
else:
print("Error removing containers!")
def getSvnUrl(self):
"""Extract the svn url from the product path."""
status, output = command(
"svn info " + self.fpath + " | grep '^URL'",
do_raise=False)
if not status and len(output) == 1:
url_line = output[0].split(':', 1)
if url_line[0] == 'URL':
return url_line[1].strip()
msg = 'Invalid product path %s, or svn url.' % self.fpath
logger.error(msg)
raise ValueError(msg)
def get_endpoints():
listing = []
endpoints = utils.command(['vpn', 'list'], silent=True).split("\n")
for endpoint in endpoints:
nb, title = endpoint.split(": ", 2)
listing.append(MenuItem(title=title, nb=nb, action='start'))
return listing
def docker_stop(container_list: List[str]) -> None:
"""Stop all docker containers
:param container_list: docker container id list
"""
cmd = "docker stop"
out, ok = command(cmd, container_list)
if ok:
print("Containers stopped!\n", out)
else:
print("Error stopping containers!")
def getSvnUrl(self):
"""Extract the svn url from the bundle path."""
status, output = command(
"svn info " + self.bundle_path + " | grep '^URL'",
do_raise=False)
if status:
return None
if len(output) == 1:
url_line = output[0].split(':', 1)
if url_line[0] == 'URL':
return url_line[1].strip()
logger.warning('URL not found, invalid output: [%s]' % output)
return None
def getSvnLastChangedRevision(self, url=None):
"""Extract from the svn repository the latest changed revision."""
if url is None:
url = self.url
status, output = command("svn info %s | grep '^Last Changed Rev: '" %
url, do_raise=False)
if status:
return None
if len(output) == 1:
rev_line = output[0].split(':')
return rev_line[1].strip()
logger.warning('Invalid output: [%s]' % output)
return None
def isValidBundlePath(bundle_path):
"""Check that the bundle_path is valid."""
if not bundle_path:
return False
if not os.path.exists(os.path.join(bundle_path, '.svn')):
return False
ret, output = command('svn pl %s | grep "svn:externals"' % bundle_path,
do_raise=False)
if ret:
return False
if len(output) != 1:
return False
return True
def docker_cleaner() -> None:
"""Stop and remove all docker containers"""
cmd = "docker ps -aq"
out, ok = command(cmd)
if ok:
container_list = out.split()
print("Stopping all running docker containers...")
docker_stop(container_list)
print("Removing all docker containers...")
docker_rm(container_list)
else:
print("Error getting container id list!")
def getSvnLastChangedRevision(self, url=None):
"""Extract from the svn repository the latest changed revision."""
if url is None:
url = self.url
status, output = command("svn info %s | grep '^Last Changed Rev: '" %
url,
do_raise=False)
if status:
return None
if len(output) == 1:
rev_line = output[0].split(':')
return rev_line[1].strip()
logger.warning('Invalid output: [%s]' % output)
return None
def get_existing_package( workdir ):
msg = "Listing rpm files"
cmd = "rpm -qp --queryformat '%{NAME}'"
packages = set()
index = 0
rpm_list = glob( workdir + "/*.rpm" )
stdout = None
for rpm in rpm_list:
stdout = command( cmd + rpm, pass_exception = True )
packages.add( stdout )
print(progress( index, len(rpm_list), msg )),
index += 1
print(progress( index, len(rpm_list), msg ))
return packages
def docker_host_port(container: str) -> dict:
"""Get docker container host port
:param container: docker container id
:return: docker container host port
"""
host_post = dict()
cmd = "docker inspect -f"
out, ok = command(cmd, [r"{{json .NetworkSettings.Ports}}", container])
if ok:
port_dict = json.loads(out)
for port in port_dict.keys():
local_port = port.split('/')[0]
host_post[local_port] = port_dict[port][0]['HostPort']
return host_post
def branch(self):
"""Create a branch for all products from a bundle tag."""
release_tag = self.release_tag
hash_tag = getHashTag(release_tag)
bundle_tag_url = computeTagUrl(self.bundle_url, release_tag)
if not bundle_tag_url:
logger.error('Invalid source url: %s' % self.bundle_url)
return -1
bundle_branch_url = bundle_tag_url.replace('/tags/', '/branches/')
logger.info('Branching %s -> %s hash_tag: %s.' %
(bundle_tag_url, bundle_branch_url, hash_tag))
ret, output = command('svn ls %s' % bundle_tag_url, do_raise=False)
if ret:
logger.error('Tag not found. You need to release %s first.' %
release_tag)
return -1
ret, output = command('svn ls %s' % bundle_branch_url, do_raise=False)
if not ret:
logger.error('Branch %s already exists.' % bundle_branch_url)
return -1
products = self.listProducts(bundle_tag_url)
branch_products = []
# create a branch for each product
for product in products:
product_url = product['url']
parent = os.path.dirname(product_url)
ret = -1
if os.path.basename(parent) == 'tags':
ret, output = command('svn ls %s/CHANGES' % product_url,
do_raise=False)
if ret:
# not a versionned product keep it asis
branch_products.append(product)
continue
branch_url = os.path.dirname(parent) + '/branches/' + hash_tag
ret, output = command('svn ls %s' % branch_url, do_raise=False)
if not ret:
logger.warning('Branch %s already exists.' % branch_url)
else:
command('svn copy -m"bundleman branch product %s release %s"'
'-r%s %s %s' %
(product['path'], hash_tag, product['revision'],
product_url, branch_url))
branch_products.append({'path': product['path'],
'url': branch_url})
# create a bundle branch
logger.info('Creating bundle %s' % bundle_branch_url)
createBundle(bundle_branch_url, branch_products, release_tag)
return bundle_branch_url
def get_branch_list(dir):
'''
获取分支列表
'''
# os.system("git pull")
name = "refs/remotes/origin/"
cmd = "git --git-dir " + dir + "/.git for-each-ref --format='%(refname)'"
print(cmd)
print(dir)
l = utils.command(cmd)
branch = l.split()
branch = filter(
lambda x: name in x and "HEAD" not in x,
branch
)
branch = map(lambda x: x[1: -1], branch)
branch = map(lambda x: x.replace(name, ""), branch)
return list(branch)
def main():
text = "what the f**k mate"
print(text)
print("pico")
command("pico2wave -w say.wav \"" + text + "\" && mplayer say.wav")
print("festival")
command("echo \"" + text + "\" | festival --tts")
print("espeak")
command("espeak -ven+f3 -k5 -s150 \"" + text + "\"")
print("google")
say_text(text, 'en')
def getSvnRevision(self):
"""Extract the svn revision of the product path.
Check that all the product is sync with the same revision.
"""
status, output = command("svn info -R " + self.fpath +
" | grep '^Revision' | sort -u",
do_raise=False)
if status:
return None
if len(output) == 1:
rev_line = output[0].split(':')
if rev_line[0] == 'Revision':
return rev_line[1].strip()
if len(output) > 1:
logger.debug('Product [%s] is out of date' % self.path)
else:
logger.warning('Invalid output: [%s]' % output)
return None
def getSvnRevision(self):
"""Extract the svn revision of the product path.
Check that all the product is sync with the same revision.
"""
status, output = command(
"svn info -R " + self.fpath + " | grep '^Revision' | sort -u",
do_raise=False)
if status:
return None
if len(output) == 1:
rev_line = output[0].split(':')
if rev_line[0] == 'Revision':
return rev_line[1].strip()
if len(output) > 1:
logger.debug('Product [%s] is out of date' % self.path)
else:
logger.warning('Invalid output: [%s]' % output)
return None
def docker_run(image: str,
image_cmd: str = "",
network: str = "",
options: str = "") -> Tuple[str, bool]:
"""Run a container docker
:param image: docker image
:param image_cmd: command for image
:param network: docker network config
:param options: extra options
:return: docker container id
"""
cmd = f"docker run -d -P { network } { options } { image } { image_cmd }"
out, ok = command(cmd)
if out:
print("Container created!\n", out)
return out[:12], True
else:
print("Error creating container!")
return "", False
def _buildArchive(self, archive_dir, url, name, version):
"""Create a tar gz."""
archive_name = '%s-%s.tgz' % (name, version)
archive_path = os.path.join(archive_dir, archive_name)
logger.info('Creating archive: %s' % archive_name)
# extract tag
tmpdir = mkdtemp()
product_path = os.path.join(tmpdir, name)
command('svn -q export %s %s' % (url, product_path))
# cleaning archive
self.prepareProductArchive(product_path, version)
# add MD5SUMS
command('cd %s; find . -type f -not -name MD5SUMS -print0 '
'| xargs -0 md5sum > MD5SUMS' % product_path)
# tarball
command('cd %s; tar czf %s %s' % (tmpdir, archive_path, name))
command('rm -rf %s' % tmpdir)
logger.info('Archive: %s' % archive_path)
return 0
def _buildArchive(self, archive_dir, url, name, version):
"""Create a tar gz."""
archive_name = '%s-%s.tgz' % (name, version)
archive_path = os.path.join(archive_dir, archive_name)
logger.info('Creating archive: %s' % archive_name)
# extract tag
tmpdir = mkdtemp()
product_path = os.path.join(tmpdir, name)
command('svn -q export %s %s' % (url, product_path))
# cleaning archive
self.prepareProductArchive(product_path, version)
# add MD5SUMS
command('cd %s; find . -type f -not -name MD5SUMS -print0 '
'| xargs -0 md5sum > MD5SUMS' % product_path)
# tarball
command('cd %s; tar czf %s %s' % (tmpdir, archive_path, name))
command('rm -rf %s' % tmpdir)
logger.info('Archive: %s' % archive_path)
return 0
def get_version(rel_url, ext_path, refs):
if 'tag' in refs:
return
bare_svn = False
if git_externals[rel_url]["vcs"] == "svn":
revision = command('svnversion', '-c').strip()
match = re_from_svnversion.search(revision)
if match:
revision = "svn:r" + match.group(2) # 565:56555 -> svn:r56555
bare_svn = True
else:
message = git("log", "--format=%b", "--grep", "git-svn-id:",
"-1")
match = re_from_git_svn_id.search(message)
if match:
revision = "svn:r" + match.group(1)
else:
here = os.path.relpath(os.getcwd(), repo_root)
error(
"Unsupported external format, svn or git-svn repo expected:\n\t{}"
.format(here))
else:
branch_name = current_branch()
remote_name = git("config", "branch.%s.remote" % branch_name)
revision = git("log", "%s/%s" % (remote_name, branch_name), "-1",
"--format=%H")
info("Freeze {0} at {1}".format(rel_url, revision))
if messages and not bare_svn:
old = resolve_revision(git_externals[rel_url]["ref"])
new = resolve_revision(revision)
git("log",
"--format=- %h %s",
"{}..{}".format(old, new),
capture=False)
git_externals[rel_url]["ref"] = revision
def tag(self):
"""Create a bundle tags/release_tag."""
release_tag = self.release_tag
logger.info('Create a bundle tag %s' % release_tag)
# check bundle url
bundle_url = self.bundle_url
tag_url = computeTagUrl(bundle_url, release_tag)
if tag_url is None:
logger.error('Invalid bundle url: %s' % bundle_url)
return -1
ret, output = command('svn ls %s' % tag_url, do_raise=False)
if not ret:
logger.error('Bundle tag %s already exists.' % tag_url)
return -1
# analyze products
self.analyze(force=True)
bad_products = []
products = []
for product in self.products:
if product.status != 'use_tag':
bad_products.append(product)
products.append({'path': product.rpath,
'revision': product.revision,
'url': product.tag_url})
if bad_products:
logger.warning('Sorry found product(s) not ready:')
for product in bad_products:
print str(product)
return -1
# create bundle
logger.info('Creating bundle %s' % tag_url)
createBundle(tag_url, products, release_tag, bundle_url)
return 0
def listProducts(self, bundle_path=None):
"""return the list of products of a bundle.
format: [{'path':value, 'url':value, 'revision':value}, ...]
"""
if bundle_path is None:
bundle_path = self.bundle_path
logger.debug('listProducts of bundle: %s' % bundle_path)
status, output = command('svn pg svn:externals ' + bundle_path)
products = []
if status:
return products
for line in output:
if not line or line.startswith('#'):
continue
ret = line.split()
if not ret:
continue
path = ret[0]
url = ret[-1]
if not (url.startswith('http') or url.startswith('file')):
if line[0] != '#':
logger.warning('Skip invalid svn return line: [%s]' % line)
continue
revision = None
if len(ret) == 3:
revision = ret[1]
if revision.startswith('-r'):
revision = revision[2:]
else:
logger.warning(
'Skipping invalid revision [%s] in line: [%s]' %
(revision, line))
revision = None
products.append({'path':path, 'url':url, 'revision':revision})
return products
def __init__(self, res_dir):
self.sys_dir = self._get_sysdir(res_dir)
self._messages_size = None
self._messages_inode = None
self.loggab = set()
self.boot_log = set()
for cmd in _LOG_PER_BOOT:
self.boot_log.add(utils.command(cmd))
for filename in _FILES_LOG_PER_BOOT:
self.boot_log.add(utils.logfile(filename))
self.before_iter_log = set()
for fname in _FILES_LOG_BEFORE_ITER:
self.before_iter_log.add(
utils.logfile(fname, log=os.path.basename(fname) + '.before'))
self.after_iter_log = set()
for fname in _DEFAULT_FILES_TO_LOG_AFTER_ITERATION:
self.after_iter_log.add(
utils.logfile(fname, log=os.path.basename(fname) + '.after'))
self.loggab.add(utils.command("df -mP", log="df"))
self.loggab.add(utils.command("dmesg -c", log="dmesg", compress=True))
self.loggab.add(
utils.command("cat /tmp/sph.log", log="sph_log", compress=True))
self.loggab.add(utils.command("ls /dev/disk/by-id", log="usb_disk"))
self.boot_log.add(utils.logfile("/proc/cmdline", keyval=True))
self.boot_log.add(
utils.logfile('/sys/class/dmi/id/bios_version',
log='bios_version'))
self.boot_log.add(utils.logfile('/etc/issue', log='host_version'))
self.boot_log.add(utils.logfile('/proc/mounts', log='proc_mounts'))
self.boot_log.add(utils.command("uname -a", log="uname", keyval=True))
def init(self):
"""Create and commit default files."""
logger.info('Initialize bundle %s.' % self.bundle_path)
# check svn tree
url = self.bundle_url
path = self.bundle_path
if not url or os.path.basename(url) != 'trunk':
logger.error(
'Expecting a bundle WCPATH that point to a trunk url.')
return -1
bundle_url = os.path.dirname(url)
for folder in ('tags', 'branches'):
ret, output = command('svn ls %s/%s' % (bundle_url, folder),
do_raise=False)
if ret:
logger.error('Missing folder %s/%s.' % (bundle_url, folder))
return -1
# check files
externals_path = os.path.join(path, SVN_EXTERNALS)
if os.path.exists(externals_path):
logger.warning('Bundle already initialized')
else:
self.promptInitialize()
status, output = command('svn pg svn:externals %s' % path)
if status or not output:
logger.error('No svn:externals defined on %s.' % path)
return -1
logger.info('Creating %s file.' % SVN_EXTERNALS)
f = open(externals_path, 'w+')
f.write('\n'.join(output))
f.close()
command('svn add %s' % externals_path)
command('svn commit -m"bundleman adding %s" %s' % (
SVN_EXTERNALS, externals_path))
return 0
def tagVersion(self):
"""Create a product tag and flush CHANGES, update VERSION."""
if self.status not in ('new_version', 'tag_not_found'):
return 0
logger.info('Tag product %s version %s-%s' %
tuple(self.version_new))
url = self.url
revision = self.revision
tag_url = self.tag_url
prod_name = self.version_new[0]
prod_version = self.version_new[1]
fpath = self.fpath
# check if the tag exists
status, output = command("svn ls %s" % tag_url, do_raise=False)
if not status:
if self.release_again or self.force:
logger.info('Removing exising tag %s' % tag_url)
command("svn remove -m'bundleman replace tag' %s" % tag_url)
else:
logger.error('The new tag %s already exists.' % tag_url)
return -1
# copy src to tag
command("svn copy -m'bundleman tag creation' -r%s %s %s" %
(revision, url, tag_url))
# switch to tag
command("svn switch %s %s" % (tag_url, fpath))
# update VERSION, CHANGES, HISTORY
changes = open(os.path.join(fpath, 'CHANGES')).read()
history = open(os.path.join(fpath, 'HISTORY')).read()
f = open(os.path.join(fpath, 'HISTORY'), 'w+')
header = """===========================================================
Package: %s %s
===========================================================
First release built by: %s at: %s
SVN Tag: %s
Build from: %s@%s
""" % (prod_name, prod_version, os.getenv('USER'),
datetime.now().isoformat()[:19], tag_url, self.url, revision)
f.write(header)
f.write(changes)
f.write('\n')
f.write(history)
f.close()
f = open(os.path.join(fpath, 'CHANGES'), 'w+')
f.write(self.tpl_changes % '')
f.close()
# set up VERSION
f = open(os.path.join(fpath, 'VERSION'), 'w+')
f.write(self.tpl_version % tuple(self.version_new))
f.close()
# commit files
command("svn commit -m'bundleman edit product %s %s in tag' "
"%s/CHANGES %s/HISTORY %s/VERSION" % (
prod_name, prod_version, fpath, fpath, fpath))
# switch back to src
command("svn switch %s %s" % (url, fpath))
# merge stuff
command("svn merge %s@%s %s %s" % (url, revision, tag_url, fpath))
# commit
command("svn commit -m'merging changes from %s' "
"%s/CHANGES %s/HISTORY %s/VERSION" % (
tag_url, fpath, fpath, fpath))
command("svn up %s" % fpath)
logger.info('Tag %s done.' % tag_url)
# update status
self.status = 'use_tag'
self.version = self.version_new
self.version_new = None
self.analyzed = False
self.revision = None
return 0
def init(self):
"""Create and commit default files."""
self.analyze()
logger.info('Initialize product %s.' % self.path)
fpath = self.fpath
# check svn tree
url = self.url
if not url or os.path.basename(self.url) != 'trunk':
logger.error(
'Expecting a working copy path that point to a trunk url.')
return -1
prod_url = os.path.dirname(url)
for folder in ('tags', 'branches'):
ret, output = command('svn ls %s/%s' % (prod_url, folder),
do_raise=False)
if ret:
logger.error('Missing folder %s/%s.' % (prod_url, folder))
return -1
# check files
changes_path = os.path.join(fpath, 'CHANGES')
files_to_add = []
if os.path.exists(changes_path):
logger.debug('CHANGES file already exist.')
else:
self.promptInitialize()
logger.info('Creating CHANGES file.')
f = open(changes_path, 'w+')
msg_first_package = 'First package'
f.write(self.tpl_changes % msg_first_package)
f.close()
files_to_add.append(changes_path)
version_path = os.path.join(fpath, 'VERSION')
if os.path.exists(version_path):
logger.debug('VERSION file already exists.')
else:
self.promptInitialize()
logger.info('Creating VERSION file.')
f = open(version_path, 'w+')
name = self.version[0]
version = self.version[1]
release = self.version[2]
if not name:
name = self.path
self.version[0] = name
if not version:
version = '0.0.0'
self.version[1] = version
if not release:
release = '1'
self.version[2] = release
f.write(self.tpl_version % tuple(self.version))
f.close()
files_to_add.append(version_path)
history_path = os.path.join(fpath, 'HISTORY')
if os.path.exists(history_path):
logger.debug('HISTORY file already exists.')
else:
self.promptInitialize()
logger.info('Creating HISTORY file.')
f = open(history_path, 'w+')
f.write('')
f.close()
files_to_add.append(history_path)
if files_to_add:
command('svn add ' + ' '.join(files_to_add))
if not self.bm_versioned:
logger.info('Add a bundleman svn property.')
command("svn ps bundleman 'manages this product.' %s" % fpath)
files_to_add.append(fpath)
if files_to_add:
command('svn commit -N -m"bundleman init." %s' %
' '.join(files_to_add))
command('svn up %s' % fpath)
self.analyzed = False
self.revision = None
else:
logger.info('Product was already initialized.')
self.bm_versioned = True
return 0
def getStatus(self):
"""Return a status:
new_version: ready to be packaged
use_tag: already tagged
svn_not_uptodate: require a svn up
missing_changes: changes is empty but src differ from last package
tag_not_found: the version tag does not exist
invalid: svn repo not accessible
"""
if not self.revision:
return 'svn_not_uptodate'
if not self.force:
# do not accept locally modified by default, do not check externals
status, output = command("svn status --ignore-externals %s "
" | grep -v '^[?X]'" %
self.fpath, do_raise=False)
if output:
logger.debug('Found some locally modified: %s' % output)
return 'svn_not_uptodate'
if not self.bm_versioned:
# do not touch a product not versioned with bundleman
return 'use_tag'
co_url = self.url
package_url = self.computeTagUrl()
if co_url != package_url:
try:
last_changed_rev = int(self.getSvnLastChangedRevision())
revision = int(self.revision)
except (TypeError, ValueError):
return 'invalid'
if last_changed_rev > revision:
# not in sync with the trunk or branch
# prevent to release with a wc not up to date
logger.debug("Svn repository contains changes at r%d "
"the wc is at r%d and requires a svn up." % (
last_changed_rev, revision))
return 'svn_not_uptodate'
if co_url != package_url and filter(None, self.changes):
return 'new_version'
if co_url == package_url:
return 'use_tag'
# check if the tag exists
status, output = command("svn ls %s" % package_url, do_raise=False)
if status:
return 'tag_not_found'
# check if the latest package is identical to our co
status, output = command("svn diff %s@%s %s" %
(co_url, self.revision,
package_url), do_raise=False)
if status:
return 'invalid'
if output:
if self.release_again:
return 'new_version'
return 'missing_changes'
return 'use_tag'
def put_directory(source, dest, container):
docker_exec('mkdir -p {}'.format(dest), container, raises=True)
with utils.cd(source):
ret = utils.command('tar zc * | docker exec -i {} tar zx -C {}'.format(container, dest))
if ret:
raise RuntimeError("Error while copying {} to {}:{}".format(source, container, dest))
def container_stop(*container):
ret = True
for cont in container:
ret &= not utils.command('docker stop ' + cont)
return ret
def container_delete(*container):
ret = True
for cont in container:
ret &= not utils.command('docker rm ' + cont)
return ret
def image_delete(image):
return not utils.command('docker rmi ' + image)
