def test_execute_shell_handles_errors(check_output_mock):
check_output_mock.side_effect = check_output_error_func
try:
utils.execute_shell(['expecting error'])
assert False, 'expected CalledProcessError'
except CalledProcessError:
pass
def get_update_date(self):
dates = execute_shell('cd %s; git log -n 1 --pretty=format:%%ci%%n' % self._dpath)
try:
date = datetime.datetime.strptime(dates[-1], '%Y-%m-%d %H:%M:%S %z')
except:
date = self._get_date_zone(dates[-1])
return date
def check_for_uncommitted_files():
print('Checking for uncommitted files...')
result = execute_shell(['git', 'status'])
if not 'nothing to commit' in result:
raise ValueError(
'There are uncommitted files in the workspace. Commit or stash them before trying to publish.'
)
def get_update_date(self):
dates = execute_shell('cd %s; git log -n 1 --pretty=format:%%ci%%n' % self._dpath)
try:
date = datetime.datetime.strptime(dates[-1], '%Y-%m-%d %H:%M:%S %z')
except:
date = self._get_date_zone(dates[-1])
return date
def main(timed_run_params:TimedRunParams, tool_run:ToolRunParams) -> (RunStats, int):
# TODO: add memory limit
logging.info('timed_run.main')
stats_file_name = get_tmp_file_name()
exec_log_file = get_tmp_file_name()
rc, out, err = \
execute_shell('{runsolver} -o {tool_log} -v {stats_file} -w {exec_log} -W {time_limit} '
'{tool_cmd}'
.format(runsolver=RUN_SOLVER_EXEC,
tool_log=tool_run.log_file,
stats_file=stats_file_name,
exec_log=exec_log_file,
time_limit=str(timed_run_params.time_limit_sec),
tool_cmd=tool_run.to_cmd_str()))
logging.info(readfile(exec_log_file))
# TODO: this should also be logged in the DB
assert rc == 0, 'timed run failed: rc={rc}, \nout={out}, \nerr={err}'\
.format(rc=str(rc), out=out, err=err)
tool_rc = get_tool_rc(readfile(exec_log_file))
stats = parse_stats(readfile(stats_file_name))
os.remove(stats_file_name)
os.remove(exec_log_file)
return stats, tool_rc
def calc_sha256(filename):
print 'Calculating the sha256 of the tarball...'
result = execute_shell(['shasum', '-a', '256', filename])
print result
sha256 = result.split(' ')[0]
print sha256
return sha256
def upload_new_brew_formula(content, version, sha):
print 'Uploading the new macprefs formula to https://github.com/clintmod/homebrew-formulas'
token = os.environ['MACPREFS_TOKEN']
auth_header = 'Authorization: token ' + token
json_header = 'Content-Type: application/json'
data = '{"path": "Formula/macprefs.rb", "message": "Updating to version ' + version + '", '
data += '"committer": {"name": "Clint M", "email": "*****@*****.**"}, '
data += '"content": "' + content + '", "branch": "master", "sha":"' + sha + '"}'
with open('github_request.json', 'w') as f:
f.write(data)
commands = [
'curl',
'-i',
'-X',
'PUT',
'-H',
auth_header,
'-H',
json_header,
'-d',
'@github_request.json',
'https://api.github.com/repos/clintmod/homebrew-formulas/contents/Formula/macprefs.rb'
]
result = execute_shell(commands)
if 'Status: 200 OK' not in result:
raise ValueError('Error uploading new brew formula to github - result\n', result)
return data
def generate(self, in_base_path, out_base_path):
self.in_base_path = in_base_path;
self.out_base_path = out_base_path;
utils.makedirs(out_base_path);
imgutils.init(in_base_path);
utils.init(in_base_path);
self.blog = Struct(json.load(utils.open_file(self.in_base_path + "/blog.json")));
# copy static content
cmd = "cp -rf " + in_base_path + "/static/* " + out_base_path;
print("copy static content: " + cmd)
proc = utils.execute_shell(cmd);
# 'dynamic' content
for c in ["sticky", "posts"]:
setattr(self, c, []);
self.generate_content(c);
# home page
self.generate_home();
# feed
self.generate_feed();
def check_answer_with_mc(test_file, result_file, rc, out, err):
if check_answer(test_file, result_file, rc, out, err)[0] != 0:
return check_answer(test_file, result_file, rc, out, err)
if not is_realizable(test_file):
return 0, None, None
tmp_file_name = result_file + '.aag' # because iimc doesn't understand non-aag extensions
assert 0 == execute_shell('cat {result_file} >> {result_file_aag}'.format(result_file=result_file,
result_file_aag=tmp_file_name),
cmd_uses_shell_tricks=True)[0]
rc, out, err = execute_shell(SPEC_FRAMEWORK_DIR + '/check_model.sh ' + tmp_file_name)
os.remove(tmp_file_name)
return rc, out, err
def _get_remote_master(self):
#_rfile = re.sub('git://git.kernel.org', 'http://www.kernel.org', self._url)
#_rfile = _rfile + '/refs/heads/master'
#_tfile = "/tmp/%s-master--" % re.sub('\.', '-', self._name)
#execute_shell('wget -O %s %s --no-check-certificate' % (_tfile, _rfile))
commits = execute_shell("git ls-remote -h %s master | awk '{print $1;}'" % self._url)
if len(commits) == 0:
return None
return commits[0]
def _get_remote_master(self):
#_rfile = re.sub('git://git.kernel.org', 'http://www.kernel.org', self._url)
#_rfile = _rfile + '/refs/heads/master'
#_tfile = "/tmp/%s-master--" % re.sub('\.', '-', self._name)
#execute_shell('wget -O %s %s --no-check-certificate' % (_tfile, _rfile))
commits = execute_shell("git ls-remote -h %s master | awk '{print $1;}'" % self._url)
if len(commits) == 0:
return None
return commits[0]
def is_change_obsoleted(self, fname, diff):
dates = []
days = read_config('patch.obsoleted.days', 30)
for line in find_remove_lines(diff):
dates = execute_shell("cd %s; git log -n 1 -S '%s' --pretty=format:%%ci%%n %s" % (self._dpath, line, fname))
if len(dates) == 0:
continue
dt = datetime.datetime.strptime(' '.join(dates[0].split(' ')[:-1]), "%Y-%m-%d %H:%M:%S")
delta = datetime.datetime.now() - dt
if delta.days < days:
return False
return True
def model_check(file_to_check) -> bool:
# Below 'pi' specifies the property to model check
cmd = "{IIMC} {file} --pi 0".format(IIMC=IIMC_EXEC,
file=file_to_check)
logger.debug('executing ' + cmd)
rc, out, err = execute_shell(cmd)
assert rc == 0, rc_out_err_to_str(rc, out, err)
last_line = tuple(l.strip() for l in out.splitlines() if l.strip())[-1]
assert last_line in ['0', '1'], rc_out_err_to_str(rc, out, err)
return "0" == last_line # the last line "0" means the circuit is correct
def get_changelist(self, scommit, ecommit, update, delta = False):
if scommit == ecommit and len(scommit) != 0:
return []
dateusing = read_config('git.diff.using.datetime', False)
daysdelta = read_config('git.diff.datetime.delta', 1)
if self.is_linux_next():
if dateusing is True and delta is False:
if not isinstance(update, datetime.datetime):
stime = strftime("%Y-%m-%d %H:%M:%S", localtime(time() - 2 * 24 * 60 * 60))
else:
stime = update - datetime.timedelta(days=daysdelta)
lines = execute_shell('cd %s; git log --after="%s" --name-only --format="%%" | sort -u | grep "\w"' % (self._dpath, stime))
return lines
else:
scommit = self.get_stable()
lines = execute_shell('cd %s; git diff --name-only %s...%s' % (self._dpath, scommit, ecommit))
return lines
else:
if len(scommit) == 0 or scommit is None:
scommit = '1da177e4c3f41524e886b7f1b8a0c1fc7321cac2'
lines = execute_shell('cd %s; git diff --name-only %s...%s' % (self._dpath, scommit, ecommit))
return lines
def get_changelist(self, scommit, ecommit, update, delta = False):
if scommit == ecommit and len(scommit) != 0:
return []
dateusing = read_config('git.diff.using.datetime', False)
daysdelta = read_config('git.diff.datetime.delta', 1)
if self.is_linux_next():
if dateusing is True and delta is False:
if not isinstance(update, datetime.datetime):
stime = strftime("%Y-%m-%d %H:%M:%S", localtime(time() - 2 * 24 * 60 * 60))
else:
stime = update - datetime.timedelta(days=daysdelta)
lines = execute_shell('cd %s; git log --after="%s" --name-only --format="%%" | sort -u | grep "\w"' % (self._dpath, stime))
return lines
else:
scommit = self.get_stable()
lines = execute_shell('cd %s; git diff --name-only %s...%s' % (self._dpath, scommit, ecommit))
return lines
else:
if len(scommit) == 0 or scommit is None:
scommit = '1da177e4c3f41524e886b7f1b8a0c1fc7321cac2'
lines = execute_shell('cd %s; git diff --name-only %s...%s' % (self._dpath, scommit, ecommit))
return lines
def do_GET(self):
#print("got request for " + self.path);
#pdb.set_trace();
path = site_dir + self.path;
if (os.path.isdir(path)):
if (path[-1] != "/"):
path += "/"
self.send_response(301);
self.send_header("Location", self.path + "/");
self.end_headers();
return;out
if (self.path == "/feed/"):
path += "feed.xml";
else:
path += "index.html"
if (path.find("index.html") != -1):
print("refresh...");
lock.acquire();
cmd = script_dir + "/generate.py " + skeleton_dir + " " + site_dir;
proc = utils.execute_shell(cmd);
if (proc.returncode != 0):
self.send_response(500)
self.end_headers();
self.wfile.write("out:\n")
self.wfile.write(proc.out)
self.wfile.write("err:\n")
self.wfile.write(proc.err)
lock.release()
return;
lock.release()
try:
fd = open(path);
except:
fd = None
if (fd):
data = fd.read();
self.send_response(200);
self.send_header("Content-type", self.guess_type(path));
self.send_header("Content-length", len(data));
self.end_headers();
self.wfile.write(data);
else:
self.send_response(404);
self.end_headers();
self.wfile.write("cannot find " + path + "\n");
if (self.path == "/"):
self.wfile.write("try reloading...");
def generate_thumbnail(_in, _out_dir):
# XXX: not nice but much faster if we have them generated already
if (os.path.isfile(_out_dir + "/thumbnail_green.png")):
return;
_tmp_square = tempfile.mkstemp(suffix=".png", prefix="mbonnin.net")[1];
_tmp_round = tempfile.mkstemp(suffix=".png", prefix="mbonnin.net")[1];
_tmp_round_bw = tempfile.mkstemp(suffix=".png", prefix="mbonnin.net")[1];
# make input square
proc = utils.execute_shell("convert -resize 256x256^ -gravity center -extent 256x256 " + _in + " " + _tmp_square);
if (proc.returncode != 0):
utils.fatal("could not square thumbnail " + _in +": " + proc.err);
# clip
proc = utils.execute_shell("convert " + in_dir + "/thumbnail_clip_mask.png " + _tmp_square + " -compose src-in -composite " + _tmp_round);
if (proc.returncode != 0):
utils.fatal("could not generate_thumbnail " + _in +": " + proc.err);
# desaturate
desaturate(_tmp_round, _tmp_round_bw);
for color in ["green", "yellow"]:
for bw in ["", "_bw"]:
if (bw == "_bw"):
r = _tmp_round_bw;
else:
r = _tmp_round;
_out = _out_dir + "/thumbnail_" + color + bw + ".png";
proc = utils.execute_shell("convert " + in_dir + "/thumbnail_border_" + color + ".png " + r + " -composite " + _out);
if (proc.returncode != 0):
utils.fatal("could not generate_thumbnail " + _in +": " + proc.err);
os.remove(_tmp_square);
os.remove(_tmp_round);
os.remove(_tmp_round_bw);
def is_change_obsoleted(self, fname, diff):
dates = []
days = read_config('patch.obsoleted.days', 30)
try:
for line in find_remove_lines(diff):
dates = execute_shell("cd %s; git log -n 1 -S '%s' --pretty=format:%%ci%%n %s" % (self._dpath, line, fname))
if len(dates) == 0:
continue
dt = datetime.datetime.strptime(' '.join(dates[0].split(' ')[:-1]), "%Y-%m-%d %H:%M:%S")
delta = datetime.datetime.now() - dt
if delta.days < days:
return False
return True
except:
return True
def download_attachment(attachment):
if (string.find(attachment, "wp-content/upload") != -1):
item_link = item.findtext("link");
attachment = urlparse.urljoin(item_link, attachment);
attachment_name = attachment[string.rfind(attachment, "/"):];
af = self.dir + attachment_name;
#print("download " + attachment + " to " + f);
if (not os.path.isfile(af)):
print("downloading... " + attachment);
proc = utils.execute_shell('wget -O ' + af + ' ' + attachment);
if (proc.returncode != 0):
print("cannot download " + attachment);
print(proc.err);
os.remove(af);
sys.exit(2);
return attachment_name[1:];
def update(self):
if not os.path.exists(self._dpath):
rpath = os.path.dirname(self._dpath)
execute_shell('cd %s; git clone %s' % (rpath, self._url))
else:
if self.is_linux_next():
rmaster = self._get_remote_master()
if rmaster == self._commit:
return False
execute_shell('cd %s; git reset --hard %s' % (self._dpath, self._stable))
execute_shell('cd %s; git pull' % self._dpath)
return False
def update(self):
if not os.path.exists(self._dpath):
rpath = os.path.dirname(self._dpath)
execute_shell('cd %s; git clone %s' % (rpath, self._url))
else:
if self.is_linux_next():
rmaster = self._get_remote_master()
if rmaster == self._commit:
return False
execute_shell('cd %s; git reset --hard %s' % (self._dpath, self._stable))
execute_shell('cd %s; git pull' % self._dpath)
return False
def write_header(self, item, tree):
header = {};
header['title'] = item.findtext("title");
# XPath bullshit: selects the text of the sibling of meta_key element whose textContent is _thumbnail_id
# I cannot use find() as it does not seem to recognize predicates like [.='foo']
thumbnail_ids = item.xpath("*//*[local-name()='meta_key'][.='_thumbnail_id']/following-sibling::*/text()");
if (thumbnail_ids):
#print("select thumbnail " + thumbnail_ids[0]);
thumbnail_urls = tree.xpath("//*[local-name()='post_id'][.='" + thumbnail_ids[0] + "']/parent::*/*[local-name()='attachment_url']/text()");
if (thumbnail_urls):
ext = os.path.splitext(thumbnail_urls[0])[1].lower();
proc = utils.execute_shell('wget -O ' + self.dir + '/thumbnail' + ext + ' ' + thumbnail_urls[0]);
if (proc.returncode != 0):
print("cannot download " + thumbnail_urls[0]);
print(proc.err);
print(proc.out);
header['thumbnail'] = os.path.basename(thumbnail_urls[0]);
self.f.write(json.dumps(header, indent=4));
self.f.write("\n\n");
def get_stable(self):
if self._ncommit is None:
#if os.path.exists(os.path.join(self._dpath, '.git/refs/remotes/origin/stable')):
# commits = execute_shell('cd %s ; cat .git/refs/remotes/origin/stable' % self._dpath)
# self._ncommit = commits[0]
#else:
if True:
logcmd = "git log --author='Linus Torvalds' --pretty='format:%H %s' -n 50"
#print execute_shell("cd %s ; %s" % (self._dpath, logcmd))
for line in execute_shell("cd %s ; %s" % (self._dpath, logcmd)):
_commit = line.split(' ')
if len(_commit) < 2:
continue
if _commit[1] == 'Merge' or _commit[1] == 'Linux':
self._ncommit = _commit[0]
# fake a stable to make build happy
fp = open(os.path.join(self._dpath, '.git/refs/remotes/origin/stable'), "w")
fp.write(self._ncommit)
fp.close()
break
return self._ncommit
def get_stable(self):
if self._ncommit is None:
#if os.path.exists(os.path.join(self._dpath, '.git/refs/remotes/origin/stable')):
# commits = execute_shell('cd %s ; cat .git/refs/remotes/origin/stable' % self._dpath)
# self._ncommit = commits[0]
#else:
if True:
logcmd = "git log --author='Linus Torvalds' --pretty='format:%H %s' -n 50"
#print execute_shell("cd %s ; %s" % (self._dpath, logcmd))
for line in execute_shell("cd %s ; %s" % (self._dpath, logcmd)):
_commit = line.split(' ')
if len(_commit) < 2:
continue
if _commit[1] == 'Merge' or _commit[1] == 'Linux':
self._ncommit = _commit[0]
# fake a stable to make build happy
fp = open(os.path.join(self._dpath, '.git/refs/remotes/origin/stable'), "w")
fp.write(self._ncommit)
fp.close()
break
return self._ncommit
def get_tag(self):
if not os.path.exists(self._dpath):
return None
tags = execute_shell('cd %s; git tag' % self._dpath)
stag = tags[-1]
if re.search(r'-rc\d+$', stag) != None:
tag = re.sub('-rc\d+$', '', stag)
if tags.count(tag) > 0:
stag = tag
for ltag in tags[::-1]:
if re.search(r'v\d.\d\d+', ltag) != None:
lversions = re.sub('-rc\d+$', '', ltag).split('.')
sversions = re.sub('-rc\d+$', '', stag).split('.')
if lversions[0] != sversions[0] or int(sversions[-1]) > int(lversions[-1]):
return stag
else:
tag = re.sub('-rc\d+$', '', ltag)
if tags.count(tag) > 0:
ltag = tag
return ltag
return stag
def get_tag(self):
if not os.path.exists(self._dpath):
return None
tags = execute_shell('cd %s; git tag' % self._dpath)
stag = tags[-1]
if re.search(r'-rc\d+$', stag) != None:
tag = re.sub('-rc\d+$', '', stag)
if tags.count(tag) > 0:
stag = tag
for ltag in tags[::-1]:
if re.search(r'v\d.\d\d+', ltag) != None:
lversions = re.sub('-rc\d+$', '', ltag).split('.')
sversions = re.sub('-rc\d+$', '', stag).split('.')
if lversions[0] != sversions[0] or int(sversions[-1]) > int(lversions[-1]):
return stag
else:
tag = re.sub('-rc\d+$', '', ltag)
if tags.count(tag) > 0:
ltag = tag
return ltag
return stag
def get_tag(self):
if not os.path.exists(self._dpath):
return None
tags = execute_shell("cd %s; git tag" % self._dpath)
stag = tags[-1]
if re.search(r"-rc\d+$", stag) != None:
tag = re.sub("-rc\d+$", "", stag)
if tags.count(tag) > 0:
stag = tag
for ltag in tags[::-1]:
if re.search(r"v\d.\d\d+", ltag) != None:
lversions = re.sub("-rc\d+$", "", ltag).split(".")
sversions = re.sub("-rc\d+$", "", stag).split(".")
if lversions[0] != sversions[0] or int(sversions[-1]) > int(lversions[-1]):
return stag
else:
tag = re.sub("-rc\d+$", "", ltag)
if tags.count(tag) > 0:
ltag = tag
return ltag
return stag
def verify_macprefs():
result = execute_shell(['macprefs', '--version'])
message = '\nworkspace:\t' + __version__ + '\ninstalled:\t' + result
assert __version__ in result, message
print 'version check verified' + message
def test_execute_shell_handles_verbose(check_output_mock):
utils.execute_shell(['asdf'], False, '.', False, True)
def get_commit_by_tag(self, tag):
commits = execute_shell('cd %s; git log -n 1 %s --pretty=format:%%H%%n' % (self._dpath, tag))
return commits[-1]
def get_commit(self):
commits = execute_shell('cd %s; git log -n 1 --pretty=format:%%H%%n' % self._dpath)
return commits[-1]
def run_tool(test_file, result_file):
return execute_shell('./aisy.py ' + test_file + ' --out ' + result_file)
def desaturate(_in, _out):
proc = utils.execute_shell("convert " + _in + " -colorspace Gray " + _out);
if (proc.returncode != 0):
utils.fatal("could not desaturate " + _in +": " + proc.err);
def get_commit(self):
commits = execute_shell('cd %s; git log -n 1 --pretty=format:%%H%%n' % self._dpath)
return commits[-1]
def get_commit_by_tag(self, tag):
commits = execute_shell('cd %s; git log -n 1 %s --pretty=format:%%H%%n' % (self._dpath, tag))
return commits[-1]
def build_file_list():
source = get_app_store_preferences_dir()
command = 'find ' + source + '*/Data/Library/Preferences -type f -name "*.plist"'
result = execute_shell(command, is_shell=True)
files = result.split('\n')
return files
def create_version_tag_and_push(tag):
print 'Tagging git repository with version ' + tag
execute_shell(['git', 'tag', tag])
print 'Pushing the new tag to github...'
execute_shell(['git', 'push', 'origin', 'HEAD', '--tags'])
def ensure_less_than_one_megapixel(_in, _out):
proc = utils.execute_shell("convert -resize \@1000000\> " + _in + " " + _out);
if (proc.returncode != 0):
print("could not convert " + _in +": " + proc.err + ":" + proc.out);
sys.exit(1);
def test_execute_shell(check_output_mock):
check_output_mock.side_effect = check_output_func
utils.execute_shell(['asdf'])
def download_macprefs():
print 'Running brew update macprefs to verify version...'
result = execute_shell(['brew', 'upgrade', 'macprefs'], False, '.', True)
if not is_none_or_empty_string(result):
print result
def run_tool(test_file, result_file, tool_args:str):
cmd_run = TOOL_EXEC + ' ' + test_file + ' -o ' + result_file + ' ' + tool_args
logger.debug('executing: ' + cmd_run)
return execute_shell(cmd_run)