def build(self, build, verbose=True, signed=False, dev=False):
if signed:
sign_str = '--sign'
else:
sign_str = '--nosign'
if dev:
dev_str = '-dev'
else:
dev_str = ''
with cd(build):
mkdir(self.name + '-clone')
with cd(self.name + '-clone'):
call([
'git', 'clone',
'https://github.com/Apricity-OS/apricity-packages' +
dev_str
])
copytree('apricity-packages' + dev_str + '/' + self.name,
'../' + self.name)
rmtree(self.name + '-clone')
with cd(self.name):
if verbose:
call(['makepkg', sign_str, '--syncdeps'])
else:
call(['makepkg', sign_str, '--syncdeps'],
stdout=open(devnull, 'w'))
def build(self, build, verbose=True, signed=False, dev=False):
if signed:
sign_str = '--sign'
else:
sign_str = '--nosign'
with cd(build):
if verbose:
call(['yaourt', '-G', self.name])
else:
call(['yaourt', '-G', self.name], stdout=open(devnull, 'w'))
with cd(self.name):
if verbose:
call(['makepkg', sign_str, '--syncdeps'])
else:
call(['makepkg', sign_str, '--syncdeps'],
stdout=open(devnull, 'w'))
def _launch_avd(path_info):
with cd(path.join(path.pardir, path.pardir)):
run_ctrlc_proof([path.join(path_info.sdk, "tools", "emulator"), "-avd", "forge"])
LOG.info("Started emulator, waiting for device to boot")
_run_adb([path_info.adb, 'wait-for-device'], 120, path_info)
_run_adb([path_info.adb, "shell", "pm", "path", "android"], 120, path_info)
def main():
args = get_args()
print('Verbosity: ' + str(args.verbose))
with cd('~/Apricity-OS/apricity-repo'):
packages = get_packages(yaourt_spot_fix=args.aur_spot_fix,
apricity_spot_fix=args.apricity_spot_fix)
failed_packages = []
if args.nonsigned:
failed_packages += build_core_nonsigned(packages,
args.install_makedeps)
if args.signed:
failed_packages += build_core_signed(packages,
args.install_makedeps)
if args.dev:
failed_packages += build_core_dev(packages, args.install_makedeps)
if args.nonsigned:
sync_core_nonsigned()
if args.signed:
sync_core_signed()
if args.dev:
sync_core_dev()
if len(failed_packages) > 0:
print('Failed packages:')
for package_name in failed_packages:
print(package_name)
def run_web(build):
# run Node locally
# TODO: port should be a parameter/configuration
port = 3000
def show_local_server():
LOG.info("Attempting to open browser at http://localhost:%d/" % port)
_open_url("http://localhost:%d/" % port)
with cd(path.join("development", "web")):
timer = None
try:
_npm("install")
attempts = 0
while not _port_available(port):
LOG.info('Port still in use, attempting to send a kill signal')
#TODO: appropriate timeout and handling
requests.post('http://localhost:%d/_forge/kill/' % port)
time.sleep(1)
attempts += 1
if attempts > 5:
raise WebError("Port %d seems to be in use, you should specify a different port to use" % port)
timer = threading.Timer(3, show_local_server).start()
_npm("start", command_log_level=logging.INFO, env=dict(os.environ, PORT=str(port), FORGE_DEBUG='1'))
finally:
if timer:
timer.cancel()
def package_ie(build, root_dir, **kw):
'Run NSIS'
nsis_check = lib.PopenWithoutNewConsole('makensis -VERSION', shell=True, stdout=PIPE, stderr=STDOUT)
stdout, stderr = nsis_check.communicate()
if nsis_check.returncode != 0:
raise CouldNotLocate("Make sure the 'makensis' executable is in your path")
# JCB: need to check nsis version in stdout here?
with cd(path.join(root_dir, 'ie')):
for arch in ('x86', 'x64'):
nsi_filename = "setup-{arch}.nsi".format(arch=arch)
package = lib.PopenWithoutNewConsole('makensis {nsi}'.format(nsi=path.join("dist", nsi_filename)),
stdout=PIPE, stderr=STDOUT, shell=True
)
out, err = package.communicate()
if package.returncode != 0:
raise IEError("problem running {arch} IE build: {stdout}".format(arch=arch, stdout=out))
# move output to root of IE directory
for exe in glob.glob(path.join("dist/*.exe")):
shutil.move(exe, "{name}-{version}-{arch}.exe".format(
name=build.config.get('name', 'Forge App'),
version=build.config.get('version', '0.1'),
arch=arch
))
def git_ignore(root, patterns):
classified_patterns = []
with cd(root):
for pattern in patterns:
if pattern:
if '/' in pattern[:-1]:
ignored_paths = (Pattern('path', match) for match in glob.glob(pattern))
classified_patterns.extend(ignored_paths)
else:
classified_patterns.append(Pattern('file', pattern))
def git_ignorer(src, names):
relative_src = src[len(root):].lstrip(r"""\/""")
ignored = []
for name in names:
for pattern in classified_patterns:
if pattern.type == 'path':
if path.join(relative_src, name) == os.path.normpath(pattern.value):
ignored.append(name)
elif pattern.type == 'file':
ignore_name = pattern.value
if pattern.value[-1] in ('/', '\\'):
if path.isdir(path.join(src, name)):
ignore_name = ignore_name[:-1]
if fnmatch.fnmatch(name, ignore_name):
ignored.append(name)
return set(ignored)
return git_ignorer
def git_ignore(root, patterns):
classified_patterns = []
with cd(root):
for pattern in patterns:
if pattern:
if '/' in pattern[:-1]:
ignored_paths = (Pattern('path', match)
for match in glob.glob(pattern))
classified_patterns.extend(ignored_paths)
else:
classified_patterns.append(Pattern('file', pattern))
def git_ignorer(src, names):
relative_src = src[len(root):].lstrip(r"""\/""")
ignored = []
for name in names:
for pattern in classified_patterns:
if pattern.type == 'path':
if path.join(relative_src,
name) == os.path.normpath(pattern.value):
ignored.append(name)
elif pattern.type == 'file':
ignore_name = pattern.value
if pattern.value[-1] in ('/', '\\'):
if path.isdir(path.join(src, name)):
ignore_name = ignore_name[:-1]
if fnmatch.fnmatch(name, ignore_name):
ignored.append(name)
return set(ignored)
return git_ignorer
def _locate_ios_app(self, error_message):
ios_build_dir = path.join(self.path_to_ios_build, 'ios')
with lib.cd(ios_build_dir):
possible_apps = glob('device-*.app/')
if not possible_apps:
raise IOError(error_message)
return possible_apps[0]
def _set_dotted_attribute(path_to_app, attribute_name, value):
from forge import build_config
LOG.info('Saving %s as %s in local_config.json' % (value, attribute_name))
with cd(path_to_app):
local_config = build_config.load_local()
current_level = local_config
crumbs = attribute_name.split('.')
for k in crumbs[:-1]:
if k not in current_level:
current_level[k] = {}
current_level = current_level[k]
current_level[crumbs[-1]] = value
build_config.save_local(local_config)
def build_repo(packages,
install_makedeps=True,
max_attempts=10,
signed=False,
dest_dir='apricity-core',
repo_name='apricity-core',
build_dir='build',
repo_dir='repo',
dev=False):
prepare(dest_dir, repo_name, build_dir, repo_dir)
failed = []
for package in packages:
attempts = 0
while attempts < max_attempts:
try:
if install_makedeps:
package.install_makedeps(verbose=True)
package.build(build_dir, verbose=True, signed=signed, dev=dev)
pkgs = glob(build_dir + '/' + package.name + '/*.pkg.tar.xz')
if len(pkgs) > 0:
for file in pkgs:
copy(file, repo_dir)
else:
raise Exception('Makepkg failed')
if signed:
sigs = glob(build_dir + '/' + package.name +
'/*.pkg.tar.xz.sig')
if len(sigs) > 0:
for file in sigs:
copy(file, repo_dir)
else:
raise Exception('Makepkg signing failed')
break
except Exception as e:
print('Unexpected Error:' + str(e))
attempts += 1
if attempts < max_attempts:
print('Trying again...')
rmtree(build_dir + '/' + package.name, ignore_errors=True)
else:
failed.append(package.name)
with cd(repo_dir):
if signed:
call('repo-add --sign ' + repo_name + '.db.tar.gz *.pkg.tar.xz',
shell=True)
else:
call('repo-add ' + repo_name + '.db.tar.gz *.pkg.tar.xz',
shell=True)
return failed
def run_web(build):
"""Run an instance of Node locally"""
# TODO: port should be a parameter/configuration
port = 3000
_update_path_for_node(build)
def show_local_server():
LOG.info("Attempting to open browser at http://localhost:%d/" % port)
_open_url("http://localhost:%d/" % port)
with cd(path.join("development", "web")):
timer = None
try:
# TODO: annoyingly difficult to kill npm processes on windows - npm.cmd actually
# launches an instance of node as a subprocess which is the real thing you need to kill!
# might be possible to kill npm.cmd in a nicer way, e.g. sending a CTRL_C event, needs
# a bit of experimentation
_npm(build, "install")
attempts = 0
while not _port_available(port):
LOG.info("Port still in use, attempting to send a kill signal")
# TODO: appropriate timeout and handling
requests.post("http://localhost:%d/_forge/kill/" % port)
time.sleep(1)
attempts += 1
if attempts > 5:
raise WebError("Port %d seems to be in use, you should specify a different port to use" % port)
timer = threading.Timer(3, show_local_server).start()
_node(
build,
"./web.js",
command_log_level=logging.INFO,
check_for_interrupt=True,
env=dict(os.environ, PORT=str(port), FORGE_DEBUG="1"),
)
finally:
if timer:
timer.cancel()
def run_web(build):
"""Run an instance of Node locally"""
# TODO: port should be a parameter/configuration
port = 3000
_update_path_for_node(build)
def show_local_server():
LOG.info("Attempting to open browser at http://localhost:%d/" % port)
_open_url("http://localhost:%d/" % port)
with cd(path.join("development", "web")):
timer = None
try:
# TODO: annoyingly difficult to kill npm processes on windows - npm.cmd actually
# launches an instance of node as a subprocess which is the real thing you need to kill!
# might be possible to kill npm.cmd in a nicer way, e.g. sending a CTRL_C event, needs
# a bit of experimentation
_npm(build, "install")
attempts = 0
while not _port_available(port):
LOG.info('Port still in use, attempting to send a kill signal')
#TODO: appropriate timeout and handling
requests.post('http://localhost:%d/_forge/kill/' % port)
time.sleep(1)
attempts += 1
if attempts > 5:
raise WebError(
"Port %d seems to be in use, you should specify a different port to use"
% port)
timer = threading.Timer(3, show_local_server).start()
_node(build,
"./web.js",
command_log_level=logging.INFO,
check_for_interrupt=True,
env=dict(os.environ, PORT=str(port), FORGE_DEBUG='1'))
finally:
if timer:
timer.cancel()
def package_firefox(build):
development_dir = path.join('development', 'firefox')
release_dir = path.join('release', 'firefox')
if not path.isdir(release_dir):
os.makedirs(release_dir)
xpi_filename = '{name}.xpi'.format(name=build.config['xml_safe_name'])
IGNORED_FILES = ['.hgignore', '.DS_Store', 'install.rdf',
'application.ini', xpi_filename]
with cd(development_dir):
zipf = zipfile.ZipFile(xpi_filename, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk('.'):
filenames = list(filter_filenames(filenames, IGNORED_FILES))
dirnames[:] = filter_dirnames(dirnames)
for filename in filenames:
abspath = os.path.join(dirpath, filename)
LOG.info('Adding: %s' % abspath)
zipf.write(abspath)
zipf.close()
shutil.move(path.join(development_dir, xpi_filename), path.join(release_dir, xpi_filename))
def package_firefox(build):
development_dir = path.join('development', 'firefox')
release_dir = path.join('release', 'firefox')
if not path.isdir(release_dir):
os.makedirs(release_dir)
xpi_filename = '{name}.xpi'.format(name=build.config['xml_safe_name'])
IGNORED_FILES = ['.hgignore', '.DS_Store',
'application.ini', xpi_filename]
with cd(development_dir):
zipf = zipfile.ZipFile(xpi_filename, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk('.'):
filenames = list(filter_filenames(filenames, IGNORED_FILES))
dirnames[:] = filter_dirnames(dirnames)
for filename in filenames:
abspath = os.path.join(dirpath, filename)
LOG.info('Adding: %s' % abspath)
zipf.write(abspath)
zipf.close()
shutil.move(path.join(development_dir, xpi_filename), path.join(release_dir, xpi_filename))
def build_repo(packages, install_makedeps=True, max_attempts=10, signed=False, dest_dir='apricity-core', repo_name='apricity-core', build_dir='build', repo_dir='repo', dev=False):
prepare(dest_dir, repo_name, build_dir, repo_dir)
failed = []
for package in packages:
attempts = 0
while attempts < max_attempts:
try:
if install_makedeps:
package.install_makedeps(verbose=True)
package.build(build_dir, verbose=True, signed=signed, dev=dev)
pkgs = glob(build_dir + '/' + package.name + '/*.pkg.tar.xz')
if len(pkgs) > 0:
for file in pkgs:
copy(file, repo_dir)
else:
raise Exception('Makepkg failed')
if signed:
sigs = glob(build_dir + '/' + package.name + '/*.pkg.tar.xz.sig')
if len(sigs) > 0:
for file in sigs:
copy(file, repo_dir)
else:
raise Exception('Makepkg signing failed')
break
except Exception as e:
print('Unexpected Error:' + str(e))
attempts += 1
if attempts < max_attempts:
print('Trying again...')
rmtree(build_dir + '/' + package.name, ignore_errors=True)
else:
failed.append(package.name)
with cd(repo_dir):
if signed:
call('repo-add --sign ' + repo_name + '.db.tar.gz *.pkg.tar.xz', shell=True)
else:
call('repo-add ' + repo_name + '.db.tar.gz *.pkg.tar.xz', shell=True)
return failed
def main():
args = get_args()
print('Verbosity: ' + str(args.verbose))
with cd('~/Apricity-OS/apricity-repo'):
packages = get_packages(yaourt_spot_fix=args.aur_spot_fix,
apricity_spot_fix=args.apricity_spot_fix)
failed_packages = []
if args.nonsigned:
failed_packages += build_core_nonsigned(packages, args.install_makedeps)
if args.signed:
failed_packages += build_core_signed(packages, args.install_makedeps)
if args.dev:
failed_packages += build_core_dev(packages, args.install_makedeps)
if args.nonsigned:
sync_core_nonsigned()
if args.signed:
sync_core_signed()
if args.dev:
sync_core_dev()
if len(failed_packages) > 0:
print('Failed packages:')
for package_name in failed_packages:
print(package_name)
def package_web(build):
interactive = build.tool_config.get('general.interactive', True)
development = lib.expand_relative_path(build, 'development/web')
output = lib.expand_relative_path(build, 'release/web/heroku')
# deploy to Heroku
with cd(development):
api_key = _get_heroku_api_key(build)
chosen_app = _get_app_to_push_to(build, api_key)
if not path.isdir(output):
os.makedirs(output)
with cd(output):
if not path.isdir('.git'):
LOG.debug('Creating git repo')
_git('init')
LOG.debug('Create dummy first commit')
with open('.forge.txt', 'w') as forge_file:
forge_file.write('')
_git('add', '.')
_git('commit', '-am', 'first commit')
# remove all previous files/folders except for .git!
with cd(output):
for f in os.listdir('.'):
if not f == '.git':
if path.isfile(f):
os.remove(f)
elif path.isdir(f):
shutil.rmtree(f)
# copy code from development to release!
with cd(development):
for f in os.listdir('.'):
if path.isfile(f):
shutil.copy2(f, output)
elif path.isdir(f) and path.basename(f) != '.git':
shutil.copytree(f,
path.join(output, f),
ignore=shutil.ignore_patterns('.git'))
with cd(output):
# setup with the specified remote
LOG.debug('Setting up git remote for %s' % chosen_app)
# remove any previous remote
try:
_git('remote', 'rm', 'heroku')
except WebError:
pass
_git('remote', 'add', 'heroku',
'[email protected]:%s.git' % chosen_app)
# commit
_git('add', '.')
diff = _git('diff', 'HEAD')
if not diff.strip():
# not interactive basically means we're using the trigger toolkit, where 'forge build'
# doesn't really make sense
if interactive:
LOG.warning(
"No app changes detected: did you forget to forge build?"
)
else:
LOG.warning(
"No app changes detected, pushing to heroku anyway")
else:
_git('commit', '-am', 'forge package web')
# push
LOG.info('Deploying to %s.herokuapp.com' % chosen_app)
# TODO: when running a packaged up toolkit there is no commandline... need to make sure
# we can use the ssh key earlier somehow and ask for passphrase if not
# also provide docs on how to use ssh-agent/pageant
if not interactive:
LOG.warning(
'If the packaging process hangs here, you need to set up ssh-agent or Pageant'
)
# TODO: show our own one line progress bar when running non-verbosely
push_output = _git('push',
'heroku',
'--all',
'--force',
command_log_level=logging.INFO)
if push_output.startswith('Everything up-to-date'):
remote_output = _git('remote', '-v')
remote_pattern = re.compile(
r'[email protected]:(.*?).git \(fetch\)')
remote_match = remote_pattern.search(remote_output)
if remote_match:
app_url = 'http://%s.herokuapp.com' % remote_match.group(1)
_open_url(app_url)
LOG.info('Deployed at %s' % app_url)
else:
deploy_pattern = re.compile(
r'(http://[^ ]+) deployed to Heroku')
deploy_match = deploy_pattern.search(push_output)
if deploy_match:
_open_url(deploy_match.group(1))
LOG.info('Deployed at %s' % deploy_match.group(1))
def package_chrome(build):
development_dir = path.join("development", "chrome")
release_dir = path.join("release", "chrome")
if not path.isdir(release_dir):
os.makedirs(release_dir)
# Make sure we have openssl otherwise show manual instructions
openssl_check = lib.PopenWithoutNewConsole('openssl version', shell=True, stdout=PIPE, stderr=STDOUT)
if openssl_check.wait() != 0:
msg = """Packaging a Chrome extensions requires openssl.
Alternatively you can manually package it. The required steps are:
1) Go to chrome:extensions in the Chrome browser
2) Make sure "developer mode" is on (top right corner)')
3) Use "Pack extension" and use use this for the root:
<app dir>/{chrome_folder}
More information on packaging Chrome extensions can be found here:
http://legacy-docs.trigger.io/en/v1.4/best_practice/release_browser.html#chrome
http://code.google.com/chrome/extensions/packaging.html
""".format(chrome_folder="development/chrome")
LOG.info(msg)
raise CouldNotLocate("Make sure 'openssl' is in your path")
# We need a private key to package chrome extensions
crx_key = build.tool_config.get('chrome.profile.crx_key')
crx_key_path = build.tool_config.get('chrome.profile.crx_key_path')
if not crx_key:
key_msg = """Packaging a Chrome extension requires a private key.
You can generate this key with openssl:
openssl genrsa -out crxkey.pem 2048
Keep this key safe and secure. It is needed to sign all future versions
of the extension. Add the following to <app dir>/local_config.json to use the key:
"chrome": {
"profile": {
"crx_key": "crxkey.pem",
"crx_key_path": "<path to crxkey.pem>",
}
}
"""
LOG.info(key_msg)
return
# HACK: build.usercode seems to be the only way to get a reference to the app directory.
crx_key_file = path.realpath(path.join(build.usercode, '..', crx_key_path, crx_key))
crx_filename = '{name}.crx'.format(name=build.config['xml_safe_name'])
IGNORED_FILES = ['.hgignore', '.DS_Store',
'application.ini', crx_filename]
# Create standard zip file
with cd(development_dir):
zipf = zipfile.ZipFile(crx_filename, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk('.'):
filenames = list(filter_filenames(filenames, IGNORED_FILES))
dirnames[:] = filter_dirnames(dirnames)
for filename in filenames:
abspath = os.path.join(dirpath, filename)
LOG.info('Adding: %s' % abspath)
zipf.write(abspath)
zipf.close()
# Generate signature
signature, pubkey = _generate_signature(path.join(development_dir, crx_filename), crx_key_file)
# Combine magic, public key and signature into header and prepend to zip file
magic = 'Cr24'
version = struct.pack('<I', 2)
pubkey_len = struct.pack('<I', len(pubkey))
signature_len = struct.pack('<I', len(signature))
with open(path.join(development_dir, crx_filename), 'rb') as crx:
zip_data = crx.read()
with open(path.join(development_dir, crx_filename), 'wb') as crx:
data = [magic, version, pubkey_len, signature_len, pubkey, signature, zip_data]
for d in data:
crx.write(d)
shutil.move(path.join(development_dir, crx_filename), path.join(release_dir, crx_filename))
def buildAndTest(submissionpath, sourceTestPath, no_remove):
points = 0
script_path = os.path.dirname(os.path.realpath(__file__))
# create temporary directory so that previous students' results will not affect subsequent tests
testCasePath = sourceTestPath
testCases = glob.glob(os.path.join(testCasePath, "*.simplec"))
#print(f"testCases {testCases}")
if not no_remove:
for i in glob.glob(os.path.join(submissionpath, "*.o")):
if os.path.exists(i):
os.remove(i)
progname = os.path.join(submissionpath, "simplec")
if os.path.exists(progname):
os.remove(progname)
if len(testCases) == 0:
print("# no tests found. double-check your path: " + testCasePath)
sys.exit()
if os.path.exists(submissionpath + "/simplec"):
os.remove(submissionpath + "/simplec")
out = subprocess.run(['make'],
cwd=submissionpath,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
output = ""
err = ""
if out.returncode != 0:
output += "Make failed."
print(output +
" Do you have a Makefile?") # can't even compile the compiler
return 0, output
else:
print("Build succeeded!")
points += build_points # points to build. tentative
# simpleC compilers lives so lets go through every test case now
for case in testCases:
base_name = os.path.basename(case)
ground_truth = case.replace(".simplec", ".ast")
output_file = base_name.replace(".simplec", ".out")
diff_file = base_name.replace(".simplec", ".diff")
print(f"Testing {base_name}:", end=" ")
with cd(submissionpath):
cmd = f"cat \"{case}\" | ./simplec > {output_file}"
#print(f"Running command: {cmd}")
return_code, stdout_, stderr_ = run_cmd(cmd)
cmd = f"diff -w -B \"{ground_truth}\" {output_file}"
#print(f"Running command: {cmd}")
return_code, stdout_, stderr_ = run_cmd(cmd, False)
if return_code == 0 and len(stdout_) == 0:
print("Success!")
points += test_case_points
if return_code == 1 and len(stdout_) > 0:
print(
f"Failure. See {diff_file} for diff and {output_file} for output."
)
diff_out = open(diff_file, "w")
diff_out.write(stdout_)
diff_out.close()
return_code, stdout_, stderr_ = run_cmd(cmd, False)
if return_code > 1:
print(
f"diff exited with an unknown return code. This shouldn't happen. Here is the stderr: {stderr_}"
)
print(
f"{int((points - build_points) / test_case_points)} / {len(testCases)} test casing passing. "
)
return points, output
def package_chrome(build):
development_dir = path.join("development", "chrome")
release_dir = path.join("release", "chrome")
if not path.isdir(release_dir):
os.makedirs(release_dir)
# Make sure we have openssl otherwise show manual instructions
openssl_check = lib.PopenWithoutNewConsole('openssl version',
shell=True,
stdout=PIPE,
stderr=STDOUT)
if openssl_check.wait() != 0:
msg = """Packaging a Chrome extensions requires openssl.
Alternatively you can manually package it. The required steps are:
1) Go to chrome:extensions in the Chrome browser
2) Make sure "developer mode" is on (top right corner)')
3) Use "Pack extension" and use use this for the root:
<app dir>/{chrome_folder}
More information on packaging Chrome extensions can be found here:
http://legacy-docs.trigger.io/en/v1.4/best_practice/release_browser.html#chrome
http://code.google.com/chrome/extensions/packaging.html
""".format(chrome_folder="development/chrome")
LOG.info(msg)
raise CouldNotLocate("Make sure 'openssl' is in your path")
# We need a private key to package chrome extensions
crx_key = build.tool_config.get('chrome.profile.crx_key')
crx_key_path = build.tool_config.get('chrome.profile.crx_key_path')
if not crx_key:
key_msg = """Packaging a Chrome extension requires a private key.
You can generate this key with openssl:
openssl genrsa -out crxkey.pem 2048
Keep this key safe and secure. It is needed to sign all future versions
of the extension. Add the following to <app dir>/local_config.json to use the key:
"chrome": {
"profile": {
"crx_key": "crxkey.pem",
"crx_key_path": "<path to crxkey.pem>",
}
}
"""
LOG.info(key_msg)
return
# HACK: build.usercode seems to be the only way to get a reference to the app directory.
crx_key_file = path.realpath(
path.join(build.usercode, '..', crx_key_path, crx_key))
crx_filename = '{name}.crx'.format(name=build.config['xml_safe_name'])
IGNORED_FILES = ['.hgignore', '.DS_Store', 'application.ini', crx_filename]
# Create standard zip file
with cd(development_dir):
zipf = zipfile.ZipFile(crx_filename, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk('.'):
filenames = list(filter_filenames(filenames, IGNORED_FILES))
dirnames[:] = filter_dirnames(dirnames)
for filename in filenames:
abspath = os.path.join(dirpath, filename)
LOG.info('Adding: %s' % abspath)
zipf.write(abspath)
zipf.close()
# Generate signature
signature, pubkey = _generate_signature(
path.join(development_dir, crx_filename), crx_key_file)
# Combine magic, public key and signature into header and prepend to zip file
magic = 'Cr24'
version = struct.pack('<I', 2)
pubkey_len = struct.pack('<I', len(pubkey))
signature_len = struct.pack('<I', len(signature))
with open(path.join(development_dir, crx_filename), 'rb') as crx:
zip_data = crx.read()
with open(path.join(development_dir, crx_filename), 'wb') as crx:
data = [
magic, version, pubkey_len, signature_len, pubkey, signature,
zip_data
]
for d in data:
crx.write(d)
shutil.move(path.join(development_dir, crx_filename),
path.join(release_dir, crx_filename))
def package_web(build):
interactive = build.tool_config.get('general.interactive', True)
development = lib.expand_relative_path(build, 'development/web')
output = lib.expand_relative_path(build, 'release/web/heroku')
# deploy to Heroku
with cd(development):
api_key = _get_heroku_api_key(build)
chosen_app = _get_app_to_push_to(build, api_key)
if not path.isdir(output):
os.makedirs(output)
with cd(output):
if not path.isdir('.git'):
LOG.debug('Creating git repo')
_git('init')
LOG.debug('Create dummy first commit')
with open('.forge.txt', 'w') as forge_file:
forge_file.write('')
_git('add', '.')
_git('commit', '-am', 'first commit')
# remove all previous files/folders except for .git!
with cd(output):
for f in os.listdir('.'):
if not f == '.git':
if path.isfile(f):
os.remove(f)
elif path.isdir(f):
shutil.rmtree(f)
# copy code from development to release!
with cd(development):
for f in os.listdir('.'):
if path.isfile(f):
shutil.copy2(f, output)
elif path.isdir(f) and path.basename(f) != '.git':
shutil.copytree(f, path.join(output, f), ignore=shutil.ignore_patterns('.git'))
with cd(output):
# setup with the specified remote
LOG.debug('Setting up git remote for %s' % chosen_app)
# remove any previous remote
try:
_git('remote', 'rm', 'heroku')
except WebError:
pass
_git('remote', 'add', 'heroku', '[email protected]:%s.git' % chosen_app)
# commit
_git('add', '.')
diff = _git('diff', 'HEAD')
if not diff.strip():
# not interactive basically means we're using the trigger toolkit, where 'forge build'
# doesn't really make sense
if interactive:
LOG.warning("No app changes detected: did you forget to forge build?")
else:
LOG.warning("No app changes detected, pushing to heroku anyway")
else:
_git('commit', '-am', 'forge package web')
# push
LOG.info('Deploying to %s.herokuapp.com' % chosen_app)
# TODO: when running a packaged up toolkit there is no commandline... need to make sure
# we can use the ssh key earlier somehow and ask for passphrase if not
# also provide docs on how to use ssh-agent/pageant
if not interactive:
LOG.warning('If the packaging process hangs here, you need to set up ssh-agent or Pageant')
# TODO: show our own one line progress bar when running non-verbosely
push_output = _git('push', 'heroku', '--all', '--force', command_log_level=logging.INFO)
if push_output.startswith('Everything up-to-date'):
remote_output = _git('remote', '-v')
remote_pattern = re.compile(r'[email protected]:(.*?).git \(fetch\)')
remote_match = remote_pattern.search(remote_output)
if remote_match:
app_url = 'http://%s.herokuapp.com' % remote_match.group(1)
_open_url(app_url)
LOG.info('Deployed at %s' % app_url)
else:
deploy_pattern = re.compile(r'(http://[^ ]+) deployed to Heroku')
deploy_match = deploy_pattern.search(push_output)
if deploy_match:
_open_url(deploy_match.group(1))
LOG.info('Deployed at %s' % deploy_match.group(1))
def package_web(build):
path_to_app = path.abspath('')
interactive = build.tool_config.get('general.interactive', True)
development = path.abspath(path.join('development', 'web'))
output = path.abspath(path.join('release', 'web', 'heroku'))
# deploy to Heroku
if sys.platform.startswith("win"):
heroku = "heroku.bat"
else:
heroku = "heroku"
with cd(development):
username = None
api_key = build.tool_config.get('web.profile.heroku_api_key')
while api_key is None:
if not interactive:
raise WebError("You need to specify an API Key for interaction with heroku")
# TODO: may want to check the api key is actually valid by hitting the api?
heroku_username, heroku_password = _request_heroku_credentials()
api_key = _heroku_get_api_key(heroku_username, heroku_password)
if api_key is not None:
_set_dotted_attribute(
path_to_app,
'web.profiles.%s.heroku_api_key' % build.tool_config.profile(),
api_key
)
chosen_app = build.tool_config.get('web.profile.heroku_app_name')
if not path.isdir(output):
os.makedirs(output)
with cd(output):
if not path.isdir('.git'):
LOG.debug('Creating git repo')
_git('init')
LOG.debug('Create dummy first commit')
with open('.forge.txt', 'w') as forge_file:
forge_file.write('')
_git('add', '.')
_git('commit', '-am', '"first commit"')
if chosen_app is None:
chosen_app = _request_app_to_push_to(build, api_key, interactive)
_set_dotted_attribute(
path_to_app,
'web.profiles.%s.heroku_app_name' % build.tool_config.profile(),
chosen_app
)
# remove all previous files/folders except for .git!
with cd(output):
for f in os.listdir('.'):
if not f == '.git':
if path.isfile(f):
os.remove(f)
elif path.isdir(f):
shutil.rmtree(f)
# copy code from development to release!
with cd(development):
for f in os.listdir('.'):
if path.isfile(f):
shutil.copy2(f, output)
elif path.isdir(f) and path.basename(f) != '.git':
shutil.copytree(f, path.join(output, f), ignore=shutil.ignore_patterns('.git'))
with cd(output):
# setup with the specified remote
LOG.debug('Setting up git remote for %s' % chosen_app)
# remove any previous remote
try:
_git('remote', 'rm', 'heroku')
except WebError:
pass
_git('remote', 'add', 'heroku', '[email protected]:%s.git' % chosen_app)
# commit
_git('add', '.')
diff = _git('diff', 'HEAD')
if not diff.strip():
if interactive:
LOG.warning("No app changes detected: did you forget to forge build?")
else:
# not interactive basically means we're using the trigger toolkit, where 'forge build'
# doesn't really make sense
LOG.warning("No app changes detected, pushing to heroku anyway")
else:
_git('commit', '-am', 'forge package web')
# push
LOG.info('Deploying to %s.herokuapp.com' % chosen_app)
if not interactive:
LOG.warning('You may need to check the commandline to enter an SSH key passphrase')
push_output = _git('push', 'heroku', '--all', '--force', command_log_level=logging.INFO)
if push_output.startswith('Everything up-to-date'):
remote_output = _git('remote', '-v')
remote_pattern = re.compile(r'[email protected]:(.*?).git \(fetch\)')
remote_match = remote_pattern.search(remote_output)
if remote_match:
app_url = 'http://%s.herokuapp.com' % remote_match.group(1)
_open_url(app_url)
LOG.info('Deployed at %s' % app_url)
else:
deploy_pattern = re.compile(r'(http://[^ ]+) deployed to Heroku')
deploy_match = deploy_pattern.search(push_output)
if deploy_match:
_open_url(deploy_match.group(1))
LOG.info('Deployed at %s' % deploy_match.group(1))
def package_web(build):
interactive = build.tool_config.get("general.interactive", True)
development = lib.expand_relative_path(build, "development/web")
output = lib.expand_relative_path(build, "release/web/heroku")
# deploy to Heroku
with cd(development):
api_key = _get_heroku_api_key(build)
chosen_app = _get_app_to_push_to(build, api_key)
if not path.isdir(output):
os.makedirs(output)
with cd(output):
if not path.isdir(".git"):
LOG.debug("Creating git repo")
_git("init")
LOG.debug("Create dummy first commit")
with open(".forge.txt", "w") as forge_file:
forge_file.write("")
_git("add", ".")
_git("commit", "-am", "first commit")
# remove all previous files/folders except for .git!
with cd(output):
for f in os.listdir("."):
if not f == ".git":
if path.isfile(f):
os.remove(f)
elif path.isdir(f):
shutil.rmtree(f)
# copy code from development to release!
with cd(development):
for f in os.listdir("."):
if path.isfile(f):
shutil.copy2(f, output)
elif path.isdir(f) and path.basename(f) != ".git":
shutil.copytree(f, path.join(output, f), ignore=shutil.ignore_patterns(".git"))
with cd(output):
# setup with the specified remote
LOG.debug("Setting up git remote for %s" % chosen_app)
# remove any previous remote
try:
_git("remote", "rm", "heroku")
except WebError:
pass
_git("remote", "add", "heroku", "[email protected]:%s.git" % chosen_app)
# commit
_git("add", ".")
diff = _git("diff", "HEAD")
if not diff.strip():
# not interactive basically means we're using the trigger toolkit, where 'forge build'
# doesn't really make sense
if interactive:
LOG.warning("No app changes detected: did you forget to forge build?")
else:
LOG.warning("No app changes detected, pushing to heroku anyway")
else:
_git("commit", "-am", "forge package web")
# push
LOG.info("Deploying to %s.herokuapp.com" % chosen_app)
# TODO: when running a packaged up toolkit there is no commandline... need to make sure
# we can use the ssh key earlier somehow and ask for passphrase if not
# also provide docs on how to use ssh-agent/pageant
if not interactive:
LOG.warning("If the packaging process hangs here, you need to set up ssh-agent or Pageant")
# TODO: show our own one line progress bar when running non-verbosely
push_output = _git("push", "heroku", "--all", "--force", command_log_level=logging.INFO)
if push_output.startswith("Everything up-to-date"):
remote_output = _git("remote", "-v")
remote_pattern = re.compile(r"[email protected]:(.*?).git \(fetch\)")
remote_match = remote_pattern.search(remote_output)
if remote_match:
app_url = "http://%s.herokuapp.com" % remote_match.group(1)
_open_url(app_url)
LOG.info("Deployed at %s" % app_url)
else:
deploy_pattern = re.compile(r"(http://[^ ]+) deployed to Heroku")
deploy_match = deploy_pattern.search(push_output)
if deploy_match:
_open_url(deploy_match.group(1))
LOG.info("Deployed at %s" % deploy_match.group(1))
def create_ipa_from_app(self, build, provisioning_profile, certificate_to_sign_with=None, relative_path_to_itunes_artwork=None):
"""Create an ipa from an app, with an embedded provisioning profile provided by the user, and
signed with a certificate provided by the user.
:param build: instance of build
:param provisioning_profile: Absolute path to the provisioning profile to embed in the ipa
:param certificate_to_sign_with: (Optional) The name of the certificate to sign the ipa with
:param relative_path_to_itunes_artwork: (Optional) A path to a 512x512 png picture for the App view in iTunes.
This should be relative to the location of the user assets.
"""
LOG.info('Starting package process for iOS')
if certificate_to_sign_with is None:
certificate_to_sign_with = 'iPhone Developer'
file_name = "{name}-{time}.ipa".format(
name=re.sub("[^a-zA-Z0-9]", "", build.config["name"].lower()),
time=str(int(time.time()))
)
output_path_for_ipa = path.abspath(path.join('release', 'ios', file_name))
directory = path.dirname(output_path_for_ipa)
if not path.isdir(directory):
os.makedirs(directory)
app_folder_name = self._locate_ios_app(error_message="Couldn't find iOS app in order to sign it")
path_to_template_app = path.abspath(path.join(self.path_to_ios_build, '..', '.template', 'ios', app_folder_name))
path_to_app = path.abspath(path.join(self.path_to_ios_build, 'ios', app_folder_name))
# Verify current signature
codesign = self._check_for_codesign()
run_shell(codesign, '--verify', '-vvvv', path_to_template_app)
LOG.info('going to package: %s' % path_to_app)
plist_str = self._grab_plist_from_binary_mess(provisioning_profile)
plist_dict = self._parse_plist(plist_str)
self.check_plist_dict(plist_dict, self.path_to_ios_build)
LOG.info("Plist OK.")
self.log_profile(plist_dict)
seed_id = self._extract_seed_id(plist_dict)
LOG.debug("extracted seed ID: {0}".format(seed_id))
temp_dir = tempfile.mkdtemp()
with lib.cd(temp_dir):
LOG.debug('Moved into tempdir: %s' % temp_dir)
LOG.debug('Making Payload directory')
os.mkdir('Payload')
path_to_payload = path.abspath(path.join(temp_dir, 'Payload'))
path_to_payload_app = path.abspath(path.join(path_to_payload, app_folder_name))
if relative_path_to_itunes_artwork is not None:
path_to_itunes_artwork = path.join(path_to_payload_app, 'assets', 'src', relative_path_to_itunes_artwork)
else:
path_to_itunes_artwork = None
self._create_entitlements_file(build, plist_dict, temp_dir)
self._sign_app(build=build,
provisioning_profile=provisioning_profile,
certificate=certificate_to_sign_with,
entitlements_file=path.join(temp_dir, 'template.entitlements'),
)
os.remove(path.join(temp_dir, 'template.entitlements'))
_copy(path_to_app, path_to_payload)
if path_to_itunes_artwork:
_copy(path_to_itunes_artwork, path.join(temp_dir, 'iTunesArtwork'))
run_shell('/usr/bin/zip', '--symlinks', '--verbose', '--recurse-paths', output_path_for_ipa, '.')
LOG.info("created IPA: {output}".format(output=output_path_for_ipa))
shutil.rmtree(temp_dir)
return output_path_for_ipa