def _sign_executable(build, signtool, target, certificate = None, certificate_path = None, certificate_password = ""): 'Sign a single executable file' LOG.info('Signing {target}'.format(target=target)) if signtool == 'signtool': command = lib.PopenWithoutNewConsole('signtool sign /f {cert} /p {password} /v /t {time} "{target}"'.format( cert=path.join(certificate_path, certificate), password=certificate_password, time='http://timestamp.comodoca.com/authenticode', target=target), stdout=PIPE, stderr=STDOUT, shell=True ) elif signtool == 'osslsigncode': command = lib.PopenWithoutNewConsole('osslsigncode -pkcs12 {cert} -pass {password} -t {time} -in "{target}" -out "{target}.signed"'.format( cert=path.join(certificate_path, certificate), password=certificate_password, time='http://timestamp.comodoca.com/authenticode', target=target), stdout=PIPE, stderr=STDOUT, shell=True ) else: raise IEError("problem signing IE build, unknown code sign tool: {signtool}".format(signtool=signtool)) out, err = command.communicate() if command.returncode != 0: raise IEError("problem signing IE build: {stdout}".format(stdout=out)) if signtool == 'osslsigncode': shutil.move(target + ".signed", target)
def _run_in_shell(queue): '''will be invoked in by a separate process, to actually run the detached command''' # setsid detaches us completely from the caller try: os.setsid() proc = lib.PopenWithoutNewConsole(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if not wait: # assume success at this point if we're not waiting for process to finish queue.put(0) for line in iter(proc.stdout.readline, ''): queue.put(line) # signify success or error queue.put(proc.wait()) except Exception as e: import traceback e._traceback = traceback.format_exc(e) queue.put(e)
def _create_avd(path_info): args = [ path_info.android, "create", "avd", "-n", "forge", "-t", "android-8", "--skin", "HVGA", "-p", path.join(path_info.sdk, 'forge-avd'), #"-a", "-c", "32M", "--force" ] proc = lib.PopenWithoutNewConsole(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT) time.sleep(0.1) proc_std = proc.communicate(input='\n')[0] if proc.returncode != 0 or (proc_std and proc_std.startswith("Error")): raise ShellError(message="Error creating Android Virtual Device", output=proc_std)
def _create_avd(path_info): LOG.info('Creating AVD') args = [ path_info.android, "create", "avd", "-n", "forge", "-t", "android-8", "--skin", "HVGA", "-p", path.join(path_info.sdk, 'forge-avd'), #"-a", "-c", "32M", "--force" ] proc = lib.PopenWithoutNewConsole(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT) time.sleep(0.1) proc_std = proc.communicate(input='\n')[0] if proc.returncode != 0: LOG.error('failed: %s' % (proc_std)) raise AndroidError LOG.debug('Output:\n' + proc_std)
def _runner(self): try: self._state.proc = lib.PopenWithoutNewConsole( self._args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=self._env) self._process_launched.put(self.PROCESS_START_SUCCESS) for line in iter(self._state.proc.stdout.readline, ''): if not self._filter or self._filter(line): self._state.output.write(line) LOG.log(self._command_log_level, line.rstrip('\r\n')) if self._fail_if and self._fail_if(line): raise ShellError( 'Detected failure based on output of subprocess "%s"' % self._args[0], self._state.output.getvalue()) except Exception as e: if self._state.proc is None: self._process_launched.put(self.PROCESS_START_FAILURE) else: self._process_launched.put(self.PROCESS_EXCEPTION) self._state.error = e finally: self._finished()
def _check_signtool(build): options = ["signtool /?", "osslsigncode -v"] # Note: The follow code can be uncommented once osslsigncode_osx has been rebuilt to work # on stock OS X. At the moment it is dynamically linked against # /opt/local/lib/libcrypto.1.0.0.dylib which is not part of the OS and probably a # homebrew library. The system libcrypto is at /usr/lib/libcrypto.dylib # # lib_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '../lib')) # if sys.platform.startswith('darwin'): # options.append('{lib_dir}/osslsigncode_osx -v'.format(lib_dir=lib_dir)) # elif sys.platform.startswith('linux'): # options.append('{lib_dir}/osslsigncode_linux -v'.format(lib_dir=lib_dir)) for option in options: LOG.info("Checking: %s", option[:-3]) check = lib.PopenWithoutNewConsole(option, shell=True, stdout=PIPE, stderr=STDOUT) stdout, stderr = check.communicate() if check.returncode == 0: return option[:-3] LOG.info("Could not find anything: %s" % stdout) return None
def package_ie(build, **kw): 'Run NSIS' nsis_check = lib.PopenWithoutNewConsole('makensis -VERSION', shell=True, stdout=PIPE, stderr=STDOUT) stdout, stderr = nsis_check.communicate() if nsis_check.returncode != 0: raise CouldNotLocate( "Make sure the 'makensis' executable is in your path") # JCB: need to check nsis version in stdout here? development_dir = path.join("development", "ie") release_dir = path.join("release", "ie") if not path.isdir(release_dir): os.makedirs(release_dir) for arch in ('x86', 'x64'): nsi_filename = "setup-{arch}.nsi".format(arch=arch) package = lib.PopenWithoutNewConsole('makensis {nsi}'.format( nsi=path.join(development_dir, "dist", nsi_filename)), stdout=PIPE, stderr=STDOUT, shell=True) out, err = package.communicate() if package.returncode != 0: raise IEError("problem running {arch} IE build: {stdout}".format( arch=arch, stdout=out)) # move output to release directory of IE directory for exe in glob.glob(development_dir + "/dist/*.exe"): shutil.move( exe, path.join( release_dir, "{name}-{version}-{arch}.exe".format( name=build.config.get('name', 'Forge App'), version=build.config.get('version', '0.1'), arch=arch)))
def _check_signtool(build): # TODO use appropriate osslsigncode in generate/lib for platform for option in ["signtool /?", "osslsigncode -v"]: LOG.info("Checking: %s", option[:-3]) check = lib.PopenWithoutNewConsole(option, shell=True, stdout=PIPE, stderr=STDOUT) stdout, stderr = check.communicate() if check.returncode == 0: return option[:-3] LOG.info("Could not find anything: %s" % stdout) return None
def ensure_lib_available(build, file): lib_dir = path.join(path.dirname(build.source_dir), '.lib') hash_path = path.join(path.dirname(build.source_dir), '.template', 'lib', 'hash.json') if not path.exists(lib_dir): os.makedirs(lib_dir) # Hide directory on windows if sys.platform == 'win32': try: lib.PopenWithoutNewConsole(['attrib', '+h', lib_dir]).wait() except Exception: # don't care if we fail to hide the templates dir pass hashes = None if path.exists(hash_path): with open(hash_path, 'r') as hash_file: hashes = json.load(hash_file) file_path = path.join(lib_dir, file) if path.exists(file_path) and file in hashes: # Check hash with open(file_path, 'rb') as cur_file: hash = hashlib.md5(cur_file.read()).hexdigest() if hash == hashes[file]: # File exists and is correct build.log.debug("File: %s, already downloaded and correct." % file) return file_path # File doesn't exist, or has the wrong hash or has no known hash - download build.log.info( "Downloading lib file: %s, this will only happen when a new file is available." % file) from forge.remote import Remote from forge import build_config config = build_config.load() remote = Remote(config) server_details = urlparse.urlparse(remote.server) url = "{protocol}://{netloc}/lib-static/{platform_version}/{file}".format( protocol=server_details.scheme, netloc=server_details.netloc, platform_version=build.config['platform_version'], file=file) remote._get_file(url, file_path) # Make file executable. os.chmod( file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) return file_path
def _java_is_in_path(): """Return True java exists on the path and can be invoked; False otherwise""" with open(os.devnull, 'w') as devnull: try: proc = lib.PopenWithoutNewConsole(['java', '-version'], stdout=devnull, stderr=devnull) proc.communicate()[0] return proc.returncode == 0 except: return False
def run_hook(build, **kw): for file in sorted(os.listdir(os.path.join('hooks', kw['hook']))): if os.path.isfile(os.path.join('hooks', kw['hook'], file)): cwd = os.getcwd() os.chdir(kw['dir']) target = iter(build.enabled_platforms).next() # Get the extension ext = os.path.splitext(file)[-1][1:] proc = None if ext == "py": build.log.info('Running (Python) hook: ' + file) proc = lib.PopenWithoutNewConsole([ "python", os.path.join(cwd, 'hooks', kw['hook'], file), target ]) elif ext == "js": build.log.info('Running (node) hook: ' + file) proc = lib.PopenWithoutNewConsole([ "node", os.path.join(cwd, 'hooks', kw['hook'], file), target ]) elif ext == "bat" and sys.platform.startswith('win'): build.log.info('Running (Windows Batch file) hook: ' + file) proc = lib.PopenWithoutNewConsole( [os.path.join(cwd, 'hooks', kw['hook'], file), target]) elif ext == "sh" and not sys.platform.startswith('win'): build.log.info('Running (shell) hook: ' + file) proc = lib.PopenWithoutNewConsole( [os.path.join(cwd, 'hooks', kw['hook'], file), target]) if proc != None: proc.wait() os.chdir(cwd) if proc != None and proc.returncode != 0: raise ConfigurationError( 'Hook script exited with a non-zero return code.')
def target(): try: runner['process'] = lib.PopenWithoutNewConsole(cmd, stdout=PIPE, stderr=STDOUT) except OSError as e: if e.errno == errno.ENOENT: # XXX: prompt to update the platform tools, then retry? raise AndroidError( NO_ADB_TEMPLATE.format(adb_location=path_info.adb)) raise runner['std_out'] = runner['process'].communicate()[0]
def runner(): LOG.debug('Running: {cmd}'.format(cmd=subprocess.list2cmdline(args))) state.proc = lib.PopenWithoutNewConsole(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=kw.get('env')) for line in iter(state.proc.stdout.readline, ''): if not filter or filter(line): state.output.write(line) LOG.log(command_log_level, line.rstrip('\r\n')) state.done = True
def _generate_signature(zip_file, key_file): # Sign the zip file with the private key openssl_sign = lib.PopenWithoutNewConsole( ['openssl', 'sha1', '-binary', '-sign', key_file, zip_file], stdout=PIPE) if openssl_sign.wait() != 0: raise ChromeError( "Problem signing Chrome package. openssl returned {}".format( openssl_sign.returncode)) signature = openssl_sign.stdout.read() # Convert the public part of the PEM key to DER format for inclusion in the CRX header openssl_der = lib.PopenWithoutNewConsole([ 'openssl', 'rsa', '-pubout', '-inform', 'PEM', '-outform', 'DER', '-in', key_file ], stdout=PIPE) if openssl_der.wait() != 0: raise ChromeError( "Problem converting PEM key to DER. openssl returned {}".format( openssl_der.returncode)) pubkey = openssl_der.stdout.read() return signature, pubkey
def target(): try: runner['process'] = lib.PopenWithoutNewConsole(cmd, stdout=PIPE, stderr=STDOUT) except Exception: LOG.error("problem finding the android debug bridge at: %s" % path_info.adb) # XXX: prompt to run the sdk manager, then retry? LOG.error( "this probably means you need to run the Android SDK manager and download the Android platform-tools." ) raise AndroidError runner['std_out'] = runner['process'].communicate()[0]
def _download_sdk_for_mac(temp_d): archive_path = path.join(temp_d, "sdk.zip") lib.download_with_progress_bar('Downloading Android SDK', _android_sdk_url(), archive_path) LOG.info('Download complete, extracting SDK') zip_process = lib.PopenWithoutNewConsole( ["unzip", archive_path, '-d', "/Applications"], stdout=PIPE, stderr=STDOUT) output = zip_process.communicate()[0] LOG.debug("unzip output") LOG.debug(output) return _create_path_info_from_sdk("/Applications/android-sdk-macosx/")
def runner(): try: preexec_fn = _required_preexec(create_process_group, os) state.proc = lib.PopenWithoutNewConsole(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=kw.get('env'), preexec_fn=preexec_fn) for line in iter(state.proc.stdout.readline, ''): if filter: line = filter(line) if line != False: state.output.write(line) LOG.log(command_log_level, line.rstrip('\r\n')) state.done = True except Exception as e: state.done = True state.error = e
def run_in_shell(queue): '''will be invoked in by a separate process, to actually run the detached command''' # setsid detaches us completely from the caller os.setsid() # os.devnull is used to ensure that no [1] foo; # lines are shown in the commandline output with open(os.devnull) as devnull: proc = lib.PopenWithoutNewConsole(args, stdout=devnull, stderr=STDOUT) if wait: proc.wait() # signal that we're finished queue.put(True)
def _download_sdk_for_linux(temp_d): archive_path = path.join(temp_d, "sdk.tgz") lib.download_with_progress_bar('Downloading Android SDK', _android_sdk_url(), archive_path) LOG.info('Download complete, extracting SDK') if not path.isdir(path.expanduser("~/.forge")): os.mkdir(path.expanduser("~/.forge")) zip_process = lib.PopenWithoutNewConsole( ["tar", "zxf", archive_path, "-C", path.expanduser("~/.forge")], stdout=PIPE, stderr=STDOUT) output = zip_process.communicate()[0] LOG.debug("unzip output") LOG.debug(output) return _create_path_info_from_sdk( path.expanduser("~/.forge/android-sdk-linux/"))
def build_wp(build, new_working_dir): original_dir = os.getcwd() build_cmd = [ 'c:\\Windows\\Microsoft.NET\\Framework\\v4.0.30319\\MSBuild.exe', 'Forge.sln', '/t:Forge', '/p:Configuration=Release' ] LOG.info('Changing dir to do Visual Studio build: %s, was in %s' % (new_working_dir, original_dir)) try: os.chdir(new_working_dir) msbuild = lib.PopenWithoutNewConsole(build_cmd, stdout=PIPE, stderr=STDOUT) out = msbuild.communicate()[0] if msbuild.returncode != 0: build.log.error('Visual Studio build error: %s' % out) raise Exception('Visual Studio build error') else: build.log.debug('Visual Studio build output: %s' % out) finally: os.chdir(original_dir)
def _update_sdk(path_info): LOG.info('Updating SDK and downloading required Android platform ' '(about 90MB, may take some time)') APPROX_UPPER_BOUND_ON_ANDROID_OUTPUT = 60 android_process = lib.PopenWithoutNewConsole( [ path_info.android, "update", "sdk", "--no-ui", "--filter", "platform-tool,tool,android-8" ], stdout=PIPE, stderr=STDOUT, ) with ProgressBar('Installing Android SDK Components') as bar: finished = [] def kill_adb_occasionally(): """When updating the android sdk, occasionally ADB will have a lock on some files causing the update to fail. Killing it here helps the update succeed. """ while not finished: time.sleep(5) try: # XXX: still time from check to use issue, but close enough if not finished: _kill_adb() except Exception: pass adb_killing_thread = threading.Thread(target=kill_adb_occasionally) adb_killing_thread.daemon = True adb_killing_thread.start() for i, line in enumerate(iter(android_process.stdout.readline, '')): bar.progress(float(i) / APPROX_UPPER_BOUND_ON_ANDROID_OUTPUT) finished.append(True)
def lint_javascript(build): if build.forge_root is None: raise BASE_EXCEPTION("We don't know where the Forge tools are") log.info("Checking JavaScript files...") if sys.platform.startswith("linux"): if platform.architecture()[0] == '64bit': command = path.join(build.forge_root, "bin", "jsl-64") else: command = path.join(build.forge_root, "bin", "jsl") elif sys.platform.startswith("darwin"): command = path.join(build.forge_root, "bin", "jsl-mac") elif sys.platform.startswith("win"): command = path.join(build.forge_root, "bin", "jsl.exe") data = lib.PopenWithoutNewConsole([ command, "-conf", path.join(build.forge_root, "jsl.conf"), "-process", path.join(os.getcwd(), "src", "*.js"), "-nologo", "-nofilelisting", "-nosummary" ], stdout=subprocess.PIPE).communicate()[0] map(log.warning, [line for line in data.split('\n') if line]) log.info("JavaScript check complete")
def package_chrome(build): development_dir = path.join("development", "chrome") release_dir = path.join("release", "chrome") if not path.isdir(release_dir): os.makedirs(release_dir) # Make sure we have openssl otherwise show manual instructions openssl_check = lib.PopenWithoutNewConsole('openssl version', shell=True, stdout=PIPE, stderr=STDOUT) if openssl_check.wait() != 0: msg = """Packaging a Chrome extensions requires openssl. Alternatively you can manually package it. The required steps are: 1) Go to chrome:extensions in the Chrome browser 2) Make sure "developer mode" is on (top right corner)') 3) Use "Pack extension" and use use this for the root: <app dir>/{chrome_folder} More information on packaging Chrome extensions can be found here: http://legacy-docs.trigger.io/en/v1.4/best_practice/release_browser.html#chrome http://code.google.com/chrome/extensions/packaging.html """.format(chrome_folder="development/chrome") LOG.info(msg) raise CouldNotLocate("Make sure 'openssl' is in your path") # We need a private key to package chrome extensions crx_key = build.tool_config.get('chrome.profile.crx_key') crx_key_path = build.tool_config.get('chrome.profile.crx_key_path') if not crx_key: key_msg = """Packaging a Chrome extension requires a private key. You can generate this key with openssl: openssl genrsa -out crxkey.pem 2048 Keep this key safe and secure. It is needed to sign all future versions of the extension. Add the following to <app dir>/local_config.json to use the key: "chrome": { "profile": { "crx_key": "crxkey.pem", "crx_key_path": "<path to crxkey.pem>", } } """ LOG.info(key_msg) return # HACK: build.usercode seems to be the only way to get a reference to the app directory. crx_key_file = path.realpath( path.join(build.usercode, '..', crx_key_path, crx_key)) crx_filename = '{name}.crx'.format(name=build.config['xml_safe_name']) IGNORED_FILES = ['.hgignore', '.DS_Store', 'application.ini', crx_filename] # Create standard zip file with cd(development_dir): zipf = zipfile.ZipFile(crx_filename, 'w', zipfile.ZIP_DEFLATED) for dirpath, dirnames, filenames in os.walk('.'): filenames = list(filter_filenames(filenames, IGNORED_FILES)) dirnames[:] = filter_dirnames(dirnames) for filename in filenames: abspath = os.path.join(dirpath, filename) LOG.info('Adding: %s' % abspath) zipf.write(abspath) zipf.close() # Generate signature signature, pubkey = _generate_signature( path.join(development_dir, crx_filename), crx_key_file) # Combine magic, public key and signature into header and prepend to zip file magic = 'Cr24' version = struct.pack('<I', 2) pubkey_len = struct.pack('<I', len(pubkey)) signature_len = struct.pack('<I', len(signature)) with open(path.join(development_dir, crx_filename), 'rb') as crx: zip_data = crx.read() with open(path.join(development_dir, crx_filename), 'wb') as crx: data = [ magic, version, pubkey_len, signature_len, pubkey, signature, zip_data ] for d in data: crx.write(d) shutil.move(path.join(development_dir, crx_filename), path.join(release_dir, crx_filename))
def package_ie(build, **kw): 'Sign executables, Run NSIS' # On OS X use the nsis executable and files we ship if sys.platform.startswith('darwin'): nsis_osx = os.path.realpath( os.path.join(os.path.dirname(__file__), '../lib/nsis_osx')) nsis_cmd = 'NSISDIR={}/share/nsis PATH={}/bin/:$PATH makensis'.format( nsis_osx, nsis_osx) else: nsis_cmd = 'makensis' LOG.debug("Using nsis command: {nsis_cmd}".format(nsis_cmd=nsis_cmd)) nsis_check = lib.PopenWithoutNewConsole( "{nsis_cmd} -VERSION".format(nsis_cmd=nsis_cmd), shell=True, stdout=PIPE, stderr=STDOUT) stdout, stderr = nsis_check.communicate() if nsis_check.returncode != 0: raise CouldNotLocate( "Make sure the 'makensis' executable is in your path") # JCB: need to check nsis version in stdout here? # Sign executables certificate = build.tool_config.get('ie.profile.developer_certificate') certificate_path = build.tool_config.get( 'ie.profile.developer_certificate_path') certificate_password = build.tool_config.get( 'ie.profile.developer_certificate_password') if certificate: # Figure out which signtool to use signtool = _check_signtool(build) if signtool == None: raise CouldNotLocate( "Make sure the 'signtool' or 'osslsigncode' executable is in your path" ) LOG.info('Signing IE executables with: {signtool}'.format( signtool=signtool)) _sign_app(build=build, signtool=signtool, certificate=certificate, certificate_path=certificate_path, certificate_password=certificate_password) development_dir = path.join("development", "ie") release_dir = path.join("release", "ie") if not path.isdir(release_dir): os.makedirs(release_dir) for arch in ('x86', 'x64'): nsi_filename = "setup-{arch}.nsi".format(arch=arch) package = lib.PopenWithoutNewConsole('{nsis_cmd} {nsi}'.format( nsis_cmd=nsis_cmd, nsi=path.join(development_dir, "dist", nsi_filename)), stdout=PIPE, stderr=STDOUT, shell=True) out, err = package.communicate() if package.returncode != 0: raise IEError("problem running {arch} IE build: {stdout}".format( arch=arch, stdout=out)) # move output to release directory of IE directory and sign it for exe in glob.glob(development_dir + "/dist/*.exe"): destination = path.join( release_dir, "{name}-{version}-{arch}.exe".format( name=build.config.get('name', 'Forge App'), version=build.config.get('version', '0.1'), arch=arch)) shutil.move(exe, destination) if certificate: _sign_executable(build=build, signtool=signtool, target=destination, certificate=certificate, certificate_path=certificate_path, certificate_password=certificate_password)
def ensure_lib_available(cookies, platform_version, file): if 'FORGE_PLATFORM_LOCATION' in os.environ: return path.abspath( path.join(os.environ['FORGE_PLATFORM_LOCATION'], 'generate', 'lib', file)) module_dynamic_path = path.split(path.abspath(__file__))[0] lib_dir = path.abspath(path.join(module_dynamic_path, '..', '..', '.lib')) hash_path = path.abspath(path.join(module_dynamic_path, '..', 'hash.json')) if not path.exists(lib_dir): os.makedirs(lib_dir) # Hide directory on windows if sys.platform == 'win32': try: lib.PopenWithoutNewConsole(['attrib', '+h', lib_dir]).wait() except Exception: # don't care if we fail to hide the templates dir pass from trigger import forge_tool remote = forge_tool.singleton.remote server_details = urlparse.urlparse(remote.server) if not path.exists(hash_path): url = "{protocol}://{netloc}/lib-static/{platform_version}/{file}".format( protocol=server_details.scheme, netloc=server_details.netloc, platform_version=platform_version, file='hash.json') remote._get_file(url, hash_path, cookies=cookies) with open(hash_path, 'r') as hash_file: hashes = json.load(hash_file) file_path = path.join(lib_dir, file) if path.exists(file_path) and file in hashes: # Check hash with open(file_path, 'rb') as cur_file: hash = hashlib.md5(cur_file.read()).hexdigest() if hash == hashes[file]: # File exists and is correct LOG.debug("File: %s, already downloaded and correct." % file) return file_path # File doesn't exist, or has the wrong hash or has no known hash - download LOG.info( "Downloading lib file: %s, this will only happen when a new file is available." % file) url = "{protocol}://{netloc}/lib-static/{platform_version}/{file}".format( protocol=server_details.scheme, netloc=server_details.netloc, platform_version=platform_version, file=file) remote._get_file(url, file_path, cookies=cookies) # Make file executable. os.chmod( file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) return file_path