def run_desugar_task(self): self.debug('========= desugar task ========') javaargs = [Builder.get_java(self._config)] arguments = ['-jar', Builder.get_desugar()] patch_classes_cache_dir = self._finder.get_patch_classes_cache_dir() arguments.append('--input') arguments.append(patch_classes_cache_dir) arguments.append('--output') arguments.append(patch_classes_cache_dir) # bootclasspath arguments.append('--bootclasspath_entry') arguments.append(os.path.join(self._config['compile_sdk_directory'], 'android.jar')) # classpath for path in self._classpaths: arguments.append('--classpath_entry') arguments.append(path) javaargs.extend(arguments) self.debug('java exec: ' + ' '.join(javaargs)) output, err, code = cexec(javaargs, callback=None) if code != 0: raise FreelineException('desugar failed.', '{}\n{}'.format(output, err))
def run_javac_task(self): if self._is_only_r_changed( ) and not self._is_other_modules_has_src_changed: self._is_need_javac = False android_tools.clean_src_changed_flag(self._cache_dir) self.debug( 'apt process do not generate new files, ignore javac task.') return extra_javac_args_enabled = not (self._is_databinding_enabled and self._should_run_databinding_apt()) javacargs = self._generate_java_compile_args( extra_javac_args_enabled=extra_javac_args_enabled) self.debug('javac exec: ' + ' '.join(javacargs)) output, err, code = cexec(javacargs, callback=None) if code != 0: raise FreelineException('incremental javac compile failed.', '{}\n{}'.format(output, err)) else: if self._is_r_file_changed: old_r_file = self._finder.get_dst_r_path(config=self._config) new_r_file = android_tools.DirectoryFinder.get_r_file_path( self._finder.get_backup_dir()) if old_r_file and new_r_file: shutil.copyfile(new_r_file, old_r_file) self.debug('copy {} to {}'.format(new_r_file, old_r_file))
def run_apt_only(self): if self._is_databinding_enabled and self._should_run_databinding_apt(): apt_args = self._generate_java_compile_args(extra_javac_args_enabled=True) self.debug('apt exec: ' + ' '.join(apt_args)) output, err, code = cexec(apt_args, callback=None) if code != 0: raise FreelineException('apt compile failed.', '{}\n{}'.format(output, err)) if self._apt_output_dir and os.path.exists(self._apt_output_dir): apt_cache_path = os.path.join(self._config['build_cache_dir'], 'apt_files_stat_cache.json') if os.path.exists(apt_cache_path): apt_cache = load_json_cache(apt_cache_path) for dirpath, dirnames, files in os.walk(self._apt_output_dir): for fn in files: fpath = os.path.join(dirpath, fn) if apt_cache and self._name in apt_cache: if fpath in apt_cache[self._name]: new_md5 = get_md5(fpath) if new_md5 != apt_cache[self._name][fpath]['md5']: self.debug('detect new md5 value, add apt file to change list: {}'.format(fpath)) self._changed_files['src'].append(fpath) else: self.debug('find new apt file, add to change list: {}'.format(fpath)) self._changed_files['src'].append(fpath) else: self.debug('apt cache not found, add to change list: {}'.format(fpath)) self._changed_files['src'].append(fpath)
def run_javac_task(self): javacargs = [self._javac, '-encoding', 'UTF-8', '-g'] if not self._is_retrolambda_enabled: javacargs.extend(['-target', '1.7', '-source', '1.7']) javacargs.append('-cp') javacargs.append(os.pathsep.join(self._classpaths)) for fpath in self._changed_files['src']: javacargs.append(fpath) javacargs.extend(self._extra_javac_args) javacargs.append('-d') javacargs.append(self._finder.get_patch_classes_cache_dir()) self.debug('javac exec: ' + ' '.join(javacargs)) output, err, code = cexec(javacargs, callback=None) if code != 0: raise FreelineException('incremental javac compile failed.', '{}\n{}'.format(output, err)) else: if self._is_r_file_changed: old_r_file = self._finder.get_dst_r_path(config=self._config) new_r_file = android_tools.DirectoryFinder.get_r_file_path(self._finder.get_backup_dir()) shutil.copyfile(new_r_file, old_r_file) self.debug('copy {} to {}'.format(new_r_file, old_r_file))
def run_retrolambda(self): if self._is_retrolambda_enabled: lambda_config = self._config['retrolambda'][self._name] target_dir = self._finder.get_patch_classes_cache_dir() jar_args = [Builder.get_java(self._config), '-Dretrolambda.inputDir={}'.format(target_dir), '-Dretrolambda.outputDir={}'.format(target_dir)] if lambda_config['supportIncludeFiles']: include_files = [] classes = [] for dirpath, dirnames, files in os.walk(target_dir): for fn in files: if fn.endswith('.class'): classes.append(os.path.relpath(os.path.join(dirpath, fn), target_dir)) src_dirs = self._config['project_source_sets'][self._name]['main_src_directory'] for fpath in self._changed_files['src']: short_path = fpath.replace('.java', '.class') for src_dir in src_dirs: if src_dir in short_path: short_path = os.path.relpath(fpath, src_dir).replace('.java', '') break for clazz in classes: if short_path + '.class' in clazz or short_path + '$' in clazz or 'R.class' in clazz \ or 'R$' in clazz or short_path + '_' in clazz: include_file = os.path.join(target_dir, clazz) if os.path.exists(include_file): self.debug('incremental build lambda file: {}'.format(include_file)) include_files.append(include_file) include_files_param = os.pathsep.join(include_files) if len(include_files_param) > 3496: include_files_path = os.path.join(self._cache_dir, self._name, 'retrolambda_inc.list') self.__save_parms_to_file(include_files_path, include_files) jar_args.append('-Dretrolambda.includedFile={}'.format(include_files_path)) else: jar_args.append('-Dretrolambda.includedFiles={}'.format(include_files_param)) lambda_classpaths = [target_dir, lambda_config['rtJar']] lambda_classpaths.extend(self._classpaths) param = os.pathsep.join(lambda_classpaths) if lambda_config['supportIncludeFiles'] and len(param) > 3496: classpath_file = os.path.join(self._cache_dir, self._name, 'retrolambda_classpaths.path') self.__save_parms_to_file(classpath_file, lambda_classpaths) jar_args.append('-Dretrolambda.classpathFile={}'.format(classpath_file)) else: jar_args.append('-Dretrolambda.classpath={}'.format(param)) jar_args.append('-cp') jar_args.append(lambda_config['targetJar']) jar_args.append(lambda_config['mainClass']) self.debug('retrolambda exec: ' + ' '.join(jar_args)) output, err, code = cexec(jar_args, callback=None) if code != 0: raise FreelineException('retrolambda compile failed.', '{}\n{}'.format(output, err))
def __init__(self, name, path, config, changed_files, module_info, is_art=False, is_other_modules_has_src_changed=False): self._name = name self._module_path = path self._config = config self._changed_files = changed_files self._module_info = module_info self._is_art = is_art self._is_other_modules_has_src_changed = is_other_modules_has_src_changed self._aapt = Builder.get_aapt() self._javac = Builder.get_javac(config=config) if self._javac is None: raise FreelineException('Please declares your JAVA_HOME to system env!', 'JAVA_HOME not found in env.') self._dx = Builder.get_dx(self._config) self._cache_dir = self._config['build_cache_dir'] self._finder = None self._res_dependencies = [] self._is_ids_changed = False self._public_xml_path = None self._ids_xml_path = None self._new_res_list = [] self._merged_xml_cache = {} self._origin_res_list = list(self._changed_files['res']) self._classpaths = [] self._is_r_file_changed = False self._is_need_javac = True self._extra_javac_args = [] self.before_execute()
def execute(self): try: self._command.execute() except FreelineException as e: raise e except Exception: raise FreelineException('incremental build task failed.', traceback.format_exc())
def run_dex_task(self): patch_classes_cache_dir = self._finder.get_patch_classes_cache_dir() # dex_path = self._finder.get_dst_dex_path() dex_path = self._finder.get_patch_dex_dir() add_path = None if is_windows_system(): add_path = str( os.path.abspath(os.path.join(self._javac, os.pardir))) dex_args = [ self._dx, '--dex', '--multi-dex', '--output=' + dex_path, patch_classes_cache_dir ] else: dex_args = [ self._dx, '--dex', '--no-optimize', '--force-jumbo', '--multi-dex', '--output=' + dex_path, patch_classes_cache_dir ] self.debug('dex exec: ' + ' '.join(dex_args)) output, err, code = cexec(dex_args, add_path=add_path) if code != 0: raise FreelineException('incremental dex compile failed.', '{}\n{}'.format(output, err)) else: mark_restart_flag(self._cache_dir)
def _install_apk(self): if self._adb: if not os.path.exists(self._apk_path): raise FreelineException( 'apk not found.', 'apk path: {}, not exists.'.format(self._apk_path)) self.debug('start to install apk to device...') install_args = [self._adb, 'install', '-r', self._apk_path] output, err, code = cexec(install_args, callback=None) if 'Failure' in output: self.debug('install apk failed, start to retry.') output, err, code = cexec(install_args, callback=None) if 'Failure' in output: raise FreelineException('install apk to device failed.', '{}\n{}'.format(output, err))
def execute(self): command = '{} -q checkBeforeCleanBuild'.format( get_gradle_executable(self._config)) output, err, code = cexec(command.split(' '), callback=None) if code != 0: from exceptions import FreelineException raise FreelineException( 'freeline failed when read project info with script: {}'. format(command), '{}\n{}'.format(output, err))
def push_full_res_pack(self): if self._is_need_sync_base_res: full_pack_path = get_base_resource_path(self._cache_dir) if os.path.exists(full_pack_path): self.debug('start to sync full resource pack...') self.debug('full pack size: {}kb'.format( os.path.getsize(full_pack_path) / 1000)) with open(full_pack_path, 'rb') as fp: url = 'http://127.0.0.1:{}/pushFullResourcePack'.format( self._port) self.debug('pushfullpack: ' + url) result, err, code = curl(url, body=fp.read()) if code != 0: raise FreelineException('push full res pack failed', err.message) else: raise FreelineException( 'You may need a clean build.', 'full resource pack not found: {}'.format(full_pack_path))
def execute(self): try: self._client.sync_incremental_res() self._client.sync_incremental_dex() self._client.sync_state(self._is_need_restart) self._client.close_connection() except FreelineException as e: raise e except Exception: raise FreelineException('sync files to your device failed', traceback.format_exc())
def execute(self): command = './gradlew -q checkBeforeCleanBuild' if is_windows_system(): command = 'gradlew.bat -q checkBeforeCleanBuild' output, err, code = cexec(command.split(' '), callback=None) if code != 0: from exceptions import FreelineException raise FreelineException('freeline failed when read project info with script: {}'.format(command), '{}\n{}'.format(output, err))
def check_base_res_exist(self): url = 'http://127.0.0.1:{}/checkResource'.format(self._port) self.debug('checkresource: ' + url) result, err, code = curl(url) if code != 0: raise FreelineException('check base res failed', err.message) if int(result) == 0: self.debug('base resource not exists, need to sync full resource pack first') self._is_need_sync_base_res = True else: self.debug('base resource exists, there is no need to sync full resource pack')
def generate_id_file_by_public(public_path, ids_path): if not os.path.exists(public_path): raise FreelineException("public file not found", "public file path: {}".format(public_path)) tree = ET.ElementTree(ET.fromstring(remove_namespace(public_path))) ids_root = ET.Element('resources') for elem in tree.iterfind('public[@type="id"]'): node = ET.SubElement(ids_root, "item") node.attrib['name'] = elem.attrib['name'] node.attrib['type'] = "id" ids_tree = ET.ElementTree(ids_root) ids_tree.write(ids_path, encoding="utf-8")
def sync_incremental_dex(self): dex_path = android_tools.get_incremental_dex_path(self._cache_dir) if os.path.exists(dex_path): self.debug('start to sync incremental dex...') with open(dex_path, 'rb') as fp: url = 'http://127.0.0.1:{}/pushDex'.format(self._port) self.debug('pushdex: ' + url) result, err, code = curl(url, body=fp.read()) if code != 0: from exceptions import FreelineException raise FreelineException('sync incremental dex failed.', err.message) else: self.debug('no {} exists.'.format(dex_path))
def symlink(base_dir, target_dir, fn): base_path = os.path.join(base_dir, fn) target_path = os.path.join(target_dir, fn) if not os.path.exists(base_path): raise FreelineException('file missing: {}'.format(base_path), ' Maybe you should sync freeline repo') if os.path.exists(target_path): os.remove(target_path) if is_windows_system(): copy(base_path, target_path) else: os.symlink(base_path, target_path)
def execute(self): if is_src_changed(self._cache_dir): pending_merge_dexes = self._get_dexes() dex_path = get_incremental_dex_path(self._cache_dir) if len(pending_merge_dexes) == 1: self.debug('just 1 dex need to sync, copy {} to {}'.format(pending_merge_dexes[0], dex_path)) shutil.copy(pending_merge_dexes[0], dex_path) elif len(pending_merge_dexes) > 1: dex_path = get_incremental_dex_path(self._cache_dir) dex_merge_args = ['java', '-jar', os.path.join('freeline', 'release-tools', 'DexMerge.jar'), dex_path] dex_merge_args.extend(pending_merge_dexes) self.debug('merge dex exec: ' + ' '.join(dex_merge_args)) output, err, code = cexec(dex_merge_args, callback=None) if code != 0: raise FreelineException('merge dex failed.', output)
def sync_state(self, is_need_restart): if self._is_need_sync_dex() or self._is_need_sync_res(): self.debug('start to sync close longlink...') restart_char = 'restart' if is_need_restart else 'no' update_last_sync_ticket(self._cache_dir) url = 'http://127.0.0.1:{}/closeLongLink?{}&lastSync={}'.format( self._port, restart_char, get_last_sync_ticket(self._cache_dir)) self.debug('closeLongLink: ' + url) result, err, code = curl(url) # self.wake_up() if code != 0: rollback_last_sync_ticket(self._cache_dir) from exceptions import FreelineException raise FreelineException('sync state failed.', err.message)
def sync_incremental_dex(self): # dex_path = android_tools.get_incremental_dex_path(self._cache_dir) dex_dir = android_tools.get_incremental_dex_dir(self._cache_dir) if os.path.isdir(dex_dir): dexes = [fn for fn in os.listdir(dex_dir) if fn.endswith('.dex')] if len(dexes) > 0: self.debug('start to sync incremental dex...') for dex_name in dexes: dex_path = os.path.join(dex_dir, dex_name) with open(dex_path, 'rb') as fp: url = 'http://127.0.0.1:{}/pushDex?dexName={}'.format(self._port, dex_name.replace('.dex', '')) self.debug('pushdex: ' + url) self.debug('dex path: {}'.format(dex_path)) result, err, code = curl(url, body=fp.read()) if code != 0: from exceptions import FreelineException raise FreelineException('sync incremental dex failed.', err.message) else: self.debug('no incremental dexes in {}'.format(dex_dir))
def execute(self): # reload config from dispatcher import read_freeline_config self._config = read_freeline_config() cwd = self._config['build_script_work_directory'].strip() if not cwd or not os.path.isdir(cwd): cwd = None command = self._config['build_script'] command += ' --stacktrace' command += ' -P freelineBuild=true' self.debug(command) self.debug("Gradle build task is running, please wait a minute...") output, err, code = cexec(command.split(' '), callback=None, cwd=cwd) if code != 0: from exceptions import FreelineException raise FreelineException('build failed with script: {}'.format(self._config['build_script']), '{}\n{}'.format(output, err))
def execute(self): # self.debug('{} start to execute...'.format(self.task.name)) self.task.start_time = time.time() self.task.status = WAITING while not self.task.is_all_parent_finished(): # self.debug('{} waiting...'.format(self.task.name)) self.task.wait() self.task.run_start_time = time.time() self.task.status = WORKING self.debug('{} start to run after waiting {}s'.format( self.task.name, round(self.task.run_start_time - self.task.start_time, 1))) # check if task need to interrupt before being executing if self.task.interrupted_exception is not None: self.task.status = FAILURE self._pass_interrupted_exception() return try: self.task.execute() self.task.status = SUCCESS except FreelineException as e: self.task.interrupted_exception = e self.task.status = FAILURE except: self.task.interrupted_exception = FreelineException( 'unexpected exception within task', traceback.format_exc()) self.task.status = FAILURE self.task.cost_time = round(time.time() - self.task.run_start_time, 1) self.debug('{} finish in {}s'.format(self.task.name, round(self.task.cost_time, 1))) # check if task need to interrupt after being executing if self.task.interrupted_exception is not None: self._pass_interrupted_exception() return for child_task in self.task.child_tasks: child_task.notify() self._check_engine_finished()
def run_aapt_task(self): self._changed_files['res'].append(self._public_xml_path) self._changed_files['res'].append(self._ids_xml_path) aapt_args, final_changed_list = self._get_aapt_args() self.debug('aapt exec: ' + ' '.join(aapt_args)) st = time.time() output, err, code = cexec(aapt_args, callback=None) if code == 0: self.debug('aapt use time: {}ms'.format((time.time() - st) * 1000)) self.debug('merged_changed_list:') self.debug(final_changed_list) self._backup_res_changed_list(final_changed_list) self._handle_with_backup_files(True) mark_res_sync_status(self._finder.get_sync_file_path()) else: clean_res_build_job_flag(self._finder.get_res_build_job_path()) self._handle_with_backup_files(False) rollback_backup_files(self._origin_res_list, self._new_res_list) raise FreelineException('incremental res build failed.', '{}\n{}'.format(output, err))
def run_kotlinc_task(self): # todo 检查R的变化 if self._is_only_r_changed() and not self._is_other_modules_has_src_changed: self._is_need_javac = False # self._is_need_kotlinc = False android_tools.clean_src_changed_flag(self._cache_dir) self.debug('apt process do not generate new files, ignore javac task.') return kotlincargs = self._generate_kotlin_compile_args() self.debug('kotlinc exec: ' + ' '.join(kotlincargs)) output, err, code = cexec(kotlincargs, callback=None) if code != 0: raise FreelineException('incremental kotlinc compile failed.', '{}\n{}'.format(output, err)) else: # todo 这个应该是和kotlin没有关系 拷贝R类用的 if self._is_r_file_changed: old_r_file = self._finder.get_dst_r_path(config=self._config) new_r_file = android_tools.DirectoryFinder.get_r_file_path(self._finder.get_backup_dir()) if old_r_file and new_r_file: shutil.copyfile(new_r_file, old_r_file) self.debug('copy {} to {}'.format(new_r_file, old_r_file))
def sync_incremental_res(self): mode = 'increment' if self._is_art else 'full' can_sync_inc_res = False for module in self._all_modules: finder = GradleDirectoryFinder(module, self._project_info[module]['path'], self._cache_dir) fpath = finder.get_dst_res_pack_path(module) sync_status = finder.get_sync_file_path() if not os.path.exists(sync_status): self.debug( '{} has no need to sync inc res pack.'.format(module)) continue if not can_sync_inc_res and self._is_art: self.check_base_res_exist() self.push_full_res_pack() can_sync_inc_res = True self.debug( 'start to sync {} incremental res pack...'.format(module)) self.debug('{} pack size: {}kb'.format( module, os.path.getsize(fpath) / 1000)) with open(fpath, 'rb') as fp: url = 'http://127.0.0.1:{}/pushResource?mode={}&bundleId={}'.format( self._port, mode, 'base-res') self.debug('pushres: ' + url) result, err, code = curl(url, body=fp.read()) if code != 0: raise FreelineException('sync incremental respack failed', err.message) android_tools.clean_res_build_job_flag( finder.get_res_build_job_path()) self.debug( 'sync {} incremental res pack finished'.format(module))
def run_retrolambda(self): if self._is_need_javac and self._is_retrolambda_enabled: lambda_config = self._config['retrolambda'][self._name] target_dir = self._finder.get_patch_classes_cache_dir() jar_args = [Builder.get_java(self._config), '-Dretrolambda.inputDir={}'.format(target_dir), '-Dretrolambda.outputDir={}'.format(target_dir)] if lambda_config['supportIncludeFiles']: files_stat_path = os.path.join(self._cache_dir, self._name, 'lambda_files_stat.json') include_files = [] if os.path.exists(files_stat_path): files_stat = load_json_cache(files_stat_path) else: files_stat = {} for dirpath, dirnames, files in os.walk(target_dir): for fn in files: fpath = os.path.join(dirpath, fn) if fpath not in files_stat: include_files.append(fpath) self.debug('incremental build new lambda file: {}'.format(fpath)) else: if os.path.getmtime(fpath) > files_stat[fpath]['mtime']: include_files.append(fpath) self.debug('incremental build lambda file: {}'.format(fpath)) include_files_param = os.pathsep.join(include_files) if len(include_files_param) > 3496: include_files_path = os.path.join(self._cache_dir, self._name, 'retrolambda_inc.list') self.__save_parms_to_file(include_files_path, include_files) jar_args.append('-Dretrolambda.includedFile={}'.format(include_files_path)) else: jar_args.append('-Dretrolambda.includedFiles={}'.format(include_files_param)) lambda_classpaths = [target_dir, lambda_config['rtJar']] lambda_classpaths.extend(self._classpaths) param = os.pathsep.join(lambda_classpaths) if lambda_config['supportIncludeFiles'] and len(param) > 3496: classpath_file = os.path.join(self._cache_dir, self._name, 'retrolambda_classpaths.path') self.__save_parms_to_file(classpath_file, lambda_classpaths) jar_args.append('-Dretrolambda.classpathFile={}'.format(classpath_file)) else: jar_args.append('-Dretrolambda.classpath={}'.format(param)) jar_args.append('-cp') jar_args.append(lambda_config['targetJar']) jar_args.append(lambda_config['mainClass']) self.debug('retrolambda exec: ' + ' '.join(jar_args)) output, err, code = cexec(jar_args, callback=None) if code != 0: raise FreelineException('retrolambda compile failed.', '{}\n{}'.format(output, err)) if lambda_config['supportIncludeFiles']: for fpath in include_files: if fpath not in files_stat: files_stat[fpath] = {} files_stat[fpath]['mtime'] = os.path.getmtime(fpath) write_json_cache(files_stat_path, files_stat) self.debug('save lambda files stat to {}'.format(files_stat_path))
def run_aapt(self): aapt_args = [Builder.get_aapt(), 'package', '-f', '-I', os.path.join(self._config['compile_sdk_directory'], 'android.jar'), '-M', self._finder.get_dst_manifest_path()] for rdir in self._config['project_source_sets'][self._main_module_name]['main_res_directory']: if os.path.exists(rdir): aapt_args.append('-S') aapt_args.append(rdir) for rdir in self._module_info['local_dep_res_path']: if os.path.exists(rdir): aapt_args.append('-S') aapt_args.append(rdir) if 'extra_dep_res_paths' in self._config and self._config['extra_dep_res_paths'] is not None: arr = self._config['extra_dep_res_paths'] for path in arr: path = path.strip() if os.path.isdir(path): aapt_args.append('-S') aapt_args.append(path) for resdir in self._module_info['dep_res_path']: if os.path.exists(resdir): aapt_args.append('-S') aapt_args.append(resdir) aapt_args.extend(['-S', self._finder.get_backup_res_dir()]) freeline_assets_dir = os.path.join(self._config['build_cache_dir'], 'freeline-assets') aapt_args.append('-A') aapt_args.append(freeline_assets_dir) for adir in self._config['project_source_sets'][self._main_module_name]['main_assets_directory']: if os.path.exists(adir): aapt_args.append('-A') aapt_args.append(adir) for m in self._module_info['local_module_dep']: if m in self._config['project_source_sets']: for adir in self._config['project_source_sets'][m]['main_assets_directory']: if os.path.exists(adir): aapt_args.append('-A') aapt_args.append(adir) base_resource_path = get_base_resource_path(self._config['build_cache_dir']) aapt_args.append('-m') aapt_args.append('-J') aapt_args.append(self._finder.get_backup_dir()) aapt_args.append('--auto-add-overlay') aapt_args.append('-F') aapt_args.append(base_resource_path) aapt_args.append('--debug-mode') aapt_args.append('--resoucres-md5-cache-path') aapt_args.append(os.path.join(self._config['build_cache_dir'], "arsc_cache.dat")) aapt_args.append('--ignore-assets') aapt_args.append('public_id.xml:public.xml:*.bak:.*') self.debug('aapt exec: ' + ' '.join(aapt_args)) output, err, code = cexec(aapt_args, callback=None) if code != 0: raise FreelineException('build base resources failed with: {}'.format(' '.join(aapt_args)), '{}\n{}'.format(output, err)) self.debug('generate base resource success: {}'.format(base_resource_path))