def build(self, process_data): Logger.i('Building repository: {0}...'.format(self.get_name())) vendor_file_data = self.load_vendor_file_data() if 'vendor' in vendor_file_data: vendor_data = vendor_file_data['vendor'] if 'build' in vendor_data: vendor_data_build = vendor_data['build'] exitcode, stderr, stdout = FileUtil.run( vendor_data_build, self.get_temp_dir(), process_data.get_environ()) if exitcode == 0: Logger.i('Build finished for repository: {0}'.format( self.get_name())) else: if stdout: Logger.i('Build output for repository: {0}'.format( self.get_name())) Logger.clean(stdout) if stderr: Logger.i( 'Error output while build repository: {0}'.format( self.get_name())) Logger.clean(stderr) Logger.f('Failed to build repository: {0}'.format( self.get_name()))
def download_from_github(self): # download Logger.i('Downloading repository: {0}...'.format(self.get_name())) download_url = self.get_download_url() download_filename = self.get_download_filename() download_dest_dir = Constants.TEMP_DIR download_dest_path = os.path.join(Constants.TEMP_DIR, download_filename) unpacked_dir = self.get_temp_dir() unpack_dir = Constants.TEMP_DIR force_download = False _, git_data_type, git_data_version = self.get_git_data() if git_data_type == Repository.GIT_TYPE_BRANCH: force_download = True # skip if exists if not force_download and os.path.isfile(download_dest_path): Logger.i('Repository already downloaded: {0}'.format( self.get_name())) else: FileUtil.remove_file(download_dest_path) DownloadUtil.download_file(download_url, download_dest_dir, download_filename) # check if file was downloaded if os.path.isfile(download_dest_path): Logger.i('Repository downloaded: {0}'.format(self.get_name())) else: Logger.f('Problems when download repository: {0}'.format( self.get_name())) # unpack Logger.i('Unpacking repository: {0}...'.format(self.get_name())) if not force_download and os.path.isdir(unpacked_dir): Logger.i('Repository already unpacked: {0}...'.format( self.get_name())) else: FileUtil.remove_dir(unpacked_dir) # untar file FileUtil.create_dir(unpack_dir) tar = tarfile.open(download_dest_path) tar.extractall(path=unpack_dir) tar.close() if os.path.isdir(unpacked_dir): Logger.i('Repository unpacked: {0}'.format(self.get_name())) else: Logger.f('Problems when unpack repository: {0}'.format( self.get_name()))
def load_target_data_file(self): Logger.d('Loading target data file...') vendor_dir = self.get_vendor_dir() target_file_path = os.path.join(vendor_dir, Constants.TARGET_DATA_FILE) try: with open(target_file_path, 'r') as stream: return yaml.load(stream) except IOError as exc: Logger.f('Error while read target file: {0}'.format(exc))
def load_vendor_file_data(self): Logger.d('Loading vendor file...') vendor_dir = self.get_temp_dir() vendor_file_path = os.path.join(vendor_dir, Constants.VENDOR_FILE) try: with open(vendor_file_path, 'r') as stream: return yaml.load(stream) except IOError as exc: Logger.f('Error while read vendor file: {0}'.format(exc))
def get_name(self): if self.name and len(self.name) > 0: return self.name if self.type == self.TYPE_COPY_FILE: return 'Copy file' elif self.type == self.TYPE_COPY_FILES: return 'Copy files' elif self.type == self.TYPE_PARSE_FILE: return 'Parse file' elif self.type == self.TYPE_RUN: return 'Run' else: Logger.f('Invalid task type')
def download_from_zip(self): # download Logger.i('Downloading repository: {0}...'.format(self.get_name())) download_url = self.get_download_url() download_filename = self.get_download_filename() download_dest_dir = Constants.TEMP_DIR download_dest_path = os.path.join(download_dest_dir, download_filename) unpacked_dir = self.get_temp_dir() unpack_dir = download_dest_dir # skip if exists if os.path.isfile(download_dest_path): Logger.i('Repository already downloaded: {0}'.format( self.get_name())) else: FileUtil.remove_file(download_dest_path) DownloadUtil.download_file(download_url, download_dest_dir, download_filename) # check if file was downloaded if os.path.isfile(download_dest_path): Logger.i('Repository downloaded: {0}'.format(self.get_name())) else: Logger.f('Problems when download repository: {0}'.format( self.get_name())) # unpack Logger.i('Unpacking repository: {0}...'.format(self.get_name())) if os.path.isdir(unpacked_dir): Logger.i('Repository already unpacked: {0}...'.format( self.get_name())) else: FileUtil.remove_dir(unpacked_dir) # unpack file FileUtil.create_dir(unpack_dir) zipref = zipfile.ZipFile(download_dest_path, 'r') zipref.extractall(path=unpack_dir) zipref.close() if os.path.isdir(unpacked_dir): Logger.i('Repository unpacked: {0}'.format(self.get_name())) else: Logger.f('Problems when unpack repository: {0}'.format( self.get_name()))
def install(self, dependency_name): from ezored.models.logger import Logger from ezored.models.project import Project project = Project.create_from_project_file() if dependency_name: Logger.i('Install dependency "{0}"'.format(dependency_name)) else: Logger.i('Install all dependencies') dependency_found = False total_deps = len(project.dependencies) if total_deps > 0: for dependency in project.dependencies: process_data = ProcessData() process_data.reset() process_data.project_name = project.get_config_value('name') can_build = False if not dependency_name: can_build = True elif dependency.get_name() == dependency_name: can_build = True if can_build: Logger.i('Installing dependency "{0}"...'.format( dependency.get_name())) dependency_found = True dependency.prepare_from_process_data(process_data) dependency.repository.download() dependency.repository.build(project=project, process_data=process_data) Logger.i('Dependency "{0}" installed'.format( dependency.get_name())) if not dependency_found: Logger.f('Dependency not found: {0}'.format(dependency_name)) else: Logger.i('Your project does not have dependencies')
def download_from_git(self): # download Logger.i('Downloading repository: {0}...'.format(self.get_name())) force_download = False rep_path, rep_type, rep_version = self.get_git_data() download_filename = self.get_download_filename() download_dest_dir = Constants.TEMP_DIR download_dest_path = os.path.join(download_dest_dir, download_filename) downloaded_version = GitUtil.get_current_downloaded_repository_version( download_dest_path) if rep_type == Constants.GIT_TYPE_BRANCH: force_download = True if downloaded_version is not None: if downloaded_version != rep_version: Logger.i( 'Repository downloaded version ({0}) is different from configured version ({1}), ' 'downloading configured version...'.format( downloaded_version.strip(), rep_version.strip())) force_download = True # skip if exists if not force_download and os.path.isdir(download_dest_path): Logger.i('Repository already downloaded: {0}'.format( self.get_name())) else: FileUtil.remove_dir(download_dest_path) GitUtil.download(rep_path, rep_type, rep_version, download_dest_path) # check if file was downloaded if os.path.isdir(download_dest_path): Logger.i('Repository downloaded: {0}'.format(self.get_name())) else: Logger.f('Problems when download repository: {0}'.format( self.get_name()))
def parse(self, process_data): if process_data: Logger.d('Parsing task: {0}...'.format(self.get_name())) if self.type == self.TYPE_COPY_FILE: if self.params and 'from' in self.params: self.params['from'] = process_data.parse_text( self.params['from']) if self.params and 'to' in self.params: self.params['to'] = process_data.parse_text( self.params['to']) elif self.type == self.TYPE_COPY_FILES: if self.params and 'from' in self.params: self.params['from'] = process_data.parse_text( self.params['from']) if self.params and 'to' in self.params: self.params['to'] = process_data.parse_text( self.params['to']) elif self.type == self.TYPE_PARSE_FILE: if self.params and 'file' in self.params: self.params['file'] = process_data.parse_text( self.params['file']) elif self.type == self.TYPE_RUN: if self.params and 'args' in self.params: self.params['args'] = process_data.parse_text_list( self.params['args']) else: Logger.f('Invalid task type') else: Logger.d('Cannot parse task params with invalid process data')
def run(self, process_data, template_data, working_dir): Logger.d('Running task: {0}...'.format(self.get_name())) if process_data: if self.type == self.TYPE_COPY_FILE: from_path = self.params['from'] if self.params['from'] else None to_path = self.params['to'] if self.params['to'] else None FileUtil.copy_file(from_path=from_path, to_path=to_path) elif self.type == self.TYPE_COPY_FILES: to_path = self.params['to'] if self.params['to'] else None file_pattern = self.params[ 'from'] if 'from' in self.params else None file_pattern = process_data.parse_text(file_pattern) found_files = FileUtil.find_files(file_pattern) for f in found_files: if f: FileUtil.copy_file(from_path=f, to_path=os.path.join( to_path, os.path.basename(f))) elif self.type == self.TYPE_PARSE_FILE: file_pattern = self.params[ 'file'] if 'file' in self.params else None file_pattern = process_data.parse_text(file_pattern) found_files = FileUtil.find_files(file_pattern) for f in found_files: if f: template_file = os.path.abspath(f) template_loader = jinja2.FileSystemLoader( searchpath=os.path.dirname(template_file)) template_env = jinja2.Environment( loader=template_loader) template = template_env.get_template( os.path.basename(template_file)) templ_result = template.render(template_data) FileUtil.write_to_file(os.path.dirname(template_file), os.path.basename(template_file), str(templ_result)) elif self.type == self.TYPE_RUN: run_args = self.params[ 'args'] if 'args' in self.params else None if run_args: exitcode, stderr, stdout = FileUtil.run( run_args, working_dir, process_data.get_merged_data_for_runner()) if exitcode == 0: Logger.i('Run finished for task: {0}'.format( self.get_name())) else: if stdout: Logger.i('Run output for task: {0}'.format( self.get_name())) Logger.clean(stdout) if stderr: Logger.i('Error output while run task: {0}'.format( self.get_name())) Logger.clean(stderr) Logger.f('Failed to run task: {0}'.format( self.get_name())) else: Logger.f('Invalid task type') else: Logger.d('Process data is invalid to run task')
def build(self, target_name): from ezored.models.logger import Logger from ezored.models.project import Project project = Project.create_from_project_file() process_data = ProcessData() process_data.reset() process_data.project_name = project.get_config_value('name') if target_name: Logger.i('Build only target: {0}'.format(target_name)) else: Logger.i('Build all targets') target_found = False for target in project.targets: can_build = False if not target_name: can_build = True elif target.get_name() == target_name: can_build = True if can_build: Logger.d('Getting target data by target name: {0}...'.format( target_name)) target_found = True # targets need be deleted to be always fresh with target data from dependencies target.remove() # build the target repository after download target.prepare_from_process_data(process_data) target.repository.download() target.repository.build(process_data) # get all target data from project dependencies target_data = TargetData() target_data.project_config = project.config for dependency in project.dependencies: dependency.prepare_from_process_data(process_data) new_target_data = dependency.get_target_data_by_target_name_and_parse( target.get_name(), process_data) target_data.merge(new_target_data) # back to target data target.prepare_from_process_data(process_data) # copy files from dependencies to target directory FileUtil.copy_files_from_list(target_data.copy_files) # parse files path and it content target_project_file_data = target.load_target_project_file_data( ) if 'target' in target_project_file_data: target_project_data = target_project_file_data['target'] # parse files if 'parse_files' in target_project_data: target_project_data_parse_files = target_project_data[ 'parse_files'] if target_project_data_parse_files: Logger.d('Files to parse from target: {0}'.format( len(target_project_data_parse_files))) target_project_data_parse_files = process_data.parse_text_list( target_project_data_parse_files) for target_project_data_parse_file in target_project_data_parse_files: template_loader = jinja2.FileSystemLoader( searchpath='/') template_env = jinja2.Environment( loader=template_loader) template_file = target_project_data_parse_file template = template_env.get_template( template_file) templ_result = template.render( target=target_data) FileUtil.write_to_file( os.path.dirname( target_project_data_parse_file), os.path.basename( target_project_data_parse_file), str(templ_result)) else: Logger.d('No files need to parse from target: {0}'. format(target.get_name())) # build target if 'build' in target_project_data: Logger.i('Building target: {0}...'.format( target.get_name())) target_project_data_build = target_project_data[ 'build'] exitcode, stderr, stdout = FileUtil.run( target_project_data_build, target.repository.get_vendor_dir(), process_data.get_environ()) if exitcode == 0: Logger.i('Build finished for target: {0}'.format( target.get_name())) else: if stdout: Logger.i('Build output for target: {0}'.format( target.get_name())) Logger.clean(stdout) if stderr: Logger.i( 'Error output while build target: {0}'. format(target.get_name())) Logger.clean(stderr) Logger.f('Failed to build target: {0}'.format( target.get_name())) if not target_found: Logger.f('Target not found: {0}'.format(target_name))
def test_fatal(self): with pytest.raises(SystemExit) as error: Logger.f('test_fatal') self.assertEqual(error.type, SystemExit) self.assertEqual(error.value.code, 1)
def execute_command(self, target_command, target_name): from ezored.models.logger import Logger from ezored.models.project import Project import importlib import sys project = Project.create_from_project_file() if target_name: Logger.i('Execute command "{0}" only on target "{1}"'.format( target_command, target_name)) else: Logger.i( 'Execute command "{0}" on all targets'.format(target_command)) target_found = False total_targets = len(project.targets) if total_targets > 0: for target in project.targets: process_data = ProcessData() process_data.reset() process_data.project_name = project.get_config_value('name') can_build = False if not target_name: can_build = True elif target.get_name() == target_name: can_build = True if can_build: Logger.d( 'Getting target data by target name "{0}"...'.format( target.get_name())) target_found = True # targets need be deleted to be always fresh with target data from dependencies target.remove() # build the target repository after download target.prepare_from_process_data(process_data) target.repository.download() target.repository.build(project=project, process_data=process_data) # get all target data from project dependencies target_data = TargetData() target_data.project_home = target.repository.get_vendor_dir( ) target_data.project_config = project.config for dependency in project.dependencies: dependency.prepare_from_process_data(process_data) new_target_data = dependency.get_target_data_by_target_name_and_parse( target.get_name(), process_data) target_data.merge(new_target_data) # back to target data target.prepare_from_process_data(process_data) # process target data and execute required command target_data_file = target.repository.load_target_data_file( ) if 'target' in target_data_file: target_project_data = target_data_file['target'] # target tasks if 'tasks' in target_project_data: target_tasks_data = target_project_data['tasks'] for target_task_data in target_tasks_data: task = Task.from_dict(target_task_data) task.parse(process_data) target_data.tasks.append(task) # run all tasks Task.run_all_tasks( tasks=target_data.tasks, process_data=process_data, template_data={'target': target_data}, working_dir=target.repository.get_vendor_dir()) # execute command on target Logger.i('Executing command "{0}" on target "{1}"...'. format(target_command, target.get_name())) sys_path = list(sys.path) original_cwd = os.getcwd() try: sys.path.insert(0, target.repository.get_vendor_dir()) target_module = importlib.import_module( Constants.TARGET_MODULE_NAME) command = getattr(target_module, 'do_' + target_command) command( params={ 'project': project, 'target': target, 'target_data': target_data, 'process_data': process_data, }) del sys.modules[Constants.TARGET_MODULE_NAME] del target_module del command Logger.i('Command "{0}" finished for target "{1}"'. format(target_command, target.get_name())) except Exception as e: Logger.e( 'Error while call "{0}" on target "{1}": {2}'. format(target_command, target.get_name(), e.message)) raise sys.path = sys_path os.chdir(original_cwd) if not target_found: Logger.f('Target not found: {0}'.format(target_name)) else: Logger.i('Your project does not have targets')