def run_py_non_docker_tasks(self, dockers: List[str]) -> int:
return_code = 0
py_num = get_python_version(dockers[0])
self._setup_dev_files_py(py_num)
if self.run_args['flake8']:
result_val = self.run_flake8(py_num)
if result_val:
return_code = result_val
if self.run_args['mypy']:
result_val = self.run_mypy(py_num)
if result_val:
return_code = result_val
if self.run_args['bandit']:
result_val = self.run_bandit(py_num)
if result_val:
return_code = result_val
if self.run_args['vulture']:
result_val = self.run_vulture(py_num)
if result_val:
return_code = result_val
return return_code
def run_dev_packages(self) -> int:
return_code = 0
# load yaml
_, yml_path = get_yml_paths_in_dir(
self.project_dir, Errors.no_yml_file(self.project_dir))
if not yml_path:
return 1
print_v('Using yaml file: {}'.format(yml_path))
with open(yml_path, 'r') as yml_file:
yml_data = yaml.safe_load(yml_file)
script_obj = yml_data
if isinstance(script_obj.get('script'), dict):
script_obj = script_obj.get('script')
script_type = script_obj.get('type')
if script_type != 'python':
if script_type == 'powershell':
# TODO powershell linting
return 0
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_type))
return 0
dockers = get_all_docker_images(script_obj)
py_num = get_python_version(dockers[0], self.log_verbose)
self.lock.acquire()
print_color(
"============ Starting process for: {} ============\n".format(
self.project_dir), LOG_COLORS.YELLOW)
if self.lock.locked():
self.lock.release()
self._setup_dev_files(py_num)
if self.run_args['flake8']:
result_val = self.run_flake8(py_num)
if result_val:
return_code = result_val
if self.run_args['mypy']:
result_val = self.run_mypy(py_num)
if result_val:
return_code = result_val
if self.run_args['bandit']:
result_val = self.run_bandit(py_num)
if result_val:
return_code = result_val
for docker in dockers:
for try_num in (1, 2):
print_v("Using docker image: {}".format(docker))
py_num = get_python_version(docker, self.log_verbose)
try:
if self.run_args['tests'] or self.run_args['pylint']:
if py_num == 2.7:
requirements = self.requirements_2
else:
requirements = self.requirements_3
docker_image_created = self._docker_image_create(
docker, requirements)
output, status_code = self._docker_run(
docker_image_created)
self.lock.acquire()
print_color(
"\n========== Running tests/pylint for: {} ========="
.format(self.project_dir), LOG_COLORS.YELLOW)
if status_code == 1:
raise subprocess.CalledProcessError(*output)
else:
print(output)
print_color(
"============ Finished process for: {} "
"with docker: {} ============\n".format(
self.project_dir, docker),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
break # all is good no need to retry
except subprocess.CalledProcessError as ex:
if ex.output:
print_color(
"=========================== ERROR IN {}==========================="
"\n{}\n".format(self.project_dir, ex.output),
LOG_COLORS.RED)
else:
print_color(
"========= Test Failed on {}, Look at the error/s above ========\n"
.format(self.project_dir), LOG_COLORS.RED)
return_code = 1
if not self.log_verbose:
sys.stderr.write(
"Need a more detailed log? try running with the -v options as so: \n{} -v\n\n"
.format(" ".join(sys.argv[:])))
if self.lock.locked():
self.lock.release()
# circle ci docker setup sometimes fails on
if try_num > 1 or not ex.output or 'read: connection reset by peer' not in ex.output:
return 2
else:
sys.stderr.write(
"Retrying as failure seems to be docker communication related...\n"
)
finally:
sys.stdout.flush()
sys.stderr.flush()
return return_code
def extract_to_package_format(self) -> int:
"""Extracts the self.input yml file into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
try:
output_path = self.get_output_path()
except ValueError as ex:
print_error(str(ex))
return 1
self.print_logs("Starting migration of: {} to dir: {}".format(self.input, output_path), log_color=LOG_COLORS.NATIVE)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path) if not self.base_name else self.base_name
code_file = "{}/{}".format(output_path, base_name)
self.extract_code(code_file)
script = self.yml_data['script']
lang_type: str = script['type'] if self.file_type == 'integration' else self.yml_data['type']
code_file = f"{code_file}{TYPE_TO_EXTENSION[lang_type]}"
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
self.print_logs("Creating yml file: {} ...".format(yaml_out), log_color=LOG_COLORS.NATIVE)
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.input, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if self.file_type == 'integration':
script_obj = yaml_obj['script']
if 'image' in yaml_obj:
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
script_obj['script'] = SingleQuotedScalarString('')
code_type = script_obj['type']
if code_type == TYPE_PWSH and not yaml_obj.get('fromversion'):
self.print_logs("Setting fromversion for PowerShell to: 5.5.0", log_color=LOG_COLORS.NATIVE)
yaml_obj['fromversion'] = "5.5.0"
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
# check if there is a README and if found, set found_readme to True
found_readme = False
if self.readme:
yml_readme = os.path.splitext(self.input)[0] + '_README.md'
readme = output_path + '/README.md'
if os.path.exists(yml_readme):
found_readme = True
self.print_logs(f"Copying {readme} to {readme}", log_color=LOG_COLORS.NATIVE)
shutil.copy(yml_readme, readme)
else:
# open an empty file
with open(readme, 'w'):
pass
# Python code formatting and dev env setup
if code_type == TYPE_PYTHON:
if self.basic_fmt:
self.print_logs("Running autopep8 on file: {} ...".format(code_file), log_color=LOG_COLORS.NATIVE)
try:
subprocess.call(["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
self.print_logs("autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW)
if self.pipenv:
if self.basic_fmt:
self.print_logs("Running isort on file: {} ...".format(code_file), LOG_COLORS.NATIVE)
try:
subprocess.call(["isort", code_file])
except FileNotFoundError:
self.print_logs("isort skipped! It doesn't seem you have isort installed.\n"
"Make sure to install it with: pip install isort.\n"
"Then run: isort {}".format(code_file), LOG_COLORS.YELLOW)
self.print_logs("Detecting python version and setting up pipenv files ...", log_color=LOG_COLORS.NATIVE)
docker = get_all_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
self.print_logs("Copying pipenv files from: {}".format(pip_env_dir), log_color=LOG_COLORS.NATIVE)
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
env = os.environ.copy()
env["PIPENV_IGNORE_VIRTUALENVS"] = "1"
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path, env=env)
self.print_logs("Installing all py requirements from docker: [{}] into pipenv".format(docker),
LOG_COLORS.NATIVE)
requirements = get_pip_requirements(docker)
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path, env=env)
except Exception:
self.print_logs("Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n", LOG_COLORS.RED)
os.unlink(fp.name)
self.print_logs("Installing flake8 for linting", log_color=LOG_COLORS.NATIVE)
subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path, env=env)
except FileNotFoundError as err:
self.print_logs("pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
f"Then run in the package dir: pipenv install --dev\n.Err: {err}", LOG_COLORS.YELLOW)
arg_path = os.path.relpath(output_path)
self.print_logs("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN)
next_steps: str = "Next steps: \n" \
"* Install additional py packages for unit testing (if needed): cd {};" \
" pipenv install <package>\n".format(arg_path) if code_type == TYPE_PYTHON else ''
next_steps += "* Create unit tests\n" \
"* Check linting and unit tests by running: demisto-sdk lint -i {}\n".format(arg_path)
next_steps += "* When ready, remove from git the old yml and/or README and add the new package:\n" \
" git rm {}\n".format(self.input)
if found_readme:
next_steps += " git rm {}\n".format(os.path.splitext(self.input)[0] + '_README.md')
next_steps += " git add {}\n".format(arg_path)
self.print_logs(next_steps, log_color=LOG_COLORS.NATIVE)
else:
self.print_logs("Skipping pipenv and requirements installation - Note: no Pipfile will be created",
log_color=LOG_COLORS.YELLOW)
self.print_logs(f"Finished splitting the yml file - you can find the split results here: {output_path}",
log_color=LOG_COLORS.GREEN)
return 0
def _docker_image_create(self, docker_base_image):
"""Create the docker image with dev dependencies. Will check if already existing.
Uses a hash of the requirements to determine the image tag
Arguments:
docker_base_image (string): docker image to use as base for installing dev deps
Returns:
string. image name to use
"""
if ':' not in docker_base_image:
docker_base_image += ':latest'
setup_script = self.container_setup_script
setup_script_name = self.container_setup_script_name
if self.script_type == TYPE_PWSH:
setup_script = self.container_setup_script_pwsh
setup_script_name = self.container_setup_script_pwsh_name
docker_input = None
else:
py_num = get_python_version(docker_base_image)
if py_num == 2.7:
requirements = self.requirements_2
else:
requirements = self.requirements_3
docker_input = requirements
with open(setup_script, "rb") as f:
setup_script_data = f.read()
if self.script_type == TYPE_PYTHON:
md5 = hashlib.md5(
requirements.encode('utf-8') + setup_script_data).hexdigest()
else:
md5 = hashlib.md5(setup_script_data).hexdigest()
target_image = 'devtest' + docker_base_image + '-' + md5
lock_file = ".lock-" + target_image.replace("/", "-")
try:
if (time.time() - os.path.getctime(lock_file)) > (60 * 5):
print("{}: Deleting old lock file: {}".format(
datetime.now(), lock_file))
os.remove(lock_file)
except Exception as ex:
print_v(
"Failed check and delete for lock file: {}. Error: {}".format(
lock_file, ex))
wait_print = True
for x in range(60):
images_ls = run_command(' '.join([
'docker', 'image', 'ls', '--format',
'{{.Repository}}:{{.Tag}}', target_image
])).strip()
if images_ls == target_image:
print('{}: Using already existing docker image: {}'.format(
datetime.now(), target_image))
return target_image
if wait_print:
print(
"{}: Existing image: {} not found will obtain lock file or wait for image"
.format(datetime.now(), target_image))
wait_print = False
print_v("Trying to obtain lock file: " + lock_file)
try:
f = open(lock_file, "x")
f.close()
print("{}: Obtained lock file: {}".format(
datetime.now(), lock_file))
break
except Exception as ex:
print_v("Failed getting lock. Will wait {}".format(str(ex)))
time.sleep(5)
try:
# try doing a pull
try:
print("{}: Trying to pull image: {}".format(
datetime.now(), target_image))
pull_res = subprocess.check_output(
['docker', 'pull', target_image],
stderr=subprocess.STDOUT,
universal_newlines=True)
print("Pull succeeded with output: {}".format(pull_res))
return target_image
except subprocess.CalledProcessError as cpe:
print_v(
"Failed docker pull (will create image) with status: {}. Output: {}"
.format(cpe.returncode, cpe.output))
print(
"{}: Creating docker image: {} (this may take a minute or two...)"
.format(datetime.now(), target_image))
update_cert = os.getenv('DEMISTO_LINT_UPDATE_CERTS', 'yes')
docker_create = [
'docker', 'create', '-e',
f'DEMISTO_LINT_UPDATE_CERTS={update_cert}', '-i',
docker_base_image, 'sh', '/' + setup_script_name
]
print_v(f'running: {docker_create}')
container_id = subprocess.check_output(
docker_create, universal_newlines=True).strip()
print_v(f'created container with id: {container_id}')
subprocess.check_call([
'docker', 'cp', setup_script,
container_id + ':/' + setup_script_name
])
if self.script_type == TYPE_PWSH:
if update_cert == 'yes':
subprocess.check_call([
'docker', 'cp', self.cert_file, container_id +
':/usr/local/share/ca-certificates/custom.crt'
])
print_v(
subprocess.check_output(
['docker', 'start', '-a', '-i', container_id],
input=docker_input,
stderr=subprocess.STDOUT,
universal_newlines=True))
print_v(
subprocess.check_output(
['docker', 'commit', container_id, target_image],
stderr=subprocess.STDOUT,
universal_newlines=True))
print_v(
subprocess.check_output(['docker', 'rm', container_id],
stderr=subprocess.STDOUT,
universal_newlines=True))
if self._docker_login():
print("{}: Pushing image: {} to docker hub".format(
datetime.now(), target_image))
print_v(
subprocess.check_output(['docker', 'push', target_image],
stderr=subprocess.STDOUT,
universal_newlines=True))
except subprocess.CalledProcessError as err:
print(
"Failed executing command with error: {} Output: \n{}".format(
err, err.output))
raise err
finally:
try:
os.remove(lock_file)
except Exception as ex:
print("{}: Error removing file: {}".format(datetime.now(), ex))
print('{}: Done creating docker image: {}'.format(
datetime.now(), target_image))
return target_image
def extract_to_package_format(self) -> int:
"""Extracts the self.yml_path into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
print("Starting migration of: {} to dir: {}".format(
self.yml_path, self.dest_path))
arg_path = self.dest_path
output_path = os.path.abspath(self.dest_path)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path)
yml_type = self.get_yml_type()
code_file = "{}/{}.py".format(output_path, base_name)
self.extract_code(code_file)
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(
output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
print("Creating yml file: {} ...".format(yaml_out))
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.yml_path, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if yml_type == INTEGRATION:
script_obj = yaml_obj['script']
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
if script_obj['type'] != 'python':
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_obj['type']))
return 1
script_obj['script'] = SingleQuotedScalarString('')
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
print("Running autopep8 on file: {} ...".format(code_file))
try:
subprocess.call(
["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
print_color(
"autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file),
LOG_COLORS.YELLOW)
print("Detecting python version and setting up pipenv files ...")
docker = get_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
print("Copying pipenv files from: {}".format(pip_env_dir))
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path)
print(
"Installing all py requirements from docker: [{}] into pipenv".
format(docker))
requirements = subprocess.check_output(
[
"docker", "run", "--rm", docker, "pip", "freeze",
"--disable-pip-version-check"
],
universal_newlines=True,
stderr=subprocess.DEVNULL).strip()
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name],
cwd=output_path)
except Exception:
print_color(
"Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n",
LOG_COLORS.RED)
os.unlink(fp.name)
print("Installing flake8 for linting")
subprocess.call(["pipenv", "install", "--dev", "flake8"],
cwd=output_path)
except FileNotFoundError:
print_color(
"pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
"Then run in the package dir: pipenv install --dev",
LOG_COLORS.YELLOW)
# check if there is a changelog
yml_changelog = os.path.splitext(self.yml_path)[0] + '_CHANGELOG.md'
changelog = arg_path + '/CHANGELOG.md'
if os.path.exists(yml_changelog):
shutil.copy(yml_changelog, changelog)
else:
with open(changelog, 'wt', encoding='utf-8') as changelog_file:
changelog_file.write("## [Unreleased]\n-\n")
print_color("\nCompleted: setting up package: {}\n".format(arg_path),
LOG_COLORS.GREEN)
print(
"Next steps: \n",
"* Install additional py packages for unit testing (if needed): cd {}; pipenv install <package>\n"
.format(arg_path),
"* Create unit tests\n",
"* Check linting and unit tests by running: ./Tests/scripts/pkg_dev_test_tasks.py -d {}\n"
.format(arg_path),
"* When ready rm from git the source yml and add the new package:\n",
" git rm {}\n".format(self.yml_path),
" git add {}\n".format(arg_path),
sep='')
return 0