def check_api_module_imports(self, py_num):
"""
Checks if the integration imports an API module and if so pastes the module in the package.
:param py_num: The python version - api modules are in python 3
"""
if py_num > 3:
unifier = Unifier(self.project_dir)
code_file_path = unifier.get_code_file('.py')
try:
# Look for an import to an API module in the code. If there is such import, we need to copy the correct
# module file to the package directory.
with io.open(code_file_path, mode='r',
encoding='utf-8') as script_file:
_, module_name = unifier.check_api_module_imports(
script_file.read())
if module_name:
module_path = os.path.join(self.configuration.env_dir,
'Packs', 'ApiModules',
'Scripts', module_name,
module_name + '.py')
print_v('Copying ' + os.path.join(
self.configuration.env_dir, 'Scripts', module_path))
if not os.path.exists(module_path):
raise ValueError(
'API Module {} not found, you might be outside of the content repository'
' or this API module does not exist'.format(
module_name))
shutil.copy(os.path.join(module_path), self.project_dir)
except Exception as e:
print_v('Unable to retrieve the module file {}: {}'.format(
module_name, str(e)))
def run_bandit(self, py_num) -> int:
"""Run bandit
Args:
py_num: The python version in use
Returns:
int. 0 on successful bandit run, 1 otherwise.
"""
lint_files = self._get_lint_files()
python_exe = 'python2' if py_num < 3 else 'python3'
output = run_command(' '.join(
[python_exe, '-m', 'bandit', '-lll', '-iii', '-q', lint_files]),
cwd=self.project_dir)
self.lock.acquire()
print("========= Running bandit on: {} ===============".format(
lint_files))
print_v('Using: {} to run bandit'.format(python_exe))
if len(output) == 0:
print_color("bandit completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
if self.lock.locked():
self.lock.release()
return 1
def run_flake8(self, py_num) -> int:
"""Runs flake8
Args:
py_num (int): The python version in use
Returns:
int. 0 if flake8 is successful, 1 otherwise.
"""
lint_files = self._get_lint_files()
python_exe = 'python2' if py_num < 3 else 'python3'
print_v('Using: {} to run flake8'.format(python_exe))
output = run_command(f'{python_exe} -m flake8 {self.project_dir}',
cwd=self.configuration.env_dir)
self.lock.acquire()
print("\n========= Running flake8 on: {}===============".format(
lint_files))
if len(output) == 0:
print_color("flake8 completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
if self.lock.locked():
self.lock.release()
return 1
def upload(self):
"""Upload the integration specified in self.infile to the remote Demisto instance.
"""
try:
if self.unify: # Create a temporary unified yml file
try:
unifier = Unifier(self.path, outdir=self.path)
self.path = unifier.merge_script_package_to_yml()[0][0]
except IndexError:
print_color(
'Error: Path input is not a valid package directory.',
LOG_COLORS.RED)
return 1
# Upload the file to Demisto
result = self.client.integration_upload(file=self.path)
# Print results
print_v(f'Result:\n{result.to_str()}', self.log_verbose)
print_color(f'Uploaded \'{result.name}\' successfully',
LOG_COLORS.GREEN)
except Exception as ex:
raise ex
finally:
if self.unify and os.path.exists(
self.path): # Remove the temporary file
os.remove(self.path)
return 0
def _get_playground_id(self):
"""Retrieves Playground ID from the remote Demisto instance.
"""
playground_filter = {'filter': {'type': [9]}}
ans = self.client.search_investigations(filter=playground_filter)
if ans.total != 1:
raise RuntimeError(
f'Got unexpected amount of results in getPlaygroundInvestigationID. '
f'Response was: {ans.total}')
result = ans.data[0].id
print_v(f'Playground ID: {result}', self.log_verbose)
return result
def _docker_login(self):
if self.docker_login_completed:
return True
docker_user = os.getenv('DOCKERHUB_USER', None)
if not docker_user:
print_v('DOCKERHUB_USER not set. Not trying to login to dockerhub')
return False
docker_pass = os.getenv('DOCKERHUB_PASSWORD', None)
# pass is optional for local testing scenario. allowing password to be passed via stdin
cmd = ['docker', 'login', '-u', docker_user]
if docker_pass:
cmd.append('--password-stdin')
res = subprocess.run(cmd,
input=docker_pass,
capture_output=True,
text=True)
if res.returncode != 0:
print("Failed docker login: {}".format(res.stderr))
return False
print_v("Completed docker login")
self.docker_login_completed = True
return True
def _setup_dev_files(self, py_num):
# copy demistomock and common server
try:
shutil.copy(
self.configuration.env_dir +
'/Tests/demistomock/demistomock.py', self.project_dir)
open(self.project_dir + '/CommonServerUserPython.py',
'a').close() # create empty file
shutil.rmtree(self.project_dir + '/__pycache__',
ignore_errors=True)
shutil.copy(
self.configuration.env_dir +
'/Tests/scripts/dev_envs/pytest/conftest.py', self.project_dir)
self.check_api_module_imports(py_num)
if "/Scripts/CommonServerPython" not in self.project_dir:
# Otherwise we already have the CommonServerPython.py file
shutil.copy(
self.configuration.env_dir +
'/Scripts/CommonServerPython/CommonServerPython.py',
self.project_dir)
except Exception as e:
print_v(
'Could not copy demistomock and CommonServer files: {}'.format(
str(e)), self.log_verbose)
def _docker_image_create(self, docker_base_image, requirements):
"""Create the docker image with dev dependencies. Will check if already existing.
Uses a hash of the requirements to determine the image tag
Arguments:
docker_base_image (string): docker image to use as base for installing dev deps
requirements (string): requirements doc
Returns:
string. image name to use
"""
if ':' not in docker_base_image:
docker_base_image += ':latest'
with open(self.container_setup_script, "rb") as f:
setup_script_data = f.read()
md5 = hashlib.md5(requirements.encode('utf-8') +
setup_script_data).hexdigest()
target_image = 'devtest' + docker_base_image + '-' + md5
lock_file = ".lock-" + target_image.replace("/", "-")
try:
if (time.time() - os.path.getctime(lock_file)) > (60 * 5):
print("{}: Deleting old lock file: {}".format(
datetime.now(), lock_file))
os.remove(lock_file)
except Exception as ex:
print_v(
"Failed check and delete for lock file: {}. Error: {}".format(
lock_file, ex))
wait_print = True
for x in range(60):
images_ls = run_command(' '.join([
'docker', 'image', 'ls', '--format',
'{{.Repository}}:{{.Tag}}', target_image
])).strip()
if images_ls == target_image:
print('{}: Using already existing docker image: {}'.format(
datetime.now(), target_image))
return target_image
if wait_print:
print(
"{}: Existing image: {} not found will obtain lock file or wait for image"
.format(datetime.now(), target_image))
wait_print = False
print_v("Trying to obtain lock file: " + lock_file)
try:
f = open(lock_file, "x")
f.close()
print("{}: Obtained lock file: {}".format(
datetime.now(), lock_file))
break
except Exception as ex:
print_v("Failed getting lock. Will wait {}".format(str(ex)))
time.sleep(5)
try:
# try doing a pull
try:
print("{}: Trying to pull image: {}".format(
datetime.now(), target_image))
pull_res = subprocess.check_output(
['docker', 'pull', target_image],
stderr=subprocess.STDOUT,
universal_newlines=True)
print("Pull succeeded with output: {}".format(pull_res))
return target_image
except subprocess.CalledProcessError as cpe:
print_v(
"Failed docker pull (will create image) with status: {}. Output: {}"
.format(cpe.returncode, cpe.output))
print(
"{}: Creating docker image: {} (this may take a minute or two...)"
.format(datetime.now(), target_image))
container_id = run_command(' '.join([
'docker', 'create', '-i', docker_base_image, 'sh',
'/' + self.container_setup_script_name
])).strip()
subprocess.check_call([
'docker', 'cp', self.container_setup_script,
container_id + ':/' + self.container_setup_script_name
])
print_v(
subprocess.check_output(
['docker', 'start', '-a', '-i', container_id],
input=requirements,
stderr=subprocess.STDOUT,
universal_newlines=True))
print_v(
subprocess.check_output(
['docker', 'commit', container_id, target_image],
stderr=subprocess.STDOUT,
universal_newlines=True))
print_v(
subprocess.check_output(['docker', 'rm', container_id],
stderr=subprocess.STDOUT,
universal_newlines=True))
if self._docker_login():
print("{}: Pushing image: {} to docker hub".format(
datetime.now(), target_image))
print_v(
subprocess.check_output(['docker', 'push', target_image],
stderr=subprocess.STDOUT,
universal_newlines=True))
except subprocess.CalledProcessError as err:
print(
"Failed executing command with error: {} Output: \n{}".format(
err, err.output))
raise err
finally:
try:
os.remove(lock_file)
except Exception as ex:
print("{}: Error removing file: {}".format(datetime.now(), ex))
print('{}: Done creating docker image: {}'.format(
datetime.now(), target_image))
return target_image
def run_dev_packages(self) -> int:
return_code = 0
# load yaml
_, yml_path = get_yml_paths_in_dir(
self.project_dir, Errors.no_yml_file(self.project_dir))
if not yml_path:
return 1
print_v('Using yaml file: {}'.format(yml_path))
with open(yml_path, 'r') as yml_file:
yml_data = yaml.safe_load(yml_file)
script_obj = yml_data
if isinstance(script_obj.get('script'), dict):
script_obj = script_obj.get('script')
script_type = script_obj.get('type')
if script_type != 'python':
if script_type == 'powershell':
# TODO powershell linting
return 0
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_type))
return 0
dockers = get_all_docker_images(script_obj)
py_num = get_python_version(dockers[0], self.log_verbose)
self.lock.acquire()
print_color(
"============ Starting process for: {} ============\n".format(
self.project_dir), LOG_COLORS.YELLOW)
if self.lock.locked():
self.lock.release()
self._setup_dev_files(py_num)
if self.run_args['flake8']:
result_val = self.run_flake8(py_num)
if result_val:
return_code = result_val
if self.run_args['mypy']:
result_val = self.run_mypy(py_num)
if result_val:
return_code = result_val
if self.run_args['bandit']:
result_val = self.run_bandit(py_num)
if result_val:
return_code = result_val
for docker in dockers:
for try_num in (1, 2):
print_v("Using docker image: {}".format(docker))
py_num = get_python_version(docker, self.log_verbose)
try:
if self.run_args['tests'] or self.run_args['pylint']:
if py_num == 2.7:
requirements = self.requirements_2
else:
requirements = self.requirements_3
docker_image_created = self._docker_image_create(
docker, requirements)
output, status_code = self._docker_run(
docker_image_created)
self.lock.acquire()
print_color(
"\n========== Running tests/pylint for: {} ========="
.format(self.project_dir), LOG_COLORS.YELLOW)
if status_code == 1:
raise subprocess.CalledProcessError(*output)
else:
print(output)
print_color(
"============ Finished process for: {} "
"with docker: {} ============\n".format(
self.project_dir, docker),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
break # all is good no need to retry
except subprocess.CalledProcessError as ex:
if ex.output:
print_color(
"=========================== ERROR IN {}==========================="
"\n{}\n".format(self.project_dir, ex.output),
LOG_COLORS.RED)
else:
print_color(
"========= Test Failed on {}, Look at the error/s above ========\n"
.format(self.project_dir), LOG_COLORS.RED)
return_code = 1
if not self.log_verbose:
sys.stderr.write(
"Need a more detailed log? try running with the -v options as so: \n{} -v\n\n"
.format(" ".join(sys.argv[:])))
if self.lock.locked():
self.lock.release()
# circle ci docker setup sometimes fails on
if try_num > 1 or not ex.output or 'read: connection reset by peer' not in ex.output:
return 2
else:
sys.stderr.write(
"Retrying as failure seems to be docker communication related...\n"
)
finally:
sys.stdout.flush()
sys.stderr.flush()
return return_code
