def get(self, **kwargs):
script_path = request.args.get('file', default=None)
script_type = request.args.get('script_type', default=None)
organization = kwargs['organization']
team = kwargs['team']
back_scripts_root = get_back_scripts_root(team=team,
organization=organization)
user_scripts_root = get_user_scripts_root(team=team,
organization=organization)
if script_path:
if script_type is None:
return error_message(EINVAL,
'Field script_type is required'), 400
if script_type == 'user_scripts':
return send_from_directory(
Path(os.getcwd()) / user_scripts_root, script_path)
elif script_type == 'backing_scripts':
return send_from_directory(
Path(os.getcwd()) / back_scripts_root, script_path)
else:
return error_message(EINVAL, 'Unsupported script type ' +
script_type), 400
elif script_type:
return error_message(EINVAL, 'Field file is required'), 400
user_scripts = path_to_dict(user_scripts_root)
back_scripts = path_to_dict(back_scripts_root)
return {'user_scripts': user_scripts, 'backing_scripts': back_scripts}
def delete(self, **kwargs):
"""Delete the script file"""
organization = kwargs['organization']
team = kwargs['team']
script = request.json.get('file', None)
if not script:
return response_message(EINVAL, 'Field file is required'), 400
if not is_path_secure(script):
return response_message(
EINVAL,
'Referencing to Upper level directory is not allowed'), 401
dirname = os.path.dirname(script)
basename = os.path.basename(script)
script_type = request.json.get('script_type', None)
if script_type is None:
return response_message(EINVAL,
'Field script_type is required'), 400
if script_type == 'test_scripts':
root = get_user_scripts_root(team=team, organization=organization)
elif script_type == 'test_libraries':
root = get_back_scripts_root(team=team, organization=organization)
else:
return response_message(EINVAL, 'Unsupported script type ' +
script_type), 400
if not os.path.exists(root / script):
return response_message(ENOENT,
'File/directory doesn\'t exist'), 404
if os.path.isfile(root / script):
try:
os.remove(root / script)
except OSError as err:
current_app.logger.exception(err)
return response_message(
EIO, 'Error happened while deleting the file'), 401
if script_type == 'test_scripts':
Test.objects(test_suite=os.path.splitext(basename)[0],
path=dirname).delete()
else:
if script_type == 'test_scripts':
cnt = Test.objects(path__startswith=dirname).delete()
if cnt == 0:
current_app.logger.error(
f'Test suite not found in the database under the path {dirname}'
)
try:
shutil.rmtree(root / script)
except OSError as err:
current_app.logger.exception(err)
return response_message(
EIO, 'Error happened while deleting the directory'), 401
return response_message(SUCCESS)
def post(self, **kwargs):
"""
Upload the scripts
Note: Files in the form request payload are tuples of file name and file content
which can't be explicitly listed here. Please check out the form data format on the web.
Usually browser will take care of it.
"""
found = False
organization = kwargs['organization']
team = kwargs['team']
user = kwargs['user']
script_type = request.form.get('script_type', None)
if script_type is None:
return response_message(EINVAL,
'Field script_type is required'), 400
if script_type == 'test_scripts':
root = get_user_scripts_root(team=team, organization=organization)
elif script_type == 'test_libraries':
root = get_back_scripts_root(team=team, organization=organization)
else:
return response_message(EINVAL, 'Unsupported script type ' +
script_type), 400
if not os.path.exists(root):
os.mkdir(root)
for name, file in request.files.items():
if not is_path_secure(file.filename):
return response_message(
EINVAL,
'Referencing to Upper level directory is not allowed'), 401
found = True
filename = root / file.filename
file.save(str(filename))
if not found:
return response_message(EINVAL,
'No files are found in the request'), 401
if script_type == 'test_scripts':
for name, file in request.files.items():
if not file.filename.endswith('.md'):
continue
ret = db_update_test(root, file.filename, user, organization,
team)
if not ret:
return response_message(UNKNOWN_ERROR,
'Failed to update test suite'), 401
def post(self, **kwargs):
found = False
organization = kwargs['organization']
team = kwargs['team']
user = kwargs['user']
script_type = request.form.get('script_type', None)
if script_type is None:
return error_message(EINVAL, 'Field script_type is required'), 400
script_type = request.form.get('script_type', None)
if script_type is None:
return error_message(EINVAL, 'Field script_type is required'), 400
if script_type == 'user_scripts':
root = get_user_scripts_root(team=team, organization=organization)
elif script_type == 'backing_scripts':
root = get_back_scripts_root(team=team, organization=organization)
else:
return error_message(EINVAL,
'Unsupported script type ' + script_type), 400
if not os.path.exists(root):
os.mkdir(root)
for name, file in request.files.items():
if '..' in file.filename:
return error_message(
EINVAL,
'Referencing to Upper level directory is not allowed'), 401
found = True
filename = root / file.filename
file.save(str(filename))
if not found:
return error_message(ENOENT,
'No files are found in the request'), 404
if script_type == 'user_scripts':
for name, file in request.files.items():
ret = db_update_test(scripts_dir=root,
script=file.filename,
user=user.email,
organization=organization,
team=team)
if ret:
return error_message(UNKNOWN_ERROR,
'Failed to update test suite'), 401
def delete(self, **kwargs):
organization = kwargs['organization']
team = kwargs['team']
script = request.json.get('file', None)
if script is None or script == '':
return error_message(EINVAL, 'Field file is required'), 400
if '..' in script:
return error_message(
EINVAL,
'Referencing to Upper level directory is not allowed'), 401
script_type = request.json.get('script_type', None)
if script_type is None:
return error_message(EINVAL, 'Field script_type is required'), 400
if script_type == 'user_scripts':
root = get_user_scripts_root(team=team, organization=organization)
elif script_type == 'backing_scripts':
root = get_back_scripts_root(team=team, organization=organization)
else:
return error_message(EINVAL,
'Unsupported script type ' + script_type), 400
if not os.path.exists(root / script):
return error_message(ENOENT, 'file/directory doesn\'t exist'), 404
if os.path.isfile(root / script):
try:
os.remove(root / script)
except OSError as err:
print(err)
return error_message(
EIO, 'Error happened while deleting a file: '), 401
else:
try:
shutil.rmtree(root / script)
except OSError as err:
print(err)
return error_message(
EIO, 'Error happened while deleting a directory'), 401
if script_type == 'user_scripts':
cnt = Test.objects(path=os.path.abspath(root / script)).delete()
if cnt == 0:
return error_message(ENOENT, 'Test suite not found'), 404
def install_test_suite(package, user, organization, team, pypi_root, proprietary, version=None, new_package=True):
if version is None:
version = package.latest_version
pkg_file = package.files[0]
else:
pkg_file = package.get_package_by_version(version)
if not pkg_file:
current_app.logger.error(f'package file not found for {package.name} with version {version}')
return False
pkg_file = pypi_root / package.package_name / pkg_file
requires = get_package_requires(str(pkg_file), organization, team, type='Test Suite')
if requires:
for pkg, ver in requires:
ret = install_test_suite(pkg, user, organization, team, pypi_root, proprietary, version=ver, new_package=False)
if not ret:
current_app.logger.error(f'Failed to install dependent package {package.name}')
return False
scripts_root = get_user_scripts_root(organization=organization, team=team)
libraries_root = get_back_scripts_root(organization=organization, team=team)
with zipfile.ZipFile(pkg_file) as zf:
for f in zf.namelist():
if f.startswith('EGG-INFO'):
continue
dirname = os.path.dirname(f)
if os.path.exists(scripts_root / dirname):
shutil.rmtree(scripts_root / dirname)
if os.path.exists(libraries_root / dirname):
shutil.rmtree(libraries_root / dirname)
with zipfile.ZipFile(pkg_file) as zf:
libraries = (f for f in zf.namelist() if not f.startswith('EGG-INFO') and '/scripts/' not in f)
for l in libraries:
zf.extract(l, libraries_root)
scripts = [f for f in zf.namelist() if '/scripts/' in f]
for s in scripts:
zf.extract(s, scripts_root)
for pkg_name in set((s.split('/', 1)[0] for s in scripts)):
for f in os.listdir(scripts_root / pkg_name / 'scripts'):
shutil.move(str(scripts_root / pkg_name / 'scripts' / f), scripts_root / pkg_name)
db_update_test(scripts_root, os.path.join(pkg_name, f), user, organization, team, package, version)
shutil.rmtree(scripts_root / pkg_name / 'scripts')
package.modify(inc__download_times=1)
return True
def get(self, **kwargs):
"""
A compound get method for returning script file list or script file content
When field file is None, return the file list as per script_type, otherwise return the specified file.
"""
script_path = request.args.get('file', default=None)
script_type = request.args.get('script_type', default=None)
organization = kwargs['organization']
team = kwargs['team']
test_libraries_root = get_back_scripts_root(team=team,
organization=organization)
test_scripts_root = get_user_scripts_root(team=team,
organization=organization)
if script_path:
if script_type is None:
return response_message(EINVAL,
'Field script_type is required'), 400
if script_type == 'test_scripts':
return send_from_directory(
Path(os.getcwd()) / test_scripts_root, script_path)
elif script_type == 'test_libraries':
return send_from_directory(
Path(os.getcwd()) / test_libraries_root, script_path)
else:
return response_message(
EINVAL, 'Unsupported script type ' + script_type), 400
elif script_type:
return response_message(EINVAL, 'Field file is required'), 400
test_scripts = path_to_dict(test_scripts_root)
test_libraries = path_to_dict(test_libraries_root)
return {'test_scripts': test_scripts, 'test_libraries': test_libraries}
def post(self, **kwargs):
"""Update the script file content"""
script = request.json.get('file', None)
if not script:
return response_message(EINVAL, 'Field file is required'), 400
if not is_path_secure(script):
return response_message(EINVAL, 'Illegal file path'), 401
new_name = request.json.get('new_name', None)
if new_name:
if not is_path_secure(new_name):
return response_message(
EINVAL,
'Referencing to Upper level directory is not allowed'), 401
script_type = request.json.get('script_type', None)
if script_type is None:
return response_message(EINVAL,
'Field script_type is required'), 400
content = request.json.get('content', None)
if content is None and new_name is None:
return response_message(EINVAL, 'Field content is required'), 400
organization = kwargs['organization']
team = kwargs['team']
user = kwargs['user']
package = None
if script_type == 'test_scripts':
root = get_user_scripts_root(team=team, organization=organization)
elif script_type == 'test_libraries':
root = get_back_scripts_root(team=team, organization=organization)
else:
return response_message(EINVAL, 'Unsupported script type ' +
script_type), 400
dirname = os.path.dirname(script)
basename = os.path.basename(script)
if script_type == 'test_scripts' and basename.endswith('.md'):
test = Test.objects(test_suite=os.path.splitext(basename)[0],
path=dirname).first()
elif script_type == 'test_libraries' and dirname:
package = Package.objects(py_packages=dirname).first()
if not package:
return response_message(ENOENT, 'package not found'), 404
try:
os.makedirs(root / dirname)
except FileExistsError:
pass
if content or content == '':
if script_type == 'test_scripts':
content = re.sub(r'\\([{}*_\.])', r'\1', content)
elif script_type == 'test_libraries':
content = re.sub(r'\r\n', '\n', content)
if basename:
with open(root / script, 'w', encoding='utf-8') as f:
f.write(content)
if new_name:
if basename:
new_path = os.path.join(dirname, new_name)
os.rename(root / script, root / new_path)
if script_type == 'test_scripts' and test:
test.modify(test_suite=os.path.splitext(new_name)[0])
else:
os.rename(root / script,
root / os.path.dirname(dirname) / new_name)
if basename and script_type == 'test_scripts':
_script = str(Path(dirname) / new_name) if new_name else script
if _script.endswith('.md'):
ret = db_update_test(root, _script, user, organization, team)
if not ret:
return response_message(UNKNOWN_ERROR,
'Failed to update test suite'), 401
if script_type == 'test_libraries' and package:
package.modify(modified=True)
return response_message(SUCCESS)
def post(self, **kwargs):
script = request.json.get('file', None)
if script is None or script == '':
return error_message(EINVAL, 'Field file is required'), 400
if '..' in script:
return error_message(
EINVAL,
'Referencing to Upper level directory is not allowed'), 401
new_name = request.json.get('new_name', None)
if new_name:
if '..' in new_name:
return error_message(
EINVAL,
'Referencing to Upper level directory is not allowed'), 401
script_type = request.json.get('script_type', None)
if script_type is None:
return error_message(EINVAL, 'Field script_type is required'), 400
content = request.json.get('content', None)
if content is None and new_name is None:
return error_message(EINVAL, 'Field content is required'), 400
organization = kwargs['organization']
team = kwargs['team']
user = kwargs['user']
if script_type == 'user_scripts':
root = get_user_scripts_root(team=team, organization=organization)
elif script_type == 'backing_scripts':
root = get_back_scripts_root(team=team, organization=organization)
else:
return error_message(EINVAL,
'Unsupported script type ' + script_type), 400
if content:
if script_type == 'user_scripts':
content = re.sub(r'\\([{}_])', r'\1', content)
elif script_type == 'backing_scripts':
content = re.sub(r'\r\n', '\n', content)
dirname = os.path.dirname(script)
try:
os.makedirs(root / dirname)
except FileExistsError:
pass
if dirname != script:
with open(root / script, 'w') as f:
f.write(content)
if new_name:
os.rename(root / script, root / os.path.dirname(script) / new_name)
if script_type == 'user_scripts':
_script = str(Path(os.path.dirname(script)) /
new_name) if new_name else script
ret = db_update_test(scripts_dir=root,
script=_script,
user=user.email,
organization=organization,
team=team)
if ret:
return error_message(UNKNOWN_ERROR,
'Failed to update test suite'), 401
def process_task_per_endpoint(app, endpoint, organization=None, team=None):
global ROBOT_PROCESSES, TASKS_CACHED
if not organization and not team:
app.logger.error('Argument organization and team must neither be None')
return
room_id = get_room_id(str(organization.id), str(team.id) if team else '')
taskqueues = TaskQueue.objects(organization=organization,
team=team,
endpoint=endpoint)
if taskqueues.count() == 0:
app.logger.error('Taskqueue not found')
return
# taskqueues = [q for q in taskqueues] # query becomes stale if the document it points to gets changed elsewhere, use document instead of query to perform deletion
taskqueue_first = taskqueues.first()
endpoint_id = str(endpoint.id)
endpoint_uid = endpoint.uid
org_name = team.organization.name + '-' + team.name if team else organization.name
while True:
taskqueue_first.reload('to_delete')
if taskqueue_first.to_delete:
taskqueues.delete()
endpoint.delete()
app.logger.info('Abort the task loop: {} @ {}'.format(
org_name, endpoint_uid))
break
# TODO: lower priority tasks will take precedence if higher priority queue is empty first
# but filled then when thread is searching for tasks in the lower priority task queues
for priority in QUEUE_PRIORITY:
exit_task = False
for taskqueue in taskqueues:
taskqueue.reload('to_delete')
if taskqueue.to_delete:
exit_task = True
break
if taskqueue.priority == priority:
break
else:
app.logger.error('Found task queue with unknown priority')
continue
if exit_task:
break
# "continue" to search for tasks in the lower priority task queues
# "break" to start over to search for tasks from the top priority task queue
task = taskqueue.pop()
if not task:
continue
task_id = str(task.id)
if isinstance(task, DBRef):
app.logger.warning(
'task {} has been deleted, ignore it'.format(task_id))
taskqueue.modify(running_task=None)
break
if task.kickedoff != 0 and not task.parallelization:
app.logger.info(
'task has been taken over by other threads, do nothing')
taskqueue.modify(running_task=None)
break
task.modify(inc__kickedoff=1)
if task.kickedoff != 1 and not task.parallelization:
app.logger.warning('a race condition happened')
taskqueue.modify(running_task=None)
break
app.logger.info('Start to run task {} in the thread {}'.format(
task_id,
threading.current_thread().name))
result_dir = get_test_result_path(task)
scripts_dir = get_user_scripts_root(task)
args = [
'robot', '--loglevel', 'debug', '--outputdir',
str(result_dir), '--extension', 'md', '--consolecolors', 'on',
'--consolemarkers', 'on'
]
os.makedirs(result_dir)
if hasattr(task, 'testcases'):
for t in task.testcases:
args.extend(['-t', t])
if hasattr(task, 'variables'):
variable_file = Path(result_dir) / 'variablefile.py'
convert_json_to_robot_variable(args, task.variables,
variable_file)
addr, port = '127.0.0.1', 8270
args.extend([
'-v', f'address_daemon:{addr}', '-v', f'port_daemon:{port}',
'-v', f'task_id:{task_id}', '-v',
f'endpoint_uid:{endpoint_uid}'
])
args.append(
os.path.join(scripts_dir, task.test.path,
task.test.test_suite + '.md'))
app.logger.info('Arguments: ' + str(args))
p = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=0,
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
if os.name == 'nt' else 0)
ROBOT_PROCESSES[task.id] = p
task.status = 'running'
task.run_date = datetime.datetime.utcnow()
task.endpoint_run = endpoint
task.save()
RPC_SOCKET.emit('task started', {'task_id': task_id}, room=room_id)
log_msg = StringIO()
if room_id not in ROOM_MESSAGES:
ROOM_MESSAGES[room_id] = {task_id: log_msg}
else:
if task_id not in ROOM_MESSAGES[room_id]:
ROOM_MESSAGES[room_id][task_id] = log_msg
ss = b''
while True:
c = p.stdout.read(1)
if not c:
break
try:
c = c.decode(encoding=sys.getdefaultencoding())
except UnicodeDecodeError:
ss += c
else:
c = '\r\n' if c == '\n' else c
log_msg.write(c)
RPC_SOCKET.emit('test report', {
'task_id': task_id,
'message': c
},
room=room_id)
del ROBOT_PROCESSES[task.id]
if ss != b'':
ss = ss.decode(chardet.detect(ss)['encoding'])
log_msg_all = StringIO()
log_msg_all.write(ss)
log_msg_all.write(log_msg.getvalue())
app.logger.info('\n' + log_msg_all.getvalue())
RPC_SOCKET.emit('test report', {
'task_id': task_id,
'message': ss
},
room=room_id)
else:
app.logger.info('\n' + log_msg.getvalue())
#app.logger.info('\n' + log_msg.getvalue().replace('\r\n', '\n'))
p.wait()
if p.returncode == 0:
task.status = 'successful'
else:
task.reload('status')
if task.status != 'cancelled':
task.status = 'failed'
task.save()
RPC_SOCKET.emit('task finished', {
'task_id': task_id,
'status': task.status
},
room=room_id)
ROOM_MESSAGES[room_id][task_id].close()
del ROOM_MESSAGES[room_id][task_id]
if task_id in TASKS_CACHED:
del TASKS_CACHED[task_id]
taskqueue.modify(running_task=None)
endpoint.modify(last_run_date=datetime.datetime.utcnow())
if task.upload_dir:
resource_dir_tmp = get_upload_files_root(task)
if os.path.exists(resource_dir_tmp):
make_tarfile_from_dir(str(result_dir / 'resource.tar.gz'),
resource_dir_tmp)
result_dir_tmp = result_dir / 'temp'
if os.path.exists(result_dir_tmp):
shutil.rmtree(result_dir_tmp)
notification_chain_call(task)
TASK_LOCK.acquire()
TASK_THREADS[endpoint_id] = 1
TASK_LOCK.release()
break
else:
TASK_LOCK.acquire()
if TASK_THREADS[endpoint_id] != 1:
TASK_THREADS[endpoint_id] = 1
TASK_LOCK.release()
app.logger.info('Run the lately scheduled task')
continue
del TASK_THREADS[endpoint_id]
TASK_LOCK.release()
break