def get(self, **kwargs): script_path = request.args.get('file', default=None) script_type = request.args.get('script_type', default=None) organization = kwargs['organization'] team = kwargs['team'] back_scripts_root = get_back_scripts_root(team=team, organization=organization) user_scripts_root = get_user_scripts_root(team=team, organization=organization) if script_path: if script_type is None: return error_message(EINVAL, 'Field script_type is required'), 400 if script_type == 'user_scripts': return send_from_directory( Path(os.getcwd()) / user_scripts_root, script_path) elif script_type == 'backing_scripts': return send_from_directory( Path(os.getcwd()) / back_scripts_root, script_path) else: return error_message(EINVAL, 'Unsupported script type ' + script_type), 400 elif script_type: return error_message(EINVAL, 'Field file is required'), 400 user_scripts = path_to_dict(user_scripts_root) back_scripts = path_to_dict(back_scripts_root) return {'user_scripts': user_scripts, 'backing_scripts': back_scripts}
def delete(self, **kwargs): """Delete the script file""" organization = kwargs['organization'] team = kwargs['team'] script = request.json.get('file', None) if not script: return response_message(EINVAL, 'Field file is required'), 400 if not is_path_secure(script): return response_message( EINVAL, 'Referencing to Upper level directory is not allowed'), 401 dirname = os.path.dirname(script) basename = os.path.basename(script) script_type = request.json.get('script_type', None) if script_type is None: return response_message(EINVAL, 'Field script_type is required'), 400 if script_type == 'test_scripts': root = get_user_scripts_root(team=team, organization=organization) elif script_type == 'test_libraries': root = get_back_scripts_root(team=team, organization=organization) else: return response_message(EINVAL, 'Unsupported script type ' + script_type), 400 if not os.path.exists(root / script): return response_message(ENOENT, 'File/directory doesn\'t exist'), 404 if os.path.isfile(root / script): try: os.remove(root / script) except OSError as err: current_app.logger.exception(err) return response_message( EIO, 'Error happened while deleting the file'), 401 if script_type == 'test_scripts': Test.objects(test_suite=os.path.splitext(basename)[0], path=dirname).delete() else: if script_type == 'test_scripts': cnt = Test.objects(path__startswith=dirname).delete() if cnt == 0: current_app.logger.error( f'Test suite not found in the database under the path {dirname}' ) try: shutil.rmtree(root / script) except OSError as err: current_app.logger.exception(err) return response_message( EIO, 'Error happened while deleting the directory'), 401 return response_message(SUCCESS)
def post(self, **kwargs): """ Upload the scripts Note: Files in the form request payload are tuples of file name and file content which can't be explicitly listed here. Please check out the form data format on the web. Usually browser will take care of it. """ found = False organization = kwargs['organization'] team = kwargs['team'] user = kwargs['user'] script_type = request.form.get('script_type', None) if script_type is None: return response_message(EINVAL, 'Field script_type is required'), 400 if script_type == 'test_scripts': root = get_user_scripts_root(team=team, organization=organization) elif script_type == 'test_libraries': root = get_back_scripts_root(team=team, organization=organization) else: return response_message(EINVAL, 'Unsupported script type ' + script_type), 400 if not os.path.exists(root): os.mkdir(root) for name, file in request.files.items(): if not is_path_secure(file.filename): return response_message( EINVAL, 'Referencing to Upper level directory is not allowed'), 401 found = True filename = root / file.filename file.save(str(filename)) if not found: return response_message(EINVAL, 'No files are found in the request'), 401 if script_type == 'test_scripts': for name, file in request.files.items(): if not file.filename.endswith('.md'): continue ret = db_update_test(root, file.filename, user, organization, team) if not ret: return response_message(UNKNOWN_ERROR, 'Failed to update test suite'), 401
def post(self, **kwargs): found = False organization = kwargs['organization'] team = kwargs['team'] user = kwargs['user'] script_type = request.form.get('script_type', None) if script_type is None: return error_message(EINVAL, 'Field script_type is required'), 400 script_type = request.form.get('script_type', None) if script_type is None: return error_message(EINVAL, 'Field script_type is required'), 400 if script_type == 'user_scripts': root = get_user_scripts_root(team=team, organization=organization) elif script_type == 'backing_scripts': root = get_back_scripts_root(team=team, organization=organization) else: return error_message(EINVAL, 'Unsupported script type ' + script_type), 400 if not os.path.exists(root): os.mkdir(root) for name, file in request.files.items(): if '..' in file.filename: return error_message( EINVAL, 'Referencing to Upper level directory is not allowed'), 401 found = True filename = root / file.filename file.save(str(filename)) if not found: return error_message(ENOENT, 'No files are found in the request'), 404 if script_type == 'user_scripts': for name, file in request.files.items(): ret = db_update_test(scripts_dir=root, script=file.filename, user=user.email, organization=organization, team=team) if ret: return error_message(UNKNOWN_ERROR, 'Failed to update test suite'), 401
def delete(self, **kwargs): organization = kwargs['organization'] team = kwargs['team'] script = request.json.get('file', None) if script is None or script == '': return error_message(EINVAL, 'Field file is required'), 400 if '..' in script: return error_message( EINVAL, 'Referencing to Upper level directory is not allowed'), 401 script_type = request.json.get('script_type', None) if script_type is None: return error_message(EINVAL, 'Field script_type is required'), 400 if script_type == 'user_scripts': root = get_user_scripts_root(team=team, organization=organization) elif script_type == 'backing_scripts': root = get_back_scripts_root(team=team, organization=organization) else: return error_message(EINVAL, 'Unsupported script type ' + script_type), 400 if not os.path.exists(root / script): return error_message(ENOENT, 'file/directory doesn\'t exist'), 404 if os.path.isfile(root / script): try: os.remove(root / script) except OSError as err: print(err) return error_message( EIO, 'Error happened while deleting a file: '), 401 else: try: shutil.rmtree(root / script) except OSError as err: print(err) return error_message( EIO, 'Error happened while deleting a directory'), 401 if script_type == 'user_scripts': cnt = Test.objects(path=os.path.abspath(root / script)).delete() if cnt == 0: return error_message(ENOENT, 'Test suite not found'), 404
def install_test_suite(package, user, organization, team, pypi_root, proprietary, version=None, new_package=True): if version is None: version = package.latest_version pkg_file = package.files[0] else: pkg_file = package.get_package_by_version(version) if not pkg_file: current_app.logger.error(f'package file not found for {package.name} with version {version}') return False pkg_file = pypi_root / package.package_name / pkg_file requires = get_package_requires(str(pkg_file), organization, team, type='Test Suite') if requires: for pkg, ver in requires: ret = install_test_suite(pkg, user, organization, team, pypi_root, proprietary, version=ver, new_package=False) if not ret: current_app.logger.error(f'Failed to install dependent package {package.name}') return False scripts_root = get_user_scripts_root(organization=organization, team=team) libraries_root = get_back_scripts_root(organization=organization, team=team) with zipfile.ZipFile(pkg_file) as zf: for f in zf.namelist(): if f.startswith('EGG-INFO'): continue dirname = os.path.dirname(f) if os.path.exists(scripts_root / dirname): shutil.rmtree(scripts_root / dirname) if os.path.exists(libraries_root / dirname): shutil.rmtree(libraries_root / dirname) with zipfile.ZipFile(pkg_file) as zf: libraries = (f for f in zf.namelist() if not f.startswith('EGG-INFO') and '/scripts/' not in f) for l in libraries: zf.extract(l, libraries_root) scripts = [f for f in zf.namelist() if '/scripts/' in f] for s in scripts: zf.extract(s, scripts_root) for pkg_name in set((s.split('/', 1)[0] for s in scripts)): for f in os.listdir(scripts_root / pkg_name / 'scripts'): shutil.move(str(scripts_root / pkg_name / 'scripts' / f), scripts_root / pkg_name) db_update_test(scripts_root, os.path.join(pkg_name, f), user, organization, team, package, version) shutil.rmtree(scripts_root / pkg_name / 'scripts') package.modify(inc__download_times=1) return True
def get(self, **kwargs): """ A compound get method for returning script file list or script file content When field file is None, return the file list as per script_type, otherwise return the specified file. """ script_path = request.args.get('file', default=None) script_type = request.args.get('script_type', default=None) organization = kwargs['organization'] team = kwargs['team'] test_libraries_root = get_back_scripts_root(team=team, organization=organization) test_scripts_root = get_user_scripts_root(team=team, organization=organization) if script_path: if script_type is None: return response_message(EINVAL, 'Field script_type is required'), 400 if script_type == 'test_scripts': return send_from_directory( Path(os.getcwd()) / test_scripts_root, script_path) elif script_type == 'test_libraries': return send_from_directory( Path(os.getcwd()) / test_libraries_root, script_path) else: return response_message( EINVAL, 'Unsupported script type ' + script_type), 400 elif script_type: return response_message(EINVAL, 'Field file is required'), 400 test_scripts = path_to_dict(test_scripts_root) test_libraries = path_to_dict(test_libraries_root) return {'test_scripts': test_scripts, 'test_libraries': test_libraries}
def post(self, **kwargs): """Update the script file content""" script = request.json.get('file', None) if not script: return response_message(EINVAL, 'Field file is required'), 400 if not is_path_secure(script): return response_message(EINVAL, 'Illegal file path'), 401 new_name = request.json.get('new_name', None) if new_name: if not is_path_secure(new_name): return response_message( EINVAL, 'Referencing to Upper level directory is not allowed'), 401 script_type = request.json.get('script_type', None) if script_type is None: return response_message(EINVAL, 'Field script_type is required'), 400 content = request.json.get('content', None) if content is None and new_name is None: return response_message(EINVAL, 'Field content is required'), 400 organization = kwargs['organization'] team = kwargs['team'] user = kwargs['user'] package = None if script_type == 'test_scripts': root = get_user_scripts_root(team=team, organization=organization) elif script_type == 'test_libraries': root = get_back_scripts_root(team=team, organization=organization) else: return response_message(EINVAL, 'Unsupported script type ' + script_type), 400 dirname = os.path.dirname(script) basename = os.path.basename(script) if script_type == 'test_scripts' and basename.endswith('.md'): test = Test.objects(test_suite=os.path.splitext(basename)[0], path=dirname).first() elif script_type == 'test_libraries' and dirname: package = Package.objects(py_packages=dirname).first() if not package: return response_message(ENOENT, 'package not found'), 404 try: os.makedirs(root / dirname) except FileExistsError: pass if content or content == '': if script_type == 'test_scripts': content = re.sub(r'\\([{}*_\.])', r'\1', content) elif script_type == 'test_libraries': content = re.sub(r'\r\n', '\n', content) if basename: with open(root / script, 'w', encoding='utf-8') as f: f.write(content) if new_name: if basename: new_path = os.path.join(dirname, new_name) os.rename(root / script, root / new_path) if script_type == 'test_scripts' and test: test.modify(test_suite=os.path.splitext(new_name)[0]) else: os.rename(root / script, root / os.path.dirname(dirname) / new_name) if basename and script_type == 'test_scripts': _script = str(Path(dirname) / new_name) if new_name else script if _script.endswith('.md'): ret = db_update_test(root, _script, user, organization, team) if not ret: return response_message(UNKNOWN_ERROR, 'Failed to update test suite'), 401 if script_type == 'test_libraries' and package: package.modify(modified=True) return response_message(SUCCESS)
def post(self, **kwargs): script = request.json.get('file', None) if script is None or script == '': return error_message(EINVAL, 'Field file is required'), 400 if '..' in script: return error_message( EINVAL, 'Referencing to Upper level directory is not allowed'), 401 new_name = request.json.get('new_name', None) if new_name: if '..' in new_name: return error_message( EINVAL, 'Referencing to Upper level directory is not allowed'), 401 script_type = request.json.get('script_type', None) if script_type is None: return error_message(EINVAL, 'Field script_type is required'), 400 content = request.json.get('content', None) if content is None and new_name is None: return error_message(EINVAL, 'Field content is required'), 400 organization = kwargs['organization'] team = kwargs['team'] user = kwargs['user'] if script_type == 'user_scripts': root = get_user_scripts_root(team=team, organization=organization) elif script_type == 'backing_scripts': root = get_back_scripts_root(team=team, organization=organization) else: return error_message(EINVAL, 'Unsupported script type ' + script_type), 400 if content: if script_type == 'user_scripts': content = re.sub(r'\\([{}_])', r'\1', content) elif script_type == 'backing_scripts': content = re.sub(r'\r\n', '\n', content) dirname = os.path.dirname(script) try: os.makedirs(root / dirname) except FileExistsError: pass if dirname != script: with open(root / script, 'w') as f: f.write(content) if new_name: os.rename(root / script, root / os.path.dirname(script) / new_name) if script_type == 'user_scripts': _script = str(Path(os.path.dirname(script)) / new_name) if new_name else script ret = db_update_test(scripts_dir=root, script=_script, user=user.email, organization=organization, team=team) if ret: return error_message(UNKNOWN_ERROR, 'Failed to update test suite'), 401
def process_task_per_endpoint(app, endpoint, organization=None, team=None): global ROBOT_PROCESSES, TASKS_CACHED if not organization and not team: app.logger.error('Argument organization and team must neither be None') return room_id = get_room_id(str(organization.id), str(team.id) if team else '') taskqueues = TaskQueue.objects(organization=organization, team=team, endpoint=endpoint) if taskqueues.count() == 0: app.logger.error('Taskqueue not found') return # taskqueues = [q for q in taskqueues] # query becomes stale if the document it points to gets changed elsewhere, use document instead of query to perform deletion taskqueue_first = taskqueues.first() endpoint_id = str(endpoint.id) endpoint_uid = endpoint.uid org_name = team.organization.name + '-' + team.name if team else organization.name while True: taskqueue_first.reload('to_delete') if taskqueue_first.to_delete: taskqueues.delete() endpoint.delete() app.logger.info('Abort the task loop: {} @ {}'.format( org_name, endpoint_uid)) break # TODO: lower priority tasks will take precedence if higher priority queue is empty first # but filled then when thread is searching for tasks in the lower priority task queues for priority in QUEUE_PRIORITY: exit_task = False for taskqueue in taskqueues: taskqueue.reload('to_delete') if taskqueue.to_delete: exit_task = True break if taskqueue.priority == priority: break else: app.logger.error('Found task queue with unknown priority') continue if exit_task: break # "continue" to search for tasks in the lower priority task queues # "break" to start over to search for tasks from the top priority task queue task = taskqueue.pop() if not task: continue task_id = str(task.id) if isinstance(task, DBRef): app.logger.warning( 'task {} has been deleted, ignore it'.format(task_id)) taskqueue.modify(running_task=None) break if task.kickedoff != 0 and not task.parallelization: app.logger.info( 'task has been taken over by other threads, do nothing') taskqueue.modify(running_task=None) break task.modify(inc__kickedoff=1) if task.kickedoff != 1 and not task.parallelization: app.logger.warning('a race condition happened') taskqueue.modify(running_task=None) break app.logger.info('Start to run task {} in the thread {}'.format( task_id, threading.current_thread().name)) result_dir = get_test_result_path(task) scripts_dir = get_user_scripts_root(task) args = [ 'robot', '--loglevel', 'debug', '--outputdir', str(result_dir), '--extension', 'md', '--consolecolors', 'on', '--consolemarkers', 'on' ] os.makedirs(result_dir) if hasattr(task, 'testcases'): for t in task.testcases: args.extend(['-t', t]) if hasattr(task, 'variables'): variable_file = Path(result_dir) / 'variablefile.py' convert_json_to_robot_variable(args, task.variables, variable_file) addr, port = '127.0.0.1', 8270 args.extend([ '-v', f'address_daemon:{addr}', '-v', f'port_daemon:{port}', '-v', f'task_id:{task_id}', '-v', f'endpoint_uid:{endpoint_uid}' ]) args.append( os.path.join(scripts_dir, task.test.path, task.test.test_suite + '.md')) app.logger.info('Arguments: ' + str(args)) p = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == 'nt' else 0) ROBOT_PROCESSES[task.id] = p task.status = 'running' task.run_date = datetime.datetime.utcnow() task.endpoint_run = endpoint task.save() RPC_SOCKET.emit('task started', {'task_id': task_id}, room=room_id) log_msg = StringIO() if room_id not in ROOM_MESSAGES: ROOM_MESSAGES[room_id] = {task_id: log_msg} else: if task_id not in ROOM_MESSAGES[room_id]: ROOM_MESSAGES[room_id][task_id] = log_msg ss = b'' while True: c = p.stdout.read(1) if not c: break try: c = c.decode(encoding=sys.getdefaultencoding()) except UnicodeDecodeError: ss += c else: c = '\r\n' if c == '\n' else c log_msg.write(c) RPC_SOCKET.emit('test report', { 'task_id': task_id, 'message': c }, room=room_id) del ROBOT_PROCESSES[task.id] if ss != b'': ss = ss.decode(chardet.detect(ss)['encoding']) log_msg_all = StringIO() log_msg_all.write(ss) log_msg_all.write(log_msg.getvalue()) app.logger.info('\n' + log_msg_all.getvalue()) RPC_SOCKET.emit('test report', { 'task_id': task_id, 'message': ss }, room=room_id) else: app.logger.info('\n' + log_msg.getvalue()) #app.logger.info('\n' + log_msg.getvalue().replace('\r\n', '\n')) p.wait() if p.returncode == 0: task.status = 'successful' else: task.reload('status') if task.status != 'cancelled': task.status = 'failed' task.save() RPC_SOCKET.emit('task finished', { 'task_id': task_id, 'status': task.status }, room=room_id) ROOM_MESSAGES[room_id][task_id].close() del ROOM_MESSAGES[room_id][task_id] if task_id in TASKS_CACHED: del TASKS_CACHED[task_id] taskqueue.modify(running_task=None) endpoint.modify(last_run_date=datetime.datetime.utcnow()) if task.upload_dir: resource_dir_tmp = get_upload_files_root(task) if os.path.exists(resource_dir_tmp): make_tarfile_from_dir(str(result_dir / 'resource.tar.gz'), resource_dir_tmp) result_dir_tmp = result_dir / 'temp' if os.path.exists(result_dir_tmp): shutil.rmtree(result_dir_tmp) notification_chain_call(task) TASK_LOCK.acquire() TASK_THREADS[endpoint_id] = 1 TASK_LOCK.release() break else: TASK_LOCK.acquire() if TASK_THREADS[endpoint_id] != 1: TASK_THREADS[endpoint_id] = 1 TASK_LOCK.release() app.logger.info('Run the lately scheduled task') continue del TASK_THREADS[endpoint_id] TASK_LOCK.release() break