def proc(self, context: CommandContext): """ メイン処理を行います。 context.get_rows()を呼び出して、データ行を取得しながら、 データ加工を行ってください。 渡されてくるデータは、イテレータの場合もあるので、次に 引き継ぐために、context.set_rows()で、データを引き渡してください。 """ rows = [] for row in context.get_rows(): if type(row) == Row: rows.append(objdict(row.raw())) else: rows.append(objdict(row)) local = { 'input_rows': rows, 'output_rows': [], 'args': self.__params.args } # スクリプトの実行を行う try: exec(self.__script, self.__modules, local) except Exception as e: logger.error(e) raise e # 次のコマンドへ引渡し context.set_rows(local['output_rows'])
def post_script(self, **param): response.headers['Content-Type'] = 'application/json' args = objdict(request.url_args) json = objdict(request.json) file_name = args.script file_name = os.path.abspath(f'./scripts/{file_name}') if not file_name.startswith(os.path.abspath('./scripts')): response.status = 404 return # ファイル書き込み text = json.script with open(file_name, 'w') as f: f.write(text) return json.dumps({})
def open_sqlite(config: dict) -> sqlite3.Connection: conf = objdict(config) try: conn = sqlite3.connect(conf.connect) conn.row_factory = sqlite3.Row return conn except RuntimeError as e: logger.error( f'database({conf.comment} was not opened. connection={conf.connect}' ) raise e
def execute_script(self, **param): response.headers['Content-Type'] = 'application/json' args = objdict(request.url_args) praqta = services.get_service('praqta')[0] parameters = {} if request.json: parameters = objdict(request.json).params parameters['script_file'] = args.script_file praqta.start(parameters) context = parameters['context'] rows = [] for row in context.get_rows(): if type(row) == Row: row = row.raw() rows.append(row) return json.dumps(rows)
def get_new_data(self, parent_row): # ファイルパスを取得 file_path = parent_row.get(self.__params.file_path_key) file_path = os.path.abspath(file_path) # タブ変換サイズの取得 tab_size = 8 if self.__params.untabify: (_, ext) = os.path.splitext(file_path) exts = objdict(self.__params.ext_tab_size) if ext in exts: tab_size = exts[ext] # ファイルオープン logger.info(f"open file {file_path}") if os.path.exists(file_path) == False: logger.error(f'file is not exists. => {file_path}') return [] # ファイルオープン with open(file_path, 'r', encoding=self.__params.encoding) as f: lines = f.readlines() rows = [] line_no = 0 for line in lines: line_no += 1 # 右側の空白は除去 line = line.rstrip() # 空チェック is_empty = line.strip() == '' # タブサイズ変換 if self.__params.untabify: line = line.expandtabs(tab_size) # 行オブジェクト生成 row = { 'file_path': file_path, 'line_no': line_no, 'is_empty': is_empty, 'line': line } # 空行スキップ判定 if self.__params.skip_empty and is_empty: continue # 親データをコピー newRow = dict(parent_row.raw()) # 指定属性のみコピーする for key in self.__params.attributes: newRow[key] = row[key] # 行を追加 rows.append(newRow) return iter(rows)
def get_scripts(self, **param): response.headers['Content-Type'] = 'application/json' ret = [] for file in glob('./scripts/*.yml'): file = os.path.abspath(file) with open(file, 'r') as f: data = yaml.load(f, Loader=yaml.FullLoader) data = objdict(data) (_, name) = os.path.split(file) ret.append({'name': name, 'comment': data.comment}) ret = sorted(ret, key=lambda x: x['name']) return json.dumps(ret)
def post_command(self, **param): response.headers['Content-Type'] = 'application/json' json = objdict(request.json) args = objdict(request.url_args) json = objdict(request.json) for ext in ['py', 'yml']: file_name = os.path.abspath( f'./packages/commands/{args.command_name}/{args.command_name}.{ext}' ) if not file_name.startswith( os.path.abspath('./packages/commands/')): response.status = 404 return # ファイル書き込み text = json.get(ext) with open(file_name, 'w') as f: f.write(text) return json.dump({})
def open_postgresql(config: dict): conf = objdict(config) try: conn = psycopg2.connect(host=conf.host, port=conf.port, user=conf.user, password=conf.password, database=conf.database, options=f"-c search_path={conf.schema}") return conn except RuntimeError as e: logger.error( f'database({conf.comment} was not opened. connection={conf.connect}' ) raise e
def get_commands(self, **param): """ commands配下のコマンドリストを返却 """ response.headers['Content-Type'] = 'application/json' ret = [] for f in glob('./packages/commands/**/*.yml'): if f.endswith('template.yml'): continue with open(f, 'r') as f: data = yaml.load(f, Loader=yaml.FullLoader) data = objdict(data) ret.append({'name': data.name, 'comment': data.comment}) ret = sorted(ret, key=lambda x: x['name']) return json.dumps(ret)
def open_mysql(config: dict): conf = objdict(config) try: connect = MySQLdb.connect(host=conf.host, port=conf.port, user=conf.user, password=conf.password, db=conf.database, use_unicode=conf.use_unicode, charset=conf.charset) return connect except RuntimeError as e: logger.error( f'database({conf.comment} was not opened. connection={conf.connect}' ) raise e
def open(self, db_name: str): if db_name in self.__dbconfig: db_conf = objdict(self.__dbconfig[db_name]) db_type = db_conf.type if db_type == 'sqlite3': return open_sqlite(db_conf) if db_type == 'postgresql': return open_postgresql(db_conf) if db_type == 'mysql': db = open_mysql(db_conf) if db: self.__has_mysql = True return db else: logger.warn(f'{db_type} was not supported.') return None
def get_script(self, **param): response.headers['Content-Type'] = 'application/json' args = objdict(request.url_args) ret = {} basePath = os.path.abspath('./scripts') file = os.path.abspath(f'./scripts/{args.script}') logger.info(f'read file={file}') if not file.startswith(basePath): response.status = 404 return if not os.path.exists(file): response.status = 404 return with open(file, 'r') as f: text = "".join(f.readlines()) ret['script'] = text return json.dumps(ret)
def init(self, config: dict): self.__dbconfig = config self.__has_mysql = False # 設定されているDB情報をもとに接続を試してみる # 接続できなかったら例外をスローして終了 logger.info("start db open check.") for db_name in self.__dbconfig: db_conf = objdict(self.__dbconfig[db_name]) try: logger.info(f"try open '{db_name}'") with Closer(self.open(db_name)) as db: if len(db_conf.init_sqls) > 0: # sql一覧を取得 for sql_file in db_conf.init_sqls: # SQLを実行(;区切りを順次実行) execute_sql_file(db, sql_file) logger.info(f"ok '{db_name}'") except: logger.error(f"can't opened '{db_name}'") pass
def get_command(self, **param): response.headers['Content-Type'] = 'application/json' args = objdict(request.url_args) ret = {} basePath = os.path.abspath('./packages/commands') for ext in ['py', 'yml']: file = os.path.abspath( f'./packages/commands/{args.command_name}/{args.command_name}.{ext}' ) logger.info(f'read file={file}') if not file.startswith(basePath): response.status = 404 return if not os.path.exists(file): response.status = 404 return with open(file, 'r') as f: text = "".join(f.readlines()) ret[ext] = text return json.dumps(ret)
def main(script_path: str, config_parameters: dict): context = CommandContext() commandList = [] # スクリプトロード with open(script_path, 'r') as f: script = yaml.load(f, Loader=yaml.FullLoader) commands = script['commands'] # スクリプトパラメータを設定 script['parameters'].update(config_parameters) context.set_script_parameters(script['parameters']) logger.info(f"start script '{script_path}'") debug_mode = False if 'debug' in script['parameters']: if type(script['parameters']['debug']) == bool: debug_mode = script['parameters']['debug'] == True logger.info(f"debug mode = {debug_mode}") # スクリプトからコマンド群を順番に生成 for command in commands: command = objdict(command) if debug_mode == False and command.debug == True: continue # コマンド名からコマンドインスタンスを作成 logger.info(f"create command {command.type}") commandInstance = factory.new_instance(command.type, logger) # コマンド引数の仕様を取得する spec = factory.get_command_arg_spec(command.type) # コマンドの仕様を記述した情報からパラメータのバリデーション情報を取得 try: logger.info(f'{command.type}:{command.comment}: start validate') argSpec = validate_spec(command.type, spec, logger) except Exception as e: logger.error(f'{command.type}:{command.comment}: {e}') context.set_parameterspec(argSpec) # 引数仕様に則ったパラメータ定義をしているか確認 argSpec.validate(command.parameters) # コマンド引数を設定 context.set_parameters(command.parameters) # コマンドの初期化 logger.info(f"init command {command.type}") commandInstance.init(context) # コマンドリストへ追加 commandList.append(commandInstance) # スクリプトパラメータを1行目のデータとして渡す context.set_rows([script['parameters']]) # コマンド実行処理 logger.info(f"start command loop") loop = 1 while context.is_stop() == False: step = 1 if loop > 1: context.set_rows([]) for commandInstance in commandList: context.set_step(step) # コマンド実行 commandInstance.proc(context) step += 1 loop += 1 # コマンド終了処理 logger.info(f"finalize command") for commandInstance in commandList: commandInstance.term(context) return context
def init(self, config: dict): self.__config = objdict(config) self.__bottle = BottleApp(config)
def static_controller(self, **param): args = objdict(request.url_args) return static_file(args.file_path, root=self.__parameters.static_root)
def __init__(self, parameters: dict): super().__init__() self.__parameters = objdict(parameters) self.init_route()