def proc(self, context: CommandContext):
"""
メイン処理を行います。
context.get_rows()を呼び出して、データ行を取得しながら、
データ加工を行ってください。
渡されてくるデータは、イテレータの場合もあるので、次に
引き継ぐために、context.set_rows()で、データを引き渡してください。
"""
rows = []
for row in context.get_rows():
if type(row) == Row:
rows.append(objdict(row.raw()))
else:
rows.append(objdict(row))
local = {
'input_rows': rows,
'output_rows': [],
'args': self.__params.args
}
# スクリプトの実行を行う
try:
exec(self.__script, self.__modules, local)
except Exception as e:
logger.error(e)
raise e
# 次のコマンドへ引渡し
context.set_rows(local['output_rows'])
def post_script(self, **param):
response.headers['Content-Type'] = 'application/json'
args = objdict(request.url_args)
json = objdict(request.json)
file_name = args.script
file_name = os.path.abspath(f'./scripts/{file_name}')
if not file_name.startswith(os.path.abspath('./scripts')):
response.status = 404
return
# ファイル書き込み
text = json.script
with open(file_name, 'w') as f:
f.write(text)
return json.dumps({})
def open_sqlite(config: dict) -> sqlite3.Connection:
conf = objdict(config)
try:
conn = sqlite3.connect(conf.connect)
conn.row_factory = sqlite3.Row
return conn
except RuntimeError as e:
logger.error(
f'database({conf.comment} was not opened. connection={conf.connect}'
)
raise e
def execute_script(self, **param):
response.headers['Content-Type'] = 'application/json'
args = objdict(request.url_args)
praqta = services.get_service('praqta')[0]
parameters = {}
if request.json:
parameters = objdict(request.json).params
parameters['script_file'] = args.script_file
praqta.start(parameters)
context = parameters['context']
rows = []
for row in context.get_rows():
if type(row) == Row:
row = row.raw()
rows.append(row)
return json.dumps(rows)
def get_new_data(self, parent_row):
# ファイルパスを取得
file_path = parent_row.get(self.__params.file_path_key)
file_path = os.path.abspath(file_path)
# タブ変換サイズの取得
tab_size = 8
if self.__params.untabify:
(_, ext) = os.path.splitext(file_path)
exts = objdict(self.__params.ext_tab_size)
if ext in exts:
tab_size = exts[ext]
# ファイルオープン
logger.info(f"open file {file_path}")
if os.path.exists(file_path) == False:
logger.error(f'file is not exists. => {file_path}')
return []
# ファイルオープン
with open(file_path, 'r', encoding=self.__params.encoding) as f:
lines = f.readlines()
rows = []
line_no = 0
for line in lines:
line_no += 1
# 右側の空白は除去
line = line.rstrip()
# 空チェック
is_empty = line.strip() == ''
# タブサイズ変換
if self.__params.untabify:
line = line.expandtabs(tab_size)
# 行オブジェクト生成
row = {
'file_path': file_path,
'line_no': line_no,
'is_empty': is_empty,
'line': line
}
# 空行スキップ判定
if self.__params.skip_empty and is_empty:
continue
# 親データをコピー
newRow = dict(parent_row.raw())
# 指定属性のみコピーする
for key in self.__params.attributes:
newRow[key] = row[key]
# 行を追加
rows.append(newRow)
return iter(rows)
def get_scripts(self, **param):
response.headers['Content-Type'] = 'application/json'
ret = []
for file in glob('./scripts/*.yml'):
file = os.path.abspath(file)
with open(file, 'r') as f:
data = yaml.load(f, Loader=yaml.FullLoader)
data = objdict(data)
(_, name) = os.path.split(file)
ret.append({'name': name, 'comment': data.comment})
ret = sorted(ret, key=lambda x: x['name'])
return json.dumps(ret)
def post_command(self, **param):
response.headers['Content-Type'] = 'application/json'
json = objdict(request.json)
args = objdict(request.url_args)
json = objdict(request.json)
for ext in ['py', 'yml']:
file_name = os.path.abspath(
f'./packages/commands/{args.command_name}/{args.command_name}.{ext}'
)
if not file_name.startswith(
os.path.abspath('./packages/commands/')):
response.status = 404
return
# ファイル書き込み
text = json.get(ext)
with open(file_name, 'w') as f:
f.write(text)
return json.dump({})
def open_postgresql(config: dict):
conf = objdict(config)
try:
conn = psycopg2.connect(host=conf.host,
port=conf.port,
user=conf.user,
password=conf.password,
database=conf.database,
options=f"-c search_path={conf.schema}")
return conn
except RuntimeError as e:
logger.error(
f'database({conf.comment} was not opened. connection={conf.connect}'
)
raise e
def get_commands(self, **param):
"""
commands配下のコマンドリストを返却
"""
response.headers['Content-Type'] = 'application/json'
ret = []
for f in glob('./packages/commands/**/*.yml'):
if f.endswith('template.yml'):
continue
with open(f, 'r') as f:
data = yaml.load(f, Loader=yaml.FullLoader)
data = objdict(data)
ret.append({'name': data.name, 'comment': data.comment})
ret = sorted(ret, key=lambda x: x['name'])
return json.dumps(ret)
def open_mysql(config: dict):
conf = objdict(config)
try:
connect = MySQLdb.connect(host=conf.host,
port=conf.port,
user=conf.user,
password=conf.password,
db=conf.database,
use_unicode=conf.use_unicode,
charset=conf.charset)
return connect
except RuntimeError as e:
logger.error(
f'database({conf.comment} was not opened. connection={conf.connect}'
)
raise e
def open(self, db_name: str):
if db_name in self.__dbconfig:
db_conf = objdict(self.__dbconfig[db_name])
db_type = db_conf.type
if db_type == 'sqlite3':
return open_sqlite(db_conf)
if db_type == 'postgresql':
return open_postgresql(db_conf)
if db_type == 'mysql':
db = open_mysql(db_conf)
if db:
self.__has_mysql = True
return db
else:
logger.warn(f'{db_type} was not supported.')
return None
def get_script(self, **param):
response.headers['Content-Type'] = 'application/json'
args = objdict(request.url_args)
ret = {}
basePath = os.path.abspath('./scripts')
file = os.path.abspath(f'./scripts/{args.script}')
logger.info(f'read file={file}')
if not file.startswith(basePath):
response.status = 404
return
if not os.path.exists(file):
response.status = 404
return
with open(file, 'r') as f:
text = "".join(f.readlines())
ret['script'] = text
return json.dumps(ret)
def init(self, config: dict):
self.__dbconfig = config
self.__has_mysql = False
# 設定されているDB情報をもとに接続を試してみる
# 接続できなかったら例外をスローして終了
logger.info("start db open check.")
for db_name in self.__dbconfig:
db_conf = objdict(self.__dbconfig[db_name])
try:
logger.info(f"try open '{db_name}'")
with Closer(self.open(db_name)) as db:
if len(db_conf.init_sqls) > 0:
# sql一覧を取得
for sql_file in db_conf.init_sqls:
# SQLを実行(；区切りを順次実行)
execute_sql_file(db, sql_file)
logger.info(f"ok '{db_name}'")
except:
logger.error(f"can't opened '{db_name}'")
pass
def get_command(self, **param):
response.headers['Content-Type'] = 'application/json'
args = objdict(request.url_args)
ret = {}
basePath = os.path.abspath('./packages/commands')
for ext in ['py', 'yml']:
file = os.path.abspath(
f'./packages/commands/{args.command_name}/{args.command_name}.{ext}'
)
logger.info(f'read file={file}')
if not file.startswith(basePath):
response.status = 404
return
if not os.path.exists(file):
response.status = 404
return
with open(file, 'r') as f:
text = "".join(f.readlines())
ret[ext] = text
return json.dumps(ret)
def main(script_path: str, config_parameters: dict):
context = CommandContext()
commandList = []
# スクリプトロード
with open(script_path, 'r') as f:
script = yaml.load(f, Loader=yaml.FullLoader)
commands = script['commands']
# スクリプトパラメータを設定
script['parameters'].update(config_parameters)
context.set_script_parameters(script['parameters'])
logger.info(f"start script '{script_path}'")
debug_mode = False
if 'debug' in script['parameters']:
if type(script['parameters']['debug']) == bool:
debug_mode = script['parameters']['debug'] == True
logger.info(f"debug mode = {debug_mode}")
# スクリプトからコマンド群を順番に生成
for command in commands:
command = objdict(command)
if debug_mode == False and command.debug == True:
continue
# コマンド名からコマンドインスタンスを作成
logger.info(f"create command {command.type}")
commandInstance = factory.new_instance(command.type, logger)
# コマンド引数の仕様を取得する
spec = factory.get_command_arg_spec(command.type)
# コマンドの仕様を記述した情報からパラメータのバリデーション情報を取得
try:
logger.info(f'{command.type}:{command.comment}: start validate')
argSpec = validate_spec(command.type, spec, logger)
except Exception as e:
logger.error(f'{command.type}:{command.comment}: {e}')
context.set_parameterspec(argSpec)
# 引数仕様に則ったパラメータ定義をしているか確認
argSpec.validate(command.parameters)
# コマンド引数を設定
context.set_parameters(command.parameters)
# コマンドの初期化
logger.info(f"init command {command.type}")
commandInstance.init(context)
# コマンドリストへ追加
commandList.append(commandInstance)
# スクリプトパラメータを１行目のデータとして渡す
context.set_rows([script['parameters']])
# コマンド実行処理
logger.info(f"start command loop")
loop = 1
while context.is_stop() == False:
step = 1
if loop > 1:
context.set_rows([])
for commandInstance in commandList:
context.set_step(step)
# コマンド実行
commandInstance.proc(context)
step += 1
loop += 1
# コマンド終了処理
logger.info(f"finalize command")
for commandInstance in commandList:
commandInstance.term(context)
return context
def init(self, config: dict):
self.__config = objdict(config)
self.__bottle = BottleApp(config)
def static_controller(self, **param):
args = objdict(request.url_args)
return static_file(args.file_path, root=self.__parameters.static_root)
def __init__(self, parameters: dict):
super().__init__()
self.__parameters = objdict(parameters)
self.init_route()
