def _generate_transform(self, resource_names: List[ResourceName]) -> DBRes:
updated = inserted = 0
connector = self.target
for resource_name in resource_names:
t_database, t_table = connector.get_transform_target_full_name(resource_name=resource_name,
connector=connector)
name = connector.get_transform_name_format(resource_name=resource_name)
source_version = self.get_source_default_version(resource_name)
if source_version is None:
return DBRes.api_error(msg="Not found resource source table {}".format(resource_name.full_name))
sink_version = self.get_sink_default_version(t_database, t_table)
if sink_version is None:
return DBRes.api_error(msg="Not found resource sink table {}".format(resource_name.full_name))
require = self.get_source_name(source_version) + ',' + self.get_source_name(sink_version)
execution = dict(planner='blink', type=self.get_flink_execution_type(),
parallelism=connector.system_execution_parallelism)
execution['restart-strategy'] = connector.system_execution_restart_strategy
transform = Transform(name=name, sql=self.build_sql(sink_version, source_version, connector),
require=require, connector_id=connector.id, yaml=dump_yaml(dict(execution=execution)))
transform, i = self.dao.upsert_transform(transform)
inserted += i
updated += not i
msg = 'update: {}\ninserted: {}'.format(updated, inserted)
return DBRes(msg=msg)
def run(cls, model: str, mode: str, pk: Union[str, int]) -> DBRes:
dao = Dao()
obj = dao.get_by_name_or_id(model, pk)
if obj:
manager = ManagerFactory.get_manager(model, mode, obj, dao)
if manager.is_support():
return manager.run()
else:
msg = "Not support {}:{} in model {} by {} in ManagerHelper".format(
obj.name, obj.id, model, mode)
return DBRes.api_error(msg)
else:
return DBRes.api_error("Not found {} in model {}".format(
pk, model))
def clean(cls, model: str, pk: int, *args, session: Session,
base: Type[DBT], **kwargs) -> DBRes:
obj = session.query(base).get(pk)
if isinstance(obj, Connector):
back = obj.as_dict()
source, target = obj.source, obj.target
session.delete(obj)
session.commit()
cls._clean(source, session, Connection)
cls._clean(target, session, Connection)
session.add(Connector(**back))
else:
if isinstance(obj, Connection):
s_names = [
x[0] for x in session.query(Connector.name).join(
Connector.source).filter(Connection.id == pk).all()
]
t_names = [
x[0] for x in session.query(Connector.name).join(
Connector.target).filter(Connection.id == pk).all()
]
msg = "please clean connector source: {} target: {}".format(
','.join(s_names), ','.join(t_names))
if s_names or t_names:
return DBRes.api_error(msg)
cls._clean(obj, session, base)
return DBRes()
def generate_transform(self) -> DBRes:
connector = self.target
need_tables = connector.need_tables
source, target = connector.source, connector.target
if not connector.source.resource_names:
return DBRes.api_error("Not Found Any Resource Name in Connection {}".format(source.name))
resource_names = [x for x in connector.source.resource_names if x.db_name in need_tables]
return self._run(resource_names)
def _handle_job(mode: str, pk: str, json_body: dict,
session: Session) -> DBRes:
handle_name = 'handle_' + mode
if mode in JobControlHandle and handle_name in JobControlHandle:
if pk.isdigit():
transform = DBDao.get_transform(pk, session=session)
if transform is None:
return DBRes.api_error(msg='job id {} not found!!!'.format(pk))
else:
transform = pk
data = json_body
run_res = getattr(JobControlHandle, handle_name)(transform, **data)
return DBRes(code=500 if run_res.startswith(FAIL_HEADER) else 200,
msg=run_res)
else:
return DBRes.api_error(msg=' {} not support!!!'.format(mode))
def get(self, path: str):
if not any(map(lambda x: path.startswith(x), support_upload)):
raise tornado.web.HTTPError(status_code=404)
full_path = os.path.join(UPLOAD_ROOT_DIR, path)
if not os.path.exists(full_path):
return self.write_res(DBRes.api_error())
mime = FileMagic.from_file(full_path)
self.set_header('content-type', mime)
self.write(open(full_path, "rb").read())
self.finish()
def post(self):
files = self.request.files
key = list(files.keys())[0]
if key not in support_upload:
return self.write_res(DBRes.api_error())
upload_file = files[key][0]
_, tem_f = mkstemp(suffix=upload_file.filename, dir=upload_dirs[key])
with open(tem_f, 'wb+') as out:
out.write(upload_file.body)
real_path = '/upload/' + key + '/' + os.path.basename(tem_f)
return self.write_res(DBRes(data={"realPath": real_path}))
def _call_(*args, **kwargs):
model = kwargs['model'] if 'model' in kwargs else args[1]
if model not in SUPPORT_MODELS:
return DBRes.api_error(msg=f'{model} not support')
base = SUPPORT_MODELS[model]
return func(*args, base=base, **kwargs)