def _generate_transform(self, resource_names: List[ResourceName]) -> DBRes: updated = inserted = 0 connector = self.target for resource_name in resource_names: t_database, t_table = connector.get_transform_target_full_name(resource_name=resource_name, connector=connector) name = connector.get_transform_name_format(resource_name=resource_name) source_version = self.get_source_default_version(resource_name) if source_version is None: return DBRes.api_error(msg="Not found resource source table {}".format(resource_name.full_name)) sink_version = self.get_sink_default_version(t_database, t_table) if sink_version is None: return DBRes.api_error(msg="Not found resource sink table {}".format(resource_name.full_name)) require = self.get_source_name(source_version) + ',' + self.get_source_name(sink_version) execution = dict(planner='blink', type=self.get_flink_execution_type(), parallelism=connector.system_execution_parallelism) execution['restart-strategy'] = connector.system_execution_restart_strategy transform = Transform(name=name, sql=self.build_sql(sink_version, source_version, connector), require=require, connector_id=connector.id, yaml=dump_yaml(dict(execution=execution))) transform, i = self.dao.upsert_transform(transform) inserted += i updated += not i msg = 'update: {}\ninserted: {}'.format(updated, inserted) return DBRes(msg=msg)
def run(cls, model: str, mode: str, pk: Union[str, int]) -> DBRes: dao = Dao() obj = dao.get_by_name_or_id(model, pk) if obj: manager = ManagerFactory.get_manager(model, mode, obj, dao) if manager.is_support(): return manager.run() else: msg = "Not support {}:{} in model {} by {} in ManagerHelper".format( obj.name, obj.id, model, mode) return DBRes.api_error(msg) else: return DBRes.api_error("Not found {} in model {}".format( pk, model))
def clean(cls, model: str, pk: int, *args, session: Session, base: Type[DBT], **kwargs) -> DBRes: obj = session.query(base).get(pk) if isinstance(obj, Connector): back = obj.as_dict() source, target = obj.source, obj.target session.delete(obj) session.commit() cls._clean(source, session, Connection) cls._clean(target, session, Connection) session.add(Connector(**back)) else: if isinstance(obj, Connection): s_names = [ x[0] for x in session.query(Connector.name).join( Connector.source).filter(Connection.id == pk).all() ] t_names = [ x[0] for x in session.query(Connector.name).join( Connector.target).filter(Connection.id == pk).all() ] msg = "please clean connector source: {} target: {}".format( ','.join(s_names), ','.join(t_names)) if s_names or t_names: return DBRes.api_error(msg) cls._clean(obj, session, base) return DBRes()
def generate_transform(self) -> DBRes: connector = self.target need_tables = connector.need_tables source, target = connector.source, connector.target if not connector.source.resource_names: return DBRes.api_error("Not Found Any Resource Name in Connection {}".format(source.name)) resource_names = [x for x in connector.source.resource_names if x.db_name in need_tables] return self._run(resource_names)
def _handle_job(mode: str, pk: str, json_body: dict, session: Session) -> DBRes: handle_name = 'handle_' + mode if mode in JobControlHandle and handle_name in JobControlHandle: if pk.isdigit(): transform = DBDao.get_transform(pk, session=session) if transform is None: return DBRes.api_error(msg='job id {} not found!!!'.format(pk)) else: transform = pk data = json_body run_res = getattr(JobControlHandle, handle_name)(transform, **data) return DBRes(code=500 if run_res.startswith(FAIL_HEADER) else 200, msg=run_res) else: return DBRes.api_error(msg=' {} not support!!!'.format(mode))
def get(self, path: str): if not any(map(lambda x: path.startswith(x), support_upload)): raise tornado.web.HTTPError(status_code=404) full_path = os.path.join(UPLOAD_ROOT_DIR, path) if not os.path.exists(full_path): return self.write_res(DBRes.api_error()) mime = FileMagic.from_file(full_path) self.set_header('content-type', mime) self.write(open(full_path, "rb").read()) self.finish()
def post(self): files = self.request.files key = list(files.keys())[0] if key not in support_upload: return self.write_res(DBRes.api_error()) upload_file = files[key][0] _, tem_f = mkstemp(suffix=upload_file.filename, dir=upload_dirs[key]) with open(tem_f, 'wb+') as out: out.write(upload_file.body) real_path = '/upload/' + key + '/' + os.path.basename(tem_f) return self.write_res(DBRes(data={"realPath": real_path}))
def _call_(*args, **kwargs): model = kwargs['model'] if 'model' in kwargs else args[1] if model not in SUPPORT_MODELS: return DBRes.api_error(msg=f'{model} not support') base = SUPPORT_MODELS[model] return func(*args, base=base, **kwargs)