예제 #1
0
파일: init.py 프로젝트: mrzhangboss/fsqlfly
    def _generate_transform(self, resource_names: List[ResourceName]) -> DBRes:
        updated = inserted = 0
        connector = self.target
        for resource_name in resource_names:
            t_database, t_table = connector.get_transform_target_full_name(resource_name=resource_name,
                                                                           connector=connector)
            name = connector.get_transform_name_format(resource_name=resource_name)
            source_version = self.get_source_default_version(resource_name)
            if source_version is None:
                return DBRes.api_error(msg="Not found resource source table {}".format(resource_name.full_name))
            sink_version = self.get_sink_default_version(t_database, t_table)
            if sink_version is None:
                return DBRes.api_error(msg="Not found resource sink table {}".format(resource_name.full_name))
            require = self.get_source_name(source_version) + ',' + self.get_source_name(sink_version)

            execution = dict(planner='blink', type=self.get_flink_execution_type(),
                             parallelism=connector.system_execution_parallelism)
            execution['restart-strategy'] = connector.system_execution_restart_strategy
            transform = Transform(name=name, sql=self.build_sql(sink_version, source_version, connector),
                                  require=require, connector_id=connector.id, yaml=dump_yaml(dict(execution=execution)))
            transform, i = self.dao.upsert_transform(transform)
            inserted += i
            updated += not i

        msg = 'update: {}\ninserted: {}'.format(updated, inserted)
        return DBRes(msg=msg)
예제 #2
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
 def clean(cls, model: str, pk: int, *args, session: Session,
           base: Type[DBT], **kwargs) -> DBRes:
     obj = session.query(base).get(pk)
     if isinstance(obj, Connector):
         back = obj.as_dict()
         source, target = obj.source, obj.target
         session.delete(obj)
         session.commit()
         cls._clean(source, session, Connection)
         cls._clean(target, session, Connection)
         session.add(Connector(**back))
     else:
         if isinstance(obj, Connection):
             s_names = [
                 x[0] for x in session.query(Connector.name).join(
                     Connector.source).filter(Connection.id == pk).all()
             ]
             t_names = [
                 x[0] for x in session.query(Connector.name).join(
                     Connector.target).filter(Connection.id == pk).all()
             ]
             msg = "please clean connector source: {} target: {}".format(
                 ','.join(s_names), ','.join(t_names))
             if s_names or t_names:
                 return DBRes.api_error(msg)
         cls._clean(obj, session, base)
     return DBRes()
예제 #3
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
 def delete(cls, model: str, pk: int, *args, session: Session,
            base: Type[Base], **kwargs) -> DBRes:
     if settings.FSQLFLY_SAVE_MODE_DISABLE:
         obj = session.query(base).get(pk)
         session.delete(obj)
         return DBRes(data=obj.id)
     else:
         return DBRes.sever_error(
             'Not Support Delete when FSQLFLY_SAVE_MODE_DISABLE not set')
예제 #4
0
    def post(self):
        files = self.request.files
        key = list(files.keys())[0]
        if key not in support_upload:
            return self.write_res(DBRes.api_error())
        upload_file = files[key][0]

        _, tem_f = mkstemp(suffix=upload_file.filename, dir=upload_dirs[key])
        with open(tem_f, 'wb+') as out:
            out.write(upload_file.body)
        real_path = '/upload/' + key + '/' + os.path.basename(tem_f)
        return self.write_res(DBRes(data={"realPath": real_path}))
예제 #5
0
 def run(cls, model: str, mode: str, pk: Union[str, int]) -> DBRes:
     dao = Dao()
     obj = dao.get_by_name_or_id(model, pk)
     if obj:
         manager = ManagerFactory.get_manager(model, mode, obj, dao)
         if manager.is_support():
             return manager.run()
         else:
             msg = "Not support {}:{} in model {} by {} in ManagerHelper".format(
                 obj.name, obj.id, model, mode)
             return DBRes.api_error(msg)
     else:
         return DBRes.api_error("Not found {} in model {}".format(
             pk, model))
예제 #6
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
    def update(cls, model: str, pk: int, obj: dict, *args, session: Session,
               base: Type[Base], **kwargs) -> DBRes:
        assert session is not None
        first = session.query(base).filter(base.id == pk).first()
        if first is None:
            return DBRes.not_found()

        if first.is_locked and obj.get('is_locked') is True:
            return DBRes.resource_locked()

        for k, v in obj.items():
            if k not in ['id', 'create_at', 'update_at'
                         ] and not cls.is_null_foreign_key(k, v):
                setattr(first, k, v)
        return DBRes(data=first.as_dict())
예제 #7
0
 def post(self):
     arg = self.json_body
     if self.is_login(arg.get('type'), arg.get('password'), arg.get('type')):
         self.set_login_status()
         self.write_json(user)
     else:
         self.write_res(DBRes.login_error())
예제 #8
0
 async def post(self, name: str):
     tm = self.terminal_manager
     if name in tm.terminals:
         await tm.terminate(name, force=True)
         self.write_res(DBRes())
     else:
         raise tornado.web.HTTPError(404, "Terminal not found: %r" % name)
예제 #9
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
    def get_require_name(cls, *args, session: Session, **kwargs) -> DBRes:
        query = session.query(ResourceVersion.full_name).join(
            ResourceVersion.connection).join(ResourceVersion.resource_name)
        version_data = [
            x[0] for x in query.filter(
                and_(Connection.is_active == true(), ResourceName.is_active ==
                     true())).all()
        ]

        v_query = session.query(ResourceVersion).join(
            ResourceVersion.connection).join(ResourceVersion.resource_name)
        default_version_data = list(
            x.template.full_name for x in v_query.filter(
                and_(Connection.is_active == true(), ResourceName.is_active ==
                     true(), ResourceVersion.is_default == true())).all())
        t_query = session.query(ResourceTemplate).join(
            ResourceTemplate.connection).join(ResourceTemplate.resource_name)
        resource_data = list(x.resource_name.full_name for x in t_query.filter(
            and_(Connection.is_active == true(), ResourceName.is_active ==
                 true(), ResourceTemplate.is_default == true())).all())

        hive_data = [
            x[0] for x in session.query(Connection.name).filter(
                Connection.is_active == true(), Connection.type ==
                'hive').all()
        ]

        return DBRes(data=hive_data + version_data + default_version_data +
                     resource_data)
예제 #10
0
def _handle_job(mode: str, pk: str, json_body: dict,
                session: Session) -> DBRes:
    handle_name = 'handle_' + mode

    if mode in JobControlHandle and handle_name in JobControlHandle:
        if pk.isdigit():
            transform = DBDao.get_transform(pk, session=session)
            if transform is None:
                return DBRes.api_error(msg='job id {} not found!!!'.format(pk))
        else:
            transform = pk
        data = json_body
        run_res = getattr(JobControlHandle, handle_name)(transform, **data)
        return DBRes(code=500 if run_res.startswith(FAIL_HEADER) else 200,
                     msg=run_res)
    else:
        return DBRes.api_error(msg=' {} not support!!!'.format(mode))
예제 #11
0
파일: init.py 프로젝트: mrzhangboss/fsqlfly
 def generate_transform(self) -> DBRes:
     connector = self.target
     need_tables = connector.need_tables
     source, target = connector.source, connector.target
     if not connector.source.resource_names:
         return DBRes.api_error("Not Found Any Resource Name in Connection {}".format(source.name))
     resource_names = [x for x in connector.source.resource_names if x.db_name in need_tables]
     return self._run(resource_names)
예제 #12
0
    def post(self, mode: str, pk: str):
        if mode == 'debug':
            term = run_debug_transform(self.json_body, self.terminal_manager)
            self.write_res(DBRes({"url": '/terminal/{}'.format(term)}))
        else:
            if not pk.isdigit():
                pk = str(DBDao.name2pk(model='transform', name=pk))

            return self.write_res(handle_job(mode, pk, self.json_body))
예제 #13
0
 def get(self, path: str):
     if not any(map(lambda x: path.startswith(x), support_upload)):
         raise tornado.web.HTTPError(status_code=404)
     full_path = os.path.join(UPLOAD_ROOT_DIR, path)
     if not os.path.exists(full_path):
         return self.write_res(DBRes.api_error())
     mime = FileMagic.from_file(full_path)
     self.set_header('content-type', mime)
     self.write(open(full_path, "rb").read())
     self.finish()
예제 #14
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
 def get(cls,
         model: str,
         *args,
         session: Session,
         base: Type[Base],
         filter_: Optional[dict] = None,
         **kwargs) -> DBRes:
     query = session.query(base)
     if filter_:
         query = cls.build_and(filter_, base, query)
     return DBRes(data=[x.as_dict() for x in query.all()])
예제 #15
0
파일: init.py 프로젝트: mrzhangboss/fsqlfly
    def create_hive_table(self, resource_names: List[ResourceName]) -> DBRes:
        connector = self.target
        engine = create_engine(connector.target.url)
        for resource_name in resource_names:
            t_database, t_table = connector.get_transform_target_full_name(resource_name=resource_name,
                                                                           connector=connector)
            version = self.get_sink_default_version(t_database, t_table)
            schemas = version.generate_version_schema()
            for sql in self.build_hive_create_sql(t_database, t_table, schemas):
                print(sql)
                engine.execute(sql)

        return DBRes()
예제 #16
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
 def _add_session(*args, **kwargs):
     session = kwargs[
         'session'] if 'session' in kwargs else DBSession.get_session()
     new_kwargs = {k: v for k, v in kwargs.items() if k != 'session'}
     try:
         res = func(*args, session=session, **new_kwargs)
         session.commit()
         return res
     except Exception as error:
         session.rollback()
         if settings.FSQLFLY_DEBUG:
             raise error
         err = traceback.format_exc()
         return DBRes.sever_error(msg=f'meet {err}')
     finally:
         if 'session' not in kwargs:
             session.close()
예제 #17
0
    def get(self):
        job_infos = get_latest_transform()
        all_jobs = list()
        for job in JobControlHandle.job_status:
            if not JobControlHandle.is_real_job(job.name):
                continue
            base = dict(**job._asdict())
            if job.name in job_infos:
                base.update(job_infos[job.name])
                base['t_id'] = job_infos[job.name]['id']
            elif job.name.startswith(TEMP_TERMINAL_HEAD):
                base['name'] = 'TEMPORARY'
                base['url'] = '/terminal/{}'.format(job.name[len(TEMP_TERMINAL_HEAD):])

            base['id'] = job.job_id
            base['detail_url'] = FSQLFLY_FINK_HOST + '/#/job/{}/overview'.format(job.job_id)

            all_jobs.append(base)

        return self.write_res(DBRes(data=all_jobs))
예제 #18
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
 def create(cls, model: str, obj: dict, *args, session: Session,
            base: Type[DBT], **kwargs) -> DBRes:
     db_obj = base(**obj)
     session.add(db_obj)
     session.commit()
     return DBRes(data=db_obj.as_dict())
예제 #19
0
 def test_not_init_session(self):
     DBSession.init_engine(None)
     with self.assertRaises(AssertionError):
         self.assertEqual(
             DBDao.update(model='connection', pk=1, obj=dict()),
             DBRes.not_found())
예제 #20
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
 def bulk_insert(cls, data: list, *args, session: Session, **kwargs):
     session.add_all(data)
     return DBRes(data=len(data))
예제 #21
0
파일: db_helper.py 프로젝트: Fathui/fsqlfly
 def _call_(*args, **kwargs):
     model = kwargs['model'] if 'model' in kwargs else args[1]
     if model not in SUPPORT_MODELS:
         return DBRes.api_error(msg=f'{model} not support')
     base = SUPPORT_MODELS[model]
     return func(*args, base=base, **kwargs)
예제 #22
0
 def get(self):
     tm = self.terminal_manager
     terms = [{'name': name, 'id': name} for name in tm.terminals]
     self.write_res(DBRes(data=terms))
예제 #23
0
 def test_not_found(self):
     self.assertEqual(DBDao.update(model='connection', pk=1, obj=dict()),
                      DBRes.not_found())
예제 #24
0
파일: init.py 프로젝트: mrzhangboss/fsqlfly
 def _run(self, resource_names: List[ResourceName]):
     res = []
     for resource_name in resource_names:
         name = self.target.get_transform_name_format(resource_name=resource_name)
         res.append(name)
     return DBRes(data=res)