def delete(source_: Source):
if source_.pipelines:
raise Exception(
f"Can't delete. Source is used by {', '.join([p.name for p in source_.pipelines])} pipelines"
)
Session.delete(source_)
Session.commit()
def create_notifications(pipeline_):
if notifies := pipeline_.config.get("notifications"):
pipeline_.notifications = PiplineNotifications()
if no_data_notification := notifies.get('no_data'):
pipeline_.notifications.no_data_notification = NoDataNotifications(
pipeline_,
no_data_notification,
)
Session.add(pipeline_.notifications.no_data_notification)
def get_by_id(pipeline_id: str) -> Pipeline:
pipeline_ = Session.query(Pipeline).filter(
Pipeline.name == pipeline_id).first()
if not pipeline_:
raise PipelineNotExistsException(
f"Pipeline {pipeline_id} doesn't exist")
return pipeline_
class SessionManager:
def __init__(self, entity):
self.entity = entity
self.session = Session()
if not self.session.object_session(self.entity):
self.session.add(self.entity)
def __enter__(self):
return self.session
def __exit__(self, exc_type, exc_value, exc_traceback):
if exc_value:
self.session.expunge(self.entity)
return False
self.session.commit()
return True
def get_all() -> List[Source]:
return Session.query(Source).all()
def save(destination_: HttpDestination):
if not Session.object_session(destination_):
Session.add(destination_)
Session.commit()
def get_by_type(source_type: str) -> List[Source]:
return Session.query(Source).filter(Source.type == source_type).all()
def exists(url: str) -> bool:
return bool(Session.query(
Session.query(StreamSets).filter(StreamSets.url == url).exists()
).scalar())
from agent.pipeline.notifications import PiplineNotifications, NoDataNotifications
from agent.modules.db import Session
def create_notifications(pipeline_):
if notifies := pipeline_.config.get("notifications"):
pipeline_.notifications = PiplineNotifications()
if no_data_notification := notifies.get('no_data'):
pipeline_.notifications.no_data_notification = NoDataNotifications(
pipeline_,
no_data_notification,
)
Session.add(pipeline_.notifications.no_data_notification)
Session.add(pipeline_.notifications)
def get_by_url(url: str) -> StreamSets:
streamsets = Session.query(StreamSets).filter(StreamSets.url == url).first()
if not streamsets:
raise StreamsetsNotExistsException(f'StreamSets with url {url} doesn\'t exist')
return streamsets
def delete(streamsets: StreamSets):
Session.delete(streamsets)
Session.commit()
def exists(source_name: str) -> bool:
return Session.query(
Session.query(Source).filter(
Source.name == source_name).exists()).scalar()
def save(source_: Source):
if not Session.object_session(source_):
Session.add(source_)
Session.commit()
def exists() -> bool:
return Session.query(Session.query(HttpDestination).exists()).scalar()
def get() -> HttpDestination:
destination_ = Session.query(HttpDestination).first()
if not destination_:
raise DestinationNotExists(f"Destination does not exist")
return destination_
def delete_auth_token(token: AuthenticationToken):
Session.delete(token)
Session.commit()
def save_auth_token(token: AuthenticationToken):
if not Session.object_session(token):
Session.add(token)
Session.commit()
def delete():
if not exists():
return
Session.delete(get())
Session.commit()
def get_last_edited(source_type: str) -> Optional[Source]:
return Session.query(Source).filter(Source.type == source_type).order_by(
Source.last_edited.desc()).first()
def add_deleted_pipeline_id(pipeline_id: str):
Session.execute(
f"INSERT INTO deleted_pipelines VALUES ('{pipeline_id}') ON CONFLICT DO NOTHING"
)
Session.commit()
def get_all() -> List[StreamSets]:
return Session.query(StreamSets).all()
def get_all_names() -> List[str]:
return list(map(lambda row: row[0], Session.query(Source.name).all()))
def save(streamsets: StreamSets):
if not Session.object_session(streamsets):
Session.add(streamsets)
Session.commit()
def get_by_streamsets_url(streamsets_url: str) -> List[Pipeline]:
return Session.query(Pipeline).filter(
Pipeline.streamsets.has(url=streamsets_url)).all()
def get_all_names() -> List[str]:
res = list(map(
lambda row: row[0],
Session.query(StreamSets.url).all()
))
return res
def get(source_id: int) -> Source:
source_ = Session.query(Source).get(source_id)
if not source_:
raise SourceNotExists(f"Source ID = {source_id} doesn't exist")
return source_
def get(id_: int) -> StreamSets:
streamsets = Session.query(StreamSets).filter(StreamSets.id == id_).first()
if not streamsets:
raise StreamsetsNotExistsException(f'StreamSets with id {id_} doesn\'t exist')
return streamsets
def get_by_name(source_name: str) -> Source:
source_ = Session.query(Source).filter(Source.name == source_name).first()
if not source_:
raise SourceNotExists(f"Source config {source_name} doesn't exist")
return source_
def find_by_name_beginning(name_part: str) -> List[Source]:
return Session.query(Source).filter(
Source.name.like(f'{name_part}%')).all()
def remove_deleted_pipeline_id(pipeline_id: str):
Session.execute(
f"DELETE FROM deleted_pipelines WHERE pipeline_id = '{pipeline_id}'")
Session.commit()
