def run_pipeline(self):
print("Running Pipeline")
ingest_process = ingest.Ingest()
ingest_process.ingest_data()
tranform_process = transform.Transform()
tranform_process.transform_data()
persist_process = persist.Persist()
persist_process.persist_data()
def run_pipeline(self):
print("Running Pipeline")
ingest_process = ingest.Ingest(self.spark)
df = ingest_process.ingest_data()
df.show()
tranform_process = transform.Transform(self.spark)
transformed_df = tranform_process.transform_data(df)
transformed_df.show()
persist_process = persist.Persist(self.spark)
persist_process.persist_data(transformed_df)
return
def run_pipeline(self):
logging.info('run_pipeline method started')
ingest_process = ingest.Ingest(self.spark)
df = ingest_process.ingest_data()
df.show()
tranform_process = transform.Transform(self.spark)
transformed_df = tranform_process.transform_data(df)
transformed_df.show()
persist_process = persist.Persist(self.spark)
persist_process.persist_data(transformed_df)
logging.info('run_pipeline method ended')
return
def run_pipeline(self):
try:
logging.info('run_pipeline method started')
ingest_process = ingest.Ingest(self.spark)
df = ingest_process.ingest_data()
df.show()
tranform_process = transform.Transform(self.spark)
transformed_df = tranform_process.transform_data(df)
transformed_df.show()
persist_process = persist.Persist(self.spark)
persist_process.persist_data(transformed_df)
logging.info('run_pipeline method ended')
except Exception as exp:
logging.error("An error occured while running the pipeline > " +
str(exp))
# send email notification
# log error to database
sys.exit(1)
return
def CreateMenuBar(self):
menubar = wx.MenuBar()
fileMenu = wx.Menu()
nitem = fileMenu.Append(wx.ID_NEW, '&New', 'New project')
oitem = fileMenu.Append(wx.ID_OPEN, '&Open', 'Open project')
self.ritem = fileMenu.Append(wx.ID_SAVEAS, '&Render\tCtrl-r', 'Render')
self.iitem = fileMenu.Append(wx.ID_ANY, '&Import\tCtrl-i',
'Import image directory')
self.qitem = fileMenu.Append(wx.ID_EXIT, '&Quit', 'Quit application')
editMenu = wx.Menu()
self.zitem = editMenu.Append(wx.ID_UNDO, '&Undo\tCtrl-z',
'Undo Delete')
#yitem = editMenu.Append(wx.ID_REDO, '&Redo', 'Redo')
self.ditem = editMenu.Append(wx.ID_DELETE, '&Delete\tDelete', 'Delete')
pitem = editMenu.Append(wx.ID_PREFERENCES, '&Preferences\tCtrl-,',
'Preferences')
helpMenu = wx.Menu()
aitem = helpMenu.Append(wx.ID_ABOUT, '&About\tCtrl-?', 'About Stopgo')
menubar.Append(fileMenu, '&File')
menubar.Append(editMenu, '&Edit')
menubar.Append(helpMenu, '&Help')
self.Bind(wx.EVT_MENU,
lambda event, args=(False): self.OpenFile(event, args),
oitem)
self.Bind(wx.EVT_MENU, self.NewFile, nitem)
self.Bind(wx.EVT_MENU, self.Pref, pitem)
self.Bind(wx.EVT_MENU,
lambda event, args=(False): ingest.Ingest(self),
self.iitem)
self.Bind(wx.EVT_MENU, self.SimpleQuit, self.qitem)
self.Bind(wx.EVT_CLOSE, self.SimpleQuit, self.qitem)
self.Bind(wx.EVT_MENU, self.About, aitem)
self.SetMenuBar(menubar)