def testExportSteps(self):
manager = Manager()
stepsStats = dict()
for project in manager.listProjects():
projectFolder = manager.getProjectPath(project.projName)
project = Project(pyworkflow.Config.getDomain(), projectFolder)
project.load()
for prot in project.getRuns():
protName = prot.getClassName()
for step in prot.loadSteps():
stepName = "-".join([
prot.getClassPackageName(), protName,
step.funcName.get()
])
stepSeconds = step.getElapsedTime().total_seconds()
if stepName not in stepsStats:
stepsStats[stepName] = []
stepsStats[stepName].append(stepSeconds)
# average steps
for name in stepsStats:
plugin, protClass, step = name.split("-")
stepStats = stepsStats[name]
mean = sum(stepStats) / len(stepStats)
# DO not send mean values if higher than a threshold to keep chart in a decent visualization range
# We are loosing long duration steps, probably due to large
if mean < 30:
bm = Benchmark(time=mean, name="-".join([protClass, step]))
codespeed.sendData(bm)
def createProjectFromWorkflow(workflow, projectName, argsList):
scipion = SCIPION_EP
scriptsPath = pw.join('project', 'scripts')
# Clean the project name as pyworkflow will do
projectName = Project.cleanProjectName(projectName)
# Create the project
print("Creating project %s" % projectName)
createProjectScript = os.path.join(scriptsPath, 'create.py')
os.system("python -m %s python %s %s %s" %
(scipion, createProjectScript, projectName, workflow))
# Wait 2 seconds to avoid activity
time.sleep(2)
if scheduleProject(argsList):
# Schedule the project
scheduleProjectScript = os.path.join(scriptsPath, 'schedule.py')
print("Scheduling project %s" % projectName)
subprocess.Popen([
"python", "-m", scipion, "python", scheduleProjectScript,
projectName
])
# Wait 5 seconds to avoid activity
time.sleep(5)
if launchGUI(argsList):
print("Showing project %s" % projectName)
# Launch scipion
subprocess.Popen(["python", "-m", scipion, MODE_PROJECT, projectName])
def loadProject(self):
self.project = Project(self.projPath)
self.project.load()
# Check if we have settings.sqlite, generate if not
settingsPath = os.path.join(self.project.path,
self.project.settingsPath)
if os.path.exists(settingsPath):
self.settings = self.project.getSettings()
else:
print(
'Warning: settings.sqlite not found! Creating default settings..'
)
self.settings = self.project.createSettings()
self.generalCfg = self.settings.getConfig()
self.protCfg = self.project.getCurrentProtocolView()
def loadProject(self):
self.project = Project(self.projPath)
self.project.load()
# Check if we have settings.sqlite, generate if not
settingsPath = os.path.join(self.project.path, self.project.settingsPath)
if os.path.exists(settingsPath):
self.settings = self.project.getSettings()
else:
print('Warning: settings.sqlite not found! Creating default settings..')
self.settings = self.project.createSettings()
self.generalCfg = self.settings.getConfig()
self.protCfg = self.project.getCurrentProtocolView()
def _select(self):
projName = self.projName.get().strip()
projLocation = self.projLocation.get().strip()
copyFiles = self.tkCheckVar.get() != 0
searchLocation = self.searchLocation.get().strip()
manager = Manager()
# If project name is empty we will use the same name as the source
if not projName:
projName = os.path.basename(projLocation)
errorMessage = ''
# Validate that project location is not empty
if not projLocation:
errorMessage = "Project location is empty\n"
# Validate that project location exists
elif not os.path.exists(projLocation):
errorMessage += "Project location does not exist\n"
# Validate that project location is a directory
elif not os.path.isdir(projLocation):
errorMessage += "Project location is not a directory\n"
# Validate that the project location is a scipion project folder
elif not os.path.exists(os.path.join(projLocation,
Project.getDbName())):
errorMessage += "Project location doesn't look like a scipion folder\n"
# Validate that there isn't already a project with the same name
if manager.hasProject(projName):
errorMessage += "Project [%s] already exists\n" % projName
# Validate that search location exists
if searchLocation:
if not os.path.exists(searchLocation):
errorMessage += "Raw files location does not exist\n"
# Validate that search location is a directory
elif not os.path.isdir(searchLocation):
errorMessage += "Raw files location is not a directory\n"
if errorMessage:
showError("Validation error", errorMessage, self.root)
else:
self.parent.importProject(projLocation, copyFiles, projName,
searchLocation)
self.close()
# sys.path.insert(1, path)
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s"
% pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(pw.Config.getDomain(), projectPath)
project.load()
runs = project.getRuns()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for prot in runs:
protClassName = prot.getClassName()
protLabelName = prot.getObjLabel()
if (protClassName not in sys.argv[3:] and
protLabelName not in sys.argv[3:]):
project.scheduleProtocol(prot)
# Wait 1 seconds to avoid concurrent activity
time.sleep(0.7)
else:
if n <> 2:
usage("This script accepts 1 parameter: the project name.")
projName = sys.argv[1]
path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui',
'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s" % pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(projectPath)
project.load()
runs = project.getRuns()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for prot in runs:
project.scheduleProtocol(prot)
projName = sys.argv[1]
path = pw.join('gui', 'no-tkinter')
sys.path.insert(1, path)
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s" % pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(path=projectPath)
project.load()
runs = project.getRuns()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for prot in runs:
if prot.isActive():
try:
project.stopProtocol(prot)
except:
print("Couldn't stop protocol %s" % prot)
class ProjectWindow(ProjectBaseWindow):
""" Main window for working in a Project. """
_OBJECT_COMMANDS = {}
def __init__(self, path, master=None):
# Load global configuration
self.projName = os.path.basename(path)
try:
projTitle = '%s (%s on %s)' % (self.projName,
pwutils.getLocalUserName(),
pwutils.getLocalHostName())
except Exception:
projTitle = self.projName
self.projPath = path
self.loadProject()
# TODO: put the menu part more nicely. From here:
menu = MenuConfig()
projMenu = menu.addSubMenu('Project')
projMenu.addSubMenu('Browse files',
'browse',
icon='fa-folder-open.png')
projMenu.addSubMenu('Remove temporary files',
'delete',
icon='fa-trash-o.png')
projMenu.addSubMenu('Manage project labels', 'labels', icon=Icon.TAGS)
projMenu.addSubMenu('Toogle color mode',
'color_mode',
shortCut="Ctrl+t",
icon=Icon.ACTION_VISUALIZE)
projMenu.addSubMenu('Select all protocols',
'select all',
shortCut="Ctrl+a")
projMenu.addSubMenu('Find protocol to add',
'find protocol',
shortCut="Ctrl+f")
projMenu.addSubMenu('', '') # add separator
projMenu.addSubMenu('Import workflow',
'load_workflow',
icon='fa-download.png')
projMenu.addSubMenu('Export tree graph', 'export_tree')
projMenu.addSubMenu('', '') # add separator
projMenu.addSubMenu('Notes', 'notes', icon='fa-pencil.png')
projMenu.addSubMenu('', '') # add separator
projMenu.addSubMenu('Exit', 'exit', icon='fa-sign-out.png')
helpMenu = menu.addSubMenu('Help')
helpMenu.addSubMenu('Online help',
'online_help',
icon='fa-external-link.png')
helpMenu.addSubMenu('About', 'about', icon='fa-question-circle.png')
helpMenu.addSubMenu('Contact support',
'contact_us',
icon='fa-question-circle.png')
self.menuCfg = menu
# TODO: up to here
if self.project.openedAsReadOnly():
self.projName += "<READ ONLY>"
# Notify about the workflow in this project
self.icon = self.generalCfg.icon.get()
self.selectedProtocol = None
self.showGraph = False
Plotter.setBackend('TkAgg')
ProjectBaseWindow.__init__(self,
projTitle,
master,
icon=self.icon,
minsize=(90, 50))
self.root.attributes("-zoomed", True)
self.switchView(VIEW_PROTOCOLS)
self.initProjectTCPServer(
) # Socket thread to communicate with clients
from notifier import ProjectNotifier
ProjectNotifier(self.project).notifyWorkflow()
def createHeaderFrame(self, parent):
"""Create the header and add the view selection frame at the right."""
header = ProjectBaseWindow.createHeaderFrame(self, parent)
self.addViewList(header)
return header
def getSettings(self):
return self.settings
def saveSettings(self):
self.settings.write()
def _onClosing(self):
try:
if not self.project.openedAsReadOnly():
self.saveSettings()
except Exception as ex:
print("%s %s" % (Message.NO_SAVE_SETTINGS, str(ex)))
ProjectBaseWindow._onClosing(self)
def loadProject(self):
self.project = Project(self.projPath)
self.project.load()
# Check if we have settings.sqlite, generate if not
settingsPath = os.path.join(self.project.path,
self.project.settingsPath)
if os.path.exists(settingsPath):
self.settings = self.project.getSettings()
else:
print(
'Warning: settings.sqlite not found! Creating default settings..'
)
self.settings = self.project.createSettings()
self.generalCfg = self.settings.getConfig()
self.protCfg = self.project.getCurrentProtocolView()
# The next functions are callbacks from the menu options.
# See how it is done in pyworkflow/gui/gui.py:Window._addMenuChilds()
#
def onBrowseFiles(self):
# Project -> Browse files
FileBrowserWindow(
"Browse Project files",
self,
self.project.getPath(''),
selectButton=None # we will select nothing
).show()
def onNotes(self):
if not all(var in os.environ for var in [
'SCIPION_NOTES_PROGRAM', 'SCIPION_NOTES_FILE',
'SCIPION_NOTES_ARGS'
]):
return self.showError(
"Missing variables SCIPION_NOTES_* under\n"
"[VARIABLES] section in the configuration file\n"
"~/.config/scipion/scipion.conf")
args = []
# Program name
program = os.environ.get('SCIPION_NOTES_PROGRAM', None)
notesFile = self.project.getPath('Logs',
os.environ['SCIPION_NOTES_FILE'])
if program:
args.append(program)
# Custom arguments
if os.environ.get('SCIPION_NOTES_ARGS', None):
args.append(os.environ['SCIPION_NOTES_ARGS'])
args.append(notesFile)
subprocess.Popen(args) #nonblocking
else:
openTextFileEditor(notesFile)
def onRemoveTemporaryFiles(self):
# Project -> Remove temporary files
tmpPath = os.path.join(self.project.path, self.project.tmpPath)
n = 0
try:
for fname in os.listdir(tmpPath):
fpath = "%s/%s" % (tmpPath, fname)
if os.path.isfile(fpath):
os.remove(fpath)
n += 1
# TODO: think what to do with directories. Delete? Report?
self.showInfo("Deleted content of %s -- %d file(s)." %
(tmpPath, n))
except Exception as e:
self.showError(str(e))
def _loadWorkflow(self, obj):
try:
self.project.loadProtocols(obj.getPath())
self.getViewWidget().updateRunsGraph(True, reorganize=True)
except Exception, ex:
self.showError(str(ex))
if n > 3:
usage("This script accepts 2 parameters: the project name (mandatory) and "
"the refresh period in seconds (optional).")
projName = sys.argv[1]
if n == 3:
wait = int(sys.argv[2])
else:
wait = 60
path = pw.join('gui', 'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s" % pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(path=projectPath)
project.load()
while True:
runs = project.getRuns()
sleep(wait)
def loadProjectFromPath(projectPath):
project = Project(projectPath)
project.load()
return project
class ProjectWindow(ProjectBaseWindow):
""" Main window for working in a Project. """
_OBJECT_COMMANDS = {}
def __init__(self, path, master=None):
# Load global configuration
self.projName = os.path.basename(path)
try:
projTitle = '%s (%s on %s)' % (self.projName,
pwutils.getLocalUserName(),
pwutils.getLocalHostName())
except Exception:
projTitle = self.projName
self.projPath = path
self.loadProject()
# TODO: put the menu part more nicely. From here:
menu = MenuConfig()
projMenu = menu.addSubMenu('Project')
projMenu.addSubMenu('Browse files', 'browse',
icon='fa-folder-open.png')
projMenu.addSubMenu('Remove temporary files', 'delete',
icon='fa-trash-o.png')
projMenu.addSubMenu('Manage project labels', 'labels',
icon=Icon.TAGS)
projMenu.addSubMenu('Toogle color mode', 'color_mode',
shortCut="Ctrl+t", icon=Icon.ACTION_VISUALIZE)
projMenu.addSubMenu('Select all protocols', 'select all',
shortCut="Ctrl+a")
projMenu.addSubMenu('Find protocol to add', 'find protocol',
shortCut="Ctrl+f")
projMenu.addSubMenu('', '') # add separator
projMenu.addSubMenu('Import workflow', 'load_workflow',
icon='fa-download.png')
projMenu.addSubMenu('Search workflow', 'search_workflow',
icon = 'fa-search.png')
projMenu.addSubMenu('Export tree graph', 'export_tree')
projMenu.addSubMenu('', '') # add separator
projMenu.addSubMenu('Notes', 'notes', icon='fa-pencil.png')
projMenu.addSubMenu('', '') # add separator
projMenu.addSubMenu('Exit', 'exit', icon='fa-sign-out.png')
helpMenu = menu.addSubMenu('Help')
helpMenu.addSubMenu('Online help', 'online_help',
icon='fa-external-link.png')
helpMenu.addSubMenu('About', 'about',
icon='fa-question-circle.png')
helpMenu.addSubMenu('Contact support', 'contact_us',
icon='fa-question-circle.png')
self.menuCfg = menu
# TODO: up to here
if self.project.openedAsReadOnly():
self.projName += "<READ ONLY>"
# Notify about the workflow in this project
self.icon = self.generalCfg.icon.get()
self.selectedProtocol = None
self.showGraph = False
Plotter.setBackend('TkAgg')
ProjectBaseWindow.__init__(self, projTitle, master,
icon=self.icon, minsize=(90,50))
self.root.attributes("-zoomed", True)
self.switchView(VIEW_PROTOCOLS)
self.initProjectTCPServer() # Socket thread to communicate with clients
ProjectWorkflowNotifier(self.project).notifyWorkflow()
def createHeaderFrame(self, parent):
"""Create the header and add the view selection frame at the right."""
header = ProjectBaseWindow.createHeaderFrame(self, parent)
self.addViewList(header)
return header
def getSettings(self):
return self.settings
def saveSettings(self):
self.settings.write()
def _onClosing(self):
try:
if not self.project.openedAsReadOnly():
self.saveSettings()
except Exception as ex:
print("%s %s" %(Message.NO_SAVE_SETTINGS, str(ex)) )
ProjectBaseWindow._onClosing(self)
def loadProject(self):
self.project = Project(self.projPath)
self.project.load()
# Check if we have settings.sqlite, generate if not
settingsPath = os.path.join(self.project.path, self.project.settingsPath)
if os.path.exists(settingsPath):
self.settings = self.project.getSettings()
else:
print('Warning: settings.sqlite not found! Creating default settings..')
self.settings = self.project.createSettings()
self.generalCfg = self.settings.getConfig()
self.protCfg = self.project.getCurrentProtocolView()
# The next functions are callbacks from the menu options.
# See how it is done in pyworkflow/gui/gui.py:Window._addMenuChilds()
#
def onBrowseFiles(self):
# Project -> Browse files
FileBrowserWindow("Browse Project files",
self, self.project.getPath(''),
selectButton=None # we will select nothing
).show()
def onNotes(self):
if not all(var in os.environ for var in ['SCIPION_NOTES_PROGRAM',
'SCIPION_NOTES_FILE',
'SCIPION_NOTES_ARGS']):
return self.showError("Missing variables SCIPION_NOTES_* under\n"
"[VARIABLES] section in the configuration file\n"
"~/.config/scipion/scipion.conf")
args = []
# Program name
program = os.environ.get('SCIPION_NOTES_PROGRAM', None)
notesFile = self.project.getPath('Logs', os.environ['SCIPION_NOTES_FILE'])
if program:
args.append(program)
# Custom arguments
if os.environ.get('SCIPION_NOTES_ARGS', None):
args.append(os.environ['SCIPION_NOTES_ARGS'])
args.append(notesFile)
subprocess.Popen(args) #nonblocking
else:
openTextFileEditor(notesFile)
def onRemoveTemporaryFiles(self):
# Project -> Remove temporary files
tmpPath = os.path.join(self.project.path, self.project.tmpPath)
n = 0
try:
for fname in os.listdir(tmpPath):
fpath = "%s/%s" % (tmpPath, fname)
if os.path.isfile(fpath):
os.remove(fpath)
n += 1
# TODO: think what to do with directories. Delete? Report?
self.showInfo("Deleted content of %s -- %d file(s)." % (tmpPath, n))
except Exception as e:
self.showError(str(e))
def _loadWorkflow(self, obj):
try:
self.project.loadProtocols(obj.getPath())
self.getViewWidget().updateRunsGraph(True, reorganize=True)
except Exception, ex:
self.showError(str(ex))
if n <>2:
usage("This script accepts 1 parameter: the project name.")
projName = sys.argv[1]
path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui', 'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s"
% pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(projectPath)
project.load()
runs = project.getRuns()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for prot in runs:
project.scheduleProtocol(prot)
def loadProjectFromPath(projectPath):
project = Project(projectPath)
project.load()
return project
def loadProjectFromPath(projectPath):
project = Project(projectPath)
project.load(chdir=False)
return project
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s"
% pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster
# so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(pw.Config.getDomain(), projectPath)
project.load()
runGraph = project.getRunsGraph()
roots = runGraph.getRootNodes()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for root in roots:
for child in root.getChilds():
workflow, _ = project._getWorkflowFromProtocol(child.run)
for prot, level in workflow.values():
protClassName = prot.getClassName()
protLabelName = prot.getObjLabel()
if (protClassName not in sys.argv[3:] and