def createInstance(id):
global CONFIG
id = 1
# set the properties
instance = rcconfig()
instance.id = id
instance.dataroot = buildinstancedir(id)
instance.status = k_status_none
instance.pid = 0
instance.data["deployment"] = getDefaultDeployment()
# Now create the necessary objects
# createDirectory
try:
os.mkdir(CONFIG.dataroot + instance.dataroot)
except Exception as e:
ERROR("cannot create directory " + CONFIG.dataroot + instance.dataroot)
return None
# create status file
putIntFile(CONFIG.dataroot + instance.dataroot + "/STATUS",
instance.status)
# create deployment file
putTextFile(CONFIG.dataroot + instance.dataroot + "/deployment.xml",
instance.data["deployment"])
return instance
def checkCliFlag(flag):
if flag == "replica": return False
if flag == "deployment": return True
if flag == "d": return True
if flag == "debug": return False
ERROR("unknown argument " + flag)
exit()
def apply_auto(self):
re_text = self.line.text()
test_string = "This is a string to check if you put in a correct regex code"
try:
match = re.search(re_text, test_string)
except Exception as e:
ERROR("This is not a valid regex string. Error is: \n" + str(e))
return
self.dialog.accept()
def writedeploymentfile (config, filespec, hostcount):
setmessage("")
if config == "":
ERROR("no XML to write as deployment file.")
return False
x = updatehostcount(config,hostcount)
if x is None: return False
try:
fid = open(filespec ,"w")
DEBUG("writing deployment file " + filespec)
result = fid.write(x)
fid.close()
return True
except Exception as e:
setmessage("INTERNAL ERROR CAN'T WRITE FILE: " + str(e))
ERROR("INTERNAL ERROR CAN'T WRITE FILE: " + str(e))
return False
def putIntFile(f, i):
try:
fid = open(f, "w")
fid.write(str(i))
fid.close()
except Exception as e:
ERROR(e)
return False
return True
def startServer(s):
global settings, state
svr = resolveServerId(s)
url = "http://" + svr + "/WEBRC?request=" + settings.action
DEBUG("URL TO START: [" + url + "]")
try:
r = urllib2.urlopen(url)
except Exception, e:
ERROR(" Cannot reach server " + s + " " + e.message)
raise
return False
def initServer(s):
global settings, state
# Get current status of server
DEBUG("init server [" + s + "]")
svr = resolveServerId(s)
url = "http://" + svr + "/WEBRC?request=init"
try:
r = urllib2.urlopen(url)
except Exception, e:
ERROR(" Cannot reach server " + s + " " + e.message)
raise
return False
def auto_label(self):
all_labels = []
# Get current labels, we want unique
for row in range(self.rowCount()):
label = self.item(row, 1).text()
all_labels.append(label)
self.dialog = QDialog()
lay = QVBoxLayout()
self.dialog.setLayout(lay)
self.line = QLineEdit()
self.line.setText(self.store_re_text)
bbox = QDialogButtonBox()
bbox.addButton("Help", QDialogButtonBox.HelpRole)
bbox.addButton("Apply", QDialogButtonBox.AcceptRole)
bbox.addButton("Cancel", QDialogButtonBox.RejectRole)
bbox.accepted.connect(self.apply_auto)
bbox.rejected.connect(self.cancel_auto)
bbox.helpRequested.connect(self.help_auto)
lay.addWidget(self.line)
lay.addWidget(bbox)
self.dialog.resize(300, 100)
ans = self.dialog.exec_() # This will block until the dialog closes
if ans != 1:
return
re_text = self.line.text()
self.store_re_text = re_text
# Assign labels
c = 0
for row in range(self.rowCount()):
label = self.item(row, 1).text()
file = self.item(row, 0).text()
if label == '' and file != '':
file_name = os.path.basename(file)
search = re.search(re_text, file_name)
if search is not None:
c += 1
# idk how to check how many captured groups, so lets do this a hacky way
for i in range(10,-1,-1):
try:
added_label = search.group(i)
break
except Exception as e:
pass
else:
added_label = '' # Do not add label if you don't know what to add
self.item(row, 1).setText(added_label)
if c == 0:
ERROR("No files matched the regex pattern")
self.auto_label()
def updatehostcount(config,hostcount):
setmessage("")
DEBUG("update host count to " + str(hostcount) + " on deployment file " + config)
try:
xml = ET.fromstring(config)
except Exception as e:
setmessage("Invalid deployment file. Cannot parse XML.")
ERROR("Invalid deployment file. Cannot parse XML.")
return None
cluster = xml.find('cluster')
DEBUG("Cluster: " + str(cluster))
if cluster is None:
setmessage("Invalid deployment file. No <cluster> element.")
ERROR("Invalid deployment file. No <cluster> element.")
return None
if ('hostcount' in cluster.attrib):
cluster.attrib['hostcount'] = str(hostcount)
else:
# Add hostcount
cluster.attrib['hostcount'] = hostcount
return ET.tostring(xml)
def time_to_nftime(self, time):
parameters = time.split(':') # Days, hours, minutes, seconds
if len(parameters) >= 5:
ERROR(f"too many values for time at {self.process}")
return
suffix = ['d', 'h', 'm', 's']
suffix = suffix[-len(parameters):]
nftime = ''
for suffix, parameter in zip(suffix, parameters):
try:
nftime += f"{int(parameter)}{suffix}"
except:
pass
return nftime
def do_PROC(s, action, args):
# determine action then do it.
# current only action is log:
if (action == "file"):
data = ""
if (args == "" or args == "/"): args = "index.html"
print "Get file: : " + args
try:
file = open(args, "r")
data = file.read()
file.close()
s.send_response(200)
s.send_header("Content-type", "text/html")
s.end_headers()
s.wfile.write(data)
return
except Exception, e:
ERROR("Cannot read file " + args + " " + str(e))
def prepServer(s):
global settings, state
svr = resolveServerId(s)
# Set the hosts and deployment (as necessary)
data = {}
data["hoststring"] = settings.hoststring
data["deployment"] = settings.deployment
data["args"] = settings.args
jdata = json.dumps(data)
urldata = urllib.urlencode({"data": jdata})
url = "http://" + svr + "/WEBRC?request=set&" + urldata
DEBUG("Prep server url: " + url)
try:
r = urllib2.urlopen(url)
except Exception, e:
ERROR(" Cannot reach server " + s + " " + e.message)
raise
return False
def run(self):
# Load settings
self.pipe_parser = custom_config_parser()
self.pipe_parser.load(self.pipe_path)
self.nf_settings_parser = custom_config_parser()
self.nf_settings_parser.load(self.nf_settings_path)
self.sh_parser = custom_config_parser()
self.sh_parser.load(self.sh_script_path)
# Read parent
parent = self.parentWidget()
# OUTPUT_DIRECTORY #
children = parent.findChildren(QLineEdit)
for child in children: # This is so I can have whatever order in widgets I want
if child.type == 'directory':
break # Will keep child
output_path = child.text()
if not os.path.isdir(output_path):
ERROR('Not a valid output path')
return 1
# Change output parameters in both nf_settings and sh
self.sh_parser.write("OUTPUT_PATH", output_path) # In sh
self.nf_settings_parser.write("params.output_path",
output_path) # In sh
# OUTPUT_LABEL #
label = self.sh_parser.get('OUTPUT_PATH_LABEL')
if label != '': # Check if label has been set
label = re.sub("_*", '', label) # Remove previous indexing
index = 0
while True:
if os.path.isdir(output_path + label + "_" + str(index)):
index += 1
else:
label = label + "_" + str(index)
self.nf_settings_parser.write("params.output_label", label)
self.sh_parser.write('OUTPUT_PATH_LABEL', label)
break
else:
self.nf_settings_parser.write("params.output_label", '')
# BATCH_FILE #
child = parent.findChildren(QTableWidget)[0]
full_table = []
for row in range(child.rowCount()):
if not child.item(row, 0).text() == '':
if not os.path.isfile(child.item(row, 0).text()):
ERROR(f"File in row {row} does not exist")
return 1
elif self.nf_settings_parser.get('params.workflow') in [
"MSconvert", "Quandenser"
] and child.item(row, 1).text() == '':
label = 'A' # Add junk labeling
elif child.item(
row, 1).text() == '' and self.nf_settings_parser.get(
'params.workflow') == "Full":
ERROR(
f"File in row {row} is missing a label (Full workflow enabled)"
)
return 1
elif child.item(row, 1).text() != '':
label = child.item(row, 1).text()
input_string = child.item(row, 0).text() + '\t' + label + '\n'
full_table.append(input_string)
if full_table == []:
ERROR('No files choosen')
return 1
with open(f"{output_path}/file_list.txt", 'w') as file:
for line in full_table:
file.write(line)
batch_file_path = f"{output_path}/file_list.txt"
self.nf_settings_parser.write("params.batch_file", batch_file_path)
# DATABASE_FILE #
children = parent.findChildren(QLineEdit)
for child in children: # This is so I can have whatever order in widgets I want
if child.type == 'file':
break # Will keep child
database_path = child.text()
self.nf_settings_parser.write("params.db", database_path)
workflow = self.nf_settings_parser.get("params.workflow")
if workflow == "Full" and not os.path.isfile(database_path):
ERROR(
"You must choose an database if you are running the full pipeline"
)
return 1
# EMAIL #
email = self.nf_settings_parser.get("params.email")
if email != '':
# Need to add -N here, since without it, nextflow will display a warning
self.sh_parser.write("EMAIL_NOTIFICATION", f"-N {email}")
else:
self.sh_parser.write("EMAIL_NOTIFICATION", f"")
# CUSTOM MOUNTS #
custom_mounts = self.pipe_parser.get('custom_mounts').replace(
'\r', '').replace('\n', '')
self.nf_settings_parser.write('params.custom_mounts', custom_mounts)
# Generate random hash for nextflow
random_hash = secrets.token_urlsafe(16)
self.nf_settings_parser.write('params.random_hash', random_hash)
# Set pipe to launch nextflow pipeline
self.pipe_parser.write('exit_code', '0', isString=False)
self.window().close()
def javaScriptConsoleMessage(self, level, message, lineNumber, sourceId):
if not "a parser-blocking" in message and level == 2: # Filter script injection erros
ERROR(f"{level}, {message}, {lineNumber}, {sourceId}")
def parseCli(conf):
global settings
flags = []
flagargs = []
arguments = []
currentflag = ""
flagargument = False
i = 0
for a in sys.argv:
i = i + 1
if i == 1: continue # skip the script name
if a[0:1] == "-":
if flagargument:
ERROR("expecting value for argument " + currentflag)
exit()
fl = ""
flarg = ""
if a[0:2] == "--":
fl = a[2:]
p = fl.find("=")
if p < 0: p = fl.find(":")
if p >= 0:
flarg = fl[p + 1:]
fl = fl[0:p]
else:
fl = a[1:]
if len(fl) > 1:
flarg = fl[1:]
fl = fl[0:1]
# Check if flag is valid and/or needs arguments
if checkCliFlag(fl):
if flarg == "":
currentflag = fl
flagargument = True
else:
flags.append(fl)
flagargs.append(flarg)
else:
flags.append(fl)
flagargs.append(flarg)
else:
# Catch arguments
if flagargument:
flags.append(currentflag)
flagargs.append(a)
flagargument = False
currentflag = ""
else:
if len(conf.action) == 0:
conf.action = a
else:
arguments.append(a)
DEBUG("Flags: " + str(len(flags)))
DEBUG("Args: " + str(len(arguments)))
for i in range(0, len(flags)):
if flags[i] == "deployment" or flags[i] == "d":
try:
deploy = ""
with open(flagargs[i], "r") as df:
for line in df:
# Compress the deployment
if len(deploy) > 0: deploy = deploy + "\n"
deploy = deploy + line.strip()
conf.deployment = deploy
except Exception, e:
ERROR("cannot read deployment file.\n" + str(e))
exit()
if flags[i] == "replica" or flags[i] == "r":
conf.args = conf.args + " --replica"
if flags[i] == "debug":
SET_DEBUG(True)
jdata = r.read()
status = checkJsonStatus(jdata)
state.server[s] = status
# If this is the first server, and no deployment file is defined,
# load the deployment file, etc.
# Validate anything but create or add (join, recover...)
#print "DEBUG INIT(), Deployment: " + checkJsonData(jdata,"deployment")
if s == settings.hoststring.split(",")[0]:
DEBUG("This is the first server in the list [" + s +
"]. Parsing data.")
if settings.deployment == "":
settings.deployment = retval = checkJsonData(jdata, "deployment")
if status == 1 or status == 2:
ERROR("a database is already running on server " + s)
return False
#else:
# DEBUG("status is " + str(status))
return True
def prepServer(s):
global settings, state
svr = resolveServerId(s)
# Set the hosts and deployment (as necessary)
data = {}
data["hoststring"] = settings.hoststring
data["deployment"] = settings.deployment
except Exception, e:
FATAL("bad port number. " + str(e))
if o in ("-d", "--data"):
DATAROOT = a
ofound = True
if o in ("-b", "--bin"):
BINROOT = a
ofound = True
if o in ("-a", "--address"):
IPADDRESS = a
ofound = True
if o in ("--debug"):
SET_DEBUG(True)
ofound = True
if not ofound:
ERROR("Unsupported option " + str(o))
usage()
sys.exit()
print "Serving at port", PORT
print "data root", DATAROOT
print "bin root", BINROOT
print "IP address", IPADDRESS
if len(DATAROOT) > 0:
if DATAROOT[len(DATAROOT) - 1:] != "/": DATAROOT = DATAROOT + "/"
print "data root", DATAROOT
###################### MAIN LOOP