def print_help(self):
"""Present out `help`."""
content = coloured(self.help, COLOURS["help"])
if self.is_default:
content += " "
content += coloured("(default)", COLOURS["default"])
return content
def shortReport(self, Session):
printStr = "\nCampaign: " + coloured(self.name, "yellow") + ": " + str(
self.jobs.count()) + " total jobs\n"
for i in self.statesDict():
stateCount = 0
for state in i[1]:
stateCount += self.jobs.filter(Job.status.like(state)).count()
if stateCount > 0:
printStr += coloured(i[0] + ": " + str(stateCount) + " jobs\n",
i[2])
return printStr
def resubmitCampaign(Session, campName, resubmit_cancelled):
QUEUE_NAME = 'ANALY_TJLAB_LQCD'
VO = 'Gluex'
# read yaml description
jobdef = None
try:
campaign = Session.query(Campaign).filter(
Campaign.name.like(campName)).first()
if (campaign is None):
print(
coloured("No campaign of name " + campName +
" found. Currently defined campaigns are: \n"), "red")
for c in Session.query(Campaign.name).all():
print(c[0])
sys.exit(1)
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
sys.exit(1)
aSrvID = None
submitStatus = ['failed']
submitStatus.append('cancelled') if resubmit_cancelled else submitStatus
print submitStatus
print resubmit_cancelled
for j in campaign.jobs.filter(Job.status.in_(submitStatus)):
print(j.id, j.status)
jobSpec = submissionTools.createJobSpec(walltime=j.wallTime,
command=j.script,
outputFile=j.outputFile,
nodes=j.nodes,
campaignID=campaign.id)
j.servername = jobSpec.jobName
s, o = Client.submitJobs([jobSpec])
try:
j.pandaID = o[0][0]
j.status = 'submitted'
j.subStatus = 'submitted'
print(
coloured(j.iterable.strip() + ", " + str(o[0][0]) + "\n",
'green'))
except Exception as e:
logging.error(traceback.format_exc())
print(
coloured(j.iterable.strip() + " job failed to submit\n",
'red'))
j.status = 'failed'
j.subStatus = 'failed'
Session.commit()
return None
def checkIterables(Session):
#Check for iterables that are repeatedly failing across campaigns/jobs - indication that input files are corrupted or missing
retStr = ""
for iterable in Session.query(Job).with_entities(Job.iterable).group_by(
Job.iterable).all():
failCount = Session.query(Job).filter(Job.iterable.isnot(None)).filter(
Job.iterable == iterable[0]).filter(
Job.status.like('failed')).count()
if failCount > 1:
retStr += coloured(
str(failCount) + " failures with iterable ", 'red') + coloured(
iterable[0], 'yellow') + coloured(
". Possibly missing or corrupt input?\n", "red")
return retStr
def ls(self):
parser = argparse.ArgumentParser(
description="List files in the specified directory")
parser.add_argument("dir", nargs="?", default=self.curdir.get_dir())
args = parser.parse_args(self.args[1:])
args.dir = self.__normalize_dir(args.dir)
try:
for f in self.dropbox.files_list_folder(args.dir).entries:
if util.is_file(f):
print(coloured(f.name, 'white'))
else:
print(coloured(f.name, 'yellow'))
except dropbox.exceptions.ApiError as e:
print_error("sink ls: no such directory")
def apply_formatting(pargs: Namespace, formatting: int, s: str) -> str:
if not use_colour(pargs):
return s
formatting_codes: str = ''
formats_to_apply: [str]
if type(formatting) == list:
formats_to_apply = formatting
else:
formats_to_apply = [formatting]
colour: str = None
on_colour: str = None
attrs: str = []
for format_to_apply in formats_to_apply:
if format_to_apply in COLOURS:
colour = format_to_apply
elif format_to_apply in HIGHLIGHTS:
on_colour = format_to_apply
elif format_to_apply in ATTRIBUTES:
attrs.append(format_to_apply)
elif format_to_apply is None:
pass
else:
print('Unknown format "%s" acceptible values %s' %
(format_to_apply,
str([None] + list(COLOURS.keys()) +
list(HIGHLIGHTS.keys()) + list(ATTRIBUTES.keys()))),
file=stderr)
exit(1)
return coloured(s, color=colour, on_color=on_colour, attrs=attrs)
def handle(self, *args, **options):
try:
print(
coloured(
"\n\nWARNING: This script is going to work directly on your "
"document originals, so\nWARNING: you probably shouldn't run "
"this unless you've got a recent backup\nWARNING: handy. It "
"*should* work without a hitch, but be safe and backup your\n"
"WARNING: stuff first.\n\nHit Ctrl+C to exit now, or Enter to "
"continue.\n\n",
"yellow",
attrs=("bold", )))
__ = input()
except KeyboardInterrupt:
return
if options["from"] == options["to"]:
raise CommandError(
'The "from" and "to" values can\'t be the same.')
passphrase = options["passphrase"] or settings.PASSPHRASE
if not passphrase:
raise CommandError(
"Passphrase not defined. Please set it with --passphrase or "
"by declaring it in your environment or your config.")
if options["from"] == "gpg" and options["to"] == "unencrypted":
self.__gpg_to_unencrypted(passphrase)
elif options["from"] == "unencrypted" and options["to"] == "gpg":
self.__unencrypted_to_gpg(passphrase)
def __gpg_to_unencrypted(passphrase):
encrypted_files = Document.objects.filter(
storage_type=Document.STORAGE_TYPE_GPG)
for document in encrypted_files:
print(coloured("Decrypting {}".format(document), "green"))
old_paths = [document.source_path, document.thumbnail_path]
raw_document = GnuPG.decrypted(document.source_file, passphrase)
raw_thumb = GnuPG.decrypted(document.thumbnail_file, passphrase)
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
with open(document.source_path, "wb") as f:
f.write(raw_document)
with open(document.thumbnail_path, "wb") as f:
f.write(raw_thumb)
document.save(update_fields=("storage_type", ))
for path in old_paths:
os.unlink(path)
def ls(self):
parser = argparse.ArgumentParser(
description="List files in the specified directory")
parser.add_argument("dir", nargs="?", default=self.curdir.get_dir())
args = parser.parse_args(self.args[1:])
args.dir = self.__normalize_dir(args.dir)
try:
for f in self.dropbox.files_list_folder(args.dir).entries:
if util.is_file(f):
print(coloured(f.name, 'white'))
else:
print(coloured(f.name, 'yellow'))
except dropbox.exceptions.ApiError as e:
print_error("sink ls: no such directory")
def play(nazwa="Nagranie.wav"):
# define stream chunk
# open a wav format music
f = wave.open(nazwa, "rb")
# instantiate PyAudio
p = pyaudio.PyAudio()
# open stream
stream = p.open(format=p.get_format_from_width(f.getsampwidth()),
channels=f.getnchannels(),
rate=f.getframerate(),
output=True)
print(
coloured(
"SNR = " + "{:.2f}".format(calc_log(f.getsampwidth() * 8)) + " dB",
'yellow'))
# read data
data = f.readframes(CHUNK)
# play stream
while data:
stream.write(data)
data = f.readframes(CHUNK)
# stop stream
stream.stop_stream()
stream.close()
# close PyAudio
p.terminate()
def statusReport(self, Session, options=None):
printStr = "\n Status Report for Campaign: " + coloured(
self.name, "yellow") + "\n"
printStr += "Last updated: " + str(self.lastUpdate) + "\n"
printStr += str(self.jobs.count()) + " total jobs\n\n"
for site in self.jobs.with_entities(
Job.computingSite, Job.status).filter(
not_(Job.status.like("cancelled"))).group_by(
Job.computingSite).all():
if site[0] and (site[0] not in ["NULL", ""]):
cS = site[0]
else:
cS = "Not yet allocated to a site"
siteCount = str(
self.jobs.filter(Job.computingSite.like(site[0])).filter(
not_(Job.status.like("cancelled"))).count())
printStr += cS + ": " + siteCount + "\n"
cancelled = self.jobs.filter(Job.status.like('cancelled')).count()
if cancelled > 0:
printStr += "Cancelled: " + str(cancelled) + "\n"
printStr += "\n"
for status in self.jobs.with_entities(Job.status,
Job.subStatus).group_by(
Job.status,
Job.subStatus).all():
if (status[0] == "finished"):
colour = 'green'
if (status[0] in ["failed", "cancelled"]):
colour = 'red'
else:
colour = 'blue'
jobsThisStatus = self.jobs.filter(Job.status.like(status[0]),
Job.subStatus.like(status[1]))
statCount = str(jobsThisStatus.count())
mostRecent = str(
jobsThisStatus.order_by(
desc('stateChangeTime')).first().stateChangeTime)
oldest = str(
jobsThisStatus.order_by(
asc('stateChangeTime')).first().stateChangeTime)
printStr += coloured(
"Status: %s \nSubStatus: %s \nCount: %s\nMost Recent: %s\nOldest: %s\n \n"
% (status[0], status[1], statCount, mostRecent, oldest),
colour)
return printStr
def syncCampaignWrap(args):
answer = raw_input(
coloured(
'Sync repopulates the local database from the panda server. It should be used only if the local database is missing jobs, otherwise use Update. Continue?: [y/n]',
'red'))
if not answer or answer[0].lower() != 'y':
print 'Aborting'
else:
print(syncCampaign(Session))
def deleteCampaignWrap(args):
answer = raw_input(
coloured(
'Really delete campaign all jobs for ' + args.CampaignName +
'?: [y/n]', 'red'))
if not answer or answer[0].lower() != 'y':
print 'Aborting'
else:
print(deleteCampaign(Session, args.CampaignName))
def execute(self):
"""Executes an external shell command"""
cwd = sh.get_cwd() + "/"
parser = argparse.ArgumentParser(
description="Executes an external shell command")
parser.add_argument("cmd")
try:
sh.sh(self.args[1:])
except:
print(coloured("shell command not found", "red"))
def repopulateDatabaseWrap(args):
print(
coloured(
"Warning: Repopulating the database from the panda monitor is slow and places a large burden on the server. It should be used with care.",
'red'))
answer = raw_input('Continue?: [y/n]')
if not answer or answer[0].lower() != 'y':
print 'Aborting'
else:
pass
def execute(self):
"""Executes an external shell command"""
cwd = sh.get_cwd() + "/"
parser = argparse.ArgumentParser(
description="Executes an external shell command")
parser.add_argument("cmd")
try:
sh.sh(self.args[1:])
except:
print(coloured("shell command not found", "red"))
def record(nazwa="Nagranie.wav"):
p = pyaudio.PyAudio()
# defaultCapability = p.get_default_host_api_info()
# print(defaultCapability)
# WAVE_OUTPUT_FILENAME = "output.wav"
if not p.is_format_supported(input_format=FORMAT,
input_channels=CHANNELS,
rate=RATE,
input_device=1):
raise pyaudio.paBadIODeviceCombination
stream = p.open(format=FORMAT,
channels=CHANNELS,
rate=RATE,
input=True,
frames_per_buffer=CHUNK)
print(coloured("Nagrywam", 'green'))
frames = []
for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
frames.append(stream.read(CHUNK))
print(coloured("Koniec nagrania", 'blue'))
stream.stop_stream()
stream.close()
p.terminate()
wf = wave.open(nazwa, 'wb')
wf.setnchannels(CHANNELS)
wf.setsampwidth(p.get_sample_size(FORMAT))
wf.setframerate(RATE)
wf.writeframes(b''.join(frames))
wf.close()
def execute(self):
"""Executes an external shell command
returns whether or not the shell command executed successfully
"""
cwd = sh.get_cwd() + "/"
parser = argparse.ArgumentParser(
description="Executes an external shell command")
parser.add_argument("cmd")
try:
sh.sh(self.args[1:])
return True
except:
print(coloured("shell command not found", "red"))
return False
def deleteCampaign(Session, campName):
try:
campaign = Session.query(Campaign).filter(
Campaign.name.like(campName)).first()
if (campaign is None):
print(
coloured("No campaign of name " + campName +
" found. Currently defined campaigns are: \n"), "red")
for c in Session.query(Campaign.name).all():
print(c[0])
sys.exit(1)
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
sys.exit(1)
return campaign.deleteJobs(Session)
def __unencrypted_to_gpg(passphrase):
unencrypted_files = Document.objects.filter(
storage_type=Document.STORAGE_TYPE_UNENCRYPTED)
for document in unencrypted_files:
print(coloured("Encrypting {}".format(document), "green"))
old_paths = [document.source_path, document.thumbnail_path]
with open(document.source_path, "rb") as raw_document:
with open(document.thumbnail_path, "rb") as raw_thumb:
document.storage_type = Document.STORAGE_TYPE_GPG
with open(document.source_path, "wb") as f:
f.write(GnuPG.encrypted(raw_document, passphrase))
with open(document.thumbnail_path, "wb") as f:
f.write(GnuPG.encrypted(raw_thumb, passphrase))
document.save(update_fields=("storage_type", ))
for path in old_paths:
os.unlink(path)
def statusReport(Session, campName, short=False):
try:
campaign = Session.query(Campaign).filter(
Campaign.name.like(campName)).first()
if (campaign is None):
print(
coloured(
"No campaign of name " + campName +
" found. Currently defined campaigns are: \n", "red"))
for c in Session.query(Campaign.name).all():
print(c[0])
sys.exit(1)
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
sys.exit(1)
campaign.updateJobs(Session)
if (short):
return campaign.shortReport(Session)
else:
return campaign.statusReport(Session)
def __gpg_to_unencrypted(passphrase):
encrypted_files = Document.objects.filter(
storage_type=Document.STORAGE_TYPE_GPG)
for document in encrypted_files:
print(coloured("Decrypting {}".format(
document).encode('utf-8'), "green"))
old_paths = [document.source_path, document.thumbnail_path]
raw_document = GnuPG.decrypted(document.source_file, passphrase)
raw_thumb = GnuPG.decrypted(document.thumbnail_file, passphrase)
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
ext = os.path.splitext(document.filename)[1]
if not ext == '.gpg':
raise CommandError(
f"Abort: encrypted file {document.source_path} does not "
f"end with .gpg")
document.filename = os.path.splitext(document.filename)[0]
with open(document.source_path, "wb") as f:
f.write(raw_document)
with open(document.thumbnail_path, "wb") as f:
f.write(raw_thumb)
Document.objects.filter(id=document.id).update(
storage_type=document.storage_type, filename=document.filename)
for path in old_paths:
os.unlink(path)
def handle(self, *args, **options) -> int:
""" run the data validation """
labels = options["labels"]
if len(labels) == 0:
# init runners first so that they can validate the inputs
runners = [
ModelValidationRunner(model) for model in REGISTRY.keys()
]
else:
runners = list(
chain(*[self.parse_label(label) for label in labels]))
totals = Counter()
for runner in runners:
summaries = runner.run(show_progress=True)
self.print_summaries(summaries)
for valinfo, summary in summaries:
totals[summary.status] += 1
result_str = (f"Total Passing: {totals[Status.PASSING]}\n"
f"Total Failing: {totals[Status.FAILING]}\n"
f"Total Exceptions: {totals[Status.EXCEPTION]}\n"
f"Total Uninitialized: {totals[Status.UNINITIALIZED]}\n")
if totals[Status.FAILING] + totals[Status.EXCEPTION] > 0:
exit_code = 1
colour = "red"
elif totals[Status.PASSING] == 0:
exit_code = 0
colour = "white"
else:
exit_code = 0
colour = "green"
logger.info("=" * 70)
logger.info(coloured(result_str, colour, attrs=["bold"]))
return exit_code
def submitCampaign(Session, jobsFile):
# read yaml description
jobdef = None
try:
campdef = submissionTools.PandaJobsJSONParser.parse(jobsFile)
campaign = Session.query(Campaign).filter(
Campaign.name.like(campdef['campaign'])).first()
if (campaign is None):
#Don't let colons into campaign names
campName = re.sub(':', '', campdef['campaign'])
campaign = Campaign(name=campName,
lastUpdate=datetime.datetime.utcnow())
Session.add(campaign)
Session.commit()
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
sys.exit(1)
aSrvID = None
for j in campdef['jobs']:
nodes = j['nodes']
walltime = j['walltime']
queuename = j['queuename']
try:
outputFile = j['outputFile'].strip()
except:
outputFile = None
command = j['command']
try:
iterable = j['iterable'].strip()
except:
iterable = None
#Check to see if this is a duplicate output file
jobsThisOF = Session.query(Job).filter(
Job.outputFile.like(outputFile)).count()
if (jobsThisOF > 0):
print(
coloured(
'Warning:' + str(jobsThisOF) +
' job(s) already exist with output file: \n' + outputFile +
'\n', 'red'))
dbJob = Job(script=command,
nodes=nodes,
wallTime=walltime,
status="To Submit",
subStatus="To Submit",
campaignID=campaign.id,
outputFile=outputFile)
dbJob.serverName = 'c:' + campaign.name + ':'
if iterable:
dbJob.serverName += 'i:' + iterable + ':'
if outputFile:
#Panda Server doesn't like slashes in its job names
dbJob.serverName += 'oF:' + re.sub('/', ';', outputFile) + ':'
dbJob.serverName += subprocess.check_output('uuidgen')
dbJob.iterable = iterable
jobSpec = submissionTools.createJobSpec(walltime=walltime,
command=command,
outputFile=outputFile,
nodes=nodes,
jobName=dbJob.serverName)
s, o = Client.submitJobs([jobSpec])
try:
print(o)
dbJob.pandaID = o[0][0]
dbJob.status = 'submitted'
dbJob.subStatus = 'submitted'
print(
coloured(iterable.strip() + ", " + str(o[0][0]) + "\n",
'green'))
except Exception as e:
logging.error(traceback.format_exc())
print(coloured(iterable.strip() + " job failed to submit\n",
'red'))
dbJob.status = 'failed'
dbJob.subStatus = 'failed'
Session.add(dbJob)
Session.commit()
return None
def print_error(msg):
print(coloured(msg, 'red'))
def print_succ(msg):
print(coloured(msg, 'green'))
def pwd(self):
parser = argparse.ArgumentParser(
description="Display the current working directory")
print(coloured(self.curdir.get_dir(), 'white'))
def submitCampaign(Session, campSpecFile, listFile):
# read yaml description
jobdef = None
try:
campdef = submissionTools.PandaJobsJSONParser.parse(campSpecFile)
campaign = Session.query(Campaign).filter(
Campaign.name.like(campdef['campaign'])).first()
if (campaign is None):
campaign = Campaign(name=campdef['campaign'],
lastUpdate=datetime.datetime.utcnow())
Session.add(campaign)
Session.commit()
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
sys.exit(1)
aSrvID = None
nodes = campdef['jobtemplate']['nodes']
walltime = campdef['jobtemplate']['walltime']
queuename = campdef['jobtemplate']['queuename']
try:
outputFile = campdef['jobtemplate']['outputFile']
except:
outputFile = None
command = campdef['jobtemplate']['command']
if (listFile):
iterList = []
with open(listFile, 'r') as f:
for i in f:
ii = re.sub("\n", "", i)
iterList.append(ii)
else:
iterList = ['']
for iterable in iterList:
if (listFile):
jobCommand = re.sub('<iter>', iterable, command)
jobOutput = re.sub('<iter>', iterable, outputFile)
else:
jobCommand = command
jobOutput = outputFile
dbJob = Job(script=jobCommand,
nodes=nodes,
wallTime=walltime,
status="To Submit",
campaignID=campaign.id,
outputFile=jobOutput)
dbJob.servername = campaign.name + subprocess.check_output('uuidgen')
if (listFile):
dbJob.iterable = iterable
Session.add(dbJob)
Session.commit()
jobSpec = submissionTools.createJobSpec(walltime=walltime,
command=jobCommand,
outputFile=jobOutput,
nodes=nodes,
jobName=dbJob.servername)
s, o = Client.submitJobs([jobSpec])
try:
dbJob.pandaID = o[0][0]
dbJob.status = 'submitted'
dbJob.subStatus = 'submitted'
print(
coloured(iterable.strip() + ", " + str(o[0][0]) + "\n",
'green'))
except Exception as e:
logging.error(traceback.format_exc())
print(coloured(iterable.strip() + " job failed to submit\n",
'red'))
dbJob.status = 'failed'
dbJob.subStatus = 'failed'
Session.commit()
return None
def print_status(self, colour: bool = True) -> str:
if not colour:
return self.status.name
return coloured(self.status.name, self.status.colour, attrs=["bold"])
def solve(x):
print(f"\nSolving for \"{x}\"...")
print(coloured(f"{solveEquation(x, True):,.2f}", "blue"))
def syncCampaign(Session):
try:
output = Client.getAllJobs()
if output[0] != 0:
raise Exception("Server error")
else:
output = json.loads(output[1])['jobs']
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
sys.exit(1)
jobsToRepopulate = []
for j in output:
try:
#Check for pre-existing job with this pandaid
#We have to evaluate these queries lazily to avoid throwing an unnecessary exception
if (j['pandaid'] and j['jobname']):
isExistingPandaID = Session.query(Job).filter(
Job.pandaID.like(j['pandaid']))
isExistingJobName = Session.query(Job).filter(
Job.serverName.like(j['jobname']))
if (isExistingPandaID.first() is None
and isExistingJobName.first() is None):
if (len(j['jobname']) > 37):
#See if the jobname fits the format
campaignName, i, oF = unpackServerName(j['jobname'])
if (campaignName):
campaign = Session.query(Campaign).filter(
Campaign.name.like(campaignName)).first()
if (campaign is None):
campaign = Campaign(
name=campaignName,
lastUpdate=datetime.datetime.utcnow())
Session.add(campaign)
Session.commit()
#We can't recover the job script from the monitor output - we do that with another query below
job = Job(script="unknown",
campaignID=campaign.id,
pandaID=j['pandaid'],
serverName=j['jobname'],
status=j['jobstatus'],
subStatus=j['jobsubstatus'])
if i:
job.iterable = i
#In some instances panda server can report a null substatus. Converting these to empty strings to fulfil database rules
if not j['jobsubstatus']:
job.subStatus = ""
Session.add(job)
Session.commit()
#Record that this campaign/job id pair was missing, but only after it's been committed
jobsToRepopulate.append((campaign.id, job.pandaID))
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
#We need to query each job individually to get its job parameters
campsToRepopulate = set([seq[0] for seq in jobsToRepopulate])
for c in campsToRepopulate:
try:
camp = Session.query(Campaign).get(c)
jobs = [seq[1] for seq in jobsToRepopulate if seq[0] == c]
#Recreate the jobs that were missing
camp.updateJobs(Session, recreate=True, jobs_to_query=jobs)
#Now update them all to make sure everything is legit
camp.updateJobs(Session)
#Now check to see if we have duplicate output files
for OF in Session.query(Job).with_entities(
Job.outputFile).group_by(Job.outputFile).all():
jobsThisOF = Session.query(Job).filter(
Job.outputFile.like(OF[0])).count()
if (jobsThisOF > 1):
print(
coloured(
'Warning:' + str(jobsThisOF) +
' job(s) have shared output file: \n' + OF[0] +
'\n', 'red'))
except Exception as e:
logging.error(traceback.format_exc())
Session.rollback()
return None
def print_succ(msg):
print(coloured(msg, 'green'))
def pwd(self):
parser = argparse.ArgumentParser(
description="Display the current working directory")
print(coloured(self.curdir.get_dir(), 'white'))
return self.curdir.get_dir()
def print_error(msg):
print(coloured(msg, 'red'))
