def print_help(self): """Present out `help`.""" content = coloured(self.help, COLOURS["help"]) if self.is_default: content += " " content += coloured("(default)", COLOURS["default"]) return content
def shortReport(self, Session): printStr = "\nCampaign: " + coloured(self.name, "yellow") + ": " + str( self.jobs.count()) + " total jobs\n" for i in self.statesDict(): stateCount = 0 for state in i[1]: stateCount += self.jobs.filter(Job.status.like(state)).count() if stateCount > 0: printStr += coloured(i[0] + ": " + str(stateCount) + " jobs\n", i[2]) return printStr
def resubmitCampaign(Session, campName, resubmit_cancelled): QUEUE_NAME = 'ANALY_TJLAB_LQCD' VO = 'Gluex' # read yaml description jobdef = None try: campaign = Session.query(Campaign).filter( Campaign.name.like(campName)).first() if (campaign is None): print( coloured("No campaign of name " + campName + " found. Currently defined campaigns are: \n"), "red") for c in Session.query(Campaign.name).all(): print(c[0]) sys.exit(1) except Exception as e: logging.error(traceback.format_exc()) Session.rollback() sys.exit(1) aSrvID = None submitStatus = ['failed'] submitStatus.append('cancelled') if resubmit_cancelled else submitStatus print submitStatus print resubmit_cancelled for j in campaign.jobs.filter(Job.status.in_(submitStatus)): print(j.id, j.status) jobSpec = submissionTools.createJobSpec(walltime=j.wallTime, command=j.script, outputFile=j.outputFile, nodes=j.nodes, campaignID=campaign.id) j.servername = jobSpec.jobName s, o = Client.submitJobs([jobSpec]) try: j.pandaID = o[0][0] j.status = 'submitted' j.subStatus = 'submitted' print( coloured(j.iterable.strip() + ", " + str(o[0][0]) + "\n", 'green')) except Exception as e: logging.error(traceback.format_exc()) print( coloured(j.iterable.strip() + " job failed to submit\n", 'red')) j.status = 'failed' j.subStatus = 'failed' Session.commit() return None
def checkIterables(Session): #Check for iterables that are repeatedly failing across campaigns/jobs - indication that input files are corrupted or missing retStr = "" for iterable in Session.query(Job).with_entities(Job.iterable).group_by( Job.iterable).all(): failCount = Session.query(Job).filter(Job.iterable.isnot(None)).filter( Job.iterable == iterable[0]).filter( Job.status.like('failed')).count() if failCount > 1: retStr += coloured( str(failCount) + " failures with iterable ", 'red') + coloured( iterable[0], 'yellow') + coloured( ". Possibly missing or corrupt input?\n", "red") return retStr
def ls(self): parser = argparse.ArgumentParser( description="List files in the specified directory") parser.add_argument("dir", nargs="?", default=self.curdir.get_dir()) args = parser.parse_args(self.args[1:]) args.dir = self.__normalize_dir(args.dir) try: for f in self.dropbox.files_list_folder(args.dir).entries: if util.is_file(f): print(coloured(f.name, 'white')) else: print(coloured(f.name, 'yellow')) except dropbox.exceptions.ApiError as e: print_error("sink ls: no such directory")
def apply_formatting(pargs: Namespace, formatting: int, s: str) -> str: if not use_colour(pargs): return s formatting_codes: str = '' formats_to_apply: [str] if type(formatting) == list: formats_to_apply = formatting else: formats_to_apply = [formatting] colour: str = None on_colour: str = None attrs: str = [] for format_to_apply in formats_to_apply: if format_to_apply in COLOURS: colour = format_to_apply elif format_to_apply in HIGHLIGHTS: on_colour = format_to_apply elif format_to_apply in ATTRIBUTES: attrs.append(format_to_apply) elif format_to_apply is None: pass else: print('Unknown format "%s" acceptible values %s' % (format_to_apply, str([None] + list(COLOURS.keys()) + list(HIGHLIGHTS.keys()) + list(ATTRIBUTES.keys()))), file=stderr) exit(1) return coloured(s, color=colour, on_color=on_colour, attrs=attrs)
def handle(self, *args, **options): try: print( coloured( "\n\nWARNING: This script is going to work directly on your " "document originals, so\nWARNING: you probably shouldn't run " "this unless you've got a recent backup\nWARNING: handy. It " "*should* work without a hitch, but be safe and backup your\n" "WARNING: stuff first.\n\nHit Ctrl+C to exit now, or Enter to " "continue.\n\n", "yellow", attrs=("bold", ))) __ = input() except KeyboardInterrupt: return if options["from"] == options["to"]: raise CommandError( 'The "from" and "to" values can\'t be the same.') passphrase = options["passphrase"] or settings.PASSPHRASE if not passphrase: raise CommandError( "Passphrase not defined. Please set it with --passphrase or " "by declaring it in your environment or your config.") if options["from"] == "gpg" and options["to"] == "unencrypted": self.__gpg_to_unencrypted(passphrase) elif options["from"] == "unencrypted" and options["to"] == "gpg": self.__unencrypted_to_gpg(passphrase)
def __gpg_to_unencrypted(passphrase): encrypted_files = Document.objects.filter( storage_type=Document.STORAGE_TYPE_GPG) for document in encrypted_files: print(coloured("Decrypting {}".format(document), "green")) old_paths = [document.source_path, document.thumbnail_path] raw_document = GnuPG.decrypted(document.source_file, passphrase) raw_thumb = GnuPG.decrypted(document.thumbnail_file, passphrase) document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED with open(document.source_path, "wb") as f: f.write(raw_document) with open(document.thumbnail_path, "wb") as f: f.write(raw_thumb) document.save(update_fields=("storage_type", )) for path in old_paths: os.unlink(path)
def play(nazwa="Nagranie.wav"): # define stream chunk # open a wav format music f = wave.open(nazwa, "rb") # instantiate PyAudio p = pyaudio.PyAudio() # open stream stream = p.open(format=p.get_format_from_width(f.getsampwidth()), channels=f.getnchannels(), rate=f.getframerate(), output=True) print( coloured( "SNR = " + "{:.2f}".format(calc_log(f.getsampwidth() * 8)) + " dB", 'yellow')) # read data data = f.readframes(CHUNK) # play stream while data: stream.write(data) data = f.readframes(CHUNK) # stop stream stream.stop_stream() stream.close() # close PyAudio p.terminate()
def statusReport(self, Session, options=None): printStr = "\n Status Report for Campaign: " + coloured( self.name, "yellow") + "\n" printStr += "Last updated: " + str(self.lastUpdate) + "\n" printStr += str(self.jobs.count()) + " total jobs\n\n" for site in self.jobs.with_entities( Job.computingSite, Job.status).filter( not_(Job.status.like("cancelled"))).group_by( Job.computingSite).all(): if site[0] and (site[0] not in ["NULL", ""]): cS = site[0] else: cS = "Not yet allocated to a site" siteCount = str( self.jobs.filter(Job.computingSite.like(site[0])).filter( not_(Job.status.like("cancelled"))).count()) printStr += cS + ": " + siteCount + "\n" cancelled = self.jobs.filter(Job.status.like('cancelled')).count() if cancelled > 0: printStr += "Cancelled: " + str(cancelled) + "\n" printStr += "\n" for status in self.jobs.with_entities(Job.status, Job.subStatus).group_by( Job.status, Job.subStatus).all(): if (status[0] == "finished"): colour = 'green' if (status[0] in ["failed", "cancelled"]): colour = 'red' else: colour = 'blue' jobsThisStatus = self.jobs.filter(Job.status.like(status[0]), Job.subStatus.like(status[1])) statCount = str(jobsThisStatus.count()) mostRecent = str( jobsThisStatus.order_by( desc('stateChangeTime')).first().stateChangeTime) oldest = str( jobsThisStatus.order_by( asc('stateChangeTime')).first().stateChangeTime) printStr += coloured( "Status: %s \nSubStatus: %s \nCount: %s\nMost Recent: %s\nOldest: %s\n \n" % (status[0], status[1], statCount, mostRecent, oldest), colour) return printStr
def syncCampaignWrap(args): answer = raw_input( coloured( 'Sync repopulates the local database from the panda server. It should be used only if the local database is missing jobs, otherwise use Update. Continue?: [y/n]', 'red')) if not answer or answer[0].lower() != 'y': print 'Aborting' else: print(syncCampaign(Session))
def deleteCampaignWrap(args): answer = raw_input( coloured( 'Really delete campaign all jobs for ' + args.CampaignName + '?: [y/n]', 'red')) if not answer or answer[0].lower() != 'y': print 'Aborting' else: print(deleteCampaign(Session, args.CampaignName))
def execute(self): """Executes an external shell command""" cwd = sh.get_cwd() + "/" parser = argparse.ArgumentParser( description="Executes an external shell command") parser.add_argument("cmd") try: sh.sh(self.args[1:]) except: print(coloured("shell command not found", "red"))
def repopulateDatabaseWrap(args): print( coloured( "Warning: Repopulating the database from the panda monitor is slow and places a large burden on the server. It should be used with care.", 'red')) answer = raw_input('Continue?: [y/n]') if not answer or answer[0].lower() != 'y': print 'Aborting' else: pass
def record(nazwa="Nagranie.wav"): p = pyaudio.PyAudio() # defaultCapability = p.get_default_host_api_info() # print(defaultCapability) # WAVE_OUTPUT_FILENAME = "output.wav" if not p.is_format_supported(input_format=FORMAT, input_channels=CHANNELS, rate=RATE, input_device=1): raise pyaudio.paBadIODeviceCombination stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK) print(coloured("Nagrywam", 'green')) frames = [] for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)): frames.append(stream.read(CHUNK)) print(coloured("Koniec nagrania", 'blue')) stream.stop_stream() stream.close() p.terminate() wf = wave.open(nazwa, 'wb') wf.setnchannels(CHANNELS) wf.setsampwidth(p.get_sample_size(FORMAT)) wf.setframerate(RATE) wf.writeframes(b''.join(frames)) wf.close()
def execute(self): """Executes an external shell command returns whether or not the shell command executed successfully """ cwd = sh.get_cwd() + "/" parser = argparse.ArgumentParser( description="Executes an external shell command") parser.add_argument("cmd") try: sh.sh(self.args[1:]) return True except: print(coloured("shell command not found", "red")) return False
def deleteCampaign(Session, campName): try: campaign = Session.query(Campaign).filter( Campaign.name.like(campName)).first() if (campaign is None): print( coloured("No campaign of name " + campName + " found. Currently defined campaigns are: \n"), "red") for c in Session.query(Campaign.name).all(): print(c[0]) sys.exit(1) except Exception as e: logging.error(traceback.format_exc()) Session.rollback() sys.exit(1) return campaign.deleteJobs(Session)
def __unencrypted_to_gpg(passphrase): unencrypted_files = Document.objects.filter( storage_type=Document.STORAGE_TYPE_UNENCRYPTED) for document in unencrypted_files: print(coloured("Encrypting {}".format(document), "green")) old_paths = [document.source_path, document.thumbnail_path] with open(document.source_path, "rb") as raw_document: with open(document.thumbnail_path, "rb") as raw_thumb: document.storage_type = Document.STORAGE_TYPE_GPG with open(document.source_path, "wb") as f: f.write(GnuPG.encrypted(raw_document, passphrase)) with open(document.thumbnail_path, "wb") as f: f.write(GnuPG.encrypted(raw_thumb, passphrase)) document.save(update_fields=("storage_type", )) for path in old_paths: os.unlink(path)
def statusReport(Session, campName, short=False): try: campaign = Session.query(Campaign).filter( Campaign.name.like(campName)).first() if (campaign is None): print( coloured( "No campaign of name " + campName + " found. Currently defined campaigns are: \n", "red")) for c in Session.query(Campaign.name).all(): print(c[0]) sys.exit(1) except Exception as e: logging.error(traceback.format_exc()) Session.rollback() sys.exit(1) campaign.updateJobs(Session) if (short): return campaign.shortReport(Session) else: return campaign.statusReport(Session)
def __gpg_to_unencrypted(passphrase): encrypted_files = Document.objects.filter( storage_type=Document.STORAGE_TYPE_GPG) for document in encrypted_files: print(coloured("Decrypting {}".format( document).encode('utf-8'), "green")) old_paths = [document.source_path, document.thumbnail_path] raw_document = GnuPG.decrypted(document.source_file, passphrase) raw_thumb = GnuPG.decrypted(document.thumbnail_file, passphrase) document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED ext = os.path.splitext(document.filename)[1] if not ext == '.gpg': raise CommandError( f"Abort: encrypted file {document.source_path} does not " f"end with .gpg") document.filename = os.path.splitext(document.filename)[0] with open(document.source_path, "wb") as f: f.write(raw_document) with open(document.thumbnail_path, "wb") as f: f.write(raw_thumb) Document.objects.filter(id=document.id).update( storage_type=document.storage_type, filename=document.filename) for path in old_paths: os.unlink(path)
def handle(self, *args, **options) -> int: """ run the data validation """ labels = options["labels"] if len(labels) == 0: # init runners first so that they can validate the inputs runners = [ ModelValidationRunner(model) for model in REGISTRY.keys() ] else: runners = list( chain(*[self.parse_label(label) for label in labels])) totals = Counter() for runner in runners: summaries = runner.run(show_progress=True) self.print_summaries(summaries) for valinfo, summary in summaries: totals[summary.status] += 1 result_str = (f"Total Passing: {totals[Status.PASSING]}\n" f"Total Failing: {totals[Status.FAILING]}\n" f"Total Exceptions: {totals[Status.EXCEPTION]}\n" f"Total Uninitialized: {totals[Status.UNINITIALIZED]}\n") if totals[Status.FAILING] + totals[Status.EXCEPTION] > 0: exit_code = 1 colour = "red" elif totals[Status.PASSING] == 0: exit_code = 0 colour = "white" else: exit_code = 0 colour = "green" logger.info("=" * 70) logger.info(coloured(result_str, colour, attrs=["bold"])) return exit_code
def submitCampaign(Session, jobsFile): # read yaml description jobdef = None try: campdef = submissionTools.PandaJobsJSONParser.parse(jobsFile) campaign = Session.query(Campaign).filter( Campaign.name.like(campdef['campaign'])).first() if (campaign is None): #Don't let colons into campaign names campName = re.sub(':', '', campdef['campaign']) campaign = Campaign(name=campName, lastUpdate=datetime.datetime.utcnow()) Session.add(campaign) Session.commit() except Exception as e: logging.error(traceback.format_exc()) Session.rollback() sys.exit(1) aSrvID = None for j in campdef['jobs']: nodes = j['nodes'] walltime = j['walltime'] queuename = j['queuename'] try: outputFile = j['outputFile'].strip() except: outputFile = None command = j['command'] try: iterable = j['iterable'].strip() except: iterable = None #Check to see if this is a duplicate output file jobsThisOF = Session.query(Job).filter( Job.outputFile.like(outputFile)).count() if (jobsThisOF > 0): print( coloured( 'Warning:' + str(jobsThisOF) + ' job(s) already exist with output file: \n' + outputFile + '\n', 'red')) dbJob = Job(script=command, nodes=nodes, wallTime=walltime, status="To Submit", subStatus="To Submit", campaignID=campaign.id, outputFile=outputFile) dbJob.serverName = 'c:' + campaign.name + ':' if iterable: dbJob.serverName += 'i:' + iterable + ':' if outputFile: #Panda Server doesn't like slashes in its job names dbJob.serverName += 'oF:' + re.sub('/', ';', outputFile) + ':' dbJob.serverName += subprocess.check_output('uuidgen') dbJob.iterable = iterable jobSpec = submissionTools.createJobSpec(walltime=walltime, command=command, outputFile=outputFile, nodes=nodes, jobName=dbJob.serverName) s, o = Client.submitJobs([jobSpec]) try: print(o) dbJob.pandaID = o[0][0] dbJob.status = 'submitted' dbJob.subStatus = 'submitted' print( coloured(iterable.strip() + ", " + str(o[0][0]) + "\n", 'green')) except Exception as e: logging.error(traceback.format_exc()) print(coloured(iterable.strip() + " job failed to submit\n", 'red')) dbJob.status = 'failed' dbJob.subStatus = 'failed' Session.add(dbJob) Session.commit() return None
def print_error(msg): print(coloured(msg, 'red'))
def print_succ(msg): print(coloured(msg, 'green'))
def pwd(self): parser = argparse.ArgumentParser( description="Display the current working directory") print(coloured(self.curdir.get_dir(), 'white'))
def submitCampaign(Session, campSpecFile, listFile): # read yaml description jobdef = None try: campdef = submissionTools.PandaJobsJSONParser.parse(campSpecFile) campaign = Session.query(Campaign).filter( Campaign.name.like(campdef['campaign'])).first() if (campaign is None): campaign = Campaign(name=campdef['campaign'], lastUpdate=datetime.datetime.utcnow()) Session.add(campaign) Session.commit() except Exception as e: logging.error(traceback.format_exc()) Session.rollback() sys.exit(1) aSrvID = None nodes = campdef['jobtemplate']['nodes'] walltime = campdef['jobtemplate']['walltime'] queuename = campdef['jobtemplate']['queuename'] try: outputFile = campdef['jobtemplate']['outputFile'] except: outputFile = None command = campdef['jobtemplate']['command'] if (listFile): iterList = [] with open(listFile, 'r') as f: for i in f: ii = re.sub("\n", "", i) iterList.append(ii) else: iterList = [''] for iterable in iterList: if (listFile): jobCommand = re.sub('<iter>', iterable, command) jobOutput = re.sub('<iter>', iterable, outputFile) else: jobCommand = command jobOutput = outputFile dbJob = Job(script=jobCommand, nodes=nodes, wallTime=walltime, status="To Submit", campaignID=campaign.id, outputFile=jobOutput) dbJob.servername = campaign.name + subprocess.check_output('uuidgen') if (listFile): dbJob.iterable = iterable Session.add(dbJob) Session.commit() jobSpec = submissionTools.createJobSpec(walltime=walltime, command=jobCommand, outputFile=jobOutput, nodes=nodes, jobName=dbJob.servername) s, o = Client.submitJobs([jobSpec]) try: dbJob.pandaID = o[0][0] dbJob.status = 'submitted' dbJob.subStatus = 'submitted' print( coloured(iterable.strip() + ", " + str(o[0][0]) + "\n", 'green')) except Exception as e: logging.error(traceback.format_exc()) print(coloured(iterable.strip() + " job failed to submit\n", 'red')) dbJob.status = 'failed' dbJob.subStatus = 'failed' Session.commit() return None
def print_status(self, colour: bool = True) -> str: if not colour: return self.status.name return coloured(self.status.name, self.status.colour, attrs=["bold"])
def solve(x): print(f"\nSolving for \"{x}\"...") print(coloured(f"{solveEquation(x, True):,.2f}", "blue"))
def syncCampaign(Session): try: output = Client.getAllJobs() if output[0] != 0: raise Exception("Server error") else: output = json.loads(output[1])['jobs'] except Exception as e: logging.error(traceback.format_exc()) Session.rollback() sys.exit(1) jobsToRepopulate = [] for j in output: try: #Check for pre-existing job with this pandaid #We have to evaluate these queries lazily to avoid throwing an unnecessary exception if (j['pandaid'] and j['jobname']): isExistingPandaID = Session.query(Job).filter( Job.pandaID.like(j['pandaid'])) isExistingJobName = Session.query(Job).filter( Job.serverName.like(j['jobname'])) if (isExistingPandaID.first() is None and isExistingJobName.first() is None): if (len(j['jobname']) > 37): #See if the jobname fits the format campaignName, i, oF = unpackServerName(j['jobname']) if (campaignName): campaign = Session.query(Campaign).filter( Campaign.name.like(campaignName)).first() if (campaign is None): campaign = Campaign( name=campaignName, lastUpdate=datetime.datetime.utcnow()) Session.add(campaign) Session.commit() #We can't recover the job script from the monitor output - we do that with another query below job = Job(script="unknown", campaignID=campaign.id, pandaID=j['pandaid'], serverName=j['jobname'], status=j['jobstatus'], subStatus=j['jobsubstatus']) if i: job.iterable = i #In some instances panda server can report a null substatus. Converting these to empty strings to fulfil database rules if not j['jobsubstatus']: job.subStatus = "" Session.add(job) Session.commit() #Record that this campaign/job id pair was missing, but only after it's been committed jobsToRepopulate.append((campaign.id, job.pandaID)) except Exception as e: logging.error(traceback.format_exc()) Session.rollback() #We need to query each job individually to get its job parameters campsToRepopulate = set([seq[0] for seq in jobsToRepopulate]) for c in campsToRepopulate: try: camp = Session.query(Campaign).get(c) jobs = [seq[1] for seq in jobsToRepopulate if seq[0] == c] #Recreate the jobs that were missing camp.updateJobs(Session, recreate=True, jobs_to_query=jobs) #Now update them all to make sure everything is legit camp.updateJobs(Session) #Now check to see if we have duplicate output files for OF in Session.query(Job).with_entities( Job.outputFile).group_by(Job.outputFile).all(): jobsThisOF = Session.query(Job).filter( Job.outputFile.like(OF[0])).count() if (jobsThisOF > 1): print( coloured( 'Warning:' + str(jobsThisOF) + ' job(s) have shared output file: \n' + OF[0] + '\n', 'red')) except Exception as e: logging.error(traceback.format_exc()) Session.rollback() return None
def pwd(self): parser = argparse.ArgumentParser( description="Display the current working directory") print(coloured(self.curdir.get_dir(), 'white')) return self.curdir.get_dir()