def script_factory(args, script_obj, script_functions, params): script_text = "" runscript_filename = fs.runscript_file_obj.file_path + fs.runscript_file_obj.file_base runscript_filename = runscript_filename + params[ 'file_extension'] + fs.runscript_file_obj.file_end runjob_filename = fs.run_job_obj.file_path + fs.run_job_obj.file_base runjob_filename = runjob_filename + params[ 'file_extension'] + fs.run_job_obj.file_end #In the below for loop, we loop through all script_generators for a certain submission script, appending the output of each function to a string gen_text = [ f( params['scard'], username=params['username'], gcard_loc=params['gcard_loc'], GcardID=params['GcardID'], lund_dir=params['lund_dir'], database_filename=params['database_filename'], file_extension=params['file_extension'], runscript_filename=runscript_filename, runjob_filename=runjob_filename, using_sqlite=args.lite, ) for f in script_functions ] script_text = script_text.join(gen_text) #This handles writing to disk and to SQL database if args.write_files: # Build path to local file for writing, and normalize it (remove ../). filename = os.path.normpath(script_obj.file_path + script_obj.file_base + params['file_extension'] + script_obj.file_end) utils.printer( "\tWriting submission file '{0}' based off of specifications of UserSubmissionID = {1}, GcardID = {2}" .format(filename, params['UserSubmissionID'], params['GcardID'])) if not os.path.exists(os.path.normpath(script_obj.file_path)): utils.printer('Creating directory: {}'.format( script_obj.file_path)) subprocess.call(['mkdir', '-p', script_obj.file_path], stdout=subprocess.PIPE) if os.path.isfile(filename): subprocess.call(['rm', filename]) subprocess.call(['touch', filename]) with open(filename, 'w') as f: f.write(script_text) str_script_db = script_text.replace( '"', "'" ) #I can't figure out a way to write "" into a sqlite field without errors # For now, we can replace " with ', which works ok, but IDK how it will run if the scripts were submitted to HTCondor strn = 'UPDATE FarmSubmissions SET {0} = "{1}" WHERE GcardID = {2};'.format( script_obj.file_text_fieldname, str_script_db, params['GcardID']) utils.db_write(strn)
def update_users_statistics(scard, params): strn = "SELECT Total_UserSubmissions FROM Users WHERE User = '******';".format( params['username']) UserSubmissions_total = utils.db_grab(strn)[0][0] UserSubmissions_total += 1 strn = "UPDATE Users SET Total_UserSubmissions = '{0}' WHERE User = '******';".format( UserSubmissions_total, params['username']) utils.db_write(strn) strn = "SELECT Total_Jobs FROM Users WHERE User = '******';".format( params['username']) jobs_total = utils.db_grab(strn)[0][0] jobs_total += int(scard.data['jobs']) strn = "UPDATE Users SET Total_Jobs = '{0}' WHERE User = '******';".format( jobs_total, params['username']) utils.db_write(strn) if 'nevents' in scard.data: strn = "SELECT Total_Events FROM Users WHERE User = '******';".format( params['username']) events_total = utils.db_grab(strn)[0][0] events_total += int(scard.data['jobs']) * int(scard.data['nevents']) strn = "UPDATE Users SET Total_Events = '{0}' WHERE User = '******';".format( events_total, params['username']) utils.db_write(strn) else: utils.printer( """Since you are using custom LUND files, we are not able to update your usage statistics. This will not affect your simulations in any way, but will affect the total number of events reported as processing through our system. """) strn = "UPDATE Users SET Most_Recent_Active_Date = '{0}' WHERE User = '******';".format( utils.gettime(), params['username']) utils.db_write(strn)
def script_factory(script_obj, gen_funcs, func_names, scard, params, file_extension): script_text = "" filename = script_obj.file_path + script_obj.file_base + file_extension + script_obj.file_end utils.printer( "\tWriting submission file '{0}' based off of specifications of BatchID = {1}, GcardID = {2}" .format(filename, params['BatchID'], params['GcardID'])) if os.path.isfile(filename): subprocess.call(['rm', filename]) for count, f in enumerate(gen_funcs): generated_text = getattr(f, func_names[count])( scard, username=params['username'], gcard_loc=params['gcard_loc'], file_extension=file_extension, runscript_filename=file_struct.runscript_file_obj.file_path + file_struct.runscript_file_obj.file_base + file_extension + file_struct.runscript_file_obj.file_end, runjob_filename=file_struct.run_job_obj.file_path + file_struct.run_job_obj.file_base + file_extension + file_struct.run_job_obj.file_end, ) with open(filename, "a") as file: file.write(generated_text) script_text += generated_text str_script_db = script_text.replace( '"', "'" ) #I can't figure out a way to write "" into a sqlite field without errors #For now, we can replace " with ', which works ok, but IDK how it will run if the scripts were submitted to HTCondor strn = 'UPDATE Submissions SET {0} = "{1}" WHERE GcardID = {2};'.format( script_obj.file_text_fieldname, str_script_db, params['GcardID']) utils.sql3_exec(strn)
def db_gcard_write(BatchID, timestamp, gcard_text): strn = "INSERT INTO Gcards(BatchID) VALUES ({0});".format(BatchID) utils.sql3_exec(strn) strn = """UPDATE Gcards SET {0} = "{1}" WHERE BatchID = {2};""".format( 'gcard_text', gcard_text, BatchID) utils.sql3_exec(strn) utils.printer( "GCard added to database corresponding to BatchID {}".format(BatchID))
def SCard_Entry(UserSubmissionID, timestamp, scard_dict): strn = """INSERT INTO Scards(UserSubmissionID,timestamp) VALUES ("{0}","{1}");""".format( UserSubmissionID, timestamp) utils.db_write(strn) for key in scard_dict: strn = "UPDATE Scards SET {0} = '{1}' WHERE UserSubmissionID = {2};".format( key, scard_dict[key], UserSubmissionID) utils.db_write(strn) utils.printer( "SCard record added to database corresponding to UserSubmissionID {0}". format(UserSubmissionID))
def SCard_Entry(BatchID, timestamp, scard_dict): strn = """INSERT INTO Scards(BatchID,timestamp) VALUES ("{0}","{1}");""".format( BatchID, timestamp) utils.sql3_exec(strn) for key in scard_dict: strn = "UPDATE Scards SET {0} = '{1}' WHERE BatchID = {2};".format( key, scard_dict[key], BatchID) utils.sql3_exec(strn) utils.printer( "SCard record added to database corresponding to BatchID {0}".format( BatchID))
def GCard_Entry(UserSubmissionID,unixtimestamp,url_dir): print("Gathering gcards from {0} ".format(url_dir)) if not 'http' in url_dir: #== fs.gcard_default: utils.printer('Using gcard from /jlab/work') gcard_text_db = url_dir db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db) elif 'http' in url_dir: utils.printer('Trying to download gcards from online repository') if '.gcard' in url_dir: utils.printer('Gcard URL name is: '+url_dir) gcard_text = html_reader.html_reader(url_dir,'')[0]#This returns a tuple, we need the contents of the tuple utils.printer2('HTML from gcard link is: {0}'.format(gcard_text)) gcard_text_db = gcard_text.replace('"',"'") print("\t Gathered gcard '{0}'".format(url_dir)) db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db) else: raw_html, gcard_urls = html_reader.html_reader(url_dir,fs.gcard_identifying_text) if len(gcard_urls) == 0: print("No gcard files found (they must end in '{0}'). Is the online repository correct?".format(fs.gcard_identifying_text)) exit() else: for url_ending in gcard_urls: utils.printer('Gcard URL name is: '+url_ending) gcard_text = html_reader.html_reader(url_dir+'/'+url_ending,'')[0]#This returns a tuple, we need the contents of the tuple utils.printer2('HTML from gcard link is: {0}'.format(gcard_text)) gcard_text_db = gcard_text.replace('"',"'") print("\t Gathered gcard '{0}'".format(url_ending)) db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db) # I don't think this block can ever be reached else: print('gcard not recognized as default option or online repository, please inspect scard') exit()
def farm_submission_manager(args, usub_id, file_extension, scard, params, db_conn, sql): timestamp = utils.gettime() farm_name = "OSG" #For now we are hardcodign OSG as it is the only farm that is currently supported if farm_name == "OSG": utils.printer("Passing to htcondor_submit") htcondor_submit.htcondor_submit(args,scard,usub_id,file_extension,params, db_conn, sql) else: raise ValueError('Unable to submit for {}'.format( farm_name ))
def validate_scard_line(self, linenum, line): if line.count("#") == 0: utils.printer("Warning: No comment in line {0}.".format(linenum + 1)) elif line.count("#") > 1: utils.printer( "ERROR: number of hashes>1 in line {0}".format(linenum + 1)) utils.printer( "# can be used only as a delimeter and only once per line. Edit scard to fix." ) exit() if line.count(":") == 0: utils.printer("ERROR: No colon in line {0}".format(linenum + 1)) utils.printer("The data cannot be interpreted. Stopped.") exit()
def farm_submission_manager(args, GcardID, file_extension, scard, params): if scard.data['farm_name'] == "MIT_Tier2" or scard.data[ 'farm_name'] == "OSG": utils.printer("Passing to htcondor_submit") htcondor_submit.htcondor_submit(args, scard, GcardID, file_extension, params) update_users_statistics(scard, params) elif scard.data['farm_name'] == "JLab": utils.printer("Passing to slurm_submit") slurm_submit.slurm_submit(args, scard, GcardID, file_extension, params) update_users_statistics(scard, params) else: print( 'Invalid farm name in scard, please check that the desired farm is spelled correctly and is supported' ) exit()
def parse_scard(self, scard_text): scard_lines = scard_text.split("\n") for linenum, line in enumerate(scard_lines): if not line: print("Reached end of scard") break pos_delimeter_colon = line.find(":") pos_delimeter_hash = line.find("#") key = line[:pos_delimeter_colon].strip() value = line[pos_delimeter_colon + 1:pos_delimeter_hash].strip() if key != file_struct.scard_key[linenum]: utils.printer( "ERROR: Line {0} of the steering card has the key '{1}''.". format(linenum + 1, key)) utils.printer("That line must have the key '{0}'.".format( file_struct.scard_key[linenum])) self.data[key] = value
def reset_world(): message = 'Are you sure you want to leave this world? (y/n) ' if (raw_input(message).lower() == 'y'): global __current_monster global game_data # log activity: exit world player = game_data['player'] create_activity(text="{} has left the world!".format(player.name), kind="world-exit") game_data = {} # this is so creepy! :-o save_game_data() # reset game data printer('okay :(') __current_monster = None else: printer("That's the spirit!")
def create_transaction(self, amount, receiver, scrooge): owned_coins = self.get_balance(scrooge.blockchain) transfer_coins = owned_coins[:amount] if len(owned_coins) < amount: transaction = Transaction(transfer_coins, receiver.public_key) printer(bcolors.FAIL, bcolors.BOLD, bcolors.UNDERLINE, BALANCE_ERROR_MESSAGE, bcolors.ENDC, bcolors.ENDC, bcolors.ENDC) printer(bcolors.FAIL, transaction, bcolors.ENDC) transfer_coins = owned_coins[:amount] transaction = Transaction(transfer_coins, receiver.public_key) hashed_transaction = object_hash(transaction) signature = signer(self.__private_key, hashed_transaction) transaction.sign_transaction(signature) scrooge.handle_transaction(transaction, self.public_key)
def Lund_Downloader(url_dir, lund_urls, lund_dir): if len(lund_urls) == 0: print( "No Lund files found (they must end in '{0}'). Is the online repository correct?" .format(fs.lund_identifying_text)) exit() else: for url_ending in lund_urls: utils.printer('Lund URL name is: ' + url_ending) lund_text = html_reader.html_reader( url_dir + '/' + url_ending, '' )[0] #This returns a tuple, we need the contents of the tuple utils.printer2('HTML from lund link is: {0}'.format(lund_text)) lund_text_db = lund_text.replace( '"', "'" ) #This isn't strictly needed but SQLite can't read " into data fields, only ' characters print("\t Gathered lund file '{0}'".format(url_ending)) filename = lund_dir + "/" + url_ending with open(filename, "a") as file: file.write(lund_text_db)
def connect(self): """ establish connection for raw serial data sending to the arduino via com_port2 """ com_port = self.config['connections']['arduino_data_port'] baud_rate = self.config['connections']['arduino_baud_rate'] utils.printer("initializing 2nd serial port to arduino: " + com_port, 'msg') try: ser = serial.Serial(port=com_port, baudrate=baud_rate, timeout=2) ser.setDTR( False ) # reset: https://stackoverflow.com/questions/21073086/wait-on-arduino-auto-reset-using-pyserial time.sleep(1) # sleep timeout length to drop all data ser.flushInput() # ser.setDTR(True) return ser except: utils.printer( "Could not open 2nd serial connection to the arduino!", 'error') sys.exit()
def enter(): global game_data try: # load existing load_game_data() game_data['player'] except (IOError, KeyError): # create new printer('Alas, for the intro. What is your name again?') name = str(raw_input()) player = Player(name=name) game_data['player'] = player # save player instance save_game_data() player = game_data['player'] if not player.registered: register(player) # log activity: enter world create_activity(text="{} has entered the world!".format(player.name), kind="world-enter") printer('{}! Our chosen one. We are pleased to meet you.'.format( player.name)) global __current_monster __current_monster = None
def handle_transaction(self, transaction, sender_pk): verified = self.verify_transaction(transaction, sender_pk) double_spending = self.is_double_spending(transaction) if not verified or double_spending: printer(bcolors.FAIL, bcolors.BOLD, bcolors.UNDERLINE, VERIFICATION_ERROR_MESSAGE, bcolors.ENDC, bcolors.ENDC, bcolors.ENDC) printer(bcolors.FAIL, transaction, bcolors.ENDC) return self.transactions_cache.append(transaction) temp_block = Block(-1, None, self.transactions_cache) printer(bcolors.WARNING, bcolors.BOLD, bcolors.UNDERLINE, UNDER_CONSTRUCTION_MESSAGE, bcolors.ENDC, bcolors.ENDC, bcolors.ENDC) printer(bcolors.WARNING, temp_block.__str__(True), bcolors.ENDC) self.check_for_new_block()
def __init__(self, datasets, lock, epoch, population, finish_tasks, device, args): self.model = resnet20().to(device) self.optim = torch.optim.SGD(self.model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) self.device = device self.args = args self.datasets = datasets self.lock = lock self.population = population self.finish_tasks = finish_tasks self.epoch_base = epoch self.printer = printer(args) self.log_dir = os.path.join(args.save_dir, 'log.txt')
def register(player): if player.registered: return True global game_data printer('Registering to server...', False) try: r = requests.post(settings.register_url, data={'username': player.name}) except Exception, e: printer(e) printer('... failed! :( . Will try again later.') return False
def GCard_Entry(BatchID, unixtimestamp, url_dir): print("Gathering gcards from {0} ".format(url_dir)) if url_dir == file_struct.gcard_default: utils.printer('Using gcard from /jlab/work') gcard_text_db = url_dir db_gcard_write(BatchID, unixtimestamp, gcard_text_db) elif 'https://' in url_dir: utils.printer('Trying to download gcards from online repository') raw_html, gcard_urls = html_reader.html_reader( url_dir, file_struct.gcard_identifying_text) for url_ending in gcard_urls: utils.printer('Gcard URL name is: ' + url_ending) gcard_text = html_reader.html_reader( url_dir + '/' + url_ending, '' )[0] #This returns a tuple, we need the contents of the tuple utils.printer2('HTML from gcard link is: {0}'.format(gcard_text)) gcard_text_db = gcard_text.replace('"', "'") print("\t Gathered gcard '{0}'".format(url_ending)) db_gcard_write(BatchID, unixtimestamp, gcard_text_db) else: print( 'gcard not recognized as default option or online repository, please inspect scard' ) exit()
import utils utils.printer(asif)
def append_block(self, transactions): previous_hash = None if len(self.chain) == 0 else object_hash( self.chain[-1]) block = Block(len(self.chain), previous_hash, transactions) self.chain.append(block) printer(self)
total_loss = sum(train_loss) / len(train_loss) # update weight optimizer.zero_grad() loss.backward() optimizer.step() # calculate accuracy acc = (pred.argmax(1) == labels ).type(torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor).mean().item() train_acc.append(acc) total_acc = sum(train_acc) / len(train_acc) # print result printer("train", e, args.num_epochs, i + 1, len(train_loader), loss.item(), total_loss, acc * 100, total_acc * 100) # tensorboard writer.add_scalar("Loss/train", loss.item(), n_iter_train) writer.add_scalar("Accuracy/train", acc, n_iter_train) n_iter_train += 1 print("") val_acc = [] val_loss = [] model.eval() for i, (datas, _) in enumerate(val_loader): datas = datas.to(device) pivot = args.way * args.shot shot, query = datas[:pivot], datas[pivot:]
import utils utils.printer(utils.default_shape) shape = utils.Shape('circle') utils.printer(shape)
# register player to server def register(player): if player.registered: return True global game_data printer('Registering to server...', False) try: r = requests.post(settings.register_url, data={'username': player.name}) except Exception, e: printer(e) printer('... failed! :( . Will try again later.') return False if r.status_code == 403: printer('... failed! with code {} :( .\ Username is already taken.'.format(r.status_code)) return False elif r.status_code != 200: printer('... failed! with code {} :( .'.format(r.status_code)) return False if 'username' in r.json() and r.json()['username'] == player.name: player.registered = True game_data['player'] = player save_game_data() # save game data printer('... success for user {} ! :) .'.format(player.name)) return True # unregister player to server def unregister(player):
datas = datas.to(device) pivot = args.way * args.shot shot, query = datas[:pivot], datas[pivot:] labels = torch.arange(args.way).repeat(args.query).to(device) # one_hot_labels = Variable(torch.zeros(args.way*args.query, args.way).scatter_(1, labels.view(-1, 1), 1)).to(device) pred = model(shot, query) # calculate loss loss = F.cross_entropy(pred, labels).item() test_loss.append(loss) total_loss = sum(test_loss) / len(test_loss) # calculate accuracy acc = 100 * (pred.argmax(1) == labels).type( torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor).mean().item() test_acc.append(acc) total_acc = sum(test_acc) / len(test_acc) printer("test", e, args.num_epochs, i + 1, len(test_loader), loss, total_loss, acc, total_acc) print("") # get mean confidence interval and mean m, h = mean_confidence_interval(test_acc, confidence=0.95) total_mean.append(m) total_interval.append(h) print("Result: {:.2f}+-{:.2f}".format( sum(total_mean) / len(total_mean), sum(total_interval) / len(total_interval)))
def main(): printer('Greetings young adventurer!\nWelcome to... The World!') enter() ask_help() reset_base_url()
def find_monster(): global __current_monster global game_data if 'player' not in game_data: printer('You are not a registered player!') printer('Use world.enter() to enter the World.') return elif game_data['player'].level > len(__monsters): printer('There are no monsters left in the world!') printer('Congratulations {}! You have finished the game.'.format( game_data['player'].name)) # log activity: finished game create_activity(text="Our hero {} finished all the monsters!".format( game_data['player'].name), kind="player-finished") return elif __current_monster: printer('You are already fighting a monster currently.') printer('Use world.attack() to try defeating the monster.') return # log activity: found monster create_activity(text="{} is searching for monsters ...".format( game_data['player'].name), kind="monster-search") game_data['player'].life = game_data['player'].level + 5 i = game_data['player'].level - 1 monster = __monsters[i] __current_monster = monster # log activity: found monster create_activity(text="{} found {}".format( game_data['player'].name, __current_monster.name), kind="monster-found") __current_monster.introduction() game_data['player'].display_stats()
def ask_help(): printer('Our hero, here are the functions you can call from this World.') printer('world.find_monster() -> Find a monster to fight.') printer('world.attack(answer) -> Attack the monster with your answer.') printer('world.ask_help() -> Print this help section.') printer('world.enter() -> Register yourself into the World.') printer('world.reset_world() -> Reset everything about you.')
def Submit_UserSubmission(args): # For debugging, we have a -b --UserSubmissionID flag that, if used (i.e. -b 15) will submit that UserSubmission only # Also, if that UserSubmission has already been marked as submitted, it will be passed through again, anyways. #First, if the -b flag is used (so UserSubmissionID is NOT equal to none), we have to make sure that the given #UserSubmissionID actually exists in the database. if args.UserSubmissionID != 'none': UserSubmissions = [] strn = "SELECT UserSubmissionID FROM UserSubmissions;" #Select all UserSubmissionIDs from the DB UserSubmissions_array = utils.db_grab(strn) for i in UserSubmissions_array: UserSubmissions.append( i[0]) #Create a list of all UserSubmissionIDs if not int( args.UserSubmissionID ) in UserSubmissions: #If the given UserSubmissionID specified does not exist, throw an error print( "The selected UserSubmission (UserSubmissionID = {0}) does not exist, exiting" .format(args.UserSubmissionID)) exit() else: #If we can find the UserSubmissionID in the database (i.e. the UserSubmissionID is valid) pass it to process_jobs() UserSubmissionID = args.UserSubmissionID submission_script_manager.process_jobs(args, UserSubmissionID) #Now we handle the case where a UserSubmissionID is not specified. This will be normal running operation. #Here we will select all UserSubmissionIDs corresponding to UserSubmissions that have not yet been simulated, and push #Then through the simulation. else: """ # There are three options for values in the run_status field in the Submissions table: # "Not Submitted", "Submission scripts generated" ,and "Submitted to __", # Consider the following cases: # 1.) When a user submits a job on the client side, the newly created entry has value "Not Sumbitted" # 2.) If the server side code runs without the -s flag, submission scripts will be generated in the DB, # but the jobs will NOT be pushed to HTCondor/Slurm, # and the run_status value will be updated to "Submission scripts generated" # 3.) If the server side code runs with the -s flag, ALL jobs that do NOT have the value "Submitted to __" # will have submission scripts generated, and the jobs will be passed to HTCondor/Slurm, # and the value of run_status for all submitted UserSubmissions will update to "Submitted to __" # Case (2) will just create submission scripts for UserSubmissions created in status (1) # Case (3) will create submission scripts and submit jobs for all UserSubmissions in status (1) and (2) """ if args.submit: #Here, we will grab ALL UserSubmissions that have NOT been simulated strn = "SELECT UserSubmissionID FROM FarmSubmissions WHERE run_status NOT LIKE '{0}';".format( "Submitted to%") UserSubmissions_to_submit = utils.db_grab(strn) if len(UserSubmissions_to_submit) == 0: print( "There are no UserSubmissions which have not yet been submitted to a farm" ) else: #Here,if the -s flag was not used, we will just generate submission scripts from UserSubmissions that have not had any generated yet strn = "SELECT UserSubmissionID FROM FarmSubmissions WHERE run_status = '{0}';".format( "Not Submitted") UserSubmissions_to_submit = utils.db_grab(strn) if len(UserSubmissions_to_submit) == 0: print( "There are no UserSubmissions which do not yet have submission scripts generated" ) #From the above we have our (non-empty) UserSubmission of jobs to submit. Now we can pass it through process_jobs for UserSubmission in UserSubmissions_to_submit: UserSubmissionID = UserSubmission[ 0] #UserSubmissionID is the first element of the tuple utils.printer( "Generating scripts for UserSubmission with UserSubmissionID = {0}" .format(str(UserSubmissionID))) submission_script_manager.process_jobs(args, UserSubmissionID)
def get_htcondor_q(): schedd = htcondor.Schedd() batch_ids = [] total_jobs_submitted = [] total_jobs_running = [] jobs_start_dates = [] unexpanded_jc = [] idle_jc = [] running_jc = [] removed_jc = [] completed_jc = [] held_jc = [] submission_err_jc = [] """#HTCondor job status codes: 0 - unexpanded 1 - idle 2 - running 3 - removed 4 - completed 5 - held 6 - submission error """ valid_job_stati = [0, 1, 2, 3, 4, 5, 6] job_counting_set = [ unexpanded_jc, idle_jc, running_jc, removed_jc, completed_jc, held_jc, submission_err_jc ] for job in schedd.xquery(): #look through all jobs in condor if job.get("owner" ) == "gemc": #look only at jobs submitted by gemcRunning batch_id = str( job.get("ClusterID") ) #get cluster id (batch ID) and convert from Long to string. Can also turn to int if want. job_status = int( job["JobStatus"]) #gets if the job is running (2) or idle (1) if job_status not in valid_job_stati: #Just error handling, should pipe to a central log print( "HTCondor returned an invalid job status of {0}, investigate more" .format(job_status)) else: if batch_id not in batch_ids: utils.printer("found new batch: {0}".format(batch_id)) total_jobs_for_batch = job.get( "TotalSubmitProcs" ) #Get total number of jobs in batch submitted start_date_unix = job.get("QDate") #Get submitted date batch_ids.append(batch_id) total_jobs_submitted.append( int(total_jobs_for_batch)) #convert from long to int jobs_start_dates.append( int(start_date_unix)) #convert from long to int total_jobs_running.append( 1) #initialzie entry for manual job counting #initialize a new counter value in job counter arrays job_counting_set[job_status].append(1) null_jc = [0, 1, 2, 3, 4, 5, 6] null_jc.remove(job_status) for jc in null_jc: #set all other jc counter arrays to 0 job_counting_set[jc].append(0) else: total_jobs_running[batch_ids.index(batch_id)] += 1 job_counting_set[job_status][batch_ids.index( batch_id)] += 1 condor_info = [ batch_ids, total_jobs_submitted, total_jobs_running, jobs_start_dates, job_counting_set ] return condor_info
def db_gcard_write(UserSubmissionID,timestamp,gcard_text): strn = "INSERT INTO Gcards(UserSubmissionID) VALUES ({0});".format(UserSubmissionID) utils.db_write(strn) strn = """UPDATE Gcards SET {0} = "{1}" WHERE UserSubmissionID = {2};""".format('gcard_text',gcard_text,UserSubmissionID) utils.db_write(strn) utils.printer("GCard added to database corresponding to UserSubmissionID {0}".format(UserSubmissionID))
def attack(answer): global __current_monster global game_data if not __current_monster: printer('You are not fighting any monster as of now.') printer('Use world.find_monster() to fight one.') return printer('You attacked {0} with your answer.'.format( __current_monster.name)) player = game_data['player'] # log activity: attacks monster create_activity(text="{} attacks {}!".format(player.name, __current_monster.name), kind="monster-attack") result = __current_monster.evaluate(answer) if result: printer('Your answer is correct!') # log activity: defeat monster create_activity(text="{} defeated {}!".format(player.name, __current_monster.name), kind="monster-defeat") __current_monster.defeat() __current_monster = None player.level_up() # log activity: level up create_activity(text="{} leveled up! Now level {}".format( player.name, player.level), kind="player-level-up") save_game_data() else: printer('Your answer is wrong!') __current_monster.attack() player.life -= __current_monster.level player.display_stats() # log activity: lost to monster create_activity(text="{} dominates {}!".format(__current_monster.name, player.name), kind="monster-success") if player.life <= 0: __current_monster = None # log activity: lost to monster create_activity(text="{} fainted!".format(player.name), kind="player-fainted") printer('You fainted because of your incompetence.') printer('What a loser.') printer('Unless you try again...') save_game_data()
def submission_script_maker(args): file_struct.DEBUG = getattr(args, file_struct.debug_long) #Grabs batch and gcards as described in respective files BatchID = grab_batchID(args) gcards = grab_gcards(BatchID) username = grab_username(BatchID) funcs_rs = (startup, initialization, run_gemc, run_evio2hipo, run_cooking, file_mover) fname_rs = ('startup', 'initialization', 'run_gemc', 'run_evio2hipo', 'run_cooking', 'file_mover') funcs_condor = (condor_startup, condor_1, condor_2) fname_condor = ('condor_startup', 'condor_1', 'condor_2') funcs_runjob = (run_job1, ) fname_runjob = ('run_job1', ) strn = "SELECT scard FROM Batches WHERE BatchID = {0};".format(BatchID) scard_text = utils.sql3_grab(strn)[0][ 0] #sql3_grab returns a list of tuples, we need the 0th element of the 0th element scard = scard_helper.scard_class(scard_text) scard.data['genExecutable'] = file_struct.genExecutable.get( scard.data.get('generator')) scard.data['genOutput'] = file_struct.genOutput.get( scard.data.get('generator')) for gcard in gcards: GcardID = gcard[0] strn = "INSERT INTO Submissions(BatchID,GcardID) VALUES ({0},{1});".format( BatchID, GcardID) utils.sql3_exec(strn) strn = "UPDATE Submissions SET submission_pool = '{0}' WHERE GcardID = '{1}';".format( scard.data['farm_name'], GcardID) utils.sql3_exec(strn) strn = "UPDATE Submissions SET run_status = 'not yet in pool' WHERE GcardID = '{0}';".format( GcardID) utils.sql3_exec(strn) if scard.data['gcards'] == file_struct.gcard_default: gcard_loc = scard.data['gcards'] elif 'https://' in scard.data['gcards']: utils.printer('Writing gcard to local file') newfile = "gcard_{0}_batch_{1}.gcard".format(GcardID, BatchID) gfile = file_struct.sub_files_path + file_struct.gcards_dir + newfile with open(gfile, "w") as file: file.write(gcard[1]) gcard_loc = 'submission_files/gcards/' + newfile else: print( 'gcard not recognized as default option or online repository, please inspect scard' ) exit() file_extension = "_gcard_{0}_batch_{1}".format(GcardID, BatchID) params = { 'table': 'Scards', 'BatchID': BatchID, 'GcardID': GcardID, 'gfile': 'gfile', 'username': username[0][0], 'gcard_loc': gcard_loc } script_factory(file_struct.runscript_file_obj, funcs_rs, fname_rs, scard, params, file_extension) script_factory(file_struct.condor_file_obj, funcs_condor, fname_condor, scard, params, file_extension) script_factory(file_struct.run_job_obj, funcs_runjob, fname_runjob, scard, params, file_extension) print( "\tSuccessfully generated submission files for Batch {0} with GcardID {1}\n" .format(BatchID, GcardID)) if args.submit: print("\tSubmitting jobs to HTCondor \n") htcondor_submit.htcondor_submit(args, GcardID, file_extension)
def process_jobs(args,UserSubmissionID): fs.DEBUG = getattr(args,fs.debug_long) # Grabs UserSubmission and gcards as described in respective files gcards = utils.db_grab("SELECT GcardID, gcard_text FROM Gcards WHERE UserSubmissionID = {0};".format(UserSubmissionID)) username = utils.db_grab("SELECT User FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0] scard = scard_helper.scard_class(utils.db_grab( "SELECT scard FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0]) #This block picks up the scard type from arguements and throws an error if it was not an int try: scard_type = int(args.scard_type) except Exception as err: print("There was an error in recognizing scard type: ") print(err) exit() #Setting sub_type to the right directory based on the scard_type if scard_type in fs.valid_scard_types: sub_type = "type_{0}".format(scard_type) print("Using scard type {0} template".format(scard_type)) elif scard_type == 0: sub_type = "type_{0}".format(type_manager.manage_type(args,scard)) else: print("Poorly defined scard_type: {0}. Below is a list of valid scard types. Exiting".format(scard_type)) for type in fs.valid_scard_types: print("Valid scard type: {0}".format(type)) exit() print("sub_type is {0}".format(sub_type)) #This is creating an array of script generating functions. script_set = [fs.runscript_file_obj,fs.condor_file_obj,fs.run_job_obj] funcs_rs, funcs_condor,funcs_runjob = [], [], [] #initialize empty function arrays script_set_funcs = [funcs_rs,funcs_condor,funcs_runjob] #Please note, the ordering of this array must match the ordering of the above scripts = ["/runscript_generators/","/clas12condor_generators/","/run_job_generators/"] #Now we will loop through directories to import the script generation functions for index, script_dir in enumerate(scripts): top_dir = os.path.dirname(os.path.abspath(__file__)) script_path = os.path.abspath(top_dir + '/../submission_files/script_generators/' + sub_type + script_dir) for function in os.listdir(script_path): if "init" not in function: if ".pyc" not in function: module_name = function[:-3] module = import_module(sub_type+'.'+script_dir[1:-1]+'.'+module_name,module_name) func = getattr(module,module_name) script_set_funcs[index].append(func) if 'http' in scard.data.get('generator'): lund_dir = lund_helper.Lund_Entry(scard.data.get('generator')) scard.data['genExecutable'] = "Null" scard.data['genOutput'] = "Null" else: lund_dir = 0 scard.data['genExecutable'] = fs.genExecutable.get(scard.data.get('generator')) scard.data['genOutput'] = fs.genOutput.get(scard.data.get('generator')) # Now we create job submissions for all jobs that were recognized for gcard in gcards: GcardID = gcard[0] if scard.data['gcards'] == fs.gcard_default: gcard_loc = scard.data['gcards'] elif 'http' in scard.data['gcards']: utils.printer('Writing gcard to local file') newfile = "gcard_{0}_UserSubmission_{1}.gcard".format(GcardID,UserSubmissionID) gfile= fs.sub_files_path+fs.gcards_dir+newfile if not os.path.exists(gfile): newdir = fs.sub_files_path+fs.gcards_dir print("newdir is {0}".format(newdir)) Popen(['mkdir','-p',newdir], stdout=PIPE) Popen(['touch',gfile], stdout=PIPE) with open(gfile,"w") as file: file.write(gcard[1]) gcard_loc = 'submission_files/gcards/'+newfile else: print('gcard not recognized as default option or online repository, please inspect scard') exit() file_extension = "_gcard_{0}_UserSubmission_{1}".format(GcardID,UserSubmissionID) if fs.use_mysql: DB_path = fs.MySQL_DB_path else: DB_path = fs.SQLite_DB_path params = {'table':'Scards','UserSubmissionID':UserSubmissionID,'GcardID':GcardID, 'database_filename':DB_path+fs.DB_name, 'username':username,'gcard_loc':gcard_loc,'lund_dir':lund_dir, 'file_extension':file_extension,'scard':scard} """ This is where we actually pass all arguements to write the scripts""" for index, script in enumerate(script_set): script_factory.script_factory(args, script, script_set_funcs[index], params) print("\tSuccessfully generated submission files for UserSubmission {0} with GcardID {1}".format(UserSubmissionID,GcardID)) submission_string = 'Submission scripts generated'.format(scard.data['farm_name']) strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID) utils.db_write(strn) if args.submit: print("\tSubmitting jobs to {0} \n".format(scard.data['farm_name'])) farm_submission_manager.farm_submission_manager(args,GcardID,file_extension,scard,params) submission_string = 'Submitted to {0}'.format(scard.data['farm_name']) strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID) utils.db_write(strn)
import utils print("Hello,world") print("I am Roshna") print("I am Zuhara") utils.printer()
import utils as utilities utilities.printer(utilities.default_shape)