def script_factory(args, script_obj, script_functions, params):
script_text = ""
runscript_filename = fs.runscript_file_obj.file_path + fs.runscript_file_obj.file_base
runscript_filename = runscript_filename + params[
'file_extension'] + fs.runscript_file_obj.file_end
runjob_filename = fs.run_job_obj.file_path + fs.run_job_obj.file_base
runjob_filename = runjob_filename + params[
'file_extension'] + fs.run_job_obj.file_end
#In the below for loop, we loop through all script_generators for a certain submission script, appending the output of each function to a string
gen_text = [
f(
params['scard'],
username=params['username'],
gcard_loc=params['gcard_loc'],
GcardID=params['GcardID'],
lund_dir=params['lund_dir'],
database_filename=params['database_filename'],
file_extension=params['file_extension'],
runscript_filename=runscript_filename,
runjob_filename=runjob_filename,
using_sqlite=args.lite,
) for f in script_functions
]
script_text = script_text.join(gen_text)
#This handles writing to disk and to SQL database
if args.write_files:
# Build path to local file for writing, and normalize it (remove ../).
filename = os.path.normpath(script_obj.file_path +
script_obj.file_base +
params['file_extension'] +
script_obj.file_end)
utils.printer(
"\tWriting submission file '{0}' based off of specifications of UserSubmissionID = {1}, GcardID = {2}"
.format(filename, params['UserSubmissionID'], params['GcardID']))
if not os.path.exists(os.path.normpath(script_obj.file_path)):
utils.printer('Creating directory: {}'.format(
script_obj.file_path))
subprocess.call(['mkdir', '-p', script_obj.file_path],
stdout=subprocess.PIPE)
if os.path.isfile(filename):
subprocess.call(['rm', filename])
subprocess.call(['touch', filename])
with open(filename, 'w') as f:
f.write(script_text)
str_script_db = script_text.replace(
'"', "'"
) #I can't figure out a way to write "" into a sqlite field without errors
# For now, we can replace " with ', which works ok, but IDK how it will run if the scripts were submitted to HTCondor
strn = 'UPDATE FarmSubmissions SET {0} = "{1}" WHERE GcardID = {2};'.format(
script_obj.file_text_fieldname, str_script_db, params['GcardID'])
utils.db_write(strn)
def update_users_statistics(scard, params):
strn = "SELECT Total_UserSubmissions FROM Users WHERE User = '******';".format(
params['username'])
UserSubmissions_total = utils.db_grab(strn)[0][0]
UserSubmissions_total += 1
strn = "UPDATE Users SET Total_UserSubmissions = '{0}' WHERE User = '******';".format(
UserSubmissions_total, params['username'])
utils.db_write(strn)
strn = "SELECT Total_Jobs FROM Users WHERE User = '******';".format(
params['username'])
jobs_total = utils.db_grab(strn)[0][0]
jobs_total += int(scard.data['jobs'])
strn = "UPDATE Users SET Total_Jobs = '{0}' WHERE User = '******';".format(
jobs_total, params['username'])
utils.db_write(strn)
if 'nevents' in scard.data:
strn = "SELECT Total_Events FROM Users WHERE User = '******';".format(
params['username'])
events_total = utils.db_grab(strn)[0][0]
events_total += int(scard.data['jobs']) * int(scard.data['nevents'])
strn = "UPDATE Users SET Total_Events = '{0}' WHERE User = '******';".format(
events_total, params['username'])
utils.db_write(strn)
else:
utils.printer(
"""Since you are using custom LUND files, we are not able to update your usage statistics.
This will not affect your simulations in any way, but will affect the total number of events reported as
processing through our system. """)
strn = "UPDATE Users SET Most_Recent_Active_Date = '{0}' WHERE User = '******';".format(
utils.gettime(), params['username'])
utils.db_write(strn)
def script_factory(script_obj, gen_funcs, func_names, scard, params,
file_extension):
script_text = ""
filename = script_obj.file_path + script_obj.file_base + file_extension + script_obj.file_end
utils.printer(
"\tWriting submission file '{0}' based off of specifications of BatchID = {1}, GcardID = {2}"
.format(filename, params['BatchID'], params['GcardID']))
if os.path.isfile(filename):
subprocess.call(['rm', filename])
for count, f in enumerate(gen_funcs):
generated_text = getattr(f, func_names[count])(
scard,
username=params['username'],
gcard_loc=params['gcard_loc'],
file_extension=file_extension,
runscript_filename=file_struct.runscript_file_obj.file_path +
file_struct.runscript_file_obj.file_base + file_extension +
file_struct.runscript_file_obj.file_end,
runjob_filename=file_struct.run_job_obj.file_path +
file_struct.run_job_obj.file_base + file_extension +
file_struct.run_job_obj.file_end,
)
with open(filename, "a") as file:
file.write(generated_text)
script_text += generated_text
str_script_db = script_text.replace(
'"', "'"
) #I can't figure out a way to write "" into a sqlite field without errors
#For now, we can replace " with ', which works ok, but IDK how it will run if the scripts were submitted to HTCondor
strn = 'UPDATE Submissions SET {0} = "{1}" WHERE GcardID = {2};'.format(
script_obj.file_text_fieldname, str_script_db, params['GcardID'])
utils.sql3_exec(strn)
def db_gcard_write(BatchID, timestamp, gcard_text):
strn = "INSERT INTO Gcards(BatchID) VALUES ({0});".format(BatchID)
utils.sql3_exec(strn)
strn = """UPDATE Gcards SET {0} = "{1}" WHERE BatchID = {2};""".format(
'gcard_text', gcard_text, BatchID)
utils.sql3_exec(strn)
utils.printer(
"GCard added to database corresponding to BatchID {}".format(BatchID))
def SCard_Entry(UserSubmissionID, timestamp, scard_dict):
strn = """INSERT INTO Scards(UserSubmissionID,timestamp) VALUES ("{0}","{1}");""".format(
UserSubmissionID, timestamp)
utils.db_write(strn)
for key in scard_dict:
strn = "UPDATE Scards SET {0} = '{1}' WHERE UserSubmissionID = {2};".format(
key, scard_dict[key], UserSubmissionID)
utils.db_write(strn)
utils.printer(
"SCard record added to database corresponding to UserSubmissionID {0}".
format(UserSubmissionID))
def SCard_Entry(BatchID, timestamp, scard_dict):
strn = """INSERT INTO Scards(BatchID,timestamp) VALUES ("{0}","{1}");""".format(
BatchID, timestamp)
utils.sql3_exec(strn)
for key in scard_dict:
strn = "UPDATE Scards SET {0} = '{1}' WHERE BatchID = {2};".format(
key, scard_dict[key], BatchID)
utils.sql3_exec(strn)
utils.printer(
"SCard record added to database corresponding to BatchID {0}".format(
BatchID))
def GCard_Entry(UserSubmissionID,unixtimestamp,url_dir):
print("Gathering gcards from {0} ".format(url_dir))
if not 'http' in url_dir: #== fs.gcard_default:
utils.printer('Using gcard from /jlab/work')
gcard_text_db = url_dir
db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db)
elif 'http' in url_dir:
utils.printer('Trying to download gcards from online repository')
if '.gcard' in url_dir:
utils.printer('Gcard URL name is: '+url_dir)
gcard_text = html_reader.html_reader(url_dir,'')[0]#This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from gcard link is: {0}'.format(gcard_text))
gcard_text_db = gcard_text.replace('"',"'")
print("\t Gathered gcard '{0}'".format(url_dir))
db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db)
else:
raw_html, gcard_urls = html_reader.html_reader(url_dir,fs.gcard_identifying_text)
if len(gcard_urls) == 0:
print("No gcard files found (they must end in '{0}'). Is the online repository correct?".format(fs.gcard_identifying_text))
exit()
else:
for url_ending in gcard_urls:
utils.printer('Gcard URL name is: '+url_ending)
gcard_text = html_reader.html_reader(url_dir+'/'+url_ending,'')[0]#This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from gcard link is: {0}'.format(gcard_text))
gcard_text_db = gcard_text.replace('"',"'")
print("\t Gathered gcard '{0}'".format(url_ending))
db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db)
# I don't think this block can ever be reached
else:
print('gcard not recognized as default option or online repository, please inspect scard')
exit()
def farm_submission_manager(args, usub_id, file_extension,
scard, params, db_conn, sql):
timestamp = utils.gettime()
farm_name = "OSG" #For now we are hardcodign OSG as it is the only farm that is currently supported
if farm_name == "OSG":
utils.printer("Passing to htcondor_submit")
htcondor_submit.htcondor_submit(args,scard,usub_id,file_extension,params,
db_conn, sql)
else:
raise ValueError('Unable to submit for {}'.format(
farm_name
))
def validate_scard_line(self, linenum, line):
if line.count("#") == 0:
utils.printer("Warning: No comment in line {0}.".format(linenum +
1))
elif line.count("#") > 1:
utils.printer(
"ERROR: number of hashes>1 in line {0}".format(linenum + 1))
utils.printer(
"# can be used only as a delimeter and only once per line. Edit scard to fix."
)
exit()
if line.count(":") == 0:
utils.printer("ERROR: No colon in line {0}".format(linenum + 1))
utils.printer("The data cannot be interpreted. Stopped.")
exit()
def farm_submission_manager(args, GcardID, file_extension, scard, params):
if scard.data['farm_name'] == "MIT_Tier2" or scard.data[
'farm_name'] == "OSG":
utils.printer("Passing to htcondor_submit")
htcondor_submit.htcondor_submit(args, scard, GcardID, file_extension,
params)
update_users_statistics(scard, params)
elif scard.data['farm_name'] == "JLab":
utils.printer("Passing to slurm_submit")
slurm_submit.slurm_submit(args, scard, GcardID, file_extension, params)
update_users_statistics(scard, params)
else:
print(
'Invalid farm name in scard, please check that the desired farm is spelled correctly and is supported'
)
exit()
def parse_scard(self, scard_text):
scard_lines = scard_text.split("\n")
for linenum, line in enumerate(scard_lines):
if not line:
print("Reached end of scard")
break
pos_delimeter_colon = line.find(":")
pos_delimeter_hash = line.find("#")
key = line[:pos_delimeter_colon].strip()
value = line[pos_delimeter_colon + 1:pos_delimeter_hash].strip()
if key != file_struct.scard_key[linenum]:
utils.printer(
"ERROR: Line {0} of the steering card has the key '{1}''.".
format(linenum + 1, key))
utils.printer("That line must have the key '{0}'.".format(
file_struct.scard_key[linenum]))
self.data[key] = value
def reset_world():
message = 'Are you sure you want to leave this world? (y/n) '
if (raw_input(message).lower() == 'y'):
global __current_monster
global game_data
# log activity: exit world
player = game_data['player']
create_activity(text="{} has left the world!".format(player.name),
kind="world-exit")
game_data = {} # this is so creepy! :-o
save_game_data() # reset game data
printer('okay :(')
__current_monster = None
else:
printer("That's the spirit!")
def create_transaction(self, amount, receiver, scrooge):
owned_coins = self.get_balance(scrooge.blockchain)
transfer_coins = owned_coins[:amount]
if len(owned_coins) < amount:
transaction = Transaction(transfer_coins, receiver.public_key)
printer(bcolors.FAIL, bcolors.BOLD, bcolors.UNDERLINE,
BALANCE_ERROR_MESSAGE, bcolors.ENDC, bcolors.ENDC,
bcolors.ENDC)
printer(bcolors.FAIL, transaction, bcolors.ENDC)
transfer_coins = owned_coins[:amount]
transaction = Transaction(transfer_coins, receiver.public_key)
hashed_transaction = object_hash(transaction)
signature = signer(self.__private_key, hashed_transaction)
transaction.sign_transaction(signature)
scrooge.handle_transaction(transaction, self.public_key)
def Lund_Downloader(url_dir, lund_urls, lund_dir):
if len(lund_urls) == 0:
print(
"No Lund files found (they must end in '{0}'). Is the online repository correct?"
.format(fs.lund_identifying_text))
exit()
else:
for url_ending in lund_urls:
utils.printer('Lund URL name is: ' + url_ending)
lund_text = html_reader.html_reader(
url_dir + '/' + url_ending, ''
)[0] #This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from lund link is: {0}'.format(lund_text))
lund_text_db = lund_text.replace(
'"', "'"
) #This isn't strictly needed but SQLite can't read " into data fields, only ' characters
print("\t Gathered lund file '{0}'".format(url_ending))
filename = lund_dir + "/" + url_ending
with open(filename, "a") as file:
file.write(lund_text_db)
def connect(self):
""" establish connection for raw serial data sending to the arduino via com_port2 """
com_port = self.config['connections']['arduino_data_port']
baud_rate = self.config['connections']['arduino_baud_rate']
utils.printer("initializing 2nd serial port to arduino: " + com_port,
'msg')
try:
ser = serial.Serial(port=com_port, baudrate=baud_rate, timeout=2)
ser.setDTR(
False
) # reset: https://stackoverflow.com/questions/21073086/wait-on-arduino-auto-reset-using-pyserial
time.sleep(1) # sleep timeout length to drop all data
ser.flushInput() #
ser.setDTR(True)
return ser
except:
utils.printer(
"Could not open 2nd serial connection to the arduino!",
'error')
sys.exit()
def enter():
global game_data
try:
# load existing
load_game_data()
game_data['player']
except (IOError, KeyError):
# create new
printer('Alas, for the intro. What is your name again?')
name = str(raw_input())
player = Player(name=name)
game_data['player'] = player # save player instance
save_game_data()
player = game_data['player']
if not player.registered:
register(player)
# log activity: enter world
create_activity(text="{} has entered the world!".format(player.name),
kind="world-enter")
printer('{}! Our chosen one. We are pleased to meet you.'.format(
player.name))
global __current_monster
__current_monster = None
def handle_transaction(self, transaction, sender_pk):
verified = self.verify_transaction(transaction, sender_pk)
double_spending = self.is_double_spending(transaction)
if not verified or double_spending:
printer(bcolors.FAIL, bcolors.BOLD, bcolors.UNDERLINE,
VERIFICATION_ERROR_MESSAGE, bcolors.ENDC, bcolors.ENDC,
bcolors.ENDC)
printer(bcolors.FAIL, transaction, bcolors.ENDC)
return
self.transactions_cache.append(transaction)
temp_block = Block(-1, None, self.transactions_cache)
printer(bcolors.WARNING, bcolors.BOLD, bcolors.UNDERLINE,
UNDER_CONSTRUCTION_MESSAGE, bcolors.ENDC, bcolors.ENDC,
bcolors.ENDC)
printer(bcolors.WARNING, temp_block.__str__(True), bcolors.ENDC)
self.check_for_new_block()
def __init__(self, datasets, lock, epoch, population, finish_tasks, device,
args):
self.model = resnet20().to(device)
self.optim = torch.optim.SGD(self.model.parameters(),
lr=0.1,
momentum=0.9,
weight_decay=1e-4)
self.device = device
self.args = args
self.datasets = datasets
self.lock = lock
self.population = population
self.finish_tasks = finish_tasks
self.epoch_base = epoch
self.printer = printer(args)
self.log_dir = os.path.join(args.save_dir, 'log.txt')
def register(player):
if player.registered:
return True
global game_data
printer('Registering to server...', False)
try:
r = requests.post(settings.register_url,
data={'username': player.name})
except Exception, e:
printer(e)
printer('... failed! :( . Will try again later.')
return False
def GCard_Entry(BatchID, unixtimestamp, url_dir):
print("Gathering gcards from {0} ".format(url_dir))
if url_dir == file_struct.gcard_default:
utils.printer('Using gcard from /jlab/work')
gcard_text_db = url_dir
db_gcard_write(BatchID, unixtimestamp, gcard_text_db)
elif 'https://' in url_dir:
utils.printer('Trying to download gcards from online repository')
raw_html, gcard_urls = html_reader.html_reader(
url_dir, file_struct.gcard_identifying_text)
for url_ending in gcard_urls:
utils.printer('Gcard URL name is: ' + url_ending)
gcard_text = html_reader.html_reader(
url_dir + '/' + url_ending, ''
)[0] #This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from gcard link is: {0}'.format(gcard_text))
gcard_text_db = gcard_text.replace('"', "'")
print("\t Gathered gcard '{0}'".format(url_ending))
db_gcard_write(BatchID, unixtimestamp, gcard_text_db)
else:
print(
'gcard not recognized as default option or online repository, please inspect scard'
)
exit()
import utils utils.printer(asif)
def append_block(self, transactions):
previous_hash = None if len(self.chain) == 0 else object_hash(
self.chain[-1])
block = Block(len(self.chain), previous_hash, transactions)
self.chain.append(block)
printer(self)
total_loss = sum(train_loss) / len(train_loss)
# update weight
optimizer.zero_grad()
loss.backward()
optimizer.step()
# calculate accuracy
acc = (pred.argmax(1) == labels
).type(torch.cuda.FloatTensor if torch.cuda.is_available(
) else torch.FloatTensor).mean().item()
train_acc.append(acc)
total_acc = sum(train_acc) / len(train_acc)
# print result
printer("train", e, args.num_epochs, i + 1, len(train_loader),
loss.item(), total_loss, acc * 100, total_acc * 100)
# tensorboard
writer.add_scalar("Loss/train", loss.item(), n_iter_train)
writer.add_scalar("Accuracy/train", acc, n_iter_train)
n_iter_train += 1
print("")
val_acc = []
val_loss = []
model.eval()
for i, (datas, _) in enumerate(val_loader):
datas = datas.to(device)
pivot = args.way * args.shot
shot, query = datas[:pivot], datas[pivot:]
import utils
utils.printer(utils.default_shape)
shape = utils.Shape('circle')
utils.printer(shape)
# register player to server
def register(player):
if player.registered:
return True
global game_data
printer('Registering to server...', False)
try:
r = requests.post(settings.register_url,
data={'username': player.name})
except Exception, e:
printer(e)
printer('... failed! :( . Will try again later.')
return False
if r.status_code == 403:
printer('... failed! with code {} :( .\
Username is already taken.'.format(r.status_code))
return False
elif r.status_code != 200:
printer('... failed! with code {} :( .'.format(r.status_code))
return False
if 'username' in r.json() and r.json()['username'] == player.name:
player.registered = True
game_data['player'] = player
save_game_data() # save game data
printer('... success for user {} ! :) .'.format(player.name))
return True
# unregister player to server
def unregister(player):
datas = datas.to(device)
pivot = args.way * args.shot
shot, query = datas[:pivot], datas[pivot:]
labels = torch.arange(args.way).repeat(args.query).to(device)
# one_hot_labels = Variable(torch.zeros(args.way*args.query, args.way).scatter_(1, labels.view(-1, 1), 1)).to(device)
pred = model(shot, query)
# calculate loss
loss = F.cross_entropy(pred, labels).item()
test_loss.append(loss)
total_loss = sum(test_loss) / len(test_loss)
# calculate accuracy
acc = 100 * (pred.argmax(1) == labels).type(
torch.cuda.FloatTensor if torch.cuda.is_available(
) else torch.FloatTensor).mean().item()
test_acc.append(acc)
total_acc = sum(test_acc) / len(test_acc)
printer("test", e, args.num_epochs, i + 1, len(test_loader), loss,
total_loss, acc, total_acc)
print("")
# get mean confidence interval and mean
m, h = mean_confidence_interval(test_acc, confidence=0.95)
total_mean.append(m)
total_interval.append(h)
print("Result: {:.2f}+-{:.2f}".format(
sum(total_mean) / len(total_mean),
sum(total_interval) / len(total_interval)))
def main():
printer('Greetings young adventurer!\nWelcome to... The World!')
enter()
ask_help()
reset_base_url()
def find_monster():
global __current_monster
global game_data
if 'player' not in game_data:
printer('You are not a registered player!')
printer('Use world.enter() to enter the World.')
return
elif game_data['player'].level > len(__monsters):
printer('There are no monsters left in the world!')
printer('Congratulations {}! You have finished the game.'.format(
game_data['player'].name))
# log activity: finished game
create_activity(text="Our hero {} finished all the monsters!".format(
game_data['player'].name), kind="player-finished")
return
elif __current_monster:
printer('You are already fighting a monster currently.')
printer('Use world.attack() to try defeating the monster.')
return
# log activity: found monster
create_activity(text="{} is searching for monsters ...".format(
game_data['player'].name),
kind="monster-search")
game_data['player'].life = game_data['player'].level + 5
i = game_data['player'].level - 1
monster = __monsters[i]
__current_monster = monster
# log activity: found monster
create_activity(text="{} found {}".format(
game_data['player'].name, __current_monster.name),
kind="monster-found")
__current_monster.introduction()
game_data['player'].display_stats()
def ask_help():
printer('Our hero, here are the functions you can call from this World.')
printer('world.find_monster() -> Find a monster to fight.')
printer('world.attack(answer) -> Attack the monster with your answer.')
printer('world.ask_help() -> Print this help section.')
printer('world.enter() -> Register yourself into the World.')
printer('world.reset_world() -> Reset everything about you.')
def Submit_UserSubmission(args):
# For debugging, we have a -b --UserSubmissionID flag that, if used (i.e. -b 15) will submit that UserSubmission only
# Also, if that UserSubmission has already been marked as submitted, it will be passed through again, anyways.
#First, if the -b flag is used (so UserSubmissionID is NOT equal to none), we have to make sure that the given
#UserSubmissionID actually exists in the database.
if args.UserSubmissionID != 'none':
UserSubmissions = []
strn = "SELECT UserSubmissionID FROM UserSubmissions;" #Select all UserSubmissionIDs from the DB
UserSubmissions_array = utils.db_grab(strn)
for i in UserSubmissions_array:
UserSubmissions.append(
i[0]) #Create a list of all UserSubmissionIDs
if not int(
args.UserSubmissionID
) in UserSubmissions: #If the given UserSubmissionID specified does not exist, throw an error
print(
"The selected UserSubmission (UserSubmissionID = {0}) does not exist, exiting"
.format(args.UserSubmissionID))
exit()
else: #If we can find the UserSubmissionID in the database (i.e. the UserSubmissionID is valid) pass it to process_jobs()
UserSubmissionID = args.UserSubmissionID
submission_script_manager.process_jobs(args, UserSubmissionID)
#Now we handle the case where a UserSubmissionID is not specified. This will be normal running operation.
#Here we will select all UserSubmissionIDs corresponding to UserSubmissions that have not yet been simulated, and push
#Then through the simulation.
else:
"""
# There are three options for values in the run_status field in the Submissions table:
# "Not Submitted", "Submission scripts generated" ,and "Submitted to __",
# Consider the following cases:
# 1.) When a user submits a job on the client side, the newly created entry has value "Not Sumbitted"
# 2.) If the server side code runs without the -s flag, submission scripts will be generated in the DB,
# but the jobs will NOT be pushed to HTCondor/Slurm,
# and the run_status value will be updated to "Submission scripts generated"
# 3.) If the server side code runs with the -s flag, ALL jobs that do NOT have the value "Submitted to __"
# will have submission scripts generated, and the jobs will be passed to HTCondor/Slurm,
# and the value of run_status for all submitted UserSubmissions will update to "Submitted to __"
# Case (2) will just create submission scripts for UserSubmissions created in status (1)
# Case (3) will create submission scripts and submit jobs for all UserSubmissions in status (1) and (2)
"""
if args.submit: #Here, we will grab ALL UserSubmissions that have NOT been simulated
strn = "SELECT UserSubmissionID FROM FarmSubmissions WHERE run_status NOT LIKE '{0}';".format(
"Submitted to%")
UserSubmissions_to_submit = utils.db_grab(strn)
if len(UserSubmissions_to_submit) == 0:
print(
"There are no UserSubmissions which have not yet been submitted to a farm"
)
else: #Here,if the -s flag was not used, we will just generate submission scripts from UserSubmissions that have not had any generated yet
strn = "SELECT UserSubmissionID FROM FarmSubmissions WHERE run_status = '{0}';".format(
"Not Submitted")
UserSubmissions_to_submit = utils.db_grab(strn)
if len(UserSubmissions_to_submit) == 0:
print(
"There are no UserSubmissions which do not yet have submission scripts generated"
)
#From the above we have our (non-empty) UserSubmission of jobs to submit. Now we can pass it through process_jobs
for UserSubmission in UserSubmissions_to_submit:
UserSubmissionID = UserSubmission[
0] #UserSubmissionID is the first element of the tuple
utils.printer(
"Generating scripts for UserSubmission with UserSubmissionID = {0}"
.format(str(UserSubmissionID)))
submission_script_manager.process_jobs(args, UserSubmissionID)
def get_htcondor_q():
schedd = htcondor.Schedd()
batch_ids = []
total_jobs_submitted = []
total_jobs_running = []
jobs_start_dates = []
unexpanded_jc = []
idle_jc = []
running_jc = []
removed_jc = []
completed_jc = []
held_jc = []
submission_err_jc = []
"""#HTCondor job status codes:
0 - unexpanded
1 - idle
2 - running
3 - removed
4 - completed
5 - held
6 - submission error
"""
valid_job_stati = [0, 1, 2, 3, 4, 5, 6]
job_counting_set = [
unexpanded_jc, idle_jc, running_jc, removed_jc, completed_jc, held_jc,
submission_err_jc
]
for job in schedd.xquery(): #look through all jobs in condor
if job.get("owner"
) == "gemc": #look only at jobs submitted by gemcRunning
batch_id = str(
job.get("ClusterID")
) #get cluster id (batch ID) and convert from Long to string. Can also turn to int if want.
job_status = int(
job["JobStatus"]) #gets if the job is running (2) or idle (1)
if job_status not in valid_job_stati: #Just error handling, should pipe to a central log
print(
"HTCondor returned an invalid job status of {0}, investigate more"
.format(job_status))
else:
if batch_id not in batch_ids:
utils.printer("found new batch: {0}".format(batch_id))
total_jobs_for_batch = job.get(
"TotalSubmitProcs"
) #Get total number of jobs in batch submitted
start_date_unix = job.get("QDate") #Get submitted date
batch_ids.append(batch_id)
total_jobs_submitted.append(
int(total_jobs_for_batch)) #convert from long to int
jobs_start_dates.append(
int(start_date_unix)) #convert from long to int
total_jobs_running.append(
1) #initialzie entry for manual job counting
#initialize a new counter value in job counter arrays
job_counting_set[job_status].append(1)
null_jc = [0, 1, 2, 3, 4, 5, 6]
null_jc.remove(job_status)
for jc in null_jc: #set all other jc counter arrays to 0
job_counting_set[jc].append(0)
else:
total_jobs_running[batch_ids.index(batch_id)] += 1
job_counting_set[job_status][batch_ids.index(
batch_id)] += 1
condor_info = [
batch_ids, total_jobs_submitted, total_jobs_running, jobs_start_dates,
job_counting_set
]
return condor_info
def db_gcard_write(UserSubmissionID,timestamp,gcard_text):
strn = "INSERT INTO Gcards(UserSubmissionID) VALUES ({0});".format(UserSubmissionID)
utils.db_write(strn)
strn = """UPDATE Gcards SET {0} = "{1}" WHERE UserSubmissionID = {2};""".format('gcard_text',gcard_text,UserSubmissionID)
utils.db_write(strn)
utils.printer("GCard added to database corresponding to UserSubmissionID {0}".format(UserSubmissionID))
def attack(answer):
global __current_monster
global game_data
if not __current_monster:
printer('You are not fighting any monster as of now.')
printer('Use world.find_monster() to fight one.')
return
printer('You attacked {0} with your answer.'.format(
__current_monster.name))
player = game_data['player']
# log activity: attacks monster
create_activity(text="{} attacks {}!".format(player.name,
__current_monster.name),
kind="monster-attack")
result = __current_monster.evaluate(answer)
if result:
printer('Your answer is correct!')
# log activity: defeat monster
create_activity(text="{} defeated {}!".format(player.name,
__current_monster.name),
kind="monster-defeat")
__current_monster.defeat()
__current_monster = None
player.level_up()
# log activity: level up
create_activity(text="{} leveled up! Now level {}".format(
player.name, player.level), kind="player-level-up")
save_game_data()
else:
printer('Your answer is wrong!')
__current_monster.attack()
player.life -= __current_monster.level
player.display_stats()
# log activity: lost to monster
create_activity(text="{} dominates {}!".format(__current_monster.name,
player.name),
kind="monster-success")
if player.life <= 0:
__current_monster = None
# log activity: lost to monster
create_activity(text="{} fainted!".format(player.name),
kind="player-fainted")
printer('You fainted because of your incompetence.')
printer('What a loser.')
printer('Unless you try again...')
save_game_data()
def submission_script_maker(args):
file_struct.DEBUG = getattr(args, file_struct.debug_long)
#Grabs batch and gcards as described in respective files
BatchID = grab_batchID(args)
gcards = grab_gcards(BatchID)
username = grab_username(BatchID)
funcs_rs = (startup, initialization, run_gemc, run_evio2hipo, run_cooking,
file_mover)
fname_rs = ('startup', 'initialization', 'run_gemc', 'run_evio2hipo',
'run_cooking', 'file_mover')
funcs_condor = (condor_startup, condor_1, condor_2)
fname_condor = ('condor_startup', 'condor_1', 'condor_2')
funcs_runjob = (run_job1, )
fname_runjob = ('run_job1', )
strn = "SELECT scard FROM Batches WHERE BatchID = {0};".format(BatchID)
scard_text = utils.sql3_grab(strn)[0][
0] #sql3_grab returns a list of tuples, we need the 0th element of the 0th element
scard = scard_helper.scard_class(scard_text)
scard.data['genExecutable'] = file_struct.genExecutable.get(
scard.data.get('generator'))
scard.data['genOutput'] = file_struct.genOutput.get(
scard.data.get('generator'))
for gcard in gcards:
GcardID = gcard[0]
strn = "INSERT INTO Submissions(BatchID,GcardID) VALUES ({0},{1});".format(
BatchID, GcardID)
utils.sql3_exec(strn)
strn = "UPDATE Submissions SET submission_pool = '{0}' WHERE GcardID = '{1}';".format(
scard.data['farm_name'], GcardID)
utils.sql3_exec(strn)
strn = "UPDATE Submissions SET run_status = 'not yet in pool' WHERE GcardID = '{0}';".format(
GcardID)
utils.sql3_exec(strn)
if scard.data['gcards'] == file_struct.gcard_default:
gcard_loc = scard.data['gcards']
elif 'https://' in scard.data['gcards']:
utils.printer('Writing gcard to local file')
newfile = "gcard_{0}_batch_{1}.gcard".format(GcardID, BatchID)
gfile = file_struct.sub_files_path + file_struct.gcards_dir + newfile
with open(gfile, "w") as file:
file.write(gcard[1])
gcard_loc = 'submission_files/gcards/' + newfile
else:
print(
'gcard not recognized as default option or online repository, please inspect scard'
)
exit()
file_extension = "_gcard_{0}_batch_{1}".format(GcardID, BatchID)
params = {
'table': 'Scards',
'BatchID': BatchID,
'GcardID': GcardID,
'gfile': 'gfile',
'username': username[0][0],
'gcard_loc': gcard_loc
}
script_factory(file_struct.runscript_file_obj, funcs_rs, fname_rs,
scard, params, file_extension)
script_factory(file_struct.condor_file_obj, funcs_condor, fname_condor,
scard, params, file_extension)
script_factory(file_struct.run_job_obj, funcs_runjob, fname_runjob,
scard, params, file_extension)
print(
"\tSuccessfully generated submission files for Batch {0} with GcardID {1}\n"
.format(BatchID, GcardID))
if args.submit:
print("\tSubmitting jobs to HTCondor \n")
htcondor_submit.htcondor_submit(args, GcardID, file_extension)
def process_jobs(args,UserSubmissionID):
fs.DEBUG = getattr(args,fs.debug_long)
# Grabs UserSubmission and gcards as described in respective files
gcards = utils.db_grab("SELECT GcardID, gcard_text FROM Gcards WHERE UserSubmissionID = {0};".format(UserSubmissionID))
username = utils.db_grab("SELECT User FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0]
scard = scard_helper.scard_class(utils.db_grab( "SELECT scard FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0])
#This block picks up the scard type from arguements and throws an error if it was not an int
try:
scard_type = int(args.scard_type)
except Exception as err:
print("There was an error in recognizing scard type: ")
print(err)
exit()
#Setting sub_type to the right directory based on the scard_type
if scard_type in fs.valid_scard_types:
sub_type = "type_{0}".format(scard_type)
print("Using scard type {0} template".format(scard_type))
elif scard_type == 0:
sub_type = "type_{0}".format(type_manager.manage_type(args,scard))
else:
print("Poorly defined scard_type: {0}. Below is a list of valid scard types. Exiting".format(scard_type))
for type in fs.valid_scard_types:
print("Valid scard type: {0}".format(type))
exit()
print("sub_type is {0}".format(sub_type))
#This is creating an array of script generating functions.
script_set = [fs.runscript_file_obj,fs.condor_file_obj,fs.run_job_obj]
funcs_rs, funcs_condor,funcs_runjob = [], [], [] #initialize empty function arrays
script_set_funcs = [funcs_rs,funcs_condor,funcs_runjob]
#Please note, the ordering of this array must match the ordering of the above
scripts = ["/runscript_generators/","/clas12condor_generators/","/run_job_generators/"]
#Now we will loop through directories to import the script generation functions
for index, script_dir in enumerate(scripts):
top_dir = os.path.dirname(os.path.abspath(__file__))
script_path = os.path.abspath(top_dir + '/../submission_files/script_generators/' + sub_type + script_dir)
for function in os.listdir(script_path):
if "init" not in function:
if ".pyc" not in function:
module_name = function[:-3]
module = import_module(sub_type+'.'+script_dir[1:-1]+'.'+module_name,module_name)
func = getattr(module,module_name)
script_set_funcs[index].append(func)
if 'http' in scard.data.get('generator'):
lund_dir = lund_helper.Lund_Entry(scard.data.get('generator'))
scard.data['genExecutable'] = "Null"
scard.data['genOutput'] = "Null"
else:
lund_dir = 0
scard.data['genExecutable'] = fs.genExecutable.get(scard.data.get('generator'))
scard.data['genOutput'] = fs.genOutput.get(scard.data.get('generator'))
# Now we create job submissions for all jobs that were recognized
for gcard in gcards:
GcardID = gcard[0]
if scard.data['gcards'] == fs.gcard_default:
gcard_loc = scard.data['gcards']
elif 'http' in scard.data['gcards']:
utils.printer('Writing gcard to local file')
newfile = "gcard_{0}_UserSubmission_{1}.gcard".format(GcardID,UserSubmissionID)
gfile= fs.sub_files_path+fs.gcards_dir+newfile
if not os.path.exists(gfile):
newdir = fs.sub_files_path+fs.gcards_dir
print("newdir is {0}".format(newdir))
Popen(['mkdir','-p',newdir], stdout=PIPE)
Popen(['touch',gfile], stdout=PIPE)
with open(gfile,"w") as file: file.write(gcard[1])
gcard_loc = 'submission_files/gcards/'+newfile
else:
print('gcard not recognized as default option or online repository, please inspect scard')
exit()
file_extension = "_gcard_{0}_UserSubmission_{1}".format(GcardID,UserSubmissionID)
if fs.use_mysql:
DB_path = fs.MySQL_DB_path
else:
DB_path = fs.SQLite_DB_path
params = {'table':'Scards','UserSubmissionID':UserSubmissionID,'GcardID':GcardID,
'database_filename':DB_path+fs.DB_name,
'username':username,'gcard_loc':gcard_loc,'lund_dir':lund_dir,
'file_extension':file_extension,'scard':scard}
""" This is where we actually pass all arguements to write the scripts"""
for index, script in enumerate(script_set):
script_factory.script_factory(args, script, script_set_funcs[index], params)
print("\tSuccessfully generated submission files for UserSubmission {0} with GcardID {1}".format(UserSubmissionID,GcardID))
submission_string = 'Submission scripts generated'.format(scard.data['farm_name'])
strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID)
utils.db_write(strn)
if args.submit:
print("\tSubmitting jobs to {0} \n".format(scard.data['farm_name']))
farm_submission_manager.farm_submission_manager(args,GcardID,file_extension,scard,params)
submission_string = 'Submitted to {0}'.format(scard.data['farm_name'])
strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID)
utils.db_write(strn)
import utils
print("Hello,world")
print("I am Roshna")
print("I am Zuhara")
utils.printer()
import utils as utilities utilities.printer(utilities.default_shape)
