def script_factory(args, script_obj, script_functions, params):
script_text = ""
runscript_filename = fs.runscript_file_obj.file_path + fs.runscript_file_obj.file_base
runscript_filename = runscript_filename + params[
'file_extension'] + fs.runscript_file_obj.file_end
runjob_filename = fs.run_job_obj.file_path + fs.run_job_obj.file_base
runjob_filename = runjob_filename + params[
'file_extension'] + fs.run_job_obj.file_end
#In the below for loop, we loop through all script_generators for a certain submission script, appending the output of each function to a string
gen_text = [
f(
params['scard'],
username=params['username'],
gcard_loc=params['gcard_loc'],
GcardID=params['GcardID'],
lund_dir=params['lund_dir'],
database_filename=params['database_filename'],
file_extension=params['file_extension'],
runscript_filename=runscript_filename,
runjob_filename=runjob_filename,
using_sqlite=args.lite,
) for f in script_functions
]
script_text = script_text.join(gen_text)
#This handles writing to disk and to SQL database
if args.write_files:
# Build path to local file for writing, and normalize it (remove ../).
filename = os.path.normpath(script_obj.file_path +
script_obj.file_base +
params['file_extension'] +
script_obj.file_end)
utils.printer(
"\tWriting submission file '{0}' based off of specifications of UserSubmissionID = {1}, GcardID = {2}"
.format(filename, params['UserSubmissionID'], params['GcardID']))
if not os.path.exists(os.path.normpath(script_obj.file_path)):
utils.printer('Creating directory: {}'.format(
script_obj.file_path))
subprocess.call(['mkdir', '-p', script_obj.file_path],
stdout=subprocess.PIPE)
if os.path.isfile(filename):
subprocess.call(['rm', filename])
subprocess.call(['touch', filename])
with open(filename, 'w') as f:
f.write(script_text)
str_script_db = script_text.replace(
'"', "'"
) #I can't figure out a way to write "" into a sqlite field without errors
# For now, we can replace " with ', which works ok, but IDK how it will run if the scripts were submitted to HTCondor
strn = 'UPDATE FarmSubmissions SET {0} = "{1}" WHERE GcardID = {2};'.format(
script_obj.file_text_fieldname, str_script_db, params['GcardID'])
utils.db_write(strn)
def user_validation():
#These next two lines are good but do not work on python < 2.7
#username = (subprocess.check_output('whoami'))[:-1]#The [:-1] is so we drop the implicit \n from the string
#domain_name = subprocess.check_output(['hostname','-d'])#socket.getfqdn() #socket.getdomain_name()
username = Popen(['whoami'], stdout=PIPE).communicate()[0].split()[0]
is_travis = 'TRAVIS' in os.environ
if is_travis == True:
print("We're at travis-ci environment")
fake_domain = "travis.dev"
domain_name = fake_domain
else:
#The following does not work on mac. This needs to get resolved, currently bridged over for testing
#domain_name = Popen(['hostname',''-d'], stdout=PIPE).communicate()[0].split()[0]
domain_name = "example_domain"
strn = """SELECT 1 FROM Users WHERE EXISTS (SELECT 1 FROM Users WHERE User ="******"
AND domain_name = "{1}")""".format(username,domain_name)
user_already_exists = utils.db_grab(strn)
if not user_already_exists:
print("""\nThis is the first time {0} from {1} has submitted jobs. Adding user to database""".format(username,domain_name))
strn = """INSERT INTO Users(User, domain_name, JoinDateStamp, Total_UserSubmissions,
Total_Jobs, Total_Events, Most_Recent_Active_Date)
VALUES ("{0}","{1}","{2}","{3}","{4}","{5}","{6}");""".format(
username,domain_name,utils.gettime(),0,0,0,"Null")
utils.db_write(strn)
return username
def test_db_write(self):
with self.mock_db_config:
self.assertEqual(utils.db_write("""INSERT INTO `test_table` (`id`, `text`, `int`) VALUES
('3', 'test_text_3', 3)"""), True)
self.assertEqual(utils.db_write("""INSERT INTO `test_table` (`id`, `text`, `int`) VALUES
('1', 'test_text_3', 3)"""), False, "Insert entry to already existing primary key")
self.assertEqual(utils.db_write("""DELETE FROM `test_table` WHERE id='1' """), True)
self.assertEqual(utils.db_write("""DELETE FROM `test_table` WHERE id='4' """), True, "Delete non-existent entry" )
def SCard_Entry(UserSubmissionID, timestamp, scard_dict):
strn = """INSERT INTO Scards(UserSubmissionID,timestamp) VALUES ("{0}","{1}");""".format(
UserSubmissionID, timestamp)
utils.db_write(strn)
for key in scard_dict:
strn = "UPDATE Scards SET {0} = '{1}' WHERE UserSubmissionID = {2};".format(
key, scard_dict[key], UserSubmissionID)
utils.db_write(strn)
utils.printer(
"SCard record added to database corresponding to UserSubmissionID {0}".
format(UserSubmissionID))
def test_db_write(self):
with self.mock_db_config:
self.assertEqual(
utils.db_write("""INSERT INTO accounts
(id,username, fullname, email,mobile,password,cpassword)
VALUES ('a', 'b', '*****@*****.**', '1234567899', 'a', 'a')"""), True)
self.assertEqual(
utils.db_write("""DELETE FROM accounts WHERE id='2' """), True)
self.assertEqual(
utils.db_write("""DELETE FROM accounts WHERE id='3' """), True,
"Delete non-existent entry")
def update_users_statistics(scard, params):
strn = "SELECT Total_UserSubmissions FROM Users WHERE User = '******';".format(
params['username'])
UserSubmissions_total = utils.db_grab(strn)[0][0]
UserSubmissions_total += 1
strn = "UPDATE Users SET Total_UserSubmissions = '{0}' WHERE User = '******';".format(
UserSubmissions_total, params['username'])
utils.db_write(strn)
strn = "SELECT Total_Jobs FROM Users WHERE User = '******';".format(
params['username'])
jobs_total = utils.db_grab(strn)[0][0]
jobs_total += int(scard.data['jobs'])
strn = "UPDATE Users SET Total_Jobs = '{0}' WHERE User = '******';".format(
jobs_total, params['username'])
utils.db_write(strn)
if 'nevents' in scard.data:
strn = "SELECT Total_Events FROM Users WHERE User = '******';".format(
params['username'])
events_total = utils.db_grab(strn)[0][0]
events_total += int(scard.data['jobs']) * int(scard.data['nevents'])
strn = "UPDATE Users SET Total_Events = '{0}' WHERE User = '******';".format(
events_total, params['username'])
utils.db_write(strn)
else:
utils.printer(
"""Since you are using custom LUND files, we are not able to update your usage statistics.
This will not affect your simulations in any way, but will affect the total number of events reported as
processing through our system. """)
strn = "UPDATE Users SET Most_Recent_Active_Date = '{0}' WHERE User = '******';".format(
utils.gettime(), params['username'])
utils.db_write(strn)
def register_user():
first_name = request.json["first_name"]
last_name = request.json["last_name"]
user_email = request.json["email"]
user_password = request.json["password"]
created = datetime.datetime.utcnow()
# user_confirm_password = request.json["confirm_password"]
if validate_user_input("authentication",
email=user_email,
password=user_password):
password_salt = generate_salt()
password_hash = generate_hash(user_password, password_salt)
if db_write(
"""INSERT INTO users (email, first_name, last_name, password_salt, password_hash, created) VALUES (%s, %s, %s, %s, %s, %s)""",
(user_email, first_name, last_name, password_salt, password_hash,
created),
):
print("Registered" + user_email)
return Response(status=201)
else:
return Response(status=409)
else:
return Response(status=400)
def dopayment(userId, paydata):
data1 = db_read(
"""SELECT nic FROM consumers WHERE name = %s AND address = %s""",
(paydata['name'], paydata['address']))
if data1:
data2 = db_read("""SELECT accno FROM fields WHERE fieldname = %s""",
(paydata['field'], ))
if data2 and str(data2[0]['accno']) == str(paydata['number']):
# print(data1)
consumernic = data1[0]['nic']
fieldaccno = data2[0]['accno']
# print(consumernic, fieldaccno)
data3 = db_read(
"""SELECT `balance` FROM billpaymenttransactions WHERE `accno` = %s AND nic = %s ORDER BY `date` DESC LIMIT 1""",
(fieldaccno, consumernic))
if data3:
billbalance = data3[0]['balance']
data4 = db_write(
"""INSERT INTO billpaymenttransactions (`accno`, `nic`, `credit`, `balance`) VALUES (%s, %s, %s, %s)""",
(fieldaccno, consumernic, paydata['amount'],
billbalance - paydata['amount']))
if data4:
dealer = getpersonal(userId)
data5 = db_write(
"""INSERT INTO transactions (`accno`, `description`, `credit`, `balance`) VALUES (%s, %s, %s, %s)""",
(dealer['accno'],
(paydata['name'] + "'s " + paydata['field'] +
"bill paid "), paydata['amount'],
getAccBalance(userId) - paydata['amount']))
if data5:
return True, "Congratulations! " + paydata[
'field'] + " Bill payment is completed to " + str(
fieldaccno) + " account by Rs." + str(
paydata['amount']), 0
return False, "Error has occured with the Banking server. Sorry about that!", 1
return False, "Error has occured with the Banking server. Sorry about that!", 1
return False, "Seems like " + paydata[
'name'] + " does not have balance to be pay", 2
return False, "There is a no such field registerd in our service. Please enter the field name", 3
return False, "There is a no such name or address in our service. Please enter the name ", 4
def htcondor_submit(args, scard, GcardID, file_extension, params):
""" if value in submission === not submitted"""
# Need to add condition here in case path is different for non-jlab
scripts_baseDir = "/group/clas12/SubMit"
condor_exec = scripts_baseDir + "/server/condor_submit.sh"
jobOutputDir = "/volatile/clas12/osg"
# don't know how to pass farmsubmissionID (4th argument), passing GcardID for now (it may be the same)
submission = Popen([
condor_exec, scripts_baseDir, jobOutputDir, params['username'],
str(GcardID)
],
stdout=PIPE).communicate()[0]
print(submission)
words = submission.split()
node_number = words[len(words) - 1] #This might only work on SubMIT
strn = "UPDATE FarmSubmissions SET run_status = 'submitted to pool' WHERE GcardID = '{0}';".format(
GcardID)
utils.db_write(strn)
timestamp = utils.gettime(
) # Can modify this if need 10ths of seconds or more resolution
strn = "UPDATE FarmSubmissions SET submission_timestamp = '{0}' WHERE GcardID = '{1}';".format(
timestamp, GcardID)
utils.db_write(strn)
strn = "UPDATE FarmSubmissions SET pool_node = '{0}' WHERE GcardID = '{1}';".format(
node_number, GcardID)
utils.db_write(strn)
def reportcomplain(complaindet):
data1 = db_read("""SELECT `id` FROM branches WHERE `branchname` = %s""",
(complaindet['branch'], ))
if data1:
brID = data1[0]['id']
type = complaindet['type']
data2 = False
if type == 'complain_behaviour':
data2 = db_write(
"""INSERT INTO banking_complaints (`brid`, `complain_behaviour`) VALUES (%s, %s)""",
(brID, complaindet['description']))
elif type == 'complain_management':
data2 = db_write(
"""INSERT INTO banking_complaints (`brid`, `complain_management`) VALUES (%s, %s)""",
(brID, complaindet['description']))
elif type == 'complain_facility':
data2 = db_write(
"""INSERT INTO banking_complaints (`brid`, `complain_facility`) VALUES (%s, %s)""",
(brID, complaindet['description']))
else:
data2 = db_write(
"""INSERT INTO banking_complaints (`brid`, `complain_wasting`) VALUES (%s, %s)""",
(brID, complaindet['description']))
if data2:
return True, "Sorry for the inconvenience !.We'll report the complaint", 0
return False, "Error has occured with the Banking server. Sorry about that!", 1
return False, "There is a no such branch in our service. Please enter the branch name", 2
# print(getTransactions(3))
# print(getpersonal('1'))
# print(dopayment(1,{'model': 'payment', 'field': 'Electricity', 'number': 11111, 'name': "K.W.Saranga", 'address': '170/A, dobagahawatta, norway', 'amount': 200.0, 'code': 182172}))
# print(reportcomplain({'model':'complain', 'type':'complain_wasting', 'branch':'galle', 'description':'billing counter accountant is not familiar with people'}))
# print(dotransfer(1,{'model': 'transfer', 'number': 456123789, 'name': 'Jorge Washinton', 'amount': 500.0, 'code': 431435}))
# print(loginaccount(5,{'number':'895623125','name':'S.C.Dhavinchi','nic':'5588124722V'}))
# print(checkjoin('6 or 1=1'))
# print(getAccBalance(2))
# print(getname("sadakan OR 1=1"))
def loginaccount(userId, regdata):
# data1 = db_read("""SELECT accno FROM accountdet WHERE accno = %s AND name = %s AND nic = %s""",(regdata['number'], regdata['name'], regdata['nic']))
data1 = db_read(
"""SELECT accno FROM accountdet WHERE accno = %s AND nic = %s""",
(regdata['number'], regdata['nic']))
if data1:
data2 = db_write(
"""INSERT into joinaccounts (user_id, accno) VALUES (%s, %s)""",
(userId, regdata['number']))
if data2:
return True, "Now, you jave joined with your bank account!\nEnjoy our service..", 0
return False, "Error has occured with the Banking server. Sorry about that!", 1
return False, "Some details may be invalid. Please re-enter Holding account number !", 2
def dotransfer(userId, transdet):
data1 = db_read(
"""SELECT `balance` FROM accountdet NATURAL JOIN transactions WHERE accno = %s AND name = %s ORDER BY `date` DESC LIMIT 1""",
(transdet['number'], transdet['name']))
if data1:
benefBalance = data1[0]['balance']
senderdet = getpersonal(userId)
data2 = db_write(
"""INSERT INTO transactions (`accno`, `description`, `debit`, `balance`) VALUES (%s, %s, %s, %s)""",
(transdet['number'], "transfered from " + senderdet['name'],
transdet['amount'], (transdet['amount'] + benefBalance)))
if data2:
data3 = db_write(
"""INSERT INTO transactions (`accno`, `description`, `credit`, `balance`) VALUES (%s, %s, %s, %s)""",
(senderdet['accno'],
"transfered to " + transdet['name'], transdet['amount'],
getAccBalance(userId) - transdet['amount']))
if data3:
return True, "Congratulations! Money transfer is completed to " + str(
transdet['number']) + " account by Rs." + str(
transdet['amount']), 0
return False, "Error has occured with the Banking server. Sorry about that!", 1
return False, "Error has occured with the Banking server. Sorry about that!", 1
return False, "There is no bank account for entered account number and name. Please re-enter the beneficiary's account number", 2
def register_user():
user_email = request.json["email"]
user_password = request.json["password"]
user_confirm_password = request.json["confirm_password"]
if user_password == user_confirm_password and validate_user_input(
"authentication", email=user_email, password=user_password):
password_salt = generate_salt()
password_hash = generate_hash(user_password, password_salt)
if db_write(
"""INSERT INTO users (email, password_salt, password_hash) VALUES (%s, %s, %s)""",
(user_email, password_salt, password_hash),
):
return Response(status=201)
else:
return Response(status=409)
else:
return Response(status=400)
def db_gcard_write(UserSubmissionID,timestamp,gcard_text):
strn = "INSERT INTO Gcards(UserSubmissionID) VALUES ({0});".format(UserSubmissionID)
utils.db_write(strn)
strn = """UPDATE Gcards SET {0} = "{1}" WHERE UserSubmissionID = {2};""".format('gcard_text',gcard_text,UserSubmissionID)
utils.db_write(strn)
utils.printer("GCard added to database corresponding to UserSubmissionID {0}".format(UserSubmissionID))
def process_jobs(args,UserSubmissionID):
fs.DEBUG = getattr(args,fs.debug_long)
# Grabs UserSubmission and gcards as described in respective files
gcards = utils.db_grab("SELECT GcardID, gcard_text FROM Gcards WHERE UserSubmissionID = {0};".format(UserSubmissionID))
username = utils.db_grab("SELECT User FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0]
scard = scard_helper.scard_class(utils.db_grab( "SELECT scard FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0])
#This block picks up the scard type from arguements and throws an error if it was not an int
try:
scard_type = int(args.scard_type)
except Exception as err:
print("There was an error in recognizing scard type: ")
print(err)
exit()
#Setting sub_type to the right directory based on the scard_type
if scard_type in fs.valid_scard_types:
sub_type = "type_{0}".format(scard_type)
print("Using scard type {0} template".format(scard_type))
elif scard_type == 0:
sub_type = "type_{0}".format(type_manager.manage_type(args,scard))
else:
print("Poorly defined scard_type: {0}. Below is a list of valid scard types. Exiting".format(scard_type))
for type in fs.valid_scard_types:
print("Valid scard type: {0}".format(type))
exit()
print("sub_type is {0}".format(sub_type))
#This is creating an array of script generating functions.
script_set = [fs.runscript_file_obj,fs.condor_file_obj,fs.run_job_obj]
funcs_rs, funcs_condor,funcs_runjob = [], [], [] #initialize empty function arrays
script_set_funcs = [funcs_rs,funcs_condor,funcs_runjob]
#Please note, the ordering of this array must match the ordering of the above
scripts = ["/runscript_generators/","/clas12condor_generators/","/run_job_generators/"]
#Now we will loop through directories to import the script generation functions
for index, script_dir in enumerate(scripts):
top_dir = os.path.dirname(os.path.abspath(__file__))
script_path = os.path.abspath(top_dir + '/../submission_files/script_generators/' + sub_type + script_dir)
for function in os.listdir(script_path):
if "init" not in function:
if ".pyc" not in function:
module_name = function[:-3]
module = import_module(sub_type+'.'+script_dir[1:-1]+'.'+module_name,module_name)
func = getattr(module,module_name)
script_set_funcs[index].append(func)
if 'http' in scard.data.get('generator'):
lund_dir = lund_helper.Lund_Entry(scard.data.get('generator'))
scard.data['genExecutable'] = "Null"
scard.data['genOutput'] = "Null"
else:
lund_dir = 0
scard.data['genExecutable'] = fs.genExecutable.get(scard.data.get('generator'))
scard.data['genOutput'] = fs.genOutput.get(scard.data.get('generator'))
# Now we create job submissions for all jobs that were recognized
for gcard in gcards:
GcardID = gcard[0]
if scard.data['gcards'] == fs.gcard_default:
gcard_loc = scard.data['gcards']
elif 'http' in scard.data['gcards']:
utils.printer('Writing gcard to local file')
newfile = "gcard_{0}_UserSubmission_{1}.gcard".format(GcardID,UserSubmissionID)
gfile= fs.sub_files_path+fs.gcards_dir+newfile
if not os.path.exists(gfile):
newdir = fs.sub_files_path+fs.gcards_dir
print("newdir is {0}".format(newdir))
Popen(['mkdir','-p',newdir], stdout=PIPE)
Popen(['touch',gfile], stdout=PIPE)
with open(gfile,"w") as file: file.write(gcard[1])
gcard_loc = 'submission_files/gcards/'+newfile
else:
print('gcard not recognized as default option or online repository, please inspect scard')
exit()
file_extension = "_gcard_{0}_UserSubmission_{1}".format(GcardID,UserSubmissionID)
if fs.use_mysql:
DB_path = fs.MySQL_DB_path
else:
DB_path = fs.SQLite_DB_path
params = {'table':'Scards','UserSubmissionID':UserSubmissionID,'GcardID':GcardID,
'database_filename':DB_path+fs.DB_name,
'username':username,'gcard_loc':gcard_loc,'lund_dir':lund_dir,
'file_extension':file_extension,'scard':scard}
""" This is where we actually pass all arguements to write the scripts"""
for index, script in enumerate(script_set):
script_factory.script_factory(args, script, script_set_funcs[index], params)
print("\tSuccessfully generated submission files for UserSubmission {0} with GcardID {1}".format(UserSubmissionID,GcardID))
submission_string = 'Submission scripts generated'.format(scard.data['farm_name'])
strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID)
utils.db_write(strn)
if args.submit:
print("\tSubmitting jobs to {0} \n".format(scard.data['farm_name']))
farm_submission_manager.farm_submission_manager(args,GcardID,file_extension,scard,params)
submission_string = 'Submitted to {0}'.format(scard.data['farm_name'])
strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID)
utils.db_write(strn)
