def run_bulk(self, t, special_input=None):
# run name
runname="%s, ni=%f, T=%f" % (self.material, self.ni, t)
self.write_log("##############################################\n")
self.write_log("### euorun: %s\n" % runname)
self.write_log("##############################################\n")
# run command
runcmd=self.mpicmd + " -np %i " % self.np
runcmd+=self.sp.get_runcmd_bulk(self.material, self.ni, t)
# add additional parameter
runcmd+=self.iteration_parameter
# add output
runoutput=self.output + self.idb.get_temp_output(t)
runcmd+=" -o %s/" % runoutput
runexists=self.run_exists(runcmd, runoutput, check_database=self.check_database)
if not runexists:
# add special input folder
if special_input!=None:
runcmd+=" -i %s" % (special_input)
# search self.input folder for suitable input folders and add it
else:
if self.inputFlag:
runcmd=database.add_input(runcmd, download_path=self.output+"/download/", path=self.input, source=self.source, input_system_name=self.input_system_name)
# run job
j=job.job(runname, self.log, self.email, [runcmd], logappend=True, verbose=self.verbose, mailcmd=self.mailcmd)
j.run()
# update database
if self.updatedbFlag and not runexists:
self.write_log("* Update bulk database\n")
updatecmd="bulk_remote.py %s" % runoutput
#subprocess.call(updatecmd, shell=True)
j=job.job("update remote bulk database", self.log, self.email, [updatecmd], logappend=True, verbose=False, mailcmd=self.mailcmd)
j.run()
self.write_log("\n")
def test_regal_create(self):
j = job(1, 0)
self.assertEqual(j.pt, 1)
self.assertEqual(j.rd, 0)
j = job()
self.assertEqual(j.pt, 1)
self.assertEqual(j.rd, 0)
s = 'pt: 1, rd: 0'
self.assertEqual(str(j), s)
def test_heapq(self):
ref_jobs = [job(1, 0), job(1, 3), job(2, 1), job(3, 1), job(4, 5)]
target_jobs = []
heapq.heappush(target_jobs, job(1, 3))
heapq.heappush(target_jobs, job(3, 1))
heapq.heappush(target_jobs, job(1, 0))
heapq.heappush(target_jobs, job(4, 5))
heapq.heappush(target_jobs, job(2, 1))
for i in range(len(target_jobs)):
self.assertEqual(ref_jobs[i], heapq.heappop(target_jobs))
def run_isolated(self, t, special_input=None):
# run name
runname = "%s, N=%i, ni=%f, T=%f" % (self.material, self.N, self.ni, t)
self.write_log("##############################################\n")
self.write_log("### euorun: %s\n" % runname)
self.write_log("##############################################\n")
# run command
runcmd = self.mpicmd + " -np %i " % self.np
runcmd += self.sp.get_runcmd_isolated(self.material, self.N, self.ni,
t)
# add additional parameter
runcmd += self.iteration_parameter
# add output
runoutput = self.output + self.idb.get_temp_output(t)
runcmd += " -o %s/" % runoutput
# check if run exists before downloading possible input
runexists = self.run_exists(runcmd,
runoutput,
check_database=self.check_database)
# run job
if not runexists:
# add special input folder
if special_input != None:
runcmd += " -i %s" % (special_input)
# search self.input folder for suitable input folders and add it
else:
if self.inputFlag:
runcmd = database.add_input(
runcmd,
download_path=self.output + "/download/",
path=self.input,
source=self.source,
input_system_name=self.input_system_name)
j = job.job(runname,
self.log,
self.email, [runcmd],
logappend=True,
verbose=self.verbose,
mailcmd=self.mailcmd)
j.run()
# update database
if self.updatedbFlag and not runexists:
self.write_log("* Update isolated database\n")
updatecmd = "isolated_remote.py %s" % runoutput
#subprocess.call(updatecmd, shell=True)
j = job.job("update remote isolated database",
self.log,
self.email, [updatecmd],
logappend=True,
verbose=False,
mailcmd=self.mailcmd)
j.run()
self.write_log("\n")
def gen_jobs(jobs, co_id):
jb = []
for x in range(0, len(jobs)):
splt = jobs[x].split(',')
new_job = job(splt[2], co_id, splt[0], splt[1])
jb.append(new_job)
return jb
def request_cpu(task, ComputeMachine):
"""
local function which is used to create a process for new/pre-empted job
"""
global jobs_list
global queue_count
if task.number_of_cpu_quantum == 0:
think_time = np.random.exponential(25)
print("New job %d is initialized at %.2f with %.2f thinking time and %.2f service time"
% (task.get_job_id(), task.env.now, think_time, task.get_service_time()))
yield task.env.timeout(think_time)
print("Job %d ready to request for CPU at %.2f." % (task.get_job_id(), task.env.now))
task.queue_entry_time = task.env.now
queue_count[task.queue_entry_time] =1
with ComputeMachine.cpu.request() as request:
print("Job %d has requested for CPU at %.2f." % (task.get_job_id(), task.env.now))
yield request
print('Job %d enters the queue at %.2f.' % (task.get_job_id(), task.env.now))
status = yield task.env.process(ComputeMachine.compute(task))
if status == 1:
print('Job %d pre-empted at %.2f.' % (task.get_job_id(), task.env.now))
task.set_service_time(task.get_service_time() - 0.1)
return task.env.process(request_cpu(task, ComputeMachine))
else:
print('Job %d has completed its execution at %.2f.' % (task.get_job_id(),
task.env.now))
task.job_complete_status = True
queue_count[task.env.now] = -1
if len(jobs_list) < 1000:
temp = job(env, job_id = len(jobs_list)+1,
service_time = np.random.exponential(0.8))
task.env.process(request_cpu(temp, ComputeMachine))
jobs_list[len(jobs_list)+1] = temp
def add_job(self, path, arg):
j = job()
j.path = path
j.args = arg
j.status = 0
j.name = "job" + str(len(self.jobs))
self.jobs.append(j)
def get_all_jobs(self):
jobs = []
jenkinsJobs = self._connection().get_jobs()
for json in jenkinsJobs:
jobs.append(job(json))
return jobs
def test_sorting(self):
ref_jobs = [job(1, 0), job(2, 1), job(3, 1), job(4, 5)]
jobs = []
jobs.append(job(3, 1))
jobs.append(job(1, 0))
jobs.append(job(4, 5))
jobs.append(job(2, 1))
# jobs.sort()
for r, t in zip(ref_jobs, sorted(jobs, key=attrgetter('rd', 'pt'))):
self.assertEqual(r, t)
def run_submit(cmd, logstring, append, email, mailcmd):
cmds = [cmd]
j = job.job(logstring,
logstring,
email,
cmds,
logappend=append,
verbose=True,
mailcmd=mailcmd)
j.run()
def getJobs():
from job import job
jobs = []
for i in range(1, 30):
respense = requests.get(url=url + '&pg=' + str(i), headers=Header)
soup = BeautifulSoup(respense.text, 'lxml')
time.sleep(3)
print("----等待3s----")
for k in soup.select('.presentation-item'):
_job = job('', '', '', '', '', '', '')
# _job.position = k.select('.fn-left.position').string
#
# _job.source = k.select('.fn-left.source').string
#
# _job.company = k.select('.fn-right.company').string
#
# _job.city = k.select('.city.fn-left').string
#
# _job.num = k.select('.num.fn-left').string
#
# _job.dateTime = k.select('.time.fn-left').string
#
# _job.industry = k.select('.industry.fn-right').string
# print(_job.company)
# print(k)
# _job = job('', '', '', '', '', '', '')
for j in k.select('.fn-left.position'):
_job.position = j.string
for j in k.select('.fn-left.source'):
_job.source = j.string
for j in k.select('.fn-right.company'):
_job.company = j.string
for j in k.select('.city.fn-left'):
_job.city = j.string
for j in k.select('.num.fn-left'):
_job.num = j.string
for j in k.select('.time.fn-left'):
_job.dateTime = j.string
for j in k.select('.industry.fn-right'):
_job.industry = j.string
jobs.append(_job)
return jobs
def sortJobs(self, Jobs):
'''
This takes a list of job references from the server, and filters them
to get the new ones. From there is passes them them to the local job
objects to be managed.
'''
newjobs = []
for j in Jobs:
if not (j in self.rawJobs):
self.rawJobs.append(j)
newjobs.append(j)
for j in newjobs:
self.jobs.append(job(self, j))
if self.currentjob == None:
self.startAJob()
def sortJobs(self, Jobs):
'''
This takes a list of job references from the server, and filters them
to get the new ones. From there is passes them them to the local job
objects to be managed.
'''
newjobs = []
for j in Jobs:
if not (j in self.rawJobs):
self.rawJobs.append(j)
newjobs.append(j)
for j in newjobs:
self.jobs.append(job(self, j))
if self.currentjob == None:
self.startAJob()
def test_irregal_create(self):
pt_error_string = 'pt must be greater than zero'
with self.assertRaises(AssertionError) as context:
job(0, 0)
self.assertTrue(pt_error_string in context.exception)
rd_error_string = 'rd must be greater than or equal to zero.'
with self.assertRaises(AssertionError) as context:
job(1, -1)
self.assertTrue(rd_error_string in context.exception)
pt_int_error_string = 'pt must be int'
with self.assertRaises(AssertionError) as context:
job(1.1, 0)
self.assertTrue(pt_int_error_string in context.exception)
rd_int_error_string = 'rd must be int'
with self.assertRaises(AssertionError) as context:
job(1, 0.2)
self.assertTrue(rd_int_error_string in context.exception)
def process_job_list(self, data):
ret = data.data
lines = ret.decode("utf-8").split("\n")
self.jobs = []
for i in range(1, len(lines)):
act = lines[i].split()
if len(act) > 1:
j = job()
j.name = act[1]
j.path = act[4]
j.ip = act[5]
j.start = act[7]
j.stop = act[9]
j.cpus = int(act[10])
j.status = int(act[3])
self.jobs.append(j)
#print(act[0], act[1], act[2], act[3],act[4], act[5], act[6], act[7])
#self.cluster_jobs.append([act[0], act[1], act[2], act[3],act[4], act[5], act[6], act[7]])
self.jobs_update.emit()
clusterName = None
# The user will provide the data in the JSON format.
# Call the function from_json to extract the contents from JSON file
Config_ = Config()
resourceObj, subScheduler = Config_.from_json("config.json", clusterName)
Schduler_ = scheduler()
if(subScheduler == "PBS"):
subScheduler = PBS(Schduler_)
elif(subScheduler == "Condor"):
subScheduler = Condor(Schduler_)
# Job object is created with initializing the parameters
Job_ = job(0, 0, inputScriptFile, resourceObj.remoteTmp, inpFiles, outFiles)
output_file = "map_jobid.csv"
# Creates the pickle object for each Job. This stores the complete information
# of the Job object.
if not os.path.isfile(output_file):
filename = 'pickle_1'
else:
with open(output_file, 'r+') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
key = row[0]
key = int(key)
key += 1
filename = 'pickle_' + str(key)
break
datadirname = sys.argv[4] # Name of data directory
plotdirname = sys.argv[5] # Name of plot directory
trajdotlammpstrj = sys.argv[6] # Trajectories
testdotout = sys.argv[7] # LAMMPS print to screen
depdotin = sys.argv[8] # Input file
edges = int(sys.argv[9]) # The edges of interest
faces = int(sys.argv[10]) # The face threshold for the edges of interest
# Loop for each path
for item in os.walk(jobs_dir):
path = item[0]
split = path.split('/')
# Filter for paths that contain jobs
if job_name not in split[-1]:
continue
run = job(path, export_dir, datadirname, plotdirname)
run.input_file(depdotin)
run.sys(testdotout)
run.box(trajdotlammpstrj)
run.ico(edges=edges, faces=faces)
print('-'*79)
def run_hetero(self, t, special_input=None):
# run name
runname="%s, N=%i, M=%i, ni=%f, ncr=%f, dW=%f, T=%f" % (self.material, self.N, self.M, self.ni, self.ncr, self.dW, t)
self.write_log("##############################################\n")
self.write_log("### euorun: %s\n" % runname)
self.write_log("##############################################\n")
# run command
runcmd=self.mpicmd + " -np %i " % self.np
runcmd+=self.sp.get_runcmd_hetero(self.material, self.N, self.M, self.ni, self.ncr, self.dW, t)
# add additional parameter
runcmd+=self.iteration_parameter
# add output
runoutput=self.output + self.hdb.get_temp_output(t)
runcmd+=" -o %s/" % runoutput
#print "check", runcmd
# check if run not already exist
runexists=self.run_exists(runcmd, runoutput, check_database=self.check_database)
if not runexists:
# add special input folder
if special_input!=None:
runcmd+=" -i %s" % (special_input)
# search self.input folder and/or remote database for suitable input folders and add it
else:
if self.inputFlag:
runcmd=database.add_input(runcmd, download_path=self.output+"/download/", path=self.input, source=self.source, input_system_name=self.input_system_name)
if self.isoDeltaFlag:
######################################################################################
####### add energy shift values for the isolated system constituents #################
######################################################################################
# check is values of energy shifts in the isolated system already exist
#print "check isodeltas:", database.get_isodelta_info(runcmd)
self.write_log("* Check isolated deltas: %s, %s, %s, %s, %s, %s, %s, %s\n" % (database.get_isodelta_info(runcmd)[:-1]))
(exists_left, material_left, N_left, nc_left, exists_right, material_right, N_right, nc_right, temp)=database.get_isodelta_info(runcmd)
# if not start isolated runs
if not exists_left or not exists_right:
if not exists_left:
# get name
runname_left="%s, N=%i, ni=%f, T=%f" % (material_left, N_left, nc_left, t)
self.write_log("* Isolated run necessary: %s\n" % runname_left)
# get run command
runcmd_left=self.mpicmd + " -np %i " % self.np
runcmd_left+=self.sp.get_runcmd_isolated(material_left, N_left, nc_left, t)
# add default additional parameter for iteration
runcmd_left+=self.get_default_iteration_parameter(material_left)
# add output
output_left=self.idb.get_output(material_left, N_left, nc_left)
runoutput_left=output_left + self.idb.get_temp_output(t)
runcmd_left+=" -o " + runoutput_left
# run left system
if not self.run_exists(runcmd_left, runoutput_left):
# add input if existent
runcmd_left=database.add_input(runcmd_left, download_path=output_left+"/download/", path=output_left, source=self.source)
j=job.job(runname_left, self.log, self.email, [runcmd_left], logappend=True, verbose=self.verbose, mailcmd=self.mailcmd)
j.run()
# update database
self.write_log("* Update isolated database\n")
#print "update isolated db"
updatecmd_left="isolated_remote.py %s" % output_left
#subprocess.call(updatecmd_left, shell=True)
j=job.job("update remote isolated database" , self.log, self.email, [updatecmd_left], logappend=True, verbose=False, mailcmd=self.mailcmd)
j.run()
if not exists_right:
# get name
runname_right="%s, N=%i, ni=%f, T=%f" % (material_right, N_right, nc_right, t)
self.write_log("* Isolated run necessary: %s\n" % runname_right)
# get run command
runcmd_right=self.mpicmd + " -np %i " % self.np
runcmd_right+=self.sp.get_runcmd_isolated(material_right, N_right, nc_right, t)
# add default additional parameter for iteration
runcmd_right+=self.get_default_iteration_parameter(material_right)
# add output
output_right=self.idb.get_output(material_right, N_right, nc_right)
runoutput_right=output_right + self.idb.get_temp_output(t)
runcmd_right+=" -o " + runoutput_right
# run right system
if not self.run_exists(runcmd_right, runoutput_right):
# add input if existent
runcmd_right=database.add_input(runcmd_right, download_path=output_right+"/download/", path=output_right, source=self.source)
j=job.job(runname_right, self.log, self.email, [runcmd_right], logappend=True, verbose=self.verbose, mailcmd=self.mailcmd)
j.run()
# update database
#print "update isolated db"
self.write_log("* Update isolated database\n")
updatecmd_right="isolated_remote.py %s" % output_right
#subprocess.call(updatecmd_right, shell=True)
j=job.job("update remote isolated database" , self.log, self.email, [updatecmd_right], logappend=True, verbose=False, mailcmd=self.mailcmd)
j.run()
# add isodeltas
runcmd=database.add_isodeltas(runcmd)
# run heterostructure job
#print "run", runcmd
j=job.job(runname, self.log, self.email, [runcmd], logappend=True, verbose=self.verbose, mailcmd=self.mailcmd)
j.run()
# update database
if self.updatedbFlag and not runexists:
self.write_log("* Update heterostructure database\n")
#print "update heterostructure db"
updatecmd="heterostructure_remote.py %s" % runoutput
#subprocess.call(updatecmd, shell=True)
j=job.job("update remote heterostructure database", self.log, self.email, [updatecmd], logappend=True, verbose=False, mailcmd=self.mailcmd)
j.run()
self.write_log("\n")
MODULENAME = "MAIN"
#Main file of our app
def custom_job(app, arg):
i = 0
while app.running:
config.log.log(str(i))
i = i + 1
if (i > 5):
break
time.sleep(5)
def main(app): #main function
while app.running: #!!!IMPORTANT!!!:life cycle should continue only while app running,otherwise app will run forever!
s = input(">")
if s == "help":
print(
"example--example showcase\nexit--quit app\nhelp--shiw this message"
)
if s == "exit":
app.event(Event("$APP_QUIT", app, {"reason": "user_exit"}))
else:
app.event(Event("UI_COMMAND_SENT", app, {"cmd": s}))
if __name__ == "__main__": #if we're running not as module
App = app("app.log", main, [job(custom_job, (10, ))]) #creating app
App.run() #and running it
from job import job
from sys import exit
# CAVEAT: CHARACTER '/' IS ASSUMED NOT TO OCCUR AT ALL IN THE DATASET
# EXAMPLES OF USE OF THE job CLASS FOR RUNNING SLATT
# use Borgelt's apriori to compute all frequent closures
# for a dataset and a support bound (in [0,1]):
# items may be strings, not just numbers
todayjob = job("lenses_recoded", 0.001)
##todayjob = job("len",0.001)
##todayjob = job("pvotes",0.30)
##todayjob = job("pumsb_star",supp=0.4)
##todayjob = job("toyNouRay",supp=0)
##anotherjob = job("e13",0.99/13)
# compute B* basis, write the rules into a file
todayjob.run("B*", 0.75, show=False, outrules=False)
# compute representative rules, show in console and write on file
todayjob.run("RR", 0.75, show=False, outrules=True)
# compute GD basis for conf 1
todayjob.run("GD", show=False)
#to apply confidence boost filter at level 1.2 to RR
todayjob.run("RR", 0.75, boost=1.2, show=True)
#now to B*, at boost 1.05, and reducing a bit the output verbosity
def run_hetero(self, t, special_input=None):
# run name
runname = "%s, N=%i, M=%i, ni=%f, ncr=%f, dW=%f, T=%f" % (
self.material, self.N, self.M, self.ni, self.ncr, self.dW, t)
self.write_log("##############################################\n")
self.write_log("### euorun: %s\n" % runname)
self.write_log("##############################################\n")
# run command
runcmd = self.mpicmd + " -np %i " % self.np
runcmd += self.sp.get_runcmd_hetero(self.material, self.N, self.M,
self.ni, self.ncr, self.dW, t)
# add additional parameter
runcmd += self.iteration_parameter
# add output
runoutput = self.output + self.hdb.get_temp_output(t)
runcmd += " -o %s/" % runoutput
#print "check", runcmd
# check if run not already exist
runexists = self.run_exists(runcmd,
runoutput,
check_database=self.check_database)
if not runexists:
# add special input folder
if special_input != None:
runcmd += " -i %s" % (special_input)
# search self.input folder and/or remote database for suitable input folders and add it
else:
if self.inputFlag:
runcmd = database.add_input(
runcmd,
download_path=self.output + "/download/",
path=self.input,
source=self.source,
input_system_name=self.input_system_name)
if self.isoDeltaFlag:
######################################################################################
####### add energy shift values for the isolated system constituents #################
######################################################################################
# check is values of energy shifts in the isolated system already exist
#print "check isodeltas:", database.get_isodelta_info(runcmd)
self.write_log(
"* Check isolated deltas: %s, %s, %s, %s, %s, %s, %s, %s\n"
% (database.get_isodelta_info(runcmd)[:-1]))
(exists_left, material_left, N_left, nc_left, exists_right,
material_right, N_right, nc_right,
temp) = database.get_isodelta_info(runcmd)
# if not start isolated runs
if not exists_left or not exists_right:
if not exists_left:
# get name
runname_left = "%s, N=%i, ni=%f, T=%f" % (
material_left, N_left, nc_left, t)
self.write_log("* Isolated run necessary: %s\n" %
runname_left)
# get run command
runcmd_left = self.mpicmd + " -np %i " % self.np
runcmd_left += self.sp.get_runcmd_isolated(
material_left, N_left, nc_left, t)
# add default additional parameter for iteration
runcmd_left += self.get_default_iteration_parameter(
material_left)
# add output
output_left = self.idb.get_output(
material_left, N_left, nc_left)
runoutput_left = output_left + self.idb.get_temp_output(
t)
runcmd_left += " -o " + runoutput_left
# run left system
if not self.run_exists(runcmd_left, runoutput_left):
# add input if existent
runcmd_left = database.add_input(
runcmd_left,
download_path=output_left + "/download/",
path=output_left,
source=self.source)
j = job.job(runname_left,
self.log,
self.email, [runcmd_left],
logappend=True,
verbose=self.verbose,
mailcmd=self.mailcmd)
j.run()
# update database
self.write_log("* Update isolated database\n")
#print "update isolated db"
updatecmd_left = "isolated_remote.py %s" % output_left
#subprocess.call(updatecmd_left, shell=True)
j = job.job("update remote isolated database",
self.log,
self.email, [updatecmd_left],
logappend=True,
verbose=False,
mailcmd=self.mailcmd)
j.run()
if not exists_right:
# get name
runname_right = "%s, N=%i, ni=%f, T=%f" % (
material_right, N_right, nc_right, t)
self.write_log("* Isolated run necessary: %s\n" %
runname_right)
# get run command
runcmd_right = self.mpicmd + " -np %i " % self.np
runcmd_right += self.sp.get_runcmd_isolated(
material_right, N_right, nc_right, t)
# add default additional parameter for iteration
runcmd_right += self.get_default_iteration_parameter(
material_right)
# add output
output_right = self.idb.get_output(
material_right, N_right, nc_right)
runoutput_right = output_right + self.idb.get_temp_output(
t)
runcmd_right += " -o " + runoutput_right
# run right system
if not self.run_exists(runcmd_right, runoutput_right):
# add input if existent
runcmd_right = database.add_input(
runcmd_right,
download_path=output_right + "/download/",
path=output_right,
source=self.source)
j = job.job(runname_right,
self.log,
self.email, [runcmd_right],
logappend=True,
verbose=self.verbose,
mailcmd=self.mailcmd)
j.run()
# update database
#print "update isolated db"
self.write_log("* Update isolated database\n")
updatecmd_right = "isolated_remote.py %s" % output_right
#subprocess.call(updatecmd_right, shell=True)
j = job.job("update remote isolated database",
self.log,
self.email, [updatecmd_right],
logappend=True,
verbose=False,
mailcmd=self.mailcmd)
j.run()
# add isodeltas
runcmd = database.add_isodeltas(runcmd)
# run heterostructure job
#print "run", runcmd
j = job.job(runname,
self.log,
self.email, [runcmd],
logappend=True,
verbose=self.verbose,
mailcmd=self.mailcmd)
j.run()
# update database
if self.updatedbFlag and not runexists:
self.write_log("* Update heterostructure database\n")
#print "update heterostructure db"
updatecmd = "heterostructure_remote.py %s" % runoutput
#subprocess.call(updatecmd, shell=True)
j = job.job("update remote heterostructure database",
self.log,
self.email, [updatecmd],
logappend=True,
verbose=False,
mailcmd=self.mailcmd)
j.run()
self.write_log("\n")
def genjobs():
nin = job('NIN', 'DPS', True, 2.4, .7, .15, True)
drg = job('DRG', 'DPS', True, 2.4, .7, .15, False)
mnk = job('MNK', 'DPS', False, 2.4, .7, .15, False)
sam = job('SAM', 'DPS', False, 2.4, .7, .15, False)
brd = job('BRD', 'DPS', False, 2.4, 0, .15, False)
mch = job('MCH', 'DPS', False, 2.4, 0, .15, False)
rdm = job('RDM', 'DPS', True, 2.4, 0, .15, False)
smn = job('SMN', 'DPS', False, 2.4, 0, .15, False)
blm = job('BLM', 'DPS', False, 2.4, 0, .15, False)
sch = job('SCH', 'HEAL', True, 2.4, 0, .15, False)
ast = job('AST', 'HEAL', True, 2.4, 0, .15, False)
whm = job('WHM', 'HEAL', False, 2.4, 0, .15, False)
gnb = job('GNB', 'TANK', True, 2.4, 0, .15, False)
war = job('WAR', 'TANK', True, 2.4, 0, .15, False)
drk = job('DRK', 'TANK', False, 2.4, 0, .15, False)
pld = job('PLD', 'TANK', False, 2.4, 0, .15, False)
table = [
nin, drg, mnk, sam, brd, mch, rdm, smn, blm, sch, ast, whm, gnb, war,
drk, pld
]
dict = {}
for i in table:
dict[i.name] = i
return dict
def test(target=None, t=None, cluster='.*', c=None, fabrun='.*', f=None,
pipeline_pattern='.*', p='.*', fablib=None, l=None):
target = t if t is not None else target
cluster = c if c is not None else cluster
re_cluster = re.compile(cluster)
fabrun = f if f is not None else fabrun
fablib = l if l is not None else fablib
pipeline_pattern = p if p is not None else pipeline_pattern
if target is None and fablib is None:
print 'Bad args.'
print 'Prease set test target (t=xxx) or fablib target (l=yyy).'
print test.__doc__
return
sys.path.remove(CONF._repo_dir)
FABTEST_DIR = os.path.dirname(os.path.abspath(__file__))
if fablib is not None:
fablib_dir = os.path.join(CONF._fablib_module_dir, fablib)
CONF._repo_dir = os.path.join(fablib_dir, 'test-repo')
else:
CONF._repo_dir = os.path.join(FABTEST_DIR, 'test-repo')
EX_CONF = cfg.ConfigOpts()
conf_file = os.path.join(CONF._repo_dir, 'fabfile.ini')
if os.path.exists(conf_file):
EX_CONF([], default_config_files=[conf_file])
else:
EX_CONF([])
from fabkit.conf import conf_base, conf_fabric, conf_web, conf_test # noqa
EX_CONF.register_opts(conf_base.default_opts)
EX_CONF.register_opts(conf_test.test_opts, group="test")
CONF._storage_dir = os.path.join(CONF._repo_dir, 'storage')
CONF._databag_dir = os.path.join(CONF._repo_dir, 'databag')
CONF._tmp_dir = os.path.join(CONF._storage_dir, 'tmp')
CONF._log_dir = os.path.join(CONF._storage_dir, 'log')
CONF._node_dir = os.path.join(CONF._repo_dir, 'nodes')
CONF._node_meta_pickle = os.path.join(CONF._node_dir, 'meta.pickle')
CONF._fabscript_module_dir = os.path.join(CONF._repo_dir, 'fabscript')
CONF._fablib_module_dir = os.path.join(CONF._repo_dir, 'fablib')
sys.path.extend([
CONF._repo_dir,
])
CONF.fablib = EX_CONF.fablib
CONF.test.fablib = EX_CONF.test.fablib
CONF.test.clusters = EX_CONF.test.clusters
util.create_required_dirs()
util.git_clone_required_fablib(is_test=True)
CONF._unittests_dir = os.path.join(FABTEST_DIR, 'unittests')
if target is None or target == 'fab':
CONF.user = CONF.test.user
CONF.password = CONF.test.password
conf_fabric.init()
env.forward_agent = False
env.disable_known_hosts = True
env.tasks.append('node:{0}'.format(cluster))
env.tasks.append("job:local,'{0}'".format(pipeline_pattern))
for cluster in CONF.test.clusters:
if re_cluster.search(cluster):
node(cluster, 'yes')
job('local', pipeline_pattern, f=fabrun)
if fablib is None:
# Test fabkit
if target == 'all':
suites = unittest.TestLoader().discover(CONF._unittests_dir,
pattern='test_*')
else:
suites = unittest.TestLoader().discover(CONF._unittests_dir,
pattern='test_{0}*'.format(target))
alltests = unittest.TestSuite(suites)
result = unittest.TextTestRunner(verbosity=2).run(alltests)
exit(len(result.errors) + len(result.failures))
def init(NUM_INSTANCE=0):
constantes.NUM_INSTANCE = NUM_INSTANCE
##path ver s les instances
constantes.INSTANCE = constantes.LISTE_INSTANCE[NUM_INSTANCE]
#chargement de la classe data correspondant au nom de l'instance
data = imp.load_source(
constantes.INSTANCE,
constantes.PATH_INSTANCE + constantes.INSTANCE + ".py")
data_2 = data.data
##definition de l'entete timestamp du fichier log
time_temp = time.ctime()
header_time = (time_temp.split(":")[0]).split(" ")
try:
header_time.remove("")
except:
pass
header_time.append(time_temp.split(":")[1])
header_time2 = ""
header_time2 += header_time[0] + "_" + header_time[
1] + "_" + header_time[2] + "_"
header_time2 += header_time[3] + ":" + header_time[4] + "_"
constantes.header_time = header_time2
###############################################################################
#########################definition constantes de l'instance###################
##extraction des constantes
constantes.nbJOB = data_2["nbJOB"]
constantes.tics = data_2["tics"]
constantes.nbTIC = data_2["nbTIC"]
constantes.speed = data_2["speed"]
constantes.nbCMP = data_2["nbCMP"]
constantes.name = data_2["name"]
constantes.jobs = data_2["jobs"]
###############################################################################
#########################initialisation des listes d'objet#####################
##liste de techniciens
#print('*** creation liste des tics')
constantes.TICS = [tic(t) for t in constantes.tics]
#for t in TICS: print(t)
##liste des jobs
#print('*** creation liste des jobs')
constantes.JOBS = [job(j) for j in constantes.jobs]
#for j in JOBS: print(j)
##dico tic pour acces rapide
constantes.dico_tic = {}
for t in constantes.TICS:
constantes.dico_tic[t.id] = t
tic.dico_tic = constantes.dico_tic
##dico job pour acces rapide
constantes.dico_job = {}
for j in constantes.JOBS:
constantes.dico_job[j.id] = j
job.dico_job = constantes.dico_job
#job.speed
job.speed = constantes.speed
###############################################################################
##########################constantes calculées#################################
##distance entre deux jobs
constantes.distance = {}
for job1 in constantes.JOBS:
for job2 in constantes.JOBS:
constantes.distance[(job1.id,
job2.id)] = job.distance(job1, job2)
##duree entre deux jobs pour faire le trajet
constantes.temps = {}
for job1 in constantes.JOBS:
for job2 in constantes.JOBS:
constantes.temps[(job1.id, job2.id)] = job.temps(job1, job2)
def execute(p):
j = None
if len(p.commands) > 1:
# iterate over all the commands that are piped
for i in range(len(p.commands)):
# first command in the pipeline
if (i == 0):
proc = None
# checks if there is input redirection
if p.input != "":
fd = os.open(p.input, os.O_RDONLY)
proc = subprocess.Popen(p.commands[i],
preexec_fn=os.setpgrp,
stdin=fd,
stdout=subprocess.PIPE)
# else runs it normally
else:
proc = subprocess.Popen(p.commands[i],
preexec_fn=os.setpgrp,
stdout=subprocess.PIPE)
j = joblist.add_job(p.fg)
j.add_process(proc)
# deals with last command in pipeline
elif (i == len(p.commands) - 1):
in_proc = j.processes[i - 1]
proc = None
# checks for output redirection and deals with it accordingly
if p.output != "":
fd_out = os.open(p.output, os.O_CREAT | os.O_WRONLY)
proc = subprocess.Popen(p.commands[i],
preexec_fn=os.setpgrp,
stdin=in_proc.subprocess.stdout,
stdout=fd_out)
# else runs last command normally
else:
proc = subprocess.Popen(p.commands[i],
preexec_fn=os.setpgrp,
stdin=in_proc.subprocess.stdout)
j.add_process(proc)
# if command is not last or first in the pipeline
else:
in_proc = j.processes[i - 1]
proc = subprocess.Popen(p.commands[i],
preexec_fn=os.setpgrp,
stdin=in_proc.subprocess.stdout,
stdout=subprocess.PIPE)
j.add_process(proc)
else:
# creates singular process within a job
proc = None
# both input and output redirection case
if p.input != "" and p.output != "":
fd_in = os.open(p.input, os.O_RDONLY)
fd_out = os.open(p.output, os.O_CREAT | os.O_WRONLY)
proc = subprocess.Popen(p.commands[0],
preexec_fn=os.setpgrp,
stdin=fd_in,
stdout=fd_out)
# just input redirection
elif p.input != "":
fd = os.open(p.input, os.O_RDONLY)
proc = subprocess.Popen(p.commands[0],
preexec_fn=os.setpgrp,
stdin=fd)
# just output redirection
elif p.output != "":
fd_out = os.open(p.output, os.O_CREAT | os.O_WRONLY)
proc = subprocess.Popen(p.commands[0],
preexec_fn=os.setpgrp,
stdout=fd_out)
# no input or output redirection
else:
proc = subprocess.Popen(p.commands[0], preexec_fn=os.setpgrp)
j = job(p.fg)
j.add_process(proc)
joblist.jobs.append(j)
while (joblist.has_fg_job()):
# repeatedly checking if any child process has finished
time.sleep(.01)
child_handler()
def test(target=None,
t=None,
cluster='.*',
c=None,
fabrun='.*',
f=None,
pipeline_pattern='.*',
p='.*',
fablib=None,
l=None):
target = t if t is not None else target
cluster = c if c is not None else cluster
re_cluster = re.compile(cluster)
fabrun = f if f is not None else fabrun
fablib = l if l is not None else fablib
pipeline_pattern = p if p is not None else pipeline_pattern
if target is None and fablib is None:
print 'Bad args.'
print 'Prease set test target (t=xxx) or fablib target (l=yyy).'
print test.__doc__
return
sys.path.remove(CONF._repo_dir)
FABTEST_DIR = os.path.dirname(os.path.abspath(__file__))
if fablib is not None:
fablib_dir = os.path.join(CONF._fablib_module_dir, fablib)
CONF._repo_dir = os.path.join(fablib_dir, 'test-repo')
else:
CONF._repo_dir = os.path.join(FABTEST_DIR, 'test-repo')
EX_CONF = cfg.ConfigOpts()
conf_file = os.path.join(CONF._repo_dir, 'fabfile.ini')
if os.path.exists(conf_file):
EX_CONF([], default_config_files=[conf_file])
else:
EX_CONF([])
from fabkit.conf import conf_base, conf_fabric, conf_web, conf_test # noqa
EX_CONF.register_opts(conf_base.default_opts)
EX_CONF.register_opts(conf_test.test_opts, group="test")
CONF._storage_dir = os.path.join(CONF._repo_dir, 'storage')
CONF._databag_dir = os.path.join(CONF._repo_dir, 'databag')
CONF._tmp_dir = os.path.join(CONF._storage_dir, 'tmp')
CONF._log_dir = os.path.join(CONF._storage_dir, 'log')
CONF._node_dir = os.path.join(CONF._repo_dir, 'nodes')
CONF._node_meta_pickle = os.path.join(CONF._node_dir, 'meta.pickle')
CONF._fabscript_module_dir = os.path.join(CONF._repo_dir, 'fabscript')
CONF._fablib_module_dir = os.path.join(CONF._repo_dir, 'fablib')
sys.path.extend([
CONF._repo_dir,
])
CONF.fablib = EX_CONF.fablib
CONF.test.fablib = EX_CONF.test.fablib
CONF.test.clusters = EX_CONF.test.clusters
util.create_required_dirs()
util.git_clone_required_fablib(is_test=True)
CONF._unittests_dir = os.path.join(FABTEST_DIR, 'unittests')
if target is None or target == 'fab':
CONF.user = CONF.test.user
CONF.password = CONF.test.password
conf_fabric.init()
env.forward_agent = False
env.disable_known_hosts = True
env.tasks.append('node:{0}'.format(cluster))
env.tasks.append("job:local,'{0}'".format(pipeline_pattern))
for cluster in CONF.test.clusters:
if re_cluster.search(cluster):
node(cluster, 'yes')
job('local', pipeline_pattern, f=fabrun)
if fablib is None:
# Test fabkit
if target == 'all':
suites = unittest.TestLoader().discover(CONF._unittests_dir,
pattern='test_*')
else:
suites = unittest.TestLoader().discover(
CONF._unittests_dir, pattern='test_{0}*'.format(target))
alltests = unittest.TestSuite(suites)
result = unittest.TextTestRunner(verbosity=2).run(alltests)
exit(len(result.errors) + len(result.failures))
task.env.now))
task.job_complete_status = True
queue_count[task.env.now] = -1
if len(jobs_list) < 1000:
temp = job(env, job_id = len(jobs_list)+1,
service_time = np.random.exponential(0.8))
task.env.process(request_cpu(temp, ComputeMachine))
jobs_list[len(jobs_list)+1] = temp
###############################################################################
"""
Execution
"""
###############################################################################
np.random.seed(RANDOM_SEED) # This helps reproducing the results
# Create an environment and start the setup process
jobs_list = {}
queue_count = {}
env = simpy.Environment()
ComputeMachine = CommmonMachine(env, NUM_CPUS, CPU_SLOT, CPU_OVERHEAD)
# Initialize first (num_terminals) jobs
for i in range(NUM_TERMINALS):
temp = job(env, job_id = i+1, service_time = np.random.exponential(0.8))
env.process(request_cpu(temp, ComputeMachine))
jobs_list[i+1] = temp
env.run()
##############################################################################
#statistics
response_mean, response_max, response_array = average_response(jobs_list)
cum_queue = queue_time_average(queue_count)
cpu_utilization = cpu_utilization(ComputeMachine, jobs_list)
def test_srpt_2(self):
jobs = [job(2, 1), job(1, 4), job(4, 1)]
ref = [[[1, 3]], [[3, 4], [5, 8]], [[4, 5]]]
sched = srpt(jobs)
for i, j in enumerate(sched.jobs):
self.assertEqual(j.ct, ref[i])
def main():
logger.info("Archiveboxmatic is running.")
args = parse_args()
config = read_config(args.config_file)
# Check if config is valid with the selected arguments.
if not validate_config(config, args):
raise Exception("Config is not valid.")
allowed_schedules = ([args.schedule] if args.schedule != "all" else
["daily", "weekly", "monthly", "yearly", "none"])
global_config = config["archivebox"]
def run_threaded(job_func, args, global_config, archive_config):
job_thread = threading.Thread(
target=job_func, args=[args, global_config, archive_config])
job_thread.start()
for i in config["archives"]:
if "schedule" in i:
if i["schedule"] in allowed_schedules:
if i["schedule"] == "daily":
schedule.every().day.at("12:00").do(
run_threaded,
job,
args=args,
global_config=global_config,
archive_config=i,
)
elif i["schedule"] == "weekly":
schedule.every().monday.at("10:00").do(
run_threaded,
job,
args=args,
global_config=global_config,
archive_config=i,
)
elif i["schedule"] == "monthly":
schedule.every().day.at("05:00").do(
run_threaded,
job_monthly,
args=args,
global_config=global_config,
archive_config=i,
)
elif i["schedule"] == "yearly":
schedule.every().day.at("01:00").do(
run_threaded,
job_yearly,
args=args,
global_config=global_config,
archive_config=i,
)
else:
job(args, global_config, i)
else:
logger.warning(f"Schedule {i['schedule']} not allowed.")
elif "none" in allowed_schedules:
job(args, global_config, i)
else:
logger.warning("Schedule none not allowed.")
while True:
schedule.run_pending()
time.sleep(600)
logger.debug(
f"Next job: {schedule.next_run() - datetime.datetime.now()}.")
def test_srpt_text_jobs(self):
jobs = [job(2, 0), job(1, 4), job(4, 1)]
ref = [[[0, 2]], [[2, 4], [5, 7]], [[4, 5]]]
sched = srpt(jobs)
for i, j in enumerate(sched.jobs):
self.assertEqual(j.ct, ref[i])
def add_job(self, fg):
self.jobs.append(job(fg))
return self.jobs[len(self.jobs) - 1]
def run_submit(cmd, logstring, append, email, mailcmd): cmds=[cmd] j=job.job(logstring, logstring, email, cmds, logappend=append, verbose=True, mailcmd=mailcmd) j.run()
from job import job
from sys import exit
# CAVEAT: CHARACTER '/' IS ASSUMED NOT TO OCCUR AT ALL IN THE DATASET
# EXAMPLES OF USE OF THE job CLASS FOR RUNNING SLATT
# FOR REPRESENTATIVE RULES MINING
# use Borgelt's apriori to compute all frequent closures
# for a dataset and a support bound (in [0,1]):
# items may be strings, not just numbers
todayjob = job("./datasets/test",0.4)
##todayjob = job(".datasets/retail",0.0005)
##todayjob = job(".datasets/retail",0.001)
##todayjob = job(".datasets/adult",0.01)
##todayjob = job(".datasets/adult",0.005)
##todayjob = job(".datasets/accidents",0.5)
##todayjob = job(".datasets/accidents",0.4)
# compute representative rules with Kryszkiewicz incomplete heuristic,
# write the rules into a file
todayjob.run("GenRR",0.7,show=False,outrules=True)
# compute B* basis, show in console and do not write on file
todayjob.run("RRGenerator",0.8,show=True,outrules=False)
# compute representative rules, show in console and write on file
todayjob.run("RRClosureGenerator",0.9,show=True,outrules=True)
path = item[0]
# Filter for paths that contain jobs
if 'job' not in path:
continue
if datadirname in path:
continue
if plotdirname in path:
continue
if 'minimization' in path:
continue
error = False
try:
run = job(path)
except Exception:
error = True
pass
try:
run.apd()
except Exception:
error = True
pass
try:
run.etg()
except Exception:
error = True
pass
else:
j = heapq.heappop(self.__stack)
j.ct.append([self.__t, self.__t + j.pt])
self.__t += j.pt
self.__stacking()
self.__print()
def draw(self, fname=None):
for j in jobs:
j.draw()
plt.gca().set_ylim([0, 2])
plt.gca().margins(0.1)
if fname:
plt.savefig(fname, bbox_inches='tight')
else:
plt.show()
if __name__ == '__main__':
jobs = []
c = mc.values()
n = len(c) - 1
random.seed(0)
jobs.append(job(2, 0, c[random.randint(0, n)]))
jobs.append(job(1, 4, c[random.randint(0, n)]))
jobs.append(job(4, 1, c[random.randint(0, n)]))
sched = srpt(jobs)
for j in sched.jobs:
print j.ct
sched.draw('srpt_for_js1.png')
