def getFile(queue, stat):
rows = mysql_query('SELECT `file` FROM `calculations` WHERE `queue` = ' +
str(queue) + ' AND `stat` = ' + str(stat))
file = []
for row in rows:
file.append(row['file'])
ID = pd.DataFrame({'file': file})
return ID
def getResults(queue, stat, keys):
rows = mysql_query(
'SELECT `file`, `results` FROM `calculations` WHERE `queue` = ' +
str(queue) + ' AND `stat` = ' + str(stat))
keys.insert(0, 'file')
resultsdict = {'file': []}
for key in keys:
resultsdict[key] = []
for row in rows:
resultsdb = json.loads(row['results'])
for key in keys:
resultsdict[key].append(resultsdb[key])
results = pd.DataFrame(resultsdict, index='file')
return results
def addElemental(df, features):
ncols = len([1 for feat in list(df) if feat.find('el') == 0])
els = set([])
for i in range(ncols):
els = els.union(set(df['el' + str(i)].unique()))
allfeats = [
'atomicnumber', 'symbol', 'mass', 'Ecoh', 'n', 's', 'p', 'd', 'V', 'r',
'EN', 'EA', 'IP'
]
atominfo = mysql_query('SELECT * FROM `elements` WHERE `symbol` IN (\'' +
'\',\''.join(els) + '\')')
atomdict = {}
for row in atominfo:
for f in allfeats:
if f not in atomdict:
atomdict[f] = []
atomdict[f].append(row[f])
atoms = pd.DataFrame(atomdict, columns=allfeats)
data = [[0 for i in range(ncols * len(features))] for j in range(len(df))]
index_df = pd.Index(df.index)
column_names = ["name" for i in range(ncols * len(features))]
for (ind, material) in df.iterrows():
for i in range(ncols):
atominfo = atoms.loc[atoms['symbol'] == material['el' + str(i)]]
row_loc = index_df.get_loc(ind)
for ind_feat, col in enumerate(features):
col_loc = i + ncols * ind_feat
column_names[col_loc] = col + str(i)
data[row_loc][col_loc] = float(atominfo[col].values[0])
new = pd.DataFrame(data, index=df.index, columns=column_names)
df = pd.concat([df, new], axis=1)
return df
def addElemental_old(df, features):
ncols = len([1 for feat in list(df) if feat.find('el') == 0])
els = set([])
for i in range(ncols):
els = els.union(set(df['el' + str(i)].unique()))
allfeats = [
'atomicnumber', 'symbol', 'mass', 'Ecoh', 'n', 's', 'p', 'd', 'V', 'r',
'EN', 'EA', 'IP'
]
atominfo = mysql_query('SELECT * FROM `elements` WHERE `symbol` IN (\'' +
'\',\''.join(els) + '\')')
atomdict = {}
for row in atominfo:
for f in allfeats:
if f not in atomdict:
atomdict[f] = []
atomdict[f].append(row[f])
atoms = pd.DataFrame(atomdict, columns=allfeats)
data = {}
for col in features:
for i in range(ncols):
data[col + str(i)] = [0 for x in range(0, len(df))]
new = pd.DataFrame(data, index=df.index)
df = pd.concat([df, new], axis=1)
for (ind, material) in df.iterrows():
for i in range(ncols):
atominfo = atoms.loc[atoms['symbol'] == material['el' + str(i)]]
for col in features:
df.loc[ind, col + str(i)] = float(atominfo[col].values[0])
return df
def resubmit(qid=None, server=None, args=None):
if qid == None:
qid = sys.argv[1]
if server == None:
server = os.getenv('VSC_INSTITUTE_CLUSTER')
script = mysql_query('SELECT `submit` FROM `queues` WHERE `id` = ' +
str(qid))['submit']
args = '' + sys.argv[2] + ' ' + sys.argv[3] + ' ' + sys.argv[4]
# args = '' + sys.argv[2] + ' ' + str(2) + ' ' + sys.argv[4]
if server != 'breniac':
execute(script + ' ' + str(qid) + ' ' + str(args))
else:
print('ssh login1 "' + script + ' ' + str(qid) + ' ' + str(args) + '"')
execute('ssh login1 "' + script + ' ' + str(qid) + ' ' + str(args) +
'"')
print('Submitted new calculation in queue ' + str(qid) + ' on server ' +
server + '.')
return True
def getComposition(queue, stat):
rows = mysql_query(
'SELECT `newdata`.`file`, `newdata`.`calcformula` FROM `calculations` INNER JOIN WHERE `queue` = '
+ str(queue) + ' AND `stat` = ' + str(stat))
compdict = {'file': []}
for row in rows:
compdict['file'].append(row['file'])
formula = row['formula'].split()
stoich = [[i for i in x if i.isdigit()] for x in formula]
els = [[i for i in x if not i.isdigit()] for x in formula]
for i in range(len(els)):
if 'el' + str(i) not in compdict:
compdict['el' + str(i)] = []
compdict['stoich' + str(i)] = []
compdict['el' + str(i)].append(els[i])
compdict['stoich' + str(i)].append(stoich[i])
composition = pd.DataFrame(compdict, index='file')
composition.fillna(0)
return composition
def eosRollback(calc, evname='EOS'):
print('Commencing EOS rollback due to detected errors. ')
step = int(np.ceil(float(calc['stat']) / 2))
crit = calc['results']['eoscheck']
edir = crit['dirs'][step].split('/')[-2]
print('The EOS dir is ' + edir)
vols = sorted([x.split('/')[-1] for x in crit['dirs'] if edir in x])
f = open(os.path.join('../', evname), 'r')
enmin = 999999999.999999999
i = 0
for line in f.readlines():
en = float(line.split()[1])
if en < enmin:
enmin = en
volmin = vols[i]
i += 1
os.chdir('../')
odir = 'old' + str(time.time())
print('Backing up old EOS in ' + str(odir))
if not os.path.isdir(odir):
mkdir(odir)
for v in vols:
if os.path.isdir(v):
shutil.copytree(v, os.path.join(odir, v))
for i in ['CHGCAR', 'CHGCAR.gz', 'WAVECAR', 'WAVECAR.gz', 'CHG']:
if os.path.isfile(os.path.join(odir, v, i)):
os.remove(os.path.join(odir, v, i))
for v in [evname, evname + '.eosout', evname + '.png']:
if os.path.isfile(v):
shutil.copy(v, os.path.join(odir, v))
os.remove(v)
print('DEBUG: volmin', volmin)
if volmin != '1.0':
for i in ['CHGCAR', 'CHGCAR.gz', 'WAVECAR', 'WAVECAR.gz', 'CONTCAR']:
if os.path.isfile(os.path.join(volmin, i)):
shutil.copy(os.path.join(volmin, i), os.path.join('1.0', i))
vols.remove('1.0')
for v in vols:
shutil.rmtree(v)
tstat = int(calc['stat']) - 2 * (len(vols) + 1) + 1
print('Rolling back to stat ' + str(tstat))
parent = mysql_query('SELECT `id` FROM `calculations` WHERE `queue` = ' +
str(calc['queue']) + ' AND `file` = ' + calc['file'] +
' AND `stat` = ' + str(tstat))
psettings = manage.getSettings(parent['id'])
if 'continue' in psettings.keys():
psettings['continue'] = str(int(psettings['continue']) + 1)
else:
psettings['continue'] = '1'
manage.modify({'settings': psettings, 'id': parent['id']})
manage.rollback(tstat, calc['id'])
resubmit()
exit()
return True
def run():
input('Welcome to the alpha test demo. This script will show you the main functionalities of the Queue Manager, once done please look inside the script to understand the various functions used. It is recommended to look through the case study in the manual beforehand. This script will reproduce what was shown there. \n\n To start please login to http://physics.epotentia.com/queue/ (only for the demo) then press any key to continue.')
print('\n\nLogging in'),
#Yes this is cheap, I know
for i in range(1,4):
sleep(1)
sys.stdout.flush()
print('.',)
#These are deep level MySQL functions, you are discouraged from using them, but if you need to you can (ask me). All MySQL functions are supported, including statistical functions. The underlying library here tries to automatically limit your access to only your data. For this it does pattern recognition on your queries, very advanced queries may conflict with this. The owner variable is automatically set and contains your user id.
name = mysql_query('SELECT `name` FROM `accounts` WHERE `id` = ' + owner)
print('\n\n Welcome ' + name['name'] + ', are you ready to begin?')
input()
print('Great! We will start by setting up a settings and results template.')
settings = {'Temperature' : 180, 'time' : 20}
print('\n\nThe settings dictionary: ')
print(settings)
settingsid = HighThroughput.manage.template.add('Baking settings',settings,'VASP','settings')
results = {'Crispiness' : '', 'Weight' : 0}
print('\n\nThe results dictionary: ')
print(results)
resultsid = HighThroughput.manage.template.add('Baking results',results,'VASP','results')
input('\n\nWe now have two templates ready to use in our workflow. Ready to add workflow?')
name = 'Baking'
button = ['Ingr. ready', 'Mixing', 'Mixed', 'Kneading', 'Kneaded', 'Rising', 'Risen', 'Baking', 'Finished']
description = ['Ingredients ready', 'Mixing [server]', 'Mixed', 'Kneading [server]', 'Kneaded', 'Rising [server]', 'Risen', 'Baking [settings:Temperature]', 'Finished [results:weight]']
buttonclass = ['warning','info','warning','info','warning','info','warning','info','success']
entries = [{'stat' : i, 'stemplate' : settingsid, 'rtemplate' : resultsid, 'description' : description[i], 'buttonname' : button[i], 'buttonclass' : buttonclass[i], 'priority' : 1} for i in range(0,9)]
print('')
print(entries)
print('')
wid = HighThroughput.manage.workflow.add(name,entries)
input('\n\nThe Baking workflow is ready. Ready to add queue?\n')
#Doesn't conserve order, I should fix that... Can use string instead a:b,c:d,...
#fields = {'ID' : 'id', 'Info' : 'text', 'Status' : 'stat', 'Start' : 'start', 'End' : 'end'}
fields = 'ID:id,Info:text,Status:stat,Start:start,End:end'
qid = HighThroughput.manage.queue.add('Jan de Bakker, Inc.', wid, fields)
input('\n\nPlease go to the website and click Queues => View queues. Currently the website does not refresh automatically when new calculations are added, but you can just click the tab again. (Updates are autorefreshed (for quick updates you can set the refresh rate to 3000ms, going lower may be overkill.)\n\nIngredients would be added as materials normally, but to avoid unnecessary fake file creation we\'ll base our ingredients off a set of COD entries starting with FCC Al (CIF ID: 9012002). Ready to add calculations?\n')
print('Adding calculations.\n')
#add(material,queue,priority = 0,settings = None,results = None), templates are gotten from workflow by default
calcs = []
#I was going to edit this but then realized it's a bad idea to start renaming COD's materials for future usage yummy = ['Bread','Muffin','Cherry Pie','Pancake','Choccie Biscuit', 'Banana eclaire', 'Michel\'s cake','Cookies!','Sinterklaasventje','You get the point by now.']
for i in range(0,10):
calcs.append(HighThroughput.manage.calculation.add(9012002+i,qid,0))
#fix the names
#HighThroughput.manage.material.modify({'id' : , 'text' : yummy[i]})
input('\n\nYou can now see the calculations on the website. Next is a demonstration of how calculations are managed. You should be able to follow this live on the website. Set your refresh rate to 3000 ms. Ready?')
print('\n\nFetching a waiting calculation from the queue',)
#Add a bit of drama, the real system is of course instant!
for i in range(1,4):
sleep(1)
sys.stdout.flush()
print('.',)
fid = HighThroughput.manage.calculation.fetch(qid)
print('\n\n Found calculation id ' + str(fid) + '. Get full calc info?\n\n')
calculation = HighThroughput.manage.calculation.get(fid)
print(calculation)
input('\n\n Start calculation?')
HighThroughput.manage.calculation.start()
input('\n\n End calculation?')
HighThroughput.manage.calculation.end()
#We're moving the the calculation back to before the last active status so 1 or 2 steps, if you want to rollback further please use the rollback function (calculations beyond that status will be deleted!)
input('\n\n Now if something went wrong... Restart calculation?')
HighThroughput.manage.calculation.restart()
input('\n\n Up the pace? Keep in mind each blue workflow is a new \'calculation\' and you\'ll have to manually click the queue currently (will fix sometime).')
for i in range(0,20):
fid = HighThroughput.manage.calculation.fetch(qid)
calculation = HighThroughput.manage.calculation.get(fid)
HighThroughput.manage.calculation.start()
print('Starting calculation ' + str(HighThroughput.manage.calculation.calcid) + ' (status ' + str(HighThroughput.manage.calculation.stat) + ').')
HighThroughput.manage.calculation.end()
#Store the results, based on the template, can be put in the workflow description with [results:weight] etc, settings is similarly accessible.
HighThroughput.manage.calculation.updateResults({'weight': (i+1)*50, 'Crispiness' : 'Just right'})
print('Ending calculation ' + str(HighThroughput.manage.calculation.calcid) + ' (status ' + str(HighThroughput.manage.calculation.stat) + ').')
input('\n\nContinuing will clean up everything added during this demo.')
HighThroughput.manage.queue.remove(qid)
HighThroughput.manage.workflow.removeAll(wid)
HighThroughput.manage.template.remove(settingsid)
HighThroughput.manage.template.remove(resultsid)
def run():
raw_input('Welcome to the alpha test demo. This script will show you the main functionalities of the Queue Manager, once done please look inside the script to understand the various functions used. It is recommended to look through the case study in the manual beforehand. This script will reproduce what was shown there. \n\n To start please login to http://physics.epotentia.com/queue/ (only for the demo) then press any key to continue.')
print '\n\nLogging in',
#Yes this is cheap, I know
for i in range(1,4):
sleep(1)
sys.stdout.flush()
print '.',
#These are deep level MySQL functions, you are discouraged from using them, but if you need to you can (ask me). All MySQL functions are supported, including statistical functions. The underlying library here tries to automatically limit your access to only your data. For this it does pattern recognition on your queries, very advanced queries may conflict with this. The owner variable is automatically set and contains your user id.
name = mysql_query('SELECT `name` FROM `accounts` WHERE `id` = ' + owner)
print '\n\n Welcome ' + name['name'] + ', are you ready to begin?'
raw_input()
print 'Great! We will start by setting up a settings and results template.'
settings = {'Temperature' : 180, 'time' : 20}
print '\n\nThe settings dictionary: '
print settings
settingsid = HighThroughput.manage.template.add('Baking settings',settings,'VASP','settings')
results = {'Crispiness' : '', 'Weight' : 0}
print '\n\nThe results dictionary: '
print results
resultsid = HighThroughput.manage.template.add('Baking results',results,'VASP','results')
raw_input('\n\nWe now have two templates ready to use in our workflow. Ready to add workflow?')
name = 'Baking'
button = ['Ingr. ready', 'Mixing', 'Mixed', 'Kneading', 'Kneaded', 'Rising', 'Risen', 'Baking', 'Finished']
description = ['Ingredients ready', 'Mixing [server]', 'Mixed', 'Kneading [server]', 'Kneaded', 'Rising [server]', 'Risen', 'Baking [settings:Temperature]', 'Finished [results:weight]']
buttonclass = ['warning','info','warning','info','warning','info','warning','info','success']
entries = [{'stat' : i, 'stemplate' : settingsid, 'rtemplate' : resultsid, 'description' : description[i], 'buttonname' : button[i], 'buttonclass' : buttonclass[i], 'priority' : 1} for i in range(0,9)]
print ''
print entries
print ''
wid = HighThroughput.manage.workflow.add(name,entries)
raw_input('\n\nThe Baking workflow is ready. Ready to add queue?\n')
#Doesn't conserve order, I should fix that... Can use string instead a:b,c:d,...
#fields = {'ID' : 'id', 'Info' : 'text', 'Status' : 'stat', 'Start' : 'start', 'End' : 'end'}
fields = 'ID:id,Info:text,Status:stat,Start:start,End:end'
qid = HighThroughput.manage.queue.add('Jan de Bakker, Inc.', wid, fields)
raw_input('\n\nPlease go to the website and click Queues => View queues. Currently the website does not refresh automatically when new calculations are added, but you can just click the tab again. (Updates are autorefreshed (for quick updates you can set the refresh rate to 3000ms, going lower may be overkill.)\n\nIngredients would be added as materials normally, but to avoid unnecessary fake file creation we\'ll base our ingredients off a set of COD entries starting with FCC Al (CIF ID: 9012002). Ready to add calculations?\n')
print 'Adding calculations.\n'
#add(material,queue,priority = 0,settings = None,results = None), templates are gotten from workflow by default
calcs = []
#I was going to edit this but then realized it's a bad idea to start renaming COD's materials for future usage yummy = ['Bread','Muffin','Cherry Pie','Pancake','Choccie Biscuit', 'Banana eclaire', 'Michel\'s cake','Cookies!','Sinterklaasventje','You get the point by now.']
for i in range(0,10):
calcs.append(HighThroughput.manage.calculation.add(9012002+i,qid,0))
#fix the names
#HighThroughput.manage.material.modify({'id' : , 'text' : yummy[i]})
raw_input('\n\nYou can now see the calculations on the website. Next is a demonstration of how calculations are managed. You should be able to follow this live on the website. Set your refresh rate to 3000 ms. Ready?')
print '\n\nFetching a waiting calculation from the queue',
#Add a bit of drama, the real system is of course instant!
for i in range(1,4):
sleep(1)
sys.stdout.flush()
print '.',
fid = HighThroughput.manage.calculation.fetch(qid)
print '\n\n Found calculation id ' + str(fid) + '. Get full calc info?\n\n'
calculation = HighThroughput.manage.calculation.get(fid)
print calculation
raw_input('\n\n Start calculation?')
HighThroughput.manage.calculation.start()
raw_input('\n\n End calculation?')
HighThroughput.manage.calculation.end()
#We're moving the the calculation back to before the last active status so 1 or 2 steps, if you want to rollback further please use the rollback function (calculations beyond that status will be deleted!)
raw_input('\n\n Now if something went wrong... Restart calculation?')
HighThroughput.manage.calculation.restart()
raw_input('\n\n Up the pace? Keep in mind each blue workflow is a new \'calculation\' and you\'ll have to manually click the queue currently (will fix sometime).')
for i in range(0,20):
fid = HighThroughput.manage.calculation.fetch(qid)
calculation = HighThroughput.manage.calculation.get(fid)
HighThroughput.manage.calculation.start()
print 'Starting calculation ' + str(HighThroughput.manage.calculation.calcid) + ' (status ' + str(HighThroughput.manage.calculation.stat) + ').'
HighThroughput.manage.calculation.end()
#Store the results, based on the template, can be put in the workflow description with [results:weight] etc, settings is similarly accessible.
HighThroughput.manage.calculation.updateResults({'weight': (i+1)*50, 'Crispiness' : 'Just right'})
print 'Ending calculation ' + str(HighThroughput.manage.calculation.calcid) + ' (status ' + str(HighThroughput.manage.calculation.stat) + ').'
raw_input('\n\nContinuing will clean up everything added during this demo.')
HighThroughput.manage.queue.remove(qid)
HighThroughput.manage.workflow.removeAll(wid)
HighThroughput.manage.template.remove(settingsid)
HighThroughput.manage.template.remove(resultsid)
# -*- coding: utf-8 -*-
from HighThroughput.communication.mysql import mysql_query
import pickle
#with open('backtest.pkl','rb') as btfile:
# newq, materials = pickle.load(btfile)
newq = 251
mysql_query('DELETE FROM `calculations` WHERE `queue` = ' + str(newq) + ' AND `stat` > 0')
mysql_query('UPDATE `calculations` SET `priority` = 0 WHERE `queue` = ' + str(newq))
mysql_query('UPDATE `calculations` SET `leaf` = 1 WHERE `queue` = ' + str(newq))
# -*- coding: utf-8 -*-
# Setup a backtesting queue
import HighThroughput.manage.queue as HTq
import HighThroughput.manage.calculation as HTc
from HighThroughput.communication.mysql import mysql_query
import pickle, sys, json
# The target queue we will try predicting
tqueue = sys.argv[1]
# The target status of the final property calculation
tstat = sys.argv[2]
# We use a default 1 step, 3 stat workflow
newq = HTq.add('Backtesting ' + str(tqueue), workflow=20)
# get all the materials
materials = mysql_query(
'SELECT `file`, `results` FROM `calculations` WHERE `queue` = ' +
str(tqueue) + ' AND `stat` = ' + str(tstat))
# Add them all to our new queue
for mat in materials:
HTc.add(mat['file'], newq, priority=1)
# Save the info
with open('backtest.pkl', 'wb') as btfile:
pickle.dump((newq, materials), btfile)
stable = 0
#target = sys.argv[1]
rdict = {}
rlist = []
limit = 0.05
N_init = 50
batch_size = 1
newq = 251
target = 'Ehull'
#with open('backtest.pkl','rb') as btfile:
# newq, materials = pickle.load(btfile)
materials = mysql_query(
'SELECT `file`, `Ehull` FROM `zintlfinal` WHERE `queue` = 239')
for mat in materials:
#result = json.loads(mat['results'])
rdict[mat['file']] = float(mat[target])
tstable = sum([1 for x in rdict.values() if float(x) < limit])
print('There are ' + str(tstable) + ' stable materials to find in this queue.')
#HTML.setMLPriority(newq,stat=2)
#HTML.setMLPriority(newq, 2, ['mass','Ecoh','EN','IP'], N_init)
#
#for i in range(N_init):
# calc = HTc.fetchgetstart(newq)
# HTc.updateResults({target: rdict[calc['file']]})
