def applyDetails(tentative=False, overwrite=False):
"""To-do: Time doing overwrite vs checking and overwriting if new """
global jobtable #make global so that functions like fromJobObject() can access correct one
jobtable = 'tentativejob' if tentative else 'job'
querytable = tentativetable if tentative else defaulttable
newjobs = [
i[0] for i in plotQuery(['jobid'], [], jobtable)
if i[0] > countDB('details')
]
for i in newjobs:
sqlexecute('INSERT INTO details (jobtableid) VALUES (%d)' % i)
for i, d in enumerate(activeDetails):
print d.name, " %d/%d" % (i + 1, len(activeDetails))
try:
addCol(d.name, d.kind, 'details')
except OperationalError:
pass
vals = d.apply(querytable)
ids = allJobIDs()
for ID, val in zip(ids, vals):
if overwrite or (val is not None and query1(
d.name, 'jobtableid', ID, 'details') is None):
updateDB(d.name, 'jobtableid', ID, val, None, 'details')
def hashAndAdd(file):
# Check if it's a valid MP3 file first by trying to get the ID3 info
try:
title, artist, album = mp3.getid3(file)
except Exception as e:
# So far the only exception is an invalid ID3 header found, so not much to grab
print(e)
return
mtime = os.path.getmtime(file)
(exists,dbmtime) = db.checkIfExists(dbcursor, unicode(str(os.path.abspath(file)).decode('utf-8')))
update = False
# Gets back a tuple with (count of rows, mtime)
# Check if the file has already been hashed
if exists > 0:
# If the file hasn't been modified since it was checked, don't bother hashing it
if dbmtime >= mtime:
return
else:
# Need to come up with an update statement...
print("Updating", file)
update = True
tempfile = mp3.stripid3(file)
strippedhash = hash.sha512file(tempfile[1])
os.close(tempfile[0])
os.remove(tempfile[1])
originalhash = hash.sha512file(file)
info = mp3info(title, artist, album, unicode(strippedhash), unicode(originalhash), unicode(str(os.path.abspath(file)).decode('utf-8')), mtime)
if not update:
print(info,"Ins")
db.insertIntoDB(dbcursor, info)
else:
#print(info,"upd")
db.updateDB(dbcursor, info)
dbconn.commit()
def setnJson(propsArray, apiRoute, oJson, title, description, update):
nJson = {}
nJson['title'] = title
nJson['description'] = description
nJson['update'] = update
nJson['header'] = setHeader(propsArray, apiRoute)
nJson['data'] = setData(nJson['header'], oJson)
db.updateDB(nJson)
print('done')
def archiveFizzled():
updateFizzled()
fizzled = [lpadIDjob(x) for x in getStatus("FIZZLED")]
for f in fizzled:
err = f.err()
if err is None:
updateDB('error', 'jobid', f.jobid, 'TIMEOUT', None, 'job')
updateStatus(f.jobid, 'fizzled', 'archived')
lpad.archive_wf(f.id)
elif any([x in err for x in archiveErrors]):
updateDB('error', 'jobid', f.jobid, err, None, 'job')
updateStatus(f.jobid, 'fizzled', 'archived')
lpad.archive_wf(f.id)
def run_task(self,fw_spec):
jobID = fw_spec['jobID']
job = db2SurfaceJob(jobID)
resultNames = ['pos','magmom','e0','f0','xcContribs','avgtime','niter']
results = [fw_spec[x][0] for x in resultNames]
resultDict = {n:fw_spec[n] for n in resultNames}
result = SurfaceResult(*([jobID]+results))
result.addToASEdb(job)
insertObject(result) # add result to surface result table
updateDB('surfacejob','status','id',jobID,'complete')
return FWAction(stored_data= resultDict)
def submit(self):
from fireworks import LaunchPad
launchpad = LaunchPad(host='suncatls2.slac.stanford.edu',
name='krisbrown',
username='******',
password='******')
if self.jobkind == 'bulkrelax': wflow = self.submitBulkRelax()
elif self.jobkind == 'relax': wflow = self.submitRelax()
elif self.jobkind == 'vib': wflow = self.submitVib()
elif self.jobkind == 'neb': wflow = self.submitNEB()
elif self.jobkind == 'dos': wflow = self.submitDOS()
print "Submitting job with ID = %d" % self.jobid
updateDB('status', 'jobid', self.jobid, 'queued', None, 'job')
launchpad.add_wf(wflow)
"""if query1('status','jobid',self.jobid,'job')=='initialized': updateStatus(self.jobid,'initialized','queued') """
def updateDB(self, dbconnection):
"""
updated the predicted relevance score & relevance flag for the unlabeled data in search results
@type dbconnection: psycopg2 connection object
@param dbconnection: The established connection object to the Google Search Result database
"""
db.updateDB(dbconnection,
value=self.AllUpdate_Score,
idlist=self.AllUpdate_Id,
type='relevanceScore',
dbtable=db._DBTABLE_,
dbfields=db._DBFIELDS_)
db.updateDB(dbconnection,
value=self.AllUpdate_Label,
idlist=self.id_test,
type='label',
dbtable=db._DBTABLE_,
dbfields=db._DBFIELDS_)
def hashAndAdd(file):
mtime = os.path.getmtime(file)
(exists,dbmtime) = db.checkIfExists(dbcursor, unicode(str(os.path.abspath(file)).decode('utf-8')))
update = False
# Gets back a tuple with (count of rows, mtime)
# Check if the file has already been hashed
if exists > 0:
# If the file hasn't been modified since it was checked, don't bother hashing it
if dbmtime >= mtime:
return
else:
# Need to come up with an update statement...
print("Updating", file)
update = True
hashresult = hash.sha512file(file)
info = fileinfo(unicode(hashresult), unicode(str(os.path.abspath(file)).decode('utf-8')), mtime)
if not update:
print(info,"Ins")
db.insertIntoDB(dbcursor, info)
else:
print(info,"upd")
db.updateDB(dbcursor, info)
dbconn.commit()
def run_task(self,fw_spec):
jobID = fw_spec['jobID']
job = db2object(jobID)
if 'relax' in job.jobkind:
pos = fw_spec['pos'][0]
cell = fw_spec['cell'][0] if job.jobkind == 'bulkrelax' else job.cellinit()
magmom = fw_spec['magmom'][0]
db = connect('/scratch/users/ksb/db/ase.db')
info = {'name': job.name()
,'emt': 0 # EMT for relaxed structures useless, only relevant for deciding when to relax something
,'relaxed': True # Could always doing this be a problem?
,'comments': 'Generated from #%d'%(job.aseidinitial)
,'parent': job.aseidinitial
,'jobid': job.jobid # Int
###Unchanged Things###
,'kind': job.kind()} # String
if job.structure() is not None: info['structure'] = job.structure()
if job.vacancies() is not None: info['vacancies'] = job.vacancies()
if job.sites() is not None: info['sites'] = job.sites()
if job.facet() is not None: info['facet'] = job.facet()
if job.xy() is not None: info['xy'] = job.xy()
if job.layers() is not None: info['layers'] = job.layers()
if job.constrained() is not None: info['constrained'] = job.constrained()
if job.symmetric() is not None: info['symmetric'] = job.symmetric()
if job.vacuum() is not None: info['vacuum'] = job.vacuum()
if job.adsorbates() is not None: info['adsorbates'] = job.adsorbates()
optAtoms = Atoms(symbols = job.symbols()
,scaled_positions = pos
,cell = cell
,magmoms = magmom
,constraint = job.constraints()
,tags = job.tags())
aseid = db.write(optAtoms,key_value_pairs=info)
else:
aseid = None
launchdir = fw_spec['_launch_dir']
energy = fw_spec['e0'][0] if 'relax' in job.jobkind else None
forces = fw_spec['f0'][0] if 'relax' in job.jobkind else None
bulkmodulus = fw_spec['b0'][0] if job.jobkind == 'bulkrelax' else None
bfit = fw_spec['quadR2'][0] if job.jobkind == 'bulkrelax' else None
xccoeffs = fw_spec['xcContribs'][0] if 'relax' in job.jobkind else None
viblist = fw_spec['vibs'][0] if job.jobkind == 'vib' else None
dos = fw_spec['dos'][0] if job.jobkind == 'dos' else None
barrier = fw_spec['barrier'][0] if job.jobkind == 'neb' else None
time = fw_spec['avgtime'][0]
niter = fw_spec['niter'][0] if 'relax' in job.jobkind else None
result = Result(jobID,launchdir,aseid=aseid,energy=energy,forces=forces
,bulkmodulus=bulkmodulus,bfit=bfit,xccoeffs=xccoeffs,viblist=viblist
,dos=dos,barrier=barrier,time=time,niter=niter)
resultDict = {'_launch_dir':launchdir,'aseid':aseid,'energy':energy
,'forces':str(forces),'bulkmodulus':bulkmodulus
,'bfit':bfit,'xccoeffs':xccoeffs,'time':time
,'niter':niter,'viblist':viblist,'barrier':barrier}
insertObject(result) # add result to bulkresult table
updateDB('status','jobid',jobID,'complete')
return FWAction(stored_data= resultDict)
def run(assetId):
ret = {}
cmdbObj = cmdbApi.getObjectById(str(assetId))['result']
if not cmdbObj:
return (False)
argv = {}
method = cmdbApi.getObjectById(
cmdbObj['requestmethod'])['result']['method']
if (method == "GET"):
argv['url'] = cmdbObj['url'] + "?" + cmdbObj['param']
argv['posts'] = ""
else:
argv['url'] = cmdbObj['url']
argv['posts'] = cmdbObj['param']
hostname = getHostName(cmdbObj['product'])
argv['name'] = getScenarioName(hostname, cmdbObj['url'])
argv['name'] = argv['name'][-64:]
argv['status_code'] = cmdbObj['responsecode']
argv['no'] = 1
argv['hostid'] = getHostId(hostname)
argv['delay'] = cmdbObj['delay']
if not argv['delay'].isdigit():
argv['delay'] = 60
argv['required'] = cmdbObj['responsedata']
argv['agent'] = agent
argv['applicationid'] = getApplicationId(argv['hostid'], argv['name'],
cmdbObj['applicationid'])
argv['status'] = getStatus(cmdbObj['status'])
argv['header'] = cmdbObj['header']
cmdbObj['curl'] = getCurlCmd(cmdbObj['url'], method, cmdbObj['header'],
cmdbObj['param'])
# 如果某个cmdb项目已经在zabbix创建过，则不以web监控名称判断web monitor是否存在(存在cmdb中修改URL的情况)
if cmdbObj['httptestid']:
scenario = web.getScenarioById(cmdbObj['httptestid'])
if not scenario:
scenario = web.getScenarioByName(argv['hostid'], argv['name'])
#如果web scenario名称变更，删除过期的触发器
elif not scenario[0]['name'] == argv['name']:
deleteTriggers(cmdbObj['triggerid'])
else:
scenario = web.getScenarioByName(argv['hostid'], argv['name'])
if scenario:
# 如果接口产品线变更（对应zabbix主机名变更）需要重新获得applicationid
if not scenario[0]['hostid'] == argv['hostid']:
application.deleteApplication(argv['applicationid'])
web.deleteScenario(scenario[0]['httptestid'])
argv['applicationid'] = getApplicationId(argv['hostid'],
argv['name'])
data = web.createScenario(argv)
argv['httptestid'] = data['httptestids'][0]
else:
argv['httptestid'] = scenario[0]['httptestid']
argv['httpstepid'] = scenario[0]['steps'][0]['httpstepid']
data = web.updateScenario(argv)
#triggername = data
else:
data = web.createScenario(argv)
argv['httptestid'] = data['httptestids'][0]
ret[assetId] = data
# trigger
if not cmdbObj['timeout'].isdigit():
cmdbObj['timeout'] = 8
if not cmdbObj['timeoutcount'].isdigit():
cmdbObj['timeoutcount'] = 10
if not cmdbObj['failcount'].isdigit():
cmdbObj['failcount'] = 3
triggers = getTriggerExps(hostname, argv['name'], argv['delay'],
cmdbObj['failcount'], cmdbObj['timeoutcount'],
cmdbObj['timeout'])
try:
triggerids = json.loads(cmdbObj['triggerid'])
except:
triggerids = None
argv['triggerid'] = json.dumps(getTriggerId(triggers, triggerids),
sort_keys=True)
#update cmdb
cmdb_argv = {}
cmdb_argv['objtype'] = "API"
cmdb_argv['objid'] = str(assetId)
cmdb_argv['objfields'] = {
'Zabbix Info': [{
'name': 'httptestid',
'value': argv['httptestid']
}, {
'name': 'applicationid',
'value': argv['applicationid']
}, {
'name': 'triggerid',
'value': argv['triggerid']
}],
"Basic": [{
"name": "url",
"label": "URL",
"type": "text",
"value": cmdbObj['url']
}],
"Extend": [{
"name": "curl",
"value": cmdbObj['curl']
}]
}
print(cmdbApi.updateObject(cmdb_argv))
if not cmdbObj['httptestid']:
cmdbObj['httptestid'] = argv['httptestid']
db.updateDB(argv['hostid'], argv['applicationid'], cmdbObj)
return (ret)
import os
from db import updateStatus, query1, updateDB
root = '/scratch/users/ksb/db/jobs'
for d in os.listdir(root):
try:
print 'trying ', d
with open('/scratch/users/ksb/db/jobs/' + d + '/myjob.err', 'r') as f:
err = f.read()
if 'TIME LIMIT' in err:
if query1('bulkjob', 'status', 'id', d) is not 'failed':
updateStatus(int(d), 'running', 'timeout')
elif 'CANCELLED AT' in err:
updateDB('bulkjob', 'status', 'id', int(d), 'cancelled')
except IOError:
pass
def sendAndSaveDocument():
print "Entre"
doc_id = request.args['doc_id']
user_type = request.args['user_type']
sent_to = request.args['sent_to']
message = request.args['message']
action = request.args['action'] if request.args['action'] else "Sent"
if action == "Authorize":
action_title = "Waiting for authorization"
if action == "Sign":
action_title = "Waiting for signature"
if action == "Verify":
action_title = "Waiting for verification"
if action == "Endose":
action_title = "Waiting for endorsement"
if action == "Sent":
action_title = "Sent"
# status = request.args['status'] ? request.args['status'] else "Sent"
sent = {'status':-1}
print doc_id, user_type, sent_to, sent, action
# If document is being sent by student
if int(user_type) == 0:
print "Soy estudiante"
documents = readDB('data/student_documents.json')["documents"]
print "documents",documents
# Getting documment
for doc in documents:
# print "Entre al doc"
print doc["id"], doc_id
if int(doc["id"]) == int(doc_id):
document = doc
# elif doc["name"] == doc_name + " Travel Request":
# document = doc
document["status"] = action_title
document["sent_status"] = "sent"
document["last_edited"] = "Student: Jessica Cotrina"
document["message"] = message
document["action"] = action
document["sent_to"] = sent_to
# document[""]
updateDB('data/student_documents.json',document)
# If document is being sent by professor
if int(user_type) == 1:
print "Soy profesor"
documents = readDB('data/professor_documents.json')["documents"]
# Getting documment
for doc in documents:
# print "Entre al doc"
print doc["id"], doc_id
if int(doc["id"]) == int(doc_id):
# print "True"
document = doc
# elif doc["name"] == doc_name + " Travel Request":
# document = doc
document["status"] = action
document["sent_status"] = "sent"
document["last_edited"] = "Professor: Nestor Rodriguez"
document["message"] = message
document["action"] = action
document["sent_to"] = sent_to
updateDB('data/professor_documents.json',document)
# If document is being sent by assistant
if int(user_type) == 2:
print "Soy Asistente"
documents = readDB('data/assistant_documents.json')["documents"]
# Getting documment
for doc in documents:
# print "Entre al doc"
print doc["id"], doc_id
if int(doc["id"]) == int(doc_id):
# print "True"
document = doc
# elif doc["name"] == doc_name + " Travel Request":
# document = doc
document["status"] = action
document["sent_status"] = "sent"
document["last_edited"] = "Assistant: Alida Minguela"
document["message"] = message
document["action"] = action
document["sent_to"] = sent_to
updateDB('data/assistant_documents.json',document)
# If document is being sent by director
if int(user_type) == 3:
print "Soy director"
documents = readDB('data/director_documents.json')["documents"]
# Getting documment
for doc in documents:
# print "Entre al doc"
print doc["id"], doc_id
if int(doc["id"]) == int(doc_id):
# print "True"
document = doc
# elif doc["name"] == doc_name + " Travel Request":
# document = doc
document["status"] = action
document["sent_status"] = "sent"
document["last_edited"] = "Director: Jose Colom"
document["message"] = message
document["action"] = action
document["sent_to"] = sent_to
updateDB('data/director_documents.json',document)
# If document being sent to professor
if sent_to == "*****@*****.**":
document["sent_status"] = "received"
documents = readDB('data/professor_documents.json')["documents"]
print "Professor Documents",documents
document_exists = False
for d in documents:
if int(d["id"]) == int(doc_id):
document_exists = True
print "Document Exists? ", document_exists
if document_exists == True:
updateDB('data/professor_documents.json',document)
else:
writeDB('data/professor_documents.json',document)
writeDB('data/document_history.json',document)
sent = {'status':0}
# If document being set to assistant
if sent_to == "*****@*****.**":
document["sent_status"] = "received"
documents = readDB('data/assistant_documents.json')["documents"]
document_exists = False
for d in documents:
if int(d["id"]) == int(doc_id):
document_exists = True
print "Document Exists? ", document_exists
if document_exists == True:
updateDB('data/assistant_documents.json',document)
else:
writeDB('data/assistant_documents.json',document)
writeDB('data/document_history.json',document)
sent = {'status':0}
# If document being sent to director
if sent_to == "*****@*****.**":
document["sent_status"] = "received"
documents = readDB('data/director_documents.json')["documents"]
document_exists = False
for d in documents:
if int(d["id"]) == int(doc_id):
document_exists = True
print "Document Exists? ", document_exists
if document_exists == True:
updateDB('data/director_documents.json',document)
else:
writeDB('data/director_documents.json',document)
writeDB('data/document_history.json',document)
sent = {'status':0}
# If document being sent to student
if sent_to == "*****@*****.**":
document["sent_status"] = "received"
documents = readDB('data/student_documents.json')["documents"]
document_exists = False
for d in documents:
if int(d["id"]) == int(doc_id):
document_exists = True
print "Document Exists? ", document_exists
if document_exists == True:
updateDB('data/student_documents.json',document)
else:
writeDB('data/student_documents.json',document)
writeDB('data/document_history.json',document)
sent = {'status':0}
return jsonify(sent)
def saveAssistantship():
user_type = request.args['user_type']
doc_id = request.args['doc_id']
project = request.args['project']
advisor = request.args['advisor']
student = request.args['student']
task = request.args['task']
saved = {'status':-1}
if int(user_type) == 0:
documents = readDB('data/student_documents.json')["documents"]
for document in documents:
if int(document["id"]) == int(doc_id):
doc = document
doc["project"] = project
doc["advisor"] = advisor
doc["student"] = student
doc["task"] = task
doc["last_edited"] = "Student: Jessica Cotrina"
print "doc", doc
updateDB('data/student_documents.json',doc)
writeDB('data/document_history.json',doc)
saved = {'status':0,'doc_id':doc_id}
if int(user_type) == 1:
documents = readDB('data/professor_documents.json')["documents"]
for document in documents:
if int(document["id"]) == int(doc_id):
doc = document
doc["project"] = project
doc["advisor"] = advisor
doc["student"] = student
doc["task"] = task
doc["last_edited"] = "Professor: Nestor Rodriguez"
updateDB('data/profssor_documents.json',doc)
writeDB('data/document_history.json',doc)
saved = {'status':0,'doc_id':doc_id}
if int(user_type) == 2:
documents = readDB('data/assistant_documents.json')["documents"]
for document in documents:
if int(document["id"]) == int(doc_id):
doc = document
doc["project"] = project
doc["advisor"] = advisor
doc["student"] = student
doc["task"] = task
doc["last_edited"] = "Assistant: Alida Minguela"
updateDB('data/assistant_documents.json',doc)
writeDB('data/document_history.json',doc)
saved = {'status':0,'doc_id':doc_id}
if int(user_type) == 3:
documents = readDB('data/director_documents.json')["documents"]
for document in documents:
if int(document["id"]) == int(doc_id):
doc = document
doc["project"] = project
doc["advisor"] = advisor
doc["student"] = student
doc["task"] = task
doc["last_edited"] = "Director: Jessica Cotrina"
updateDB('data/director_documents.json',doc)
writeDB('data/document_history.json',doc)
saved = {'status':0,'doc_id':doc_id}
return jsonify(saved)