def applyDetails(tentative=False, overwrite=False): """To-do: Time doing overwrite vs checking and overwriting if new """ global jobtable #make global so that functions like fromJobObject() can access correct one jobtable = 'tentativejob' if tentative else 'job' querytable = tentativetable if tentative else defaulttable newjobs = [ i[0] for i in plotQuery(['jobid'], [], jobtable) if i[0] > countDB('details') ] for i in newjobs: sqlexecute('INSERT INTO details (jobtableid) VALUES (%d)' % i) for i, d in enumerate(activeDetails): print d.name, " %d/%d" % (i + 1, len(activeDetails)) try: addCol(d.name, d.kind, 'details') except OperationalError: pass vals = d.apply(querytable) ids = allJobIDs() for ID, val in zip(ids, vals): if overwrite or (val is not None and query1( d.name, 'jobtableid', ID, 'details') is None): updateDB(d.name, 'jobtableid', ID, val, None, 'details')
def hashAndAdd(file): # Check if it's a valid MP3 file first by trying to get the ID3 info try: title, artist, album = mp3.getid3(file) except Exception as e: # So far the only exception is an invalid ID3 header found, so not much to grab print(e) return mtime = os.path.getmtime(file) (exists,dbmtime) = db.checkIfExists(dbcursor, unicode(str(os.path.abspath(file)).decode('utf-8'))) update = False # Gets back a tuple with (count of rows, mtime) # Check if the file has already been hashed if exists > 0: # If the file hasn't been modified since it was checked, don't bother hashing it if dbmtime >= mtime: return else: # Need to come up with an update statement... print("Updating", file) update = True tempfile = mp3.stripid3(file) strippedhash = hash.sha512file(tempfile[1]) os.close(tempfile[0]) os.remove(tempfile[1]) originalhash = hash.sha512file(file) info = mp3info(title, artist, album, unicode(strippedhash), unicode(originalhash), unicode(str(os.path.abspath(file)).decode('utf-8')), mtime) if not update: print(info,"Ins") db.insertIntoDB(dbcursor, info) else: #print(info,"upd") db.updateDB(dbcursor, info) dbconn.commit()
def setnJson(propsArray, apiRoute, oJson, title, description, update): nJson = {} nJson['title'] = title nJson['description'] = description nJson['update'] = update nJson['header'] = setHeader(propsArray, apiRoute) nJson['data'] = setData(nJson['header'], oJson) db.updateDB(nJson) print('done')
def archiveFizzled(): updateFizzled() fizzled = [lpadIDjob(x) for x in getStatus("FIZZLED")] for f in fizzled: err = f.err() if err is None: updateDB('error', 'jobid', f.jobid, 'TIMEOUT', None, 'job') updateStatus(f.jobid, 'fizzled', 'archived') lpad.archive_wf(f.id) elif any([x in err for x in archiveErrors]): updateDB('error', 'jobid', f.jobid, err, None, 'job') updateStatus(f.jobid, 'fizzled', 'archived') lpad.archive_wf(f.id)
def run_task(self,fw_spec): jobID = fw_spec['jobID'] job = db2SurfaceJob(jobID) resultNames = ['pos','magmom','e0','f0','xcContribs','avgtime','niter'] results = [fw_spec[x][0] for x in resultNames] resultDict = {n:fw_spec[n] for n in resultNames} result = SurfaceResult(*([jobID]+results)) result.addToASEdb(job) insertObject(result) # add result to surface result table updateDB('surfacejob','status','id',jobID,'complete') return FWAction(stored_data= resultDict)
def submit(self): from fireworks import LaunchPad launchpad = LaunchPad(host='suncatls2.slac.stanford.edu', name='krisbrown', username='******', password='******') if self.jobkind == 'bulkrelax': wflow = self.submitBulkRelax() elif self.jobkind == 'relax': wflow = self.submitRelax() elif self.jobkind == 'vib': wflow = self.submitVib() elif self.jobkind == 'neb': wflow = self.submitNEB() elif self.jobkind == 'dos': wflow = self.submitDOS() print "Submitting job with ID = %d" % self.jobid updateDB('status', 'jobid', self.jobid, 'queued', None, 'job') launchpad.add_wf(wflow) """if query1('status','jobid',self.jobid,'job')=='initialized': updateStatus(self.jobid,'initialized','queued') """
def updateDB(self, dbconnection): """ updated the predicted relevance score & relevance flag for the unlabeled data in search results @type dbconnection: psycopg2 connection object @param dbconnection: The established connection object to the Google Search Result database """ db.updateDB(dbconnection, value=self.AllUpdate_Score, idlist=self.AllUpdate_Id, type='relevanceScore', dbtable=db._DBTABLE_, dbfields=db._DBFIELDS_) db.updateDB(dbconnection, value=self.AllUpdate_Label, idlist=self.id_test, type='label', dbtable=db._DBTABLE_, dbfields=db._DBFIELDS_)
def hashAndAdd(file): mtime = os.path.getmtime(file) (exists,dbmtime) = db.checkIfExists(dbcursor, unicode(str(os.path.abspath(file)).decode('utf-8'))) update = False # Gets back a tuple with (count of rows, mtime) # Check if the file has already been hashed if exists > 0: # If the file hasn't been modified since it was checked, don't bother hashing it if dbmtime >= mtime: return else: # Need to come up with an update statement... print("Updating", file) update = True hashresult = hash.sha512file(file) info = fileinfo(unicode(hashresult), unicode(str(os.path.abspath(file)).decode('utf-8')), mtime) if not update: print(info,"Ins") db.insertIntoDB(dbcursor, info) else: print(info,"upd") db.updateDB(dbcursor, info) dbconn.commit()
def run_task(self,fw_spec): jobID = fw_spec['jobID'] job = db2object(jobID) if 'relax' in job.jobkind: pos = fw_spec['pos'][0] cell = fw_spec['cell'][0] if job.jobkind == 'bulkrelax' else job.cellinit() magmom = fw_spec['magmom'][0] db = connect('/scratch/users/ksb/db/ase.db') info = {'name': job.name() ,'emt': 0 # EMT for relaxed structures useless, only relevant for deciding when to relax something ,'relaxed': True # Could always doing this be a problem? ,'comments': 'Generated from #%d'%(job.aseidinitial) ,'parent': job.aseidinitial ,'jobid': job.jobid # Int ###Unchanged Things### ,'kind': job.kind()} # String if job.structure() is not None: info['structure'] = job.structure() if job.vacancies() is not None: info['vacancies'] = job.vacancies() if job.sites() is not None: info['sites'] = job.sites() if job.facet() is not None: info['facet'] = job.facet() if job.xy() is not None: info['xy'] = job.xy() if job.layers() is not None: info['layers'] = job.layers() if job.constrained() is not None: info['constrained'] = job.constrained() if job.symmetric() is not None: info['symmetric'] = job.symmetric() if job.vacuum() is not None: info['vacuum'] = job.vacuum() if job.adsorbates() is not None: info['adsorbates'] = job.adsorbates() optAtoms = Atoms(symbols = job.symbols() ,scaled_positions = pos ,cell = cell ,magmoms = magmom ,constraint = job.constraints() ,tags = job.tags()) aseid = db.write(optAtoms,key_value_pairs=info) else: aseid = None launchdir = fw_spec['_launch_dir'] energy = fw_spec['e0'][0] if 'relax' in job.jobkind else None forces = fw_spec['f0'][0] if 'relax' in job.jobkind else None bulkmodulus = fw_spec['b0'][0] if job.jobkind == 'bulkrelax' else None bfit = fw_spec['quadR2'][0] if job.jobkind == 'bulkrelax' else None xccoeffs = fw_spec['xcContribs'][0] if 'relax' in job.jobkind else None viblist = fw_spec['vibs'][0] if job.jobkind == 'vib' else None dos = fw_spec['dos'][0] if job.jobkind == 'dos' else None barrier = fw_spec['barrier'][0] if job.jobkind == 'neb' else None time = fw_spec['avgtime'][0] niter = fw_spec['niter'][0] if 'relax' in job.jobkind else None result = Result(jobID,launchdir,aseid=aseid,energy=energy,forces=forces ,bulkmodulus=bulkmodulus,bfit=bfit,xccoeffs=xccoeffs,viblist=viblist ,dos=dos,barrier=barrier,time=time,niter=niter) resultDict = {'_launch_dir':launchdir,'aseid':aseid,'energy':energy ,'forces':str(forces),'bulkmodulus':bulkmodulus ,'bfit':bfit,'xccoeffs':xccoeffs,'time':time ,'niter':niter,'viblist':viblist,'barrier':barrier} insertObject(result) # add result to bulkresult table updateDB('status','jobid',jobID,'complete') return FWAction(stored_data= resultDict)
def run(assetId): ret = {} cmdbObj = cmdbApi.getObjectById(str(assetId))['result'] if not cmdbObj: return (False) argv = {} method = cmdbApi.getObjectById( cmdbObj['requestmethod'])['result']['method'] if (method == "GET"): argv['url'] = cmdbObj['url'] + "?" + cmdbObj['param'] argv['posts'] = "" else: argv['url'] = cmdbObj['url'] argv['posts'] = cmdbObj['param'] hostname = getHostName(cmdbObj['product']) argv['name'] = getScenarioName(hostname, cmdbObj['url']) argv['name'] = argv['name'][-64:] argv['status_code'] = cmdbObj['responsecode'] argv['no'] = 1 argv['hostid'] = getHostId(hostname) argv['delay'] = cmdbObj['delay'] if not argv['delay'].isdigit(): argv['delay'] = 60 argv['required'] = cmdbObj['responsedata'] argv['agent'] = agent argv['applicationid'] = getApplicationId(argv['hostid'], argv['name'], cmdbObj['applicationid']) argv['status'] = getStatus(cmdbObj['status']) argv['header'] = cmdbObj['header'] cmdbObj['curl'] = getCurlCmd(cmdbObj['url'], method, cmdbObj['header'], cmdbObj['param']) # 如果某个cmdb项目已经在zabbix创建过,则不以web监控名称判断web monitor是否存在(存在cmdb中修改URL的情况) if cmdbObj['httptestid']: scenario = web.getScenarioById(cmdbObj['httptestid']) if not scenario: scenario = web.getScenarioByName(argv['hostid'], argv['name']) #如果web scenario名称变更,删除过期的触发器 elif not scenario[0]['name'] == argv['name']: deleteTriggers(cmdbObj['triggerid']) else: scenario = web.getScenarioByName(argv['hostid'], argv['name']) if scenario: # 如果接口产品线变更(对应zabbix主机名变更)需要重新获得applicationid if not scenario[0]['hostid'] == argv['hostid']: application.deleteApplication(argv['applicationid']) web.deleteScenario(scenario[0]['httptestid']) argv['applicationid'] = getApplicationId(argv['hostid'], argv['name']) data = web.createScenario(argv) argv['httptestid'] = data['httptestids'][0] else: argv['httptestid'] = scenario[0]['httptestid'] argv['httpstepid'] = scenario[0]['steps'][0]['httpstepid'] data = web.updateScenario(argv) #triggername = data else: data = web.createScenario(argv) argv['httptestid'] = data['httptestids'][0] ret[assetId] = data # trigger if not cmdbObj['timeout'].isdigit(): cmdbObj['timeout'] = 8 if not cmdbObj['timeoutcount'].isdigit(): cmdbObj['timeoutcount'] = 10 if not cmdbObj['failcount'].isdigit(): cmdbObj['failcount'] = 3 triggers = getTriggerExps(hostname, argv['name'], argv['delay'], cmdbObj['failcount'], cmdbObj['timeoutcount'], cmdbObj['timeout']) try: triggerids = json.loads(cmdbObj['triggerid']) except: triggerids = None argv['triggerid'] = json.dumps(getTriggerId(triggers, triggerids), sort_keys=True) #update cmdb cmdb_argv = {} cmdb_argv['objtype'] = "API" cmdb_argv['objid'] = str(assetId) cmdb_argv['objfields'] = { 'Zabbix Info': [{ 'name': 'httptestid', 'value': argv['httptestid'] }, { 'name': 'applicationid', 'value': argv['applicationid'] }, { 'name': 'triggerid', 'value': argv['triggerid'] }], "Basic": [{ "name": "url", "label": "URL", "type": "text", "value": cmdbObj['url'] }], "Extend": [{ "name": "curl", "value": cmdbObj['curl'] }] } print(cmdbApi.updateObject(cmdb_argv)) if not cmdbObj['httptestid']: cmdbObj['httptestid'] = argv['httptestid'] db.updateDB(argv['hostid'], argv['applicationid'], cmdbObj) return (ret)
import os from db import updateStatus, query1, updateDB root = '/scratch/users/ksb/db/jobs' for d in os.listdir(root): try: print 'trying ', d with open('/scratch/users/ksb/db/jobs/' + d + '/myjob.err', 'r') as f: err = f.read() if 'TIME LIMIT' in err: if query1('bulkjob', 'status', 'id', d) is not 'failed': updateStatus(int(d), 'running', 'timeout') elif 'CANCELLED AT' in err: updateDB('bulkjob', 'status', 'id', int(d), 'cancelled') except IOError: pass
def sendAndSaveDocument(): print "Entre" doc_id = request.args['doc_id'] user_type = request.args['user_type'] sent_to = request.args['sent_to'] message = request.args['message'] action = request.args['action'] if request.args['action'] else "Sent" if action == "Authorize": action_title = "Waiting for authorization" if action == "Sign": action_title = "Waiting for signature" if action == "Verify": action_title = "Waiting for verification" if action == "Endose": action_title = "Waiting for endorsement" if action == "Sent": action_title = "Sent" # status = request.args['status'] ? request.args['status'] else "Sent" sent = {'status':-1} print doc_id, user_type, sent_to, sent, action # If document is being sent by student if int(user_type) == 0: print "Soy estudiante" documents = readDB('data/student_documents.json')["documents"] print "documents",documents # Getting documment for doc in documents: # print "Entre al doc" print doc["id"], doc_id if int(doc["id"]) == int(doc_id): document = doc # elif doc["name"] == doc_name + " Travel Request": # document = doc document["status"] = action_title document["sent_status"] = "sent" document["last_edited"] = "Student: Jessica Cotrina" document["message"] = message document["action"] = action document["sent_to"] = sent_to # document[""] updateDB('data/student_documents.json',document) # If document is being sent by professor if int(user_type) == 1: print "Soy profesor" documents = readDB('data/professor_documents.json')["documents"] # Getting documment for doc in documents: # print "Entre al doc" print doc["id"], doc_id if int(doc["id"]) == int(doc_id): # print "True" document = doc # elif doc["name"] == doc_name + " Travel Request": # document = doc document["status"] = action document["sent_status"] = "sent" document["last_edited"] = "Professor: Nestor Rodriguez" document["message"] = message document["action"] = action document["sent_to"] = sent_to updateDB('data/professor_documents.json',document) # If document is being sent by assistant if int(user_type) == 2: print "Soy Asistente" documents = readDB('data/assistant_documents.json')["documents"] # Getting documment for doc in documents: # print "Entre al doc" print doc["id"], doc_id if int(doc["id"]) == int(doc_id): # print "True" document = doc # elif doc["name"] == doc_name + " Travel Request": # document = doc document["status"] = action document["sent_status"] = "sent" document["last_edited"] = "Assistant: Alida Minguela" document["message"] = message document["action"] = action document["sent_to"] = sent_to updateDB('data/assistant_documents.json',document) # If document is being sent by director if int(user_type) == 3: print "Soy director" documents = readDB('data/director_documents.json')["documents"] # Getting documment for doc in documents: # print "Entre al doc" print doc["id"], doc_id if int(doc["id"]) == int(doc_id): # print "True" document = doc # elif doc["name"] == doc_name + " Travel Request": # document = doc document["status"] = action document["sent_status"] = "sent" document["last_edited"] = "Director: Jose Colom" document["message"] = message document["action"] = action document["sent_to"] = sent_to updateDB('data/director_documents.json',document) # If document being sent to professor if sent_to == "*****@*****.**": document["sent_status"] = "received" documents = readDB('data/professor_documents.json')["documents"] print "Professor Documents",documents document_exists = False for d in documents: if int(d["id"]) == int(doc_id): document_exists = True print "Document Exists? ", document_exists if document_exists == True: updateDB('data/professor_documents.json',document) else: writeDB('data/professor_documents.json',document) writeDB('data/document_history.json',document) sent = {'status':0} # If document being set to assistant if sent_to == "*****@*****.**": document["sent_status"] = "received" documents = readDB('data/assistant_documents.json')["documents"] document_exists = False for d in documents: if int(d["id"]) == int(doc_id): document_exists = True print "Document Exists? ", document_exists if document_exists == True: updateDB('data/assistant_documents.json',document) else: writeDB('data/assistant_documents.json',document) writeDB('data/document_history.json',document) sent = {'status':0} # If document being sent to director if sent_to == "*****@*****.**": document["sent_status"] = "received" documents = readDB('data/director_documents.json')["documents"] document_exists = False for d in documents: if int(d["id"]) == int(doc_id): document_exists = True print "Document Exists? ", document_exists if document_exists == True: updateDB('data/director_documents.json',document) else: writeDB('data/director_documents.json',document) writeDB('data/document_history.json',document) sent = {'status':0} # If document being sent to student if sent_to == "*****@*****.**": document["sent_status"] = "received" documents = readDB('data/student_documents.json')["documents"] document_exists = False for d in documents: if int(d["id"]) == int(doc_id): document_exists = True print "Document Exists? ", document_exists if document_exists == True: updateDB('data/student_documents.json',document) else: writeDB('data/student_documents.json',document) writeDB('data/document_history.json',document) sent = {'status':0} return jsonify(sent)
def saveAssistantship(): user_type = request.args['user_type'] doc_id = request.args['doc_id'] project = request.args['project'] advisor = request.args['advisor'] student = request.args['student'] task = request.args['task'] saved = {'status':-1} if int(user_type) == 0: documents = readDB('data/student_documents.json')["documents"] for document in documents: if int(document["id"]) == int(doc_id): doc = document doc["project"] = project doc["advisor"] = advisor doc["student"] = student doc["task"] = task doc["last_edited"] = "Student: Jessica Cotrina" print "doc", doc updateDB('data/student_documents.json',doc) writeDB('data/document_history.json',doc) saved = {'status':0,'doc_id':doc_id} if int(user_type) == 1: documents = readDB('data/professor_documents.json')["documents"] for document in documents: if int(document["id"]) == int(doc_id): doc = document doc["project"] = project doc["advisor"] = advisor doc["student"] = student doc["task"] = task doc["last_edited"] = "Professor: Nestor Rodriguez" updateDB('data/profssor_documents.json',doc) writeDB('data/document_history.json',doc) saved = {'status':0,'doc_id':doc_id} if int(user_type) == 2: documents = readDB('data/assistant_documents.json')["documents"] for document in documents: if int(document["id"]) == int(doc_id): doc = document doc["project"] = project doc["advisor"] = advisor doc["student"] = student doc["task"] = task doc["last_edited"] = "Assistant: Alida Minguela" updateDB('data/assistant_documents.json',doc) writeDB('data/document_history.json',doc) saved = {'status':0,'doc_id':doc_id} if int(user_type) == 3: documents = readDB('data/director_documents.json')["documents"] for document in documents: if int(document["id"]) == int(doc_id): doc = document doc["project"] = project doc["advisor"] = advisor doc["student"] = student doc["task"] = task doc["last_edited"] = "Director: Jessica Cotrina" updateDB('data/director_documents.json',doc) writeDB('data/document_history.json',doc) saved = {'status':0,'doc_id':doc_id} return jsonify(saved)