def updateGridTaskStatus(gridTask): #get GridWay status (result, status, error)=drmaa_job_ps(gridTask.gwID) #error control if result != DRMAA_ERRNO_SUCCESS: print >> sys.stderr, "updatingGridTaskStatus, drmaa_job_ps() failed for task " + gridTask.gwID + ". Error: %s" % (error) gridTask.status = "DONE" #DONE does not imply success, just that it has finished else: if gridTask.status == "CLEAR": #ya se ha acabado, nothing to do here return elif status == DRMAA_PS_UNDETERMINED or status == DRMAA_PS_QUEUED_ACTIVE: gridTask.status = "SUBMITTED" elif status == DRMAA_PS_RUNNING: #if needed, update task execution start if gridTask.status == "SUBMITTED" or gridTask.status=="WAITING": gridTask.executionStartDate = datetime.now() print ("Task " + gridTask.gwID + "with type " + gridTask.type + " on host " + gridTask.host.hostname + " with ID=" + str(gridTask.host.id) + " has started its execution") gridTask.status = "RUNNING" elif status == DRMAA_PS_DONE or status == DRMAA_PS_FAILED: gridTask.status = "DONE" Session.add(gridTask)
def updateStatus(self, gridTasks): #=============================================================================== # print ("updatestatus: UPDATING INFRASTRUCTURE STATUS") # # print(" numbver of tasks to process: " + str(len(gridTasks))) #=============================================================================== for host in self.hosts: host.currentSlotCount = 0 for gridTask in gridTasks: if gridTask.status == "RUNNING": try: gridTask.host.currentSlotCount +=1 except: pass #=============================================================== # print (" task running on host " + gridTask.host.hostname + "with id " + str(gridTask.host.id) +", that makes " + str(gridTask.host.currentSlotCount) + " tasks on the host") #=============================================================== for host in self.hosts: host.maxSlotCountThisTime = max(host.currentSlotCount, host.maxSlotCountThisTime) if host.maxSlotCountThisTime > host.maxSlotCount: host.maxSlotCount = host.maxSlotCountThisTime Session.add(host)
def updateInfoAfterExecutionInPilots(self, gridTask): self.updateInfoAfterExecution(gridTask) #queremos aumentar el numero de tareas otra vez (porque lo restamos al enviarla, y el updateInfo... lo vuelve a restar) # -si ha acabado OK, para que el numero de pendientes se quede como estaba # - si NO ha acabado Ok, para que se vuelva a incrementar, ya que lo restamos al enviarla self.remainingSamples += gridTask.realSamples Session.add(self)
def load(self): pastHosts = Session.query(Host).all() print ("INFRASTRUCTURE-LOAD: LOADING INFORMATION FROM PAST EXECUTIONS") print (" Desired LRMS: " + self.lrms) #obtain present hosts prom gridway #PILOTS pastPilots = Session.query(Pilot).all() print (" Deleting all past pilots from database") for pilot in pastPilots: base.Session.delete(pilot) presentHosts = [] GWHosts = obtainGWResources() if GWHosts == []: print("Error when parsing host information file, employing information from past executions") self.hosts = pastHosts return #load XML to memory and extract data from hosts for hostInfo in GWHosts: hostName = hostInfo.getElementsByTagName("HOSTNAME")[0].firstChild.data #TODO: remove "unicode" from TEXT #for every found host, check if it existed on a previous execution for host in pastHosts: if host.hostname.strip().lower() == hostName.strip().lower(): #in the case of pilot jobs, only employ the ones with FREENODECOUNT > 0 if host.lrms =="jobmanager-pilot": try: freeNodeCount = int(hostInfo.getElementsByTagName("FREENODECOUNT")[0].firstChild.data) except: freeNodeCount = 0 if not freeNodeCount > 0: continue; presentHosts.append(host) break # Si no pones este break, y está repetido en memoria, se cargan todos #now, if only hosts with a certain LRMS are desired, we remove the rest if self.lrms != None: for host in presentHosts: if host.lrms != self.lrms: print (" Removing host " + host.hostname + ", wrong LRMS found: " + host.lrms) presentHosts.remove(host) else: print (" Keeping host " + host.hostname) #the hosts that we will first employ are the resulting ones self.hosts = presentHosts
def getTask(): newParameter = Session.query(Parameter).filter(Parameter.status=="WAITING").first() if newParameter == None: return template('{{id}}', id=-1) newParameter.status="RUNNING" newParameter.executionStartDate = datetime.now() Session.add(newParameter) Session.commit() return template('{{id}}', id=newParameter.id)
def failedTask(idInput=-1): #por si acaso try: id = int(idInput) except: return template('-1') failedParameter = Session.query(Parameter).filter(Parameter.id==id).first() failedParameter.status="WAITING" failedParameter.executionStartDate = None Session.add(failedParameter) Session.commit() return template('{{id}}', id=failedParameter.id)
def finishedTask(idInput=-1): #por si acaso try: id = int(idInput) except: return template('-1') finishedParameter = Session.query(Parameter).filter(Parameter.id==id).first() finishedParameter.status="DONE" finishedParameter.executionEndDate = datetime.now() Session.add(finishedParameter) Session.commit() #TODO: en algun sitio habria que verificar los datos de salida antes del done return template('{{id}}', id=finishedParameter.id)
def analyzeParametricJobfile(self): tmpFile = "/tmp/tmpParameterFile" process = monteraLocation + '/templateManager -c ' + self.parametricJobFile + " > " + tmpFile parameters = self.runProcess(process) numTasks = 0 for line in open(tmpFile, 'r').readlines(): print ("Parameter: " + line) myParameter = Parameter(line) Session.add(myParameter) numTasks+=1 Session.commit()
def connectDB(): logger.debug(f"connectDB(): entered") db = Session() db_url = os.environ["DB_HEROKU"] db_url = "postgres://*****:*****@ec2-107-22-162-8.compute-1.amazonaws.com:5432/delmad8ora7a1v" engine = create_engine(db_url) Base.metadata.bind = engine DBSession = sessionmaker(bind=engine) session = DBSession() session.autoflush = True return session
def sendQuestions(): session = Session() questions = session.query(Question).all() numberOfQuestions = 10 # Number of questions in the quiz chosenQuestions = random.sample(questions, numberOfQuestions) finalQuestionList = [] for eachQuestion in chosenQuestions: finalQuestionList.append({ 'id': eachQuestion.id, 'question': eachQuestion.question, 'optionA': eachQuestion.optionA, 'optionB': eachQuestion.optionB, 'optionC': eachQuestion.optionC, 'optionD': eachQuestion.optionD, 'answer': eachQuestion.answer, 'trivia': eachQuestion.trivia }) return jsonify(finalQuestionList), 200
def __init__(self, url, script): self.url = url self.session = Session.get() self.headers = { 'Referer': BaseTest.stand + self.url, 'Content-Type': 'application/x-www-form-urlencoded' } self.script = script payload = {'Login': BaseTest.login, 'Password': BaseTest.password} login_url = BaseTest.stand + '/Auth/LogOn' self.session.post(url=login_url, data=payload) self.cookies = self.session.cookies.get_dict() self.headers = {**self.headers, **self.cookies}
def insert_data(item_type, account_id, item_id, auth): Base.metadata.create_all(engine) session = Session() # def __init__(self, id_id, name, create_date, no_devices, email, account): def create_user(): item_user = kp.get_items(item_type=item_type, account_id=account_id, item_id=item_id, auth=auth) print(item_user) data = item_user['data'] user_name = data['caller_id']['internal']['name'] email = data.get('email') account = session.query(Account).get(account_id) user = User(item_id, user_name, date(2015, 4, 2), 3, email, account) session.add(user) # def __init__(self, id_1, name, device_type): def create_device(): item_device = kp.get_items(item_type=item_type, account_id=account_id, item_id=item_id, auth=auth) print(item_device) data = item_device['data'] device_type = data.get('device_type') owner_id = data.get('owner_id') name = data.get('name') device = Device(item_id, name, device_type) user = session.query(User).get(owner_id) device.user = [user] session.add(device) # def __init__(self, id_1, name, create_date): def create_account(): item_account = kp.get_items(item_type=item_type, account_id=account_id, item_id=item_id, auth=auth) print(item_account) data = item_account['data'] account_name = data.get('name') account = Account(item_id, account_name, date(2015, 4, 2)) session.add(account) if item_type == 'user': create_user() if item_type == 'account': create_account() if item_type == 'device': create_device() session.commit() session.close()
def set_password(id, password): #TODO: Maybe check if user is authenticated? session = Session() account = session.query(User).filter(User.id == id).first() if account == None: print("No account with id " + id) return False account.passwordhash = hashlib.sha512(password).hexdigest() session.commit() session.close()
def editorOferta(offer_id): email = dict(session).get('email', None) given_name = dict(session).get('given_name', None) user = User.get_by_email(email) s = Session() query = s.query(Oferta) oferta = query.filter(Oferta.id == offer_id).first() if request.method == 'POST': dictupdate = { Oferta.nombre: request.form['nombre'], Oferta.descripcion: request.form['descripcion'], Oferta.precio: request.form['precio'] } query.filter(Oferta.id == offer_id).update(dictupdate, synchronize_session=False) s.commit() return render_template("editoroferta.html", oferta=oferta, email=email, name=given_name, user=user)
def main(filename): Base.metadata.create_all(Engine) session = Session() articles = pd.read_csv(filename) for index, row in articles.iterrows(): logger.info('loading, articles uid {} into DB'.format(row['uid'])) article = Article(row['uid'], row['body'], row['host'], row['newspaper_uid'], row['m_tokens_body'], row['n_tokens_title'], row['title'], row['url'] ) session.add(article) session.commit() session.close()
def updateAdd(): requestDetails = request.json userID = requestDetails["userID"] maskID = requestDetails["maskID"] session = Session() newOrder = Order(userID, maskID) session.add(newOrder) session.commit() session.close() return '', 200 # Successfully update the masks for the user
def main(filename): Base.metadata.create_all(Engine) session = Session() articles = pd.read_csv(filename) # uid,body,title,url,newspaper_uid,host,token_title,token_body for index, row in articles.iterrows(): logger.info(f'Saving article {index}') article = Article(row['uid'], row['body'], row['title'], row['url'], row['newspaper_uid'], row['host'], row['token_title'], row['token_body']) session.add(article) session.commit() session.close()
def handler(event, context): # check authorization authorized_user_types = [UserType.ADMIN, UserType.MENTOR, UserType.PAID] success, _ = check_auth(event['headers']['Authorization'], authorized_user_types) if not success: return http_status.unauthorized() # # create a new session session = Session() jobId = event["pathParameters"]["jobId"] job = session.query(Job).get(jobId) info = json.loads(event["body"]) if job != None: #if it was a valid jobid, and the job was found keys = info.keys() for key in keys: value = info[key] logger.info('key is ' + str(key) + ' and value is ' + str(value)) if key == 'salary': if value is None or value == '': value = 0 else: value = int(value) elif key == 'deadline': value = datetime.fromtimestamp(value).replace(hour=0, minute=0, second=0, microsecond=0) elif key == 'tags': value = [] if key == 'job_applications': continue setattr(job, key, value) session.commit() session.close() if "job_status" in keys and info["job_status"] is "ACTIVE": job_title = job.title today = datetime.today().strftime("%Y-%m-%d") hiring_manager = job.posted_by subject = "[MAX Aspire] Your job is now active!" body = f"Salaam!\r\n\nWe are delighted to confirm that the job posting {job_title} is successfully posted on MAX Aspire job board on {today}. You will frequently be notified about the job applications as and when received. Keep an eye on your member portal/email.\r\n\nWe appreciate you putting your trust in MAX Aspire. We wish you luck in hiring the best possible candidate form our talented pool of aspiring professionals.\r\n\nBest regards,\nTeam MAX Aspire\r\n" send_email(to_addresses=hiring_manager, subject=subject, body_text=body) return http_status.success() else: return http_status.not_found()
def email_to_id(self, email): session = Session() try: user = session.query(User).filter(User.email == email).one() except: print("Email", id, "not found...") session.close() return None else: session.close() return user.id
def main(filename): # Generating schema in DB Base.metadata.create_all(Engine) session = Session() articles = pd.read_csv(filename) # Iterating the DataFrame for index, row in articles.iterrows(): logger.info('Loading article uid {} into DB'.format(row['uid'])) article = Article(row['uid'], row['body'], row['host'], row['newspaper_uid'], row['tokens_body'], row['tokens_title'], row['title'], row['url']) # Inserting the article into the DB session.add(article) #Po'que ajá session.commit() session.close()
def get_items_by_auction(auction_id): results = [] items = Session().query(distinct(AuctionItem.name).label('name')).filter(AuctionItem.auction_id == auction_id).all() # loop thru items and gather price data for item in items: data = Session().query(AuctionItem)\ .filter(AuctionItem.auction_id == auction_id)\ .filter(AuctionItem.name == item.name)\ .all() results.append({ 'item': item.name, 'data': [{ 'site': x.site, 'modified': x.modified, 'link': x.link, 'max': AuctionItem.max_price(x.price), 'avg': AuctionItem.avg_price(x.price) } for x in data] }) return results
def handler(event, context): # check authorization authorized_user_types = [ UserType.ADMIN, UserType.MENTOR, UserType.PAID, UserType.FREE ] success, _ = check_auth(event['headers']['Authorization'], authorized_user_types) if not success: return http_status.unauthorized() industryTagId = event["pathParameters"].get( "industryTagId") if event["pathParameters"] else None if not industryTagId: return http_status.bad_request( "missing path parameter(s): 'industryTagId'") session = Session() industry_tag = session.query(IndustryTag).get(industryTagId.lower()) session.close() if not industry_tag: return http_status.not_found() return http_status.success(json.dumps(row2dict(industry_tag)))
def main(filename): Base.metadata.create_all(engine) #genera el schema session = Session() #generamos la sesion articles = pd.read_csv(filename) #leermos el csv #iteramos por todos los articulos en los archivos limpios for index, row in articles.iterrows(): logger.info('Loading article uid {} into DB'.format(row['uid'])) article = Article(row['uid'], row['body'], row['link'], row['title'], row['newspaper_uid'], row['host'], row['n_tokens_body'], row['n_tokens_title']) session.add(article) session.commit() #genera commit session.close() #cierra la sesion
def set_query_table(): #census = Table('census2018', metadata, autoload=True) census = Table('cgy2018', metadata, Column('comm_code', String, primary_key=True), Column('name', String), Column('sector', String), Column('cclass', String), Column('res_cnt', String), Column('dwell_cnt', String), Column('comm_structure', String), Column('gcoord', String), Column('gcenter', String)) mapper(Communities, census) session = Session() return session
def parseMasks(): with open('sampleIR.json', 'r') as read_file: data = json.load(read_file) session = Session() for eachImage in data: address = data[eachImage]["address"] points = data[eachImage]["points"] newMaskImage = Mask(address, points) session.add(newMaskImage) session.commit() session.close()
def editorAccion(accion_id): email = dict(session).get('email', None) user = User.get_by_email(email) given_name = dict(session).get('given_name', None) s = Session() query = s.query(Accion) accion = query.filter(Accion.id == accion_id).first() if request.method == 'POST' and 'actualizarA' in request.form: dictupdate = { Accion.nombre: request.form['nombre'], Accion.descripcion: request.form['descripcion'], Accion.recompensa: float(request.form['recompensa']), Accion.indicadorKpi: request.form['indicadorKpi'], Accion.kpiObj: int(request.form['kpiObj']) } query.filter(Accion.id == accion_id).update(dictupdate, synchronize_session=False) s.commit() return render_template("editoraccion.html", accion=accion, email=email, name=given_name, user=user)
def __init__(self, topic, group_id): if not os.environ['KAFKA_HOST'] or not os.environ['KAFKA_PORT']: sys.exit("KAFKA_HOST and KAFKA_PORT must be configured") kafka_url = os.environ['KAFKA_HOST'] + ':' + os.environ['KAFKA_PORT'] self.__producer = KafkaProducer(bootstrap_servers=kafka_url) self.__consumer = KafkaConsumer(topic, bootstrap_servers=[kafka_url], auto_offset_reset='earliest', enable_auto_commit=True, group_id=group_id, consumer_timeout_ms=10000) self.__session = Session()
def Add_Department(): dep_name = input("Enter Department Name: ") new_dep = Department(dep_name=dep_name) session = Session() session.add(new_dep) session.commit()
class GetWithdrawDayHandler(threading.Thread): def __init__(self, threadName, username, password, trade_password="******"): super(GetWithdrawDayHandler, self).__init__(name=threadName) self.session = Session(username, password, trade_password) if not self.session.set_cookies(): print 'Cookie set failed...' def run(self): count = 0 while count < 10: if not self.session.set_cookies(): time.sleep(1) continue code = self.session.get_withdraw_day() if code == "1000": break else: time.sleep(1) count = count + 1 continue
def addEntrada(self, datos): # Añadir try: e = datos['entrada'][0] # Creamos una nueva sesion session = Session() entrada = Entradas(e['evento'], e['precio'], None, e['descripcion']) session.add(entrada) session.commit() session.close() return 201 except: return 400
def login(): try: data = request.get_json() if all(keys in data for keys in LOGIN_LST): username = data['username'].upper() password = data['password'] sess = Session() user = sess.query(UserDetail).filter_by(name=username, passwd=password).first() sess.close() if user: session["logged"] = True session.permanent = True APP.permanent_session_lifetime = timedelta(minutes=5) return redirect(url_for('profile_info', username=username)) else: return jsonify({ "status": 401, "result": "Incorrect Username or Password" }) else: return jsonify({"status": 400, "result": "Error in JSON format"}) except Exception as exception: return jsonify({"status": 500, "result": exception})
def handler(event, context): # check authorization authorized_user_types = [ UserType.ADMIN, UserType.MENTOR ] success, user = check_auth(event['headers']['Authorization'], authorized_user_types) if not success: return http_status.unauthorized() # validate body body = json.loads(event["body"]) if not body.get('senior_executive') or not body.get('chat_type') or body['chat_type'] not in ChatType.__members__: return http_status.bad_request("invalid parameter(s): 'senior_executive, chat_type'") senior_executive = body['senior_executive'] success = edit_auth(user, senior_executive) if not success: return http_status.unauthorized() chat_type = ChatType[body['chat_type']] description = body.get('description') tags = body.get('tags') fixed_date = body.get('fixed_date') if chat_type in mandatory_date and not fixed_date: return http_status.bad_request("missing body attribute { fixed_date } with chat_type { %s }" % (chat_type.name)) session = Session() chat_new = Chat( chat_type=chat_type, description=description, chat_status=ChatStatus.PENDING, tags=tags, senior_executive=senior_executive ) admin_update_declared_chats_frequency(senior_executive, 1) if fixed_date: chat_new.fixed_date = datetime.fromtimestamp(fixed_date).replace(hour=0, minute=0,second=0, microsecond=0) chat_new.chat_status = ChatStatus.ACTIVE else: admin_update_remaining_chats_frequency(senior_executive, 1) session.add(chat_new) session.commit() session.close() return http_status.success()
def main(filename): Base.metadata.create_all(engine) session = Session() articles = pd.read_csv(filename) # iterrow permite generar un loop en cada una de las filas del datagrame #portanto lo que devolvera sera el indice y la columna for index, row in articles.iterrows(): logger.info('cargando el articulo uid {} en la DB'.format(row['uid'])) article = Article(row['uid'], row['body'], row['host'], row['newspaper_uid'], row['n_tokenise_body'], row['n_tokenise_title'], row['title'], row['url']) #ingresando articulos a la base de datos session.add(article) session.commit() session.close()
def get_vaga(identificadorVaga=None): session = Session() vagas = None if identificadorVaga is not None: vagas = session.query(Vaga).filter( Vaga.identificador == identificadorVaga).first() else: vagas = session.query(Vaga).all() session.close() return vagas
class BorrowRecordsRepository: session = Session() def get_all(self): all_borrow_records = self.session.query(BorrowRecord).all() return jsonify(get_users_list_as_json(all_borrow_records)) def get_by_id(self, book_record_id): borrow_record = self.session.query(BorrowRecord).get(book_record_id) return jsonify(get_borrow_record_as_json(borrow_record)) def save(self, new_borrow_record): exist_book = self.session.query(Book).filter( Book.id == new_borrow_record["book"]["id"]).first() exist_user = self.session.query(User).filter( User.id == new_borrow_record["user"]["id"]).first() new_borrow_record = BorrowRecord(exist_book, exist_user, datetime.now()) self.session.add(new_borrow_record) self.session.commit() return jsonify(get_borrow_record_as_json(new_borrow_record)) def update(self, borrow_record_id, borrow_record): exist_borrow_record = self.session.query(BorrowRecord).filter( BorrowRecord.id == borrow_record_id).first() exist_book = self.session.query(Book).filter( Book.id == borrow_record["book"]["id"]).first() exist_user = self.session.query(User).filter( User.id == borrow_record["user"]["id"]).first() exist_borrow_record.book = exist_book exist_borrow_record.user = exist_user self.session.commit() return jsonify(get_borrow_record_as_json(exist_borrow_record)) def delete(self, borrow_record_id): result = self.session.query(BorrowRecord).filter( BorrowRecord.id == borrow_record_id).delete() self.session.commit() if result == 0: return False else: return True
def search_passwordhash(self, id): session = Session() try: user = session.query(User).filter(User.id == id).one() except: print("User", id, "not found...") session.close() return None else: print("User found") finally: session.close() return user.passwordhash
def main(filename): Base.metadata.create_all(engine) session = Session() articles = pd.read_csv(filename) for index, row in articles.iterrows(): logger.info("Loading article uid {} into DB".format(row["uid"])) article = Article( row["uid"], row["body"], row["host"], row["title"], row["newspaper_uid"], row["url"], ) session.add(article) session.commit() session.close()
def save(data): d = json_loads(data) session = Session() time_equals = Weather.dt == d["dt"] city_equals = Weather.city == d["city"] # check if data is in the DB q = session.query(Weather).filter(time_equals).filter(city_equals).count() print(q) # if data is not in db, save it if q <= 0: w = Weather(d['temp'], d['temp_min'], d['temp_max'], d['dt'], d['humidity'], d['city']) session.add(w) session.commit() session.close() print(str(data) + 'is saved') else: print('not saved') return
def updateStatus (self, gridTasks): print ("UPDATE STATUS") GWHosts = obtainGWResources() if GWHosts ==[]: print("Error when parsing host information file, employing information from past executions") return GWHosts #PILOTS pastPilots = Session.query(Pilot).all() presentPilots = [] #load XML to memory and extract data from hosts for resource in GWHosts: hostname = resource.getElementsByTagName("HOSTNAME")[0].firstChild.data.strip().lower() #TODO: remove "unicode" from TEXT try: foundLrms = resource.getElementsByTagName("LRMS_NAME")[0].firstChild.data.strip().lower() except: print ("Could not obtain resource LRMS, skipping it") continue if foundLrms != "jobmanager-pilot": continue #ahora son todos pilots try: freeNodeCount = int(resource.getElementsByTagName("FREENODECOUNT")[0].firstChild.data) except: freeNodeCount = 0 if not freeNodeCount > 0: continue; for pilot in pastPilots: if pilot.hostname == hostname: presentPilots.append(pilot) break #the pilots that we will first employ are the resulting ones self.pilots = presentPilots
from datetime import date from base import Session, engine from models import Base, Author, Book Base.metadata.create_all(engine) session = Session() author_1 = Author(name='J.R.R. Tolkien', birth=date(1892, 1, 3)) author_2 = Author(name='J.K. Rowling', birth=date(1965, 7, 31)) author_3 = Author(name='Stephen King', birth=date(1947, 9, 21)) book_1 = Book(title='The Hobbit', published_in=date(1937, 9, 21), author_id=1) book_2 = Book(title='The Lord of the Rings', published_in=date(1954, 7, 29), author_id=1) book_3 = Book(title='Harry Potter', published_in=date(1997, 6, 26), author_id=2) book_4 = Book(title='Carrie', published_in=date(1974, 1, 1), author_id=3) book_5 = Book(title='Salem Lot', published_in=date(1975, 1, 1), author_id=3) book_6 = Book(title='The Shining', published_in=date(1977, 1, 1), author_id=3) book_7 = Book(title='Rage', published_in=date(1977, 1, 1), author_id=3) session.add(author_1) session.add(author_2) session.add(author_3) session.add(book_1) session.add(book_2) session.add(book_3) session.add(book_4) session.add(book_5) session.add(book_6) session.add(book_7)
def updateInfoAfterExecution(self, gridTask): print("Updating info after exetcution of task " +gridTask.gwID + " on host " + gridTask.host.hostname + " (hostID " + str(gridTask.host.id) + ")") gridTask.status="CLEAR" #1.- abrir el archivo correspondiente a esa task hostToUpdate = gridTask.host execution_file = base.tmpExecutionFolder + "/execution_result_" + gridTask.gwID + ".xml" #1.- abrir el archivo correspondiente a esa task try: doc = xml.dom.minidom.parse(execution_file) except: print("failed when updating info after execution. File " + execution_file + " could not be found") hostToUpdate.updateInfoAfterFailedExecution() Session.add(gridTask) return #si los archivos de salida deseados no existen, también la cuento como fallida for outputFile in self.outputFiles.split(","): #JOB_ID has to be replaced by gwID as it happens along the execution splittedFile = outputFile.split('JOB_ID') output="" for pos in range(len(splittedFile)): output += splittedFile[pos] if pos < len(splittedFile) -1: output+=gridTask.gwID if not os.path.exists(base.tmpExecutionFolder + "/" + output): print("failed when updating info after execution. output file " + base.tmpExecutionFolder + "/" + output + " could not be found") hostToUpdate.updateInfoAfterFailedExecution() Session.add(gridTask) return executionInfoList = doc.getElementsByTagName('execution_info') gridTaskType = None remoteHostName = None executionTime = None dataSize = None realSamples = None for executionData in executionInfoList: try: gridTaskType = executionData.getElementsByTagName("type")[0].firstChild.data #TODO: remove "unicode" from TEXT remoteHostName = executionData.getElementsByTagName("hostname")[0].firstChild.data executionTime = float(executionData.getElementsByTagName("execution_time")[0].firstChild.data) dataSize = float(executionData.getElementsByTagName("data_size")[0].firstChild.data) realSamples = int(executionData.getElementsByTagName("real_samples")[0].firstChild.data) except: print ("Error when reading execution file, exiting" ) Session.add(gridTask) return #2.- procesar los resultados if gridTaskType != "execution": print ("ERROR when updating info from an application execution") print("Incorrect task type, expecting \"execution\"") gridTask.status = "CLEAR" Session.add(gridTask) return if remoteHostName != hostToUpdate.hostname: print ("ERROR when updating info from a application execution") print("Incorrect host name, expecting " + hostToUpdate.hostname) gridTask.status = "CLEAR" Session.add(gridTask) return if executionTime == 0: print ("ERROR when updating info from an application execution") print ("Execution time appears to be zero, and that's quite strange") gridTask.status = "CLEAR" hostToUpdate.updateInfoAfterFailedExecution() Session.add(gridTask) return totalActiveTime = InformationManager.readTotalActiveTime(gridTask.gwID) if totalActiveTime == -1: print ("ERROR when updating info from an application execution") print ("Could not read active time from GridWay log, considering that task failed") gridTask.status = "CLEAR" hostToUpdate.updateInfoAfterFailedExecution() Session.add(gridTask) return queueTime = InformationManager.readQueueTime(gridTask.gwID) if queueTime == -1: print ("ERROR when updating info from an application execution") print ("Could not read queue time from GridWay log, considering that task failed") gridTask.status = "CLEAR" hostToUpdate.updateInfoAfterFailedExecution() Session.add(gridTask) return transferTime = totalActiveTime - executionTime if transferTime > 0: bandwidth = dataSize / transferTime else: bandwidth = -1 #3.- actualizar rendimiento del host computationalEffort = self.profile.constantEffort + self.profile.sampleEffort * realSamples whetstones = computationalEffort / executionTime hostToUpdate.updateInfoFromSuccessFulExecution(whetstones, queueTime, bandwidth) hostToUpdate.failedProfilings -=1 #4 actualizar estado de la tarea y la apliación. gridTask.realSamples = realSamples gridTask.status = "CLEAR" self.remainingSamples -= realSamples print("APPLICATION UPDATE: " + str(self.remainingSamples) + "/" + str(self.desiredSamples) + " left") #5.- eliminar archivos temporales try: #os.remove(execution_file) print ("In Application.py, I would be deletign" + execution_file) print ("Execution file has been successfully deleted: " + execution_file) except: print ("Could not delete profiling file " + execution_file) #6 update info on DB gridTask.endDate = datetime.now() Session.add(gridTask) Session.add(self)
def updateInfoAfterProfiling(self, gridTask): #1.- leer la info correspondiente a esa aplicacion #tener en cuenta que es una serie de parejas de valores execution_file = base.tmpExecutionFolder + "execution_result_" + gridTask.gwID + ".xml" try: doc = xml.dom.minidom.parse(execution_file) except: print("failed when profiling host " + gridTask.host.hostname + ". File " + execution_file + " could not be found") gridTask.host.updateInfoAfterFailedProfiling() Session.add(gridTask.host) return executionInfoList = doc.getElementsByTagName('results') executionResults=[] for executionData in executionInfoList: profileInfoList = executionData.getElementsByTagName('profile') for profileInfo in profileInfoList: samples = int(profileInfo.getElementsByTagName("samples")[0].firstChild.data) #TODO: remove "unicode" from TEXT time = int(profileInfo.getElementsByTagName("time")[0].firstChild.data) if samples == 0: print ("There was an error on host " + gridTask.host.hostname + ". File " + execution_file + ", no samples have been executed") gridTask.host.updateInfoAfterFailedProfiling() Session.add(gridTask.host) return executionResults.append([samples, time]) #2.- procesar los resultados #TODO: esto es muy posible que esté mal numSimulations = len(executionResults) if numSimulations == 0: print ("There was an error on host " + gridTask.host.hostname + ". File " + execution_file + ", no simulations performed") gridTask.host.updateInfoAfterFailedProfiling() Session.add(gridTask.host) return avgSampleTime = 0 for i in range(numSimulations-1): newSamples = executionResults[i+1][0] - executionResults[i][0] newTime = executionResults[i+1][1] - executionResults[i][1] if newTime <=0: value = 0 continue else: value = newTime / newSamples ######TODO esta es la clave!!! #weighted sample time, so later executions are more important than the first ones #the reason is that they executed more samples, so their information is more valuable if i==0 or avgSampleTime == 0: avgSampleTime = value else: avgSampleTime = 0.6 * value + 0.4 * avgSampleTime acumConstantTime = 0 #this has been modifiied to avoid zero values for i in range(len(executionResults)): acumConstantTime += max(0, executionResults[i][1] - executionResults[i][0] * avgSampleTime) avgConstantTime = acumConstantTime / numSimulations normalizedSampleEffort = avgSampleTime * gridTask.host.getWhetstones() normalizedConstantEffort = avgConstantTime * gridTask.host.getWhetstones() print("RESULTADO DEL PROFILING") print ("task id: " + gridTask.gwID) print("normalizado, this_sample_effort = " + str(normalizedSampleEffort)) print("normalizado, this_constant_effort = " + str(normalizedConstantEffort)) print("STATUS DEL PROFILING") print("normalizado, sample_effort = " + str(self.profile.sampleEffort)) print("normalizado, constant_effort = " + str(self.profile.constantEffort)) print("") if (normalizedSampleEffort < 0 ) or (normalizedConstantEffort < 0): print ("A normalized effort below zero makes no sense, dismissing profile") return self.profile.updateInfoAfterProfiling(normalizedConstantEffort, normalizedSampleEffort) Session.add(self.profile)
from base import Session from models import table_class_list session = Session() if __name__ == '__main__': tables = {index: table for index, table in enumerate(table_class_list)} while True: print('Add records:', '\n') print('Tables list:', '\n') print(tables, '\n') user_response = input('Enter \'table #\' or \'x\' to exit: ') if user_response.lower() == 'x' or user_response == '': break else: table = tables.get(int(user_response), None) if table is not None: # Retrieve list of columns columns = [col.name for col in table.__table__.columns if col.name != 'id'] print(columns) # Initialize a new record data = {} for col in columns: value = input('type a value for {0}: '.format(col)) data[col] = value
from GridTask import GridTask import UserInterface if __name__ == '__main__': requirementsFile = "/soft/users/home/u5682/nfs/workspace/montera/FLUKA/fluka_example.mt" print("Starting connection with database") metadata = MetaData() myDBDesign = DBDesign() hostDB = myDBDesign.HostDBDesign(metadata) applicationDB = myDBDesign.ApplicationDesign(metadata) appProfileDB = myDBDesign.AppProfileDBDesign(metadata) gridTaskDesignDB = myDBDesign.GridTaskDBDesign(metadata) parameterDesignDB = myDBDesign.parameterDBDesign(metadata) metadata.create_all(base.engine) print("Database is correct") print("") myInfrastructure = Infrastructure.Infrastructure('lrms') myInfrastructure.createHostProfilingTasks() Session.add(myInfrastructure) Session.commit()
def createInfrastructureTasks(self, infrastructureTasks): print ("-------------------") print ("-------------------") print ("createInfrastructureTasks- NewPilotInfrastructure") # self.showHosts() hostList = obtainGWResources() hostsToProfile = [] print ("Analyzing resources ") for hostInfo in hostList: hostName = hostInfo.getElementsByTagName("HOSTNAME")[0].firstChild.data.strip().lower() #TODO: remove "unicode" from TEXT whetstones=0 try: foundArch = hostInfo.getElementsByTagName("ARCH")[0].firstChild.data except: foundArch="" try: foundCpuMHz = int(hostInfo.getElementsByTagName("CPU_MHZ")[0].firstChild.data) except: foundCpuMHz = 0 try: foundLrms = hostInfo.getElementsByTagName("LRMS_NAME")[0].firstChild.data.strip().lower() except: foundLrms = None print ("Could not find LRMS for host " + hostName + ", skipping it") continue try: freeNodeCount = int(hostInfo.getElementsByTagName("FREENODECOUNT")[0].firstChild.data) except: freeNodeCount = 0 if foundLrms == "jobmanager-pilot": #solo tenemos en cuenta los pilots con al menos un slot disponible if not freeNodeCount > 0: continue username = os.getenv("USER") genericStringArgs = hostInfo.getElementsByTagName("GENERIC_VAR_STR") for node in genericStringArgs: if node.attributes['NAME'].value =="PILOT_REAL_HOSTNAME": workerNode = node.attributes['VALUE'].value.strip().lower() if node.attributes['NAME'].value =="PILOT_REAL_RESOURCE": site = node.attributes['VALUE'].value.strip().lower() genericIntArgs = hostInfo.getElementsByTagName("GENERIC_VAR_INT") for node in genericIntArgs: if node.attributes['NAME'].value =="PILOT_" + username + "_VAR_5": whetstones = int(node.attributes['VALUE'].value.strip().lower()) if whetstones > 65534: whetstones = 0 # whetstones = 0 #if host is unknown, create a profiling task currentHost = self.getHost(hostName) if currentHost == None: print ("Host/Pilot not found. hostname: " + hostName + ", LRMS: " + foundLrms) if foundLrms == "jobmanager-pilot": #he encontrado un pilot: #primero busco e resource, y si no existe lo creo. #luego creo un pilot que utilice ese resource pilotResource = base.Session.query(PilotResource).filter(PilotResource.site == site, PilotResource.workerNode == workerNode).first() if pilotResource == None: print (" PilotResource was not found, creating a new one") pilotResource = PilotResource(site, workerNode) print (" Creating a new Pilot in NewPilotInfrastructure.createInfrastructureTasks") newHost = Pilot(hostName, arch=foundArch, cpuMHz = foundCpuMHz, pilotResource = pilotResource, whetstones = whetstones) self.pilots.append(newHost) Session.add(newHost) else: print (" Creating a new Host in NewPilotInfrastructure.createInfrastructureTasks") newHost = Host(hostName, arch=foundArch, cpuMHz = foundCpuMHz, lrms=foundLrms) self.hosts.append(newHost) Session.add(newHost) #ESTO ES PARA HACER EL PROFILING DE LOS PILOT SI NO HAN PUBLICADO LOS WHETSTONES, SI NO NO HACE FALTA #=============================================================== # if whetstones == 0 or whetstones > 65534: # whetstones = 0 # print (" Host to profile: " + hostName + ": whetstone value not initialized ") # hostsToProfile.append(newHost) # #store new host on databae (faiulre resistance # Session.add(newHost) #=============================================================== #if information has changed, update host information elif (currentHost.getWhetstones() != whetstones): #va con un set porque es una operación más complicada, así que está encapsulada en esta funcion currentHost.setWhetstones(whetstones) Session.add(currentHost) print ("Host: " + hostName + " UPDATED, new whetstones=" + str(whetstones)) elif currentHost.lrms == None: currentHost.lrms = foundLrms #pprofiling of new sites hostProfilingTasks = [ExecutionManager.createHostProfilingTask(host) for host in hostsToProfile for i in range(base.profilingTasksPerHost)] #estamos asumiento que todos los pilots publican la variable esa con su #rendimiento, con lo que no hay que hacer el profiling de nada. #AHORA, EN ESA NUEVA APROXIMACION, QUEREMOS TENER UNOS CUANTO SBENCHMARKS PARA IR ARRANCANDO PILOTS print ("creating fake profiling tasks") existingFakeTasks = len([task for task in infrastructureTasks if task.host.hostname=="" and task.status != "PENDING"]) existingGoodPilots = len (self.getGoodHosts()) existingProfilingTasks = len(hostProfilingTasks) #fakeTasksToCreate = base.maxRunningTasks - (existingFakeTasks + existingGoodPilots + existingProfilingTasks) fakeTasksToCreate = base.maxRunningTasks - existingFakeTasks print (" Desired tasks: " + str(base.maxRunningTasks)) print (" Existing fake tasks: " + str(existingFakeTasks)) print (" Existing good pilots: " + str(existingGoodPilots)) print (" created: " + str(fakeTasksToCreate)) emptyHost = FakeHost() fakeHostProfilingTasks = [ExecutionManager.createFakeHostProfilingTask(emptyHost) for i in range(fakeTasksToCreate)] hostProfilingTasks+=fakeHostProfilingTasks return hostProfilingTasks
from base import Session from models import table_class_list from list import list_all session = Session() if __name__ == '__main__': tables = {index: table for index, table in enumerate(table_class_list)} while True: print('Delete records:', '\n') print('Tables list:', '\n') print(tables, '\n') user_response = input('Enter \'table #\' or \'x\' to exit: ') if user_response.lower() == 'x' or user_response == '': break else: # Get the Table Class table = tables.get(int(user_response), None) if table is not None: # Query to DB list_all(table) record_id = input('Enter \'record #\' or \'c\' to cancel...') if record_id.lower() != 'c' and record_id != '': # Get the Record ID record = session.query(table).get(record_id) if input('Add record? y/n') == 'y' and record is not None:
def createInfrastructureTasks(self, infrastructureTasks): print ("---------------------") print ("---------------------") print ("---------------------") print ("CREATE INFRASTRUCTURE TASKS") hostsToProfile = [] hostList = obtainGWResources() for hostInfo in hostList: hostName = hostInfo.getElementsByTagName("HOSTNAME")[0].firstChild.data #TODO: remove "unicode" from TEXT try: foundArch = hostInfo.getElementsByTagName("ARCH")[0].firstChild.data except: foundArch="" try: foundCpuMHz = int(hostInfo.getElementsByTagName("CPU_MHZ")[0].firstChild.data) except: foundCpuMHz = 0 try: foundLrms = hostInfo.getElementsByTagName("LRMS_NAME")[0].firstChild.data except: foundLrms = None try: freeNodeCount = int(hostInfo.getElementsByTagName("FREENODECOUNT")[0].firstChild.data) except: freeNodeCount = 0 if foundLrms != None: if foundLrms == "jobmanager-pilot": #solo tenemos en cuenta los pilots con al menos un slot disponible if not freeNodeCount > 0: continue #if a certain LRMS is desired, remove the hosts with a different one if self.lrms != None: if foundLrms != self.lrms: continue #if host is unknown, create a profiling task currentHost = self.getHost(hostName) if currentHost == None: newHost = Host(hostName, arch=foundArch, cpuMHz = foundCpuMHz, lrms=foundLrms) self.hosts.append(newHost) hostsToProfile.append(newHost) #store new host on databae (faiulre resistance Session.add(newHost) #if information has changed, update host information elif (currentHost.arch != foundArch) or (currentHost.cpuMHz != foundCpuMHz): #TODO: pensar que hacer aqui. habria que eliminar el viejo o solo sobreescribir la información? Si se elimina el viejo, que pasa con las tareas ahí ejecutadas? No es trivial currentHost.arch = foundArch currentHost.cpuMHz = foundCpuMHz if currentHost.lrms == None: currentHost.lrms = foundLrms hostsToProfile.append(currentHost) Session.add(currentHost) elif currentHost.shouldBeProfiled(): if currentHost.lrms == None: currentHost.lrms = foundLrms hostsToProfile.append(currentHost) #print("Host profiling: submission of 1 tasks per host") hostProfilingTasks = [ExecutionManager.createHostProfilingTask(host) for host in hostsToProfile for i in range(1)] siteTasks = [] for task in hostProfilingTasks: found=False for gridTask in infrastructureTasks: if gridTask.host.hostname == task.host.hostname: found=True break if not found: siteTasks.append(task) #Esto es para el primer experimento de montera + gwpilot #queremos tener pilots funcionando, así que los arranco con esto if self.lrms=="jobmanager-pilot": print ("creating fake profiling tasks") existingFakeTasks = len([task for task in infrastructureTasks if task.host.hostname=="" and task.status != "PENDING"]) existingGoodPilots = len (self.getGoodHosts()) existingProfilingTasks = len(hostProfilingTasks) #fakeTasksToCreate = base.maxRunningTasks - (existingFakeTasks + existingGoodPilots + existingProfilingTasks) fakeTasksToCreate = base.maxRunningTasks - existingFakeTasks print (" Desired tasks: " + str(base.maxRunningTasks)) print (" Existing fake tasks: " + str(existingFakeTasks)) print (" Existing good pilots: " + str(existingGoodPilots)) print (" created: " + str(fakeTasksToCreate)) emptyHost = FakeHost() fakeHostProfilingTasks = [ExecutionManager.createWakeUpask(emptyHost) for i in range(fakeTasksToCreate)] siteTasks+=fakeHostProfilingTasks return siteTasks
def updateInfoAfterProfiling(self, gridTask): print ("Updating info after profiling site " + gridTask.host.hostname) print (" Task info:") print (" Task id: " + str(gridTask.id)) print (" GW ID: " + gridTask.gwID) print (" desired host: " + gridTask.host.hostname) print (" Host type: " + str(gridTask.host.__class__)) gridTask.status="CLEAR" #1.- abrir el archivo correspondiente a esa task execution_file = base.tmpExecutionFolder + "execution_result_" + gridTask.gwID + ".xml" try: doc = xml.dom.minidom.parse(execution_file) except: print("failed when profiling host " + gridTask.host.hostname + ". File " + execution_file + " could not be found") gridTask.host.updateInfoAfterFailedProfiling() Session.add(gridTask) return executionInfoList = doc.getElementsByTagName('execution_info') for executionData in executionInfoList: try: gridTaskType = executionData.getElementsByTagName("type")[0].firstChild.data #TODO: remove "unicode" from TEXT remoteHostName = executionData.getElementsByTagName("hostname")[0].firstChild.data whetstones = float(executionData.getElementsByTagName("whetstones")[0].firstChild.data) waitingTime = float(executionData.getElementsByTagName("execution_time")[0].firstChild.data) dataSize = float(executionData.getElementsByTagName("data_size")[0].firstChild.data) except: print("failed when profiling host " + gridTask.host.hostname + ". File " + execution_file + " could not be found") gridTask.host.updateInfoAfterFailedProfiling() Session.add(gridTask) return #2.- procesar los resultados if gridTaskType != "benchmark": print ("ERROR when updating info from a site profiling") print("Incorrect task type, readed " + gridTaskType + " and should be \"benchmark\"") print (" considering the execution as failed") gridTask.host.updateInfoAfterFailedProfiling() Session.add(gridTask) return if remoteHostName != gridTask.host.hostname: print ("ERROR when updating info from a site profiling") print("Incorrect host name, readed" + remoteHostName + " and should be " + gridTask.host.hostname) print (" considering the execution as failed") gridTask.host.updateInfoAfterFailedProfiling() Session.add(gridTask) return totalActiveTime = InformationManager.readTotalActiveTime(gridTask.gwID) transferTime = totalActiveTime - waitingTime queueTime = InformationManager.readQueueTime(gridTask.gwID) if transferTime > 0: bandwidth = dataSize / transferTime else: bandwidth = -1 #3.- suministrar esa info al host. gridTask.host.updateInfoFromSuccessFulExecution(whetstones, queueTime, bandwidth) #4.- eliminar archivos temporales try: #os.remove(execution_file) print ("IN application.py, I would be deleting " + execution_file) print ("Profiling file has been successfully deleted: " + execution_file) except: print ("Could not delete profiling file " + execution_file) gridTask.endDate = datetime.now()
def readRequirements(requirementsFile): myRequirementsFile = open(requirementsFile,'r') myname = mydesiredSamples = mymaxSupportedSamples = \ mymaxProfilingTime = mypreProcessScript = mypostProcessScript = myinputFiles = myoutputFiles = myworkingDirectory = None myRemoteMontera = myLrms = myInfrastructure = None for line in myRequirementsFile.readlines(): normalizedLine = line.lower().strip() if normalizedLine.startswith("#"): continue; if normalizedLine.count("name") > 0: myname = line.split("=")[1].strip() elif normalizedLine.count("num_samples") > 0: mydesiredSamples= int(line.split("=")[1].strip()) elif normalizedLine.count("postprocess_script") > 0: mypostProcessScript = line.split("=")[1].strip() elif normalizedLine.count("preprocessScript") > 0: mypreProcessScript = line.split("=")[1].strip() elif normalizedLine.count("max_profiling_time") > 0: mymaxProfilingTime = line.split("=")[1].strip() elif normalizedLine.count("max_supported_samples") > 0: mymaxSupportedSamples = line.split("=")[1].strip() #los ficheros de entrada estan separados por comas elif normalizedLine.count("input_files") > 0: myinputFiles = line.split("=")[1].strip().replace(' ', '') elif normalizedLine.count("output_files") > 0: myoutputFiles = line.split("=")[1].strip().replace(' ', '') elif normalizedLine.count("working_directory") > 0: myworkingDirectory = line.split("=")[1].strip() elif normalizedLine.count("schedulingalgorithm") > 0: mySchedulingAlgorithm = normalizedLine.split("=")[1].strip() elif normalizedLine.count("remoteMontera") > 0: myRemoteMontera = normalizedLine.split("=")[1].strip() elif normalizedLine.count("infrastructure") > 0: myInfrastructure = normalizedLine.split("=")[1].strip() elif normalizedLine.count("parametricjob") > 0: print("Parametric Job"); mySpecificationFile = line.split("=")[1].strip() myAnalyzer = Analyzer(mySpecificationFile) mydesiredSamples = myAnalyzer.numTasks myRemoteMontera = "parametricRemoteMontera.sh" myApplicationProfile = Session.query(AppProfile).filter(AppProfile.applicationName == myname).first() if myApplicationProfile == None: myApplicationProfile = AppProfile(myname) myApplication = Application(myname, mydesiredSamples, maxSupportedSamples = mymaxSupportedSamples, maxProfilingTime = mymaxProfilingTime, preProcessScript = mypreProcessScript, postProcessScript = mypostProcessScript, inputFiles = myinputFiles, outputFiles = myoutputFiles, workingDirectory = myworkingDirectory, profile = myApplicationProfile, schedulingAlgorithm = mySchedulingAlgorithm, remoteMontera = myRemoteMontera, infrastructure = myInfrastructure) return myApplication
from base import Session from models import table_class_list session = Session() if __name__ == '__main__': tables = {index: table for index, table in enumerate(table_class_list)} while True: print('Filter table:', '\n') print('Tables list:', '\n') print(tables, '\n') user_response = input('Enter \'table #\' or \'x\' to exit: ') if user_response.lower() == 'x' or user_response == '': break else: # Get the Table Class table = tables.get(int(user_response), None) if table is not None: columns = {index: col.name for index, col in enumerate(table.__table__.columns) if col.name != 'id'} print(columns, '\n') user_response = input('Enter \'column #\' or \'c\' to cancel: ') if user_response.lower() == 'c' or user_response == '': pass else:
from base import Session from models import table_class_list from list import list_all session = Session() if __name__ == '__main__': tables = {index: table for index, table in enumerate(table_class_list)} while True: print('Modify records:', '\n') print('Tables list:', '\n') print(tables, '\n') user_response = input('Enter \'table #\' or \'x\' to exit: ') if user_response.lower() == 'x' or user_response == '': break else: table = tables.get(int(user_response), None) if table is not None: # Retrieve list of columns columns = [col.name.replace('_', '') for col in table.__table__.columns if col.name != 'id'] # List all records list_all(table) record_id = input('Enter \'record #\' or \'c\' to cancel...') if record_id.lower() != 'c' and record_id != '': if input('Update record? y/n') == 'y' and record_id is not None:
def load(self): print ("LOAD - pilotInfrastructures") #SITES pastHosts = Session.query(Host).filter(Host.type=="hosts").all() presentHosts = [] #PILOTS pastPilots = Session.query(Pilot).all() print ("Deleting all past pilots from database") for pilot in pastPilots: base.Session.delete(pilot) presentPilots = [] GWHosts = obtainGWResources() print ("Updating information") #load XML to memory and extract data from hosts try: for resource in GWHosts: hostname = resource.getElementsByTagName("HOSTNAME")[0].firstChild.data.strip().lower() #TODO: remove "unicode" from TEXT try: foundLrms = resource.getElementsByTagName("LRMS_NAME")[0].firstChild.data except: print ("Could not obtain resource LRMS for site " + hostname + ", skipping it") continue if foundLrms == "jobmanager-pilot": #nombre de pilot genericArgs = resource.getElementsByTagName("GENERIC_VAR_STR") for node in genericArgs: if node.attributes['NAME'].value =="PILOT_REAL_HOSTNAME": workerNode = node.attributes['VALUE'].value.strip().lower() if node.attributes['NAME'].value =="PILOT_REAL_RESOURCE": site = node.attributes['VALUE'].value.strip().lower() #numero de nodos libres. Esto en los pilots funciona bien, y es lo que se emplea para saber si está #activo o apagado try: freeNodeCount = int(resource.getElementsByTagName("FREENODECOUNT")[0].firstChild.data) except: freeNodeCount = 0 if not freeNodeCount > 0: continue; #cargamos el pilot en la base de datos: #primero buscamos el pilotResource, o lo creamos si no existe #después creamos el pilot que emplee ese recurso pilotResource = base.Session.query(PilotResource).filter(PilotResource.site == site, PilotResource.workerNode == workerNode).first() if pilotResource == None: pilotResource = PilotResource(site, workerNode) pilot = Pilot(hostname, pilotResource=pilotResource) presentPilots.append(pilot) else: #it is a site for host in pastHosts: if host.hostname == hostname: presentHosts.append(host) break except: print("Error when parsing host information file, employing information from past executions") self.hosts = pastHosts self.pilots = [] return #the hosts that we will first employ are the resulting ones self.pilots = presentPilots self.hosts = presentHosts