Beispiel #1
0
def updateGridTaskStatus(gridTask):
	#get GridWay status
	(result, status, error)=drmaa_job_ps(gridTask.gwID)

#error control
	if result != DRMAA_ERRNO_SUCCESS:
		print >> sys.stderr, "updatingGridTaskStatus, drmaa_job_ps() failed for task " + gridTask.gwID + ". Error: %s" % (error)
		gridTask.status = "DONE" #DONE does not imply success, just that it has finished
	else:
		if gridTask.status == "CLEAR": #ya se ha acabado, nothing to do here
			return
		
		elif status == DRMAA_PS_UNDETERMINED or status == DRMAA_PS_QUEUED_ACTIVE:
			gridTask.status = "SUBMITTED"
			
		elif status == DRMAA_PS_RUNNING:
			#if needed, update task execution start
			if gridTask.status == "SUBMITTED" or gridTask.status=="WAITING":
				gridTask.executionStartDate = datetime.now()
				print ("Task " + gridTask.gwID + "with type " + gridTask.type + "  on host " + gridTask.host.hostname +  " with ID=" + str(gridTask.host.id) + " has started its execution")
			gridTask.status = "RUNNING"
		
		elif status == DRMAA_PS_DONE or status == DRMAA_PS_FAILED:
			gridTask.status = "DONE"
	
	Session.add(gridTask)
Beispiel #2
0
	def updateStatus(self, gridTasks):
#===============================================================================
# 		print ("updatestatus: UPDATING INFRASTRUCTURE STATUS")
# 
# 		print("	numbver of tasks to process: " + str(len(gridTasks)))
#===============================================================================

		for host in self.hosts:
			host.currentSlotCount = 0
			
		for gridTask in gridTasks:
			if gridTask.status == "RUNNING":
				try:
					gridTask.host.currentSlotCount +=1
				except:
					pass
				#===============================================================
				# print ("	task running on host " + gridTask.host.hostname  + "with id " + str(gridTask.host.id) +", that makes " + str(gridTask.host.currentSlotCount) + " tasks on the host") 				
				#===============================================================

		for host in self.hosts:
			host.maxSlotCountThisTime = max(host.currentSlotCount, host.maxSlotCountThisTime)
			if host.maxSlotCountThisTime > host.maxSlotCount:
				host.maxSlotCount = host.maxSlotCountThisTime
				Session.add(host)
Beispiel #3
0
	def updateInfoAfterExecutionInPilots(self, gridTask):


		self.updateInfoAfterExecution(gridTask)

		#queremos aumentar el numero de tareas otra vez (porque lo restamos al enviarla, y el updateInfo... lo vuelve a restar)
		# -si ha acabado OK, para que el numero de pendientes se quede como estaba
		# - si NO ha acabado Ok, para que se vuelva a incrementar, ya que lo restamos al enviarla
		self.remainingSamples += gridTask.realSamples
		Session.add(self)
Beispiel #4
0
	def load(self):
		pastHosts = Session.query(Host).all()

		print ("INFRASTRUCTURE-LOAD: LOADING INFORMATION FROM PAST EXECUTIONS")
		print ("	Desired LRMS: " + self.lrms)
		#obtain present hosts prom gridway
		
				#PILOTS
		pastPilots = Session.query(Pilot).all()
		print ("	Deleting all past pilots from database")
		for pilot in pastPilots:
			base.Session.delete(pilot)
			
			
		presentHosts = []
		GWHosts =  obtainGWResources()

		if GWHosts == []:
			print("Error when parsing host information file, employing information from past executions")	
			self.hosts = pastHosts
			return

	
		#load XML to memory and extract data from hosts
		for hostInfo in GWHosts:
			hostName = hostInfo.getElementsByTagName("HOSTNAME")[0].firstChild.data #TODO: remove "unicode" from TEXT
			
			#for every found host, check if it existed on a previous execution
			for host in pastHosts:
				if host.hostname.strip().lower() == hostName.strip().lower():
					
					#in the case of pilot jobs, only employ the ones with FREENODECOUNT > 0
					if host.lrms =="jobmanager-pilot":
						try:	
							freeNodeCount = int(hostInfo.getElementsByTagName("FREENODECOUNT")[0].firstChild.data)
						except:
							freeNodeCount = 0
						if not freeNodeCount > 0:
							continue;
					
					presentHosts.append(host)
					break  # Si no pones este break, y está repetido en memoria, se cargan todos

		
		#now, if only hosts with a certain LRMS are desired, we remove the rest
		if self.lrms != None:
			for host in presentHosts:
				if host.lrms != self.lrms:
					print ("	Removing host " + host.hostname + ", wrong LRMS found: " + host.lrms)
					presentHosts.remove(host)
				else:
					print ("	Keeping host " + host.hostname)
		
		#the hosts that we will first employ are the resulting ones
		self.hosts = presentHosts
Beispiel #5
0
def getTask():
	newParameter = Session.query(Parameter).filter(Parameter.status=="WAITING").first()
	if newParameter == None:
		return template('{{id}}', id=-1)

	
	newParameter.status="RUNNING"
	newParameter.executionStartDate = datetime.now()
	Session.add(newParameter)
	Session.commit()
	return template('{{id}}', id=newParameter.id)
Beispiel #6
0
def failedTask(idInput=-1):
	#por si acaso
	try:
		id = int(idInput)
	except:
		return template('-1')

	failedParameter = Session.query(Parameter).filter(Parameter.id==id).first()
	failedParameter.status="WAITING"
	failedParameter.executionStartDate = None
	Session.add(failedParameter)
	Session.commit()
	return template('{{id}}', id=failedParameter.id)
Beispiel #7
0
def finishedTask(idInput=-1):
	#por si acaso
	try:
		id = int(idInput)
	except:
		return template('-1')

	finishedParameter = Session.query(Parameter).filter(Parameter.id==id).first()
	finishedParameter.status="DONE"
	finishedParameter.executionEndDate = datetime.now()
	Session.add(finishedParameter)
	Session.commit()
	#TODO: en algun sitio habria que verificar los datos de salida antes del done
	return template('{{id}}', id=finishedParameter.id)
	def analyzeParametricJobfile(self):
		
		tmpFile = "/tmp/tmpParameterFile"
		process = monteraLocation + '/templateManager -c ' +  self.parametricJobFile + " > " + tmpFile
		parameters = self.runProcess(process)
		numTasks = 0
		
		for line in open(tmpFile, 'r').readlines():
			print ("Parameter: " + line)
			myParameter = Parameter(line) 
			Session.add(myParameter)
			numTasks+=1
			
		Session.commit()		
Beispiel #9
0
def connectDB():
    logger.debug(f"connectDB(): entered")
    db = Session()
    db_url = os.environ["DB_HEROKU"]
    db_url = "postgres://*****:*****@ec2-107-22-162-8.compute-1.amazonaws.com:5432/delmad8ora7a1v"

    engine = create_engine(db_url)
    Base.metadata.bind = engine

    DBSession = sessionmaker(bind=engine)
    session = DBSession()
    session.autoflush = True
    return session
def sendQuestions():
    session = Session()
    questions = session.query(Question).all()

    numberOfQuestions = 10  # Number of questions in the quiz
    chosenQuestions = random.sample(questions, numberOfQuestions)

    finalQuestionList = []
    for eachQuestion in chosenQuestions:

        finalQuestionList.append({
            'id': eachQuestion.id,
            'question': eachQuestion.question,
            'optionA': eachQuestion.optionA,
            'optionB': eachQuestion.optionB,
            'optionC': eachQuestion.optionC,
            'optionD': eachQuestion.optionD,
            'answer': eachQuestion.answer,
            'trivia': eachQuestion.trivia
        })

    return jsonify(finalQuestionList), 200
Beispiel #11
0
 def __init__(self, url, script):
     self.url = url
     self.session = Session.get()
     self.headers = {
         'Referer': BaseTest.stand + self.url,
         'Content-Type': 'application/x-www-form-urlencoded'
     }
     self.script = script
     payload = {'Login': BaseTest.login, 'Password': BaseTest.password}
     login_url = BaseTest.stand + '/Auth/LogOn'
     self.session.post(url=login_url, data=payload)
     self.cookies = self.session.cookies.get_dict()
     self.headers = {**self.headers, **self.cookies}
Beispiel #12
0
def insert_data(item_type, account_id, item_id, auth):

    Base.metadata.create_all(engine)
    session = Session()

    # def __init__(self, id_id, name, create_date, no_devices, email, account):
    def create_user():
        item_user = kp.get_items(item_type=item_type,
                                 account_id=account_id,
                                 item_id=item_id,
                                 auth=auth)
        print(item_user)
        data = item_user['data']
        user_name = data['caller_id']['internal']['name']
        email = data.get('email')
        account = session.query(Account).get(account_id)
        user = User(item_id, user_name, date(2015, 4, 2), 3, email, account)
        session.add(user)

    # def __init__(self, id_1,  name, device_type):
    def create_device():
        item_device = kp.get_items(item_type=item_type,
                                   account_id=account_id,
                                   item_id=item_id,
                                   auth=auth)
        print(item_device)
        data = item_device['data']
        device_type = data.get('device_type')
        owner_id = data.get('owner_id')
        name = data.get('name')

        device = Device(item_id, name, device_type)
        user = session.query(User).get(owner_id)
        device.user = [user]
        session.add(device)

    # def __init__(self, id_1, name, create_date):
    def create_account():
        item_account = kp.get_items(item_type=item_type,
                                    account_id=account_id,
                                    item_id=item_id,
                                    auth=auth)
        print(item_account)
        data = item_account['data']
        account_name = data.get('name')
        account = Account(item_id, account_name, date(2015, 4, 2))
        session.add(account)

    if item_type == 'user':
        create_user()

    if item_type == 'account':
        create_account()

    if item_type == 'device':
        create_device()

    session.commit()
    session.close()
def set_password(id, password):
    #TODO: Maybe check if user is authenticated?
    session = Session()
    account = session.query(User).filter(User.id == id).first()
    if account == None:
        print("No account with id " + id)
        return False

    account.passwordhash = hashlib.sha512(password).hexdigest()
    session.commit()
    session.close()
Beispiel #14
0
def editorOferta(offer_id):
    email = dict(session).get('email', None)
    given_name = dict(session).get('given_name', None)
    user = User.get_by_email(email)

    s = Session()
    query = s.query(Oferta)
    oferta = query.filter(Oferta.id == offer_id).first()
    if request.method == 'POST':
        dictupdate = {
            Oferta.nombre: request.form['nombre'],
            Oferta.descripcion: request.form['descripcion'],
            Oferta.precio: request.form['precio']
        }
        query.filter(Oferta.id == offer_id).update(dictupdate,
                                                   synchronize_session=False)
        s.commit()
    return render_template("editoroferta.html",
                           oferta=oferta,
                           email=email,
                           name=given_name,
                           user=user)
Beispiel #15
0
def main(filename):
    Base.metadata.create_all(Engine)
    session = Session()
    articles = pd.read_csv(filename)

    for index, row in articles.iterrows():
        logger.info('loading, articles uid {} into DB'.format(row['uid']))
        article = Article(row['uid'], row['body'], row['host'],
                          row['newspaper_uid'], row['m_tokens_body'],
                          row['n_tokens_title'], row['title'], row['url']
                          )
        session.add(article)

    session.commit()
    session.close()
def updateAdd():
    requestDetails = request.json
    userID = requestDetails["userID"]
    maskID = requestDetails["maskID"]

    session = Session()

    newOrder = Order(userID, maskID)

    session.add(newOrder)

    session.commit()
    session.close()

    return '', 200  # Successfully update the masks for the user
Beispiel #17
0
def main(filename):
    Base.metadata.create_all(Engine)
    session = Session()
    articles = pd.read_csv(filename)
    # uid,body,title,url,newspaper_uid,host,token_title,token_body
    for index, row in articles.iterrows():
        logger.info(f'Saving article {index}')
        article = Article(row['uid'], row['body'], row['title'], row['url'],
                          row['newspaper_uid'], row['host'],
                          row['token_title'], row['token_body'])

        session.add(article)

    session.commit()
    session.close()
def handler(event, context):
    # check authorization
    authorized_user_types = [UserType.ADMIN, UserType.MENTOR, UserType.PAID]
    success, _ = check_auth(event['headers']['Authorization'],
                            authorized_user_types)
    if not success:
        return http_status.unauthorized()

    # # create a new session
    session = Session()
    jobId = event["pathParameters"]["jobId"]
    job = session.query(Job).get(jobId)
    info = json.loads(event["body"])

    if job != None:  #if it was a valid jobid, and the job was found
        keys = info.keys()
        for key in keys:
            value = info[key]
            logger.info('key is ' + str(key) + ' and value is ' + str(value))
            if key == 'salary':
                if value is None or value == '':
                    value = 0
                else:
                    value = int(value)
            elif key == 'deadline':
                value = datetime.fromtimestamp(value).replace(hour=0,
                                                              minute=0,
                                                              second=0,
                                                              microsecond=0)
            elif key == 'tags':
                value = []

            if key == 'job_applications':
                continue

            setattr(job, key, value)

        session.commit()
        session.close()

        if "job_status" in keys and info["job_status"] is "ACTIVE":
            job_title = job.title
            today = datetime.today().strftime("%Y-%m-%d")
            hiring_manager = job.posted_by
            subject = "[MAX Aspire] Your job is now active!"
            body = f"Salaam!\r\n\nWe are delighted to confirm that the job posting {job_title} is successfully posted on MAX Aspire job board on {today}. You will frequently be notified about the job applications as and when received. Keep an eye on your member portal/email.\r\n\nWe appreciate you putting your trust in MAX Aspire. We wish you luck in hiring the best possible candidate form our talented pool of aspiring professionals.\r\n\nBest regards,\nTeam MAX Aspire\r\n"
            send_email(to_addresses=hiring_manager,
                       subject=subject,
                       body_text=body)

        return http_status.success()
    else:
        return http_status.not_found()
    def email_to_id(self, email):
        session = Session()

        try:
            user = session.query(User).filter(User.email == email).one()
        except:
            print("Email", id, "not found...")
            session.close()
            return None
        else:
            session.close()
            return user.id
Beispiel #20
0
def main(filename):
    # Generating schema in DB
    Base.metadata.create_all(Engine)
    session = Session()
    articles = pd.read_csv(filename)
    # Iterating the DataFrame
    for index, row in articles.iterrows():
        logger.info('Loading article uid {} into DB'.format(row['uid']))
        article = Article(row['uid'], row['body'], row['host'],
                          row['newspaper_uid'], row['tokens_body'],
                          row['tokens_title'], row['title'], row['url'])
        # Inserting the article into the DB
        session.add(article)
    #Po'que ajá
    session.commit()
    session.close()
Beispiel #21
0
    def get_items_by_auction(auction_id):
        results = []
        items = Session().query(distinct(AuctionItem.name).label('name')).filter(AuctionItem.auction_id == auction_id).all()

        # loop thru items and gather price data
        for item in items:
            data = Session().query(AuctionItem)\
                .filter(AuctionItem.auction_id == auction_id)\
                .filter(AuctionItem.name == item.name)\
                .all()

            results.append({
                'item': item.name, 
                'data': [{ 
                    'site': x.site,
                    'modified': x.modified,
                    'link': x.link,
                    'max': AuctionItem.max_price(x.price),
                    'avg': AuctionItem.avg_price(x.price)
                } for x in data]
            })

        return results
def handler(event, context):
    # check authorization
    authorized_user_types = [
        UserType.ADMIN, UserType.MENTOR, UserType.PAID, UserType.FREE
    ]
    success, _ = check_auth(event['headers']['Authorization'],
                            authorized_user_types)
    if not success:
        return http_status.unauthorized()

    industryTagId = event["pathParameters"].get(
        "industryTagId") if event["pathParameters"] else None
    if not industryTagId:
        return http_status.bad_request(
            "missing path parameter(s): 'industryTagId'")

    session = Session()
    industry_tag = session.query(IndustryTag).get(industryTagId.lower())
    session.close()
    if not industry_tag:
        return http_status.not_found()

    return http_status.success(json.dumps(row2dict(industry_tag)))
def main(filename):
    Base.metadata.create_all(engine)  #genera el schema
    session = Session()  #generamos la sesion
    articles = pd.read_csv(filename)  #leermos el csv

    #iteramos por todos los articulos en los archivos limpios
    for index, row in articles.iterrows():
        logger.info('Loading article uid {} into DB'.format(row['uid']))
        article = Article(row['uid'], row['body'], row['link'], row['title'],
                          row['newspaper_uid'], row['host'],
                          row['n_tokens_body'], row['n_tokens_title'])

        session.add(article)

    session.commit()  #genera commit
    session.close()  #cierra la sesion
Beispiel #24
0
def set_query_table():

    #census = Table('census2018', metadata, autoload=True)
    census = Table('cgy2018', metadata,
                   Column('comm_code', String, primary_key=True),
                   Column('name', String), Column('sector', String),
                   Column('cclass', String), Column('res_cnt', String),
                   Column('dwell_cnt', String), Column('comm_structure',
                                                       String),
                   Column('gcoord', String), Column('gcenter', String))

    mapper(Communities, census)
    session = Session()
    return session
Beispiel #25
0
def parseMasks():
    with open('sampleIR.json', 'r') as read_file:
        data = json.load(read_file)

    session = Session()

    for eachImage in data:

        address = data[eachImage]["address"]
        points = data[eachImage]["points"]

        newMaskImage = Mask(address, points)
        session.add(newMaskImage)

    session.commit()
    session.close()
Beispiel #26
0
def editorAccion(accion_id):
    email = dict(session).get('email', None)
    user = User.get_by_email(email)
    given_name = dict(session).get('given_name', None)
    s = Session()
    query = s.query(Accion)
    accion = query.filter(Accion.id == accion_id).first()
    if request.method == 'POST' and 'actualizarA' in request.form:
        dictupdate = {
            Accion.nombre: request.form['nombre'],
            Accion.descripcion: request.form['descripcion'],
            Accion.recompensa: float(request.form['recompensa']),
            Accion.indicadorKpi: request.form['indicadorKpi'],
            Accion.kpiObj: int(request.form['kpiObj'])
        }
        query.filter(Accion.id == accion_id).update(dictupdate,
                                                    synchronize_session=False)
        s.commit()
    return render_template("editoraccion.html",
                           accion=accion,
                           email=email,
                           name=given_name,
                           user=user)
Beispiel #27
0
    def __init__(self, topic, group_id):
        if not os.environ['KAFKA_HOST'] or not os.environ['KAFKA_PORT']:
            sys.exit("KAFKA_HOST and KAFKA_PORT must be configured")

        kafka_url = os.environ['KAFKA_HOST'] + ':' + os.environ['KAFKA_PORT']
        self.__producer = KafkaProducer(bootstrap_servers=kafka_url)
        self.__consumer = KafkaConsumer(topic,
                                        bootstrap_servers=[kafka_url],
                                        auto_offset_reset='earliest',
                                        enable_auto_commit=True,
                                        group_id=group_id,
                                        consumer_timeout_ms=10000)

        self.__session = Session()
Beispiel #28
0
def Add_Department():
    dep_name = input("Enter Department Name: ")

    new_dep = Department(dep_name=dep_name)
    session = Session()

    session.add(new_dep)

    session.commit()
Beispiel #29
0
class GetWithdrawDayHandler(threading.Thread):
    def __init__(self,
                 threadName,
                 username,
                 password,
                 trade_password="******"):
        super(GetWithdrawDayHandler, self).__init__(name=threadName)
        self.session = Session(username, password, trade_password)
        if not self.session.set_cookies():
            print 'Cookie set failed...'

    def run(self):
        count = 0
        while count < 10:
            if not self.session.set_cookies():
                time.sleep(1)
                continue
            code = self.session.get_withdraw_day()
            if code == "1000":
                break
            else:
                time.sleep(1)
                count = count + 1
                continue
Beispiel #30
0
    def addEntrada(self, datos):
        # Añadir
        try:

            e = datos['entrada'][0]
            # Creamos una nueva sesion
            session = Session()
            entrada = Entradas(e['evento'], e['precio'], None,
                               e['descripcion'])

            session.add(entrada)
            session.commit()
            session.close()

            return 201
        except:
            return 400
Beispiel #31
0
def login():
    try:
        data = request.get_json()
        if all(keys in data for keys in LOGIN_LST):
            username = data['username'].upper()
            password = data['password']
            sess = Session()
            user = sess.query(UserDetail).filter_by(name=username,
                                                    passwd=password).first()
            sess.close()
            if user:
                session["logged"] = True
                session.permanent = True
                APP.permanent_session_lifetime = timedelta(minutes=5)
                return redirect(url_for('profile_info', username=username))
            else:
                return jsonify({
                    "status": 401,
                    "result": "Incorrect Username or Password"
                })
        else:
            return jsonify({"status": 400, "result": "Error in JSON format"})
    except Exception as exception:
        return jsonify({"status": 500, "result": exception})
Beispiel #32
0
def handler(event, context):
    # check authorization
    authorized_user_types = [
        UserType.ADMIN,
        UserType.MENTOR
    ]
    success, user = check_auth(event['headers']['Authorization'], authorized_user_types)
    if not success:
        return http_status.unauthorized()

    # validate body
    body = json.loads(event["body"])
    if not body.get('senior_executive') or not body.get('chat_type') or body['chat_type'] not in ChatType.__members__:
        return http_status.bad_request("invalid parameter(s): 'senior_executive, chat_type'")

    senior_executive = body['senior_executive']
    success = edit_auth(user, senior_executive)
    if not success:
        return http_status.unauthorized()

    chat_type = ChatType[body['chat_type']]
    description = body.get('description')
    tags = body.get('tags')
    fixed_date = body.get('fixed_date')
    if chat_type in mandatory_date and not fixed_date:
        return http_status.bad_request("missing body attribute { fixed_date } with chat_type { %s }" % (chat_type.name))
 
    session = Session()
    chat_new = Chat(
        chat_type=chat_type, description=description,
        chat_status=ChatStatus.PENDING, tags=tags,
        senior_executive=senior_executive
    )

    admin_update_declared_chats_frequency(senior_executive, 1)
    if fixed_date:
        chat_new.fixed_date = datetime.fromtimestamp(fixed_date).replace(hour=0, minute=0,second=0, microsecond=0)
        chat_new.chat_status = ChatStatus.ACTIVE
    else:
        admin_update_remaining_chats_frequency(senior_executive, 1)

    session.add(chat_new)
    session.commit()
    session.close()

    return http_status.success()
Beispiel #33
0
def main(filename):
    Base.metadata.create_all(engine)
    session = Session()

    articles = pd.read_csv(filename)
    # iterrow permite generar un loop en cada una de las filas del datagrame
    #portanto lo que devolvera sera el indice y la columna
    for index, row in articles.iterrows():
        logger.info('cargando el articulo uid {} en la DB'.format(row['uid']))
        article = Article(row['uid'], row['body'], row['host'],
                          row['newspaper_uid'], row['n_tokenise_body'],
                          row['n_tokenise_title'], row['title'], row['url'])
        #ingresando articulos a la base de datos
        session.add(article)

    session.commit()
    session.close()
Beispiel #34
0
def get_vaga(identificadorVaga=None):
    session = Session()
    vagas = None
    if identificadorVaga is not None:
        vagas = session.query(Vaga).filter(
            Vaga.identificador == identificadorVaga).first()
    else:
        vagas = session.query(Vaga).all()
    session.close()
    return vagas
Beispiel #35
0
class BorrowRecordsRepository:
    session = Session()

    def get_all(self):
        all_borrow_records = self.session.query(BorrowRecord).all()
        return jsonify(get_users_list_as_json(all_borrow_records))

    def get_by_id(self, book_record_id):
        borrow_record = self.session.query(BorrowRecord).get(book_record_id)
        return jsonify(get_borrow_record_as_json(borrow_record))

    def save(self, new_borrow_record):
        exist_book = self.session.query(Book).filter(
            Book.id == new_borrow_record["book"]["id"]).first()
        exist_user = self.session.query(User).filter(
            User.id == new_borrow_record["user"]["id"]).first()

        new_borrow_record = BorrowRecord(exist_book, exist_user,
                                         datetime.now())
        self.session.add(new_borrow_record)
        self.session.commit()

        return jsonify(get_borrow_record_as_json(new_borrow_record))

    def update(self, borrow_record_id, borrow_record):
        exist_borrow_record = self.session.query(BorrowRecord).filter(
            BorrowRecord.id == borrow_record_id).first()

        exist_book = self.session.query(Book).filter(
            Book.id == borrow_record["book"]["id"]).first()
        exist_user = self.session.query(User).filter(
            User.id == borrow_record["user"]["id"]).first()

        exist_borrow_record.book = exist_book
        exist_borrow_record.user = exist_user

        self.session.commit()
        return jsonify(get_borrow_record_as_json(exist_borrow_record))

    def delete(self, borrow_record_id):
        result = self.session.query(BorrowRecord).filter(
            BorrowRecord.id == borrow_record_id).delete()
        self.session.commit()

        if result == 0:
            return False
        else:
            return True
    def search_passwordhash(self, id):
        session = Session()

        try:
            user = session.query(User).filter(User.id == id).one()
        except:
            print("User", id, "not found...")
            session.close()
            return None
        else:
            print("User found")
        finally:
            session.close()
            return user.passwordhash
Beispiel #37
0
def main(filename):
    Base.metadata.create_all(engine)
    session = Session()
    articles = pd.read_csv(filename)

    for index, row in articles.iterrows():
        logger.info("Loading article uid {} into DB".format(row["uid"]))
        article = Article(
            row["uid"],
            row["body"],
            row["host"],
            row["title"],
            row["newspaper_uid"],
            row["url"],
        )
        session.add(article)
    session.commit()
    session.close()
def save(data):
    d = json_loads(data)
    session = Session()

    time_equals = Weather.dt == d["dt"]
    city_equals = Weather.city == d["city"]

    # check if data is in the DB
    q = session.query(Weather).filter(time_equals).filter(city_equals).count()
    print(q)

    # if data is not in db, save it
    if q <= 0:
        w = Weather(d['temp'], d['temp_min'], d['temp_max'], d['dt'],
                    d['humidity'], d['city'])
        session.add(w)
        session.commit()
        session.close()
        print(str(data) + 'is saved')
    else:
        print('not saved')
    return
	def updateStatus (self, gridTasks):
		print ("UPDATE STATUS")
		GWHosts = obtainGWResources()
		if GWHosts ==[]:
			print("Error when parsing host information file, employing information from past executions")	
			return GWHosts
		#PILOTS
		pastPilots = Session.query(Pilot).all()
		presentPilots = []
	
		#load XML to memory and extract data from hosts
		for resource in GWHosts:
			hostname = resource.getElementsByTagName("HOSTNAME")[0].firstChild.data.strip().lower() #TODO: remove "unicode" from TEXT

			try:	
				foundLrms = resource.getElementsByTagName("LRMS_NAME")[0].firstChild.data.strip().lower()
			except:
				print ("Could not obtain resource LRMS, skipping it")
				continue
			
			if foundLrms != "jobmanager-pilot":
				continue
		
			#ahora son todos pilots
			try:	
				freeNodeCount = int(resource.getElementsByTagName("FREENODECOUNT")[0].firstChild.data)
			except:
				freeNodeCount = 0
			if not freeNodeCount > 0:
				continue;

			for pilot in pastPilots:
				if pilot.hostname == hostname:
					presentPilots.append(pilot)
					break	
		#the pilots that we will first employ are the resulting ones
		self.pilots = presentPilots
Beispiel #40
0
from datetime import date
from base import Session, engine
from models import Base, Author, Book

Base.metadata.create_all(engine)
session = Session()

author_1 = Author(name='J.R.R. Tolkien', birth=date(1892, 1, 3))
author_2 = Author(name='J.K. Rowling', birth=date(1965, 7, 31))
author_3 = Author(name='Stephen King', birth=date(1947, 9, 21))

book_1 = Book(title='The Hobbit', published_in=date(1937, 9, 21), author_id=1)
book_2 = Book(title='The Lord of the Rings', published_in=date(1954, 7, 29), author_id=1)
book_3 = Book(title='Harry Potter', published_in=date(1997, 6, 26), author_id=2)
book_4 = Book(title='Carrie', published_in=date(1974, 1, 1), author_id=3)
book_5 = Book(title='Salem Lot', published_in=date(1975, 1, 1), author_id=3)
book_6 = Book(title='The Shining', published_in=date(1977, 1, 1), author_id=3)
book_7 = Book(title='Rage', published_in=date(1977, 1, 1), author_id=3)

session.add(author_1)
session.add(author_2)
session.add(author_3)

session.add(book_1)
session.add(book_2)
session.add(book_3)
session.add(book_4)
session.add(book_5)
session.add(book_6)
session.add(book_7)
Beispiel #41
0
	def updateInfoAfterExecution(self, gridTask):

		print("Updating info after exetcution of task " +gridTask.gwID + " on host " + gridTask.host.hostname + " (hostID " + str(gridTask.host.id) + ")")
		gridTask.status="CLEAR"

		#1.- abrir el archivo correspondiente a esa task
		hostToUpdate = gridTask.host

		execution_file = base.tmpExecutionFolder + "/execution_result_" + gridTask.gwID + ".xml"

		#1.- abrir el archivo correspondiente a esa task
		try:
			doc = xml.dom.minidom.parse(execution_file)
		except:
			print("failed when updating info after execution. File " + execution_file + " could not be found")
			hostToUpdate.updateInfoAfterFailedExecution()
			Session.add(gridTask)
			return

		#si los archivos de salida deseados no existen, también la cuento como fallida
		for outputFile in self.outputFiles.split(","):

			#JOB_ID has to be replaced by gwID as it happens along the execution
			splittedFile = outputFile.split('JOB_ID')
			output=""
			for pos in range(len(splittedFile)):
				output += splittedFile[pos]
				if pos < len(splittedFile) -1:
					output+=gridTask.gwID


			if not os.path.exists(base.tmpExecutionFolder + "/" + output):
					print("failed when updating info after execution. output file " + base.tmpExecutionFolder + "/"  + output + " could not be found")
					hostToUpdate.updateInfoAfterFailedExecution()
					Session.add(gridTask)
					return

		executionInfoList = doc.getElementsByTagName('execution_info')

		gridTaskType = None
		remoteHostName = None
		executionTime = None
		dataSize = None
		realSamples = None

		for executionData in executionInfoList:

			try:
				gridTaskType = executionData.getElementsByTagName("type")[0].firstChild.data #TODO: remove "unicode" from TEXT
				remoteHostName = executionData.getElementsByTagName("hostname")[0].firstChild.data
				executionTime = float(executionData.getElementsByTagName("execution_time")[0].firstChild.data)
				dataSize = float(executionData.getElementsByTagName("data_size")[0].firstChild.data)
				realSamples = int(executionData.getElementsByTagName("real_samples")[0].firstChild.data)
			except:
				print ("Error when reading execution file, exiting" )
				Session.add(gridTask)
				return

		#2.- procesar los resultados

		if gridTaskType != "execution":
			print ("ERROR when updating info from an application execution")
			print("Incorrect task type, expecting \"execution\"")
			gridTask.status = "CLEAR"
			Session.add(gridTask)
			return

		if remoteHostName != hostToUpdate.hostname:
			print ("ERROR when updating info from a application execution")
			print("Incorrect host name, expecting " + hostToUpdate.hostname)
			gridTask.status = "CLEAR"
			Session.add(gridTask)
			return


		if executionTime == 0:
			print ("ERROR when updating info from an application execution")
			print ("Execution time appears to be zero, and that's quite strange")
			gridTask.status = "CLEAR"
			hostToUpdate.updateInfoAfterFailedExecution()
			Session.add(gridTask)
			return

		totalActiveTime = InformationManager.readTotalActiveTime(gridTask.gwID)
		if totalActiveTime == -1:
			print ("ERROR when updating info from an application execution")
			print ("Could not read active time from GridWay log, considering that task failed")
			gridTask.status = "CLEAR"
			hostToUpdate.updateInfoAfterFailedExecution()
			Session.add(gridTask)
			return

		queueTime = InformationManager.readQueueTime(gridTask.gwID)
		if queueTime == -1:
			print ("ERROR when updating info from an application execution")
			print ("Could not read queue time from GridWay log, considering that task failed")
			gridTask.status = "CLEAR"
			hostToUpdate.updateInfoAfterFailedExecution()
			Session.add(gridTask)
			return


		transferTime = totalActiveTime - executionTime
		if transferTime > 0:
			bandwidth = dataSize / transferTime
		else:
			bandwidth = -1
		#3.- actualizar rendimiento del host

		computationalEffort = self.profile.constantEffort + self.profile.sampleEffort * realSamples
		whetstones = computationalEffort / executionTime

		hostToUpdate.updateInfoFromSuccessFulExecution(whetstones, queueTime, bandwidth)
		hostToUpdate.failedProfilings -=1

		#4 actualizar estado de la tarea y la apliación.
		gridTask.realSamples = realSamples
		gridTask.status = "CLEAR"
		self.remainingSamples -= realSamples

		print("APPLICATION UPDATE: " + str(self.remainingSamples) + "/" + str(self.desiredSamples) + " left")

		#5.- eliminar archivos temporales
		try:
			#os.remove(execution_file)
			print ("In Application.py, I would be deletign" + execution_file)
			print ("Execution file has been successfully deleted: " + execution_file)
		except:
			print ("Could not delete profiling file " + execution_file)

		#6 update info on DB
		gridTask.endDate = datetime.now()

		Session.add(gridTask)
		Session.add(self)
Beispiel #42
0
	def updateInfoAfterProfiling(self, gridTask):

		#1.- leer la info  correspondiente a esa aplicacion
		#tener en cuenta que es una serie de parejas de valores
		execution_file = base.tmpExecutionFolder + "execution_result_" + gridTask.gwID + ".xml"
		try:
			doc = xml.dom.minidom.parse(execution_file)
		except:
			print("failed when profiling host " + gridTask.host.hostname + ". File " + execution_file + " could not be found")
			gridTask.host.updateInfoAfterFailedProfiling()
			Session.add(gridTask.host)
			return

		executionInfoList = doc.getElementsByTagName('results')

		executionResults=[]
		for executionData in executionInfoList:
			profileInfoList = executionData.getElementsByTagName('profile')
			for profileInfo in profileInfoList:
				samples = int(profileInfo.getElementsByTagName("samples")[0].firstChild.data) #TODO: remove "unicode" from TEXT
				time = int(profileInfo.getElementsByTagName("time")[0].firstChild.data)

				if samples == 0:
					print ("There was an error on host " + gridTask.host.hostname + ". File " + execution_file + ", no samples have been executed")
					gridTask.host.updateInfoAfterFailedProfiling()
					Session.add(gridTask.host)
					return
				executionResults.append([samples, time])


		#2.- procesar los resultados
		#TODO: esto es muy posible que esté mal
		numSimulations = len(executionResults)
		if numSimulations == 0:
			print ("There was an error on host " + gridTask.host.hostname + ". File " + execution_file + ", no simulations performed")
			gridTask.host.updateInfoAfterFailedProfiling()
			Session.add(gridTask.host)
			return
		avgSampleTime = 0
		for i in range(numSimulations-1):
			newSamples = executionResults[i+1][0] - executionResults[i][0]
			newTime = executionResults[i+1][1] - executionResults[i][1]

			if newTime <=0:
				value = 0
				continue
			else:
				value = newTime / newSamples ######TODO esta es la clave!!!

			#weighted sample time, so later executions are more important than the first ones
			#the reason is that they executed more samples, so their information is more valuable
			if i==0 or avgSampleTime == 0:
				avgSampleTime = value
			else:
				avgSampleTime = 0.6 * value + 0.4 * avgSampleTime


		acumConstantTime = 0

		#this has been modifiied to avoid zero values
		for i in range(len(executionResults)):
			acumConstantTime += max(0, executionResults[i][1] - executionResults[i][0] *  avgSampleTime)
		avgConstantTime = acumConstantTime / numSimulations


		normalizedSampleEffort = avgSampleTime * gridTask.host.getWhetstones()
		normalizedConstantEffort = avgConstantTime * gridTask.host.getWhetstones()


		print("RESULTADO DEL PROFILING")
		print ("task id: " + gridTask.gwID)
		print("normalizado, this_sample_effort = " + str(normalizedSampleEffort))
		print("normalizado, this_constant_effort = " + str(normalizedConstantEffort))

		print("STATUS DEL PROFILING")
		print("normalizado, sample_effort = " + str(self.profile.sampleEffort))
		print("normalizado, constant_effort = " + str(self.profile.constantEffort))
		print("")


		if (normalizedSampleEffort < 0 ) or (normalizedConstantEffort < 0):
			print ("A normalized effort below zero makes no sense, dismissing profile")
			return
		self.profile.updateInfoAfterProfiling(normalizedConstantEffort, normalizedSampleEffort)

		Session.add(self.profile)
Beispiel #43
0
from base import Session
from models import table_class_list

session = Session()

if __name__ == '__main__':
    tables = {index: table for index, table in enumerate(table_class_list)}

    while True:
        print('Add records:', '\n')
        print('Tables list:', '\n')
        print(tables, '\n')
        user_response = input('Enter \'table #\' or \'x\' to exit: ')

        if user_response.lower() == 'x' or user_response == '':
            break
        else:
            table = tables.get(int(user_response), None)

            if table is not None:
                # Retrieve list of columns
                columns = [col.name for col in table.__table__.columns if col.name != 'id']
                print(columns)

                # Initialize a new record
                data = {}

                for col in columns:
                    value = input('type a value for {0}: '.format(col))
                    data[col] = value
Beispiel #44
0
from GridTask import GridTask
import UserInterface

if __name__ == '__main__':
	requirementsFile = "/soft/users/home/u5682/nfs/workspace/montera/FLUKA/fluka_example.mt"

	print("Starting connection with database")
			
	metadata = MetaData()
	
	myDBDesign = DBDesign()
	
	hostDB = myDBDesign.HostDBDesign(metadata)
	applicationDB = myDBDesign.ApplicationDesign(metadata)
	appProfileDB = myDBDesign.AppProfileDBDesign(metadata)
	gridTaskDesignDB = myDBDesign.GridTaskDBDesign(metadata)
	parameterDesignDB = myDBDesign.parameterDBDesign(metadata)

	metadata.create_all(base.engine)
	
	print("Database is correct")
	print("")
	
	
	myInfrastructure = Infrastructure.Infrastructure('lrms')
	myInfrastructure.createHostProfilingTasks()
	

	Session.add(myInfrastructure)
	Session.commit()
	def createInfrastructureTasks(self, infrastructureTasks):	
		
		print ("-------------------")
		print ("-------------------")

		print ("createInfrastructureTasks- NewPilotInfrastructure")

	#	self.showHosts()
					
		hostList = obtainGWResources()
		
		hostsToProfile = []

		print ("Analyzing resources ")
		for hostInfo in hostList:
			hostName = hostInfo.getElementsByTagName("HOSTNAME")[0].firstChild.data.strip().lower() #TODO: remove "unicode" from TEXT
			whetstones=0

			try:
				foundArch = hostInfo.getElementsByTagName("ARCH")[0].firstChild.data
			except:
				foundArch=""
				
			try:	
				foundCpuMHz = int(hostInfo.getElementsByTagName("CPU_MHZ")[0].firstChild.data)
			except:
				foundCpuMHz = 0
			
			try:	
				foundLrms = hostInfo.getElementsByTagName("LRMS_NAME")[0].firstChild.data.strip().lower()
			except:
				foundLrms = None
				print ("Could not find LRMS for host " + hostName + ", skipping it")
				continue
			
			try:	
				freeNodeCount = int(hostInfo.getElementsByTagName("FREENODECOUNT")[0].firstChild.data)
			except:
				freeNodeCount = 0	

			if foundLrms == "jobmanager-pilot":			
				#solo tenemos en cuenta los pilots con al menos un slot disponible
				if not freeNodeCount > 0:
					continue
				
				username = os.getenv("USER")
				genericStringArgs = hostInfo.getElementsByTagName("GENERIC_VAR_STR")
				for node in genericStringArgs:
					if node.attributes['NAME'].value =="PILOT_REAL_HOSTNAME":
						workerNode = node.attributes['VALUE'].value.strip().lower()
					if node.attributes['NAME'].value =="PILOT_REAL_RESOURCE":
						site = node.attributes['VALUE'].value.strip().lower()
				
				genericIntArgs = hostInfo.getElementsByTagName("GENERIC_VAR_INT")
				for node in genericIntArgs:
					if node.attributes['NAME'].value =="PILOT_" + username + "_VAR_5":
						whetstones = int(node.attributes['VALUE'].value.strip().lower())
						if whetstones > 65534: 
							whetstones = 0
				# 	whetstones = 0
			#if host is unknown, create a profiling task
			currentHost = self.getHost(hostName)
			if  currentHost == None:
				print ("Host/Pilot  not found. hostname: " + hostName + ", LRMS: " + foundLrms)
				if foundLrms == "jobmanager-pilot":
					#he encontrado un pilot:
					#primero busco e resource, y si no existe lo creo.
					#luego creo un pilot que utilice ese resource

					pilotResource = base.Session.query(PilotResource).filter(PilotResource.site == site, PilotResource.workerNode == workerNode).first()
					if pilotResource == None:
						print ("	PilotResource was not found, creating a new one")
						pilotResource = PilotResource(site, workerNode)
					print ("	Creating a new Pilot in NewPilotInfrastructure.createInfrastructureTasks")
					newHost = Pilot(hostName, arch=foundArch, cpuMHz = foundCpuMHz, pilotResource = pilotResource, whetstones = whetstones)
					self.pilots.append(newHost)
					Session.add(newHost)

				else:
					print ("	Creating a new Host in NewPilotInfrastructure.createInfrastructureTasks")
					newHost = Host(hostName, arch=foundArch, cpuMHz = foundCpuMHz, lrms=foundLrms)
					self.hosts.append(newHost)
					Session.add(newHost)

				#ESTO ES PARA HACER EL PROFILING DE LOS PILOT SI NO HAN PUBLICADO LOS WHETSTONES, SI NO NO HACE FALTA	
				#===============================================================
				# if whetstones == 0 or whetstones > 65534: 
				# 	whetstones = 0
				# 	print ("	Host to profile: " + hostName + ": whetstone value not initialized ")
				# 	hostsToProfile.append(newHost)
				# 	#store new host on databae (faiulre resistance
				# 	Session.add(newHost)
				#===============================================================
				
			#if information has changed, update host information
			elif (currentHost.getWhetstones() != whetstones):
				#va con un set porque es una operación más complicada, así que está encapsulada en esta funcion
				currentHost.setWhetstones(whetstones)	
				Session.add(currentHost)
				print ("Host: " + hostName + " UPDATED, new whetstones=" + str(whetstones))

			elif currentHost.lrms == None:
				currentHost.lrms = foundLrms


		#pprofiling of new sites		
		hostProfilingTasks = [ExecutionManager.createHostProfilingTask(host) 
							for host in hostsToProfile
							for i in range(base.profilingTasksPerHost)]
		
		

	
		#estamos asumiento que todos los pilots publican la variable esa con su 
		#rendimiento, con lo que no hay que hacer el profiling de nada. 		
				
		#AHORA, EN ESA NUEVA APROXIMACION, QUEREMOS TENER UNOS CUANTO SBENCHMARKS PARA IR ARRANCANDO PILOTS 
		print ("creating fake profiling tasks")
		
		existingFakeTasks = len([task for task in infrastructureTasks if task.host.hostname=="" and task.status != "PENDING"])
		existingGoodPilots = len (self.getGoodHosts())
		existingProfilingTasks = len(hostProfilingTasks)
		#fakeTasksToCreate = base.maxRunningTasks - (existingFakeTasks + existingGoodPilots + existingProfilingTasks)
		fakeTasksToCreate = base.maxRunningTasks - existingFakeTasks
		
		print ("	Desired tasks: " + str(base.maxRunningTasks))
		print ("	Existing fake tasks: " + str(existingFakeTasks))
		print ("	Existing good pilots: " + str(existingGoodPilots))
		print ("	created: " + str(fakeTasksToCreate))
		
		emptyHost = FakeHost()
		fakeHostProfilingTasks = [ExecutionManager.createFakeHostProfilingTask(emptyHost) 
					for i in range(fakeTasksToCreate)]

		hostProfilingTasks+=fakeHostProfilingTasks
		
		
		return hostProfilingTasks
Beispiel #46
0
from base import Session
from models import table_class_list
from list import list_all

session = Session()

if __name__ == '__main__':
    tables = {index: table for index, table in enumerate(table_class_list)}

    while True:
        print('Delete records:', '\n')
        print('Tables list:', '\n')
        print(tables, '\n')
        user_response = input('Enter \'table #\' or \'x\' to exit: ')

        if user_response.lower() == 'x' or user_response == '':
            break
        else:
            # Get the Table Class
            table = tables.get(int(user_response), None)

            if table is not None:
                # Query to DB
                list_all(table)

                record_id = input('Enter \'record #\' or \'c\' to cancel...')
                if record_id.lower() != 'c' and record_id != '':
                    # Get the Record ID
                    record = session.query(table).get(record_id)

                    if input('Add record? y/n') == 'y' and record is not None:
Beispiel #47
0
	def createInfrastructureTasks(self, infrastructureTasks):	
		
		print ("---------------------")
		print ("---------------------")
		print ("---------------------")

		print ("CREATE INFRASTRUCTURE TASKS")
		
					
					
		hostsToProfile = []
		
		hostList =  obtainGWResources()
		for hostInfo in hostList:
			hostName = hostInfo.getElementsByTagName("HOSTNAME")[0].firstChild.data #TODO: remove "unicode" from TEXT
			
			try:
				foundArch = hostInfo.getElementsByTagName("ARCH")[0].firstChild.data
			except:
				foundArch=""
				
			try:	
				foundCpuMHz = int(hostInfo.getElementsByTagName("CPU_MHZ")[0].firstChild.data)
			except:
				foundCpuMHz = 0
			
			try:	
				foundLrms = hostInfo.getElementsByTagName("LRMS_NAME")[0].firstChild.data
			except:
				foundLrms = None
			
			try:	
				freeNodeCount = int(hostInfo.getElementsByTagName("FREENODECOUNT")[0].firstChild.data)
			except:
				freeNodeCount = 0	

			if foundLrms != None:
				if foundLrms == "jobmanager-pilot":			
					#solo tenemos en cuenta los pilots con al menos un slot disponible
					if not freeNodeCount > 0:
						continue
			
			#if a certain LRMS is desired, remove the hosts with a different one
			if self.lrms != None:
				if foundLrms != self.lrms:
					continue
				
			#if host is unknown, create a profiling task
			currentHost = self.getHost(hostName)
			if  currentHost == None:
				newHost = Host(hostName, arch=foundArch, cpuMHz = foundCpuMHz, lrms=foundLrms)
				self.hosts.append(newHost)
				hostsToProfile.append(newHost)
				#store new host on databae (faiulre resistance
				Session.add(newHost)
			#if information has changed, update host information
			elif (currentHost.arch != foundArch) or (currentHost.cpuMHz != foundCpuMHz):
				#TODO: pensar que hacer aqui. habria que eliminar el viejo o solo sobreescribir la información? Si se elimina el viejo, que pasa con las tareas ahí ejecutadas? No es trivial
				currentHost.arch = foundArch
				currentHost.cpuMHz = foundCpuMHz
				if currentHost.lrms == None:
					currentHost.lrms = foundLrms
				hostsToProfile.append(currentHost)
				Session.add(currentHost)
					
			elif currentHost.shouldBeProfiled():
				if currentHost.lrms == None:
					currentHost.lrms = foundLrms
				hostsToProfile.append(currentHost)

				
		#print("Host profiling: submission of 1 tasks per host")		
		hostProfilingTasks = [ExecutionManager.createHostProfilingTask(host) 
							for host in hostsToProfile
							for i in range(1)]
		
		
		
		siteTasks = []
		for task in hostProfilingTasks:
			found=False
			for gridTask in infrastructureTasks:
				if gridTask.host.hostname == task.host.hostname:
					found=True
					break
			if not found:
				siteTasks.append(task)
				
				
		#Esto es para el primer experimento de montera + gwpilot
		#queremos tener pilots funcionando, así que los arranco con esto 
		if self.lrms=="jobmanager-pilot":
			print ("creating fake profiling tasks")
			
			existingFakeTasks = len([task for task in infrastructureTasks if task.host.hostname=="" and task.status != "PENDING"])
			existingGoodPilots = len (self.getGoodHosts())
			existingProfilingTasks = len(hostProfilingTasks)
			#fakeTasksToCreate = base.maxRunningTasks - (existingFakeTasks + existingGoodPilots + existingProfilingTasks)
			fakeTasksToCreate = base.maxRunningTasks - existingFakeTasks
			
			print ("	Desired tasks: " + str(base.maxRunningTasks))
			print ("	Existing fake tasks: " + str(existingFakeTasks))
			print ("	Existing good pilots: " + str(existingGoodPilots))
			print ("	created: " + str(fakeTasksToCreate))
			
			emptyHost = FakeHost()
			fakeHostProfilingTasks = [ExecutionManager.createWakeUpask(emptyHost) 
						for i in range(fakeTasksToCreate)]
	
			siteTasks+=fakeHostProfilingTasks
		
				

		return siteTasks
Beispiel #48
0
	def updateInfoAfterProfiling(self, gridTask):
		print ("Updating info after profiling site " + gridTask.host.hostname)
		print ("	Task info:")
		print  ("	Task id: " + str(gridTask.id))
		print ("	GW ID: " + gridTask.gwID)
		print ("	desired host: " + gridTask.host.hostname)
		print ("	Host type: " + str(gridTask.host.__class__))

		gridTask.status="CLEAR"
		
		#1.- abrir el archivo correspondiente a esa task
		execution_file = base.tmpExecutionFolder + "execution_result_" + gridTask.gwID + ".xml"
		try:
			doc = xml.dom.minidom.parse(execution_file)
		except:
			print("failed when profiling host " + gridTask.host.hostname + ". File " + execution_file + " could not be found")
			gridTask.host.updateInfoAfterFailedProfiling()
			Session.add(gridTask)
			return
		
		executionInfoList = doc.getElementsByTagName('execution_info')
		for executionData in executionInfoList:
			try:
				gridTaskType = executionData.getElementsByTagName("type")[0].firstChild.data #TODO: remove "unicode" from TEXT
				remoteHostName = executionData.getElementsByTagName("hostname")[0].firstChild.data
				whetstones = float(executionData.getElementsByTagName("whetstones")[0].firstChild.data)
				waitingTime = float(executionData.getElementsByTagName("execution_time")[0].firstChild.data)
				dataSize = float(executionData.getElementsByTagName("data_size")[0].firstChild.data)
			except:
				print("failed when profiling host " + gridTask.host.hostname + ". File " + execution_file + " could not be found")
				gridTask.host.updateInfoAfterFailedProfiling()
				Session.add(gridTask)
				return
			
		#2.- procesar los resultados
	
		if gridTaskType != "benchmark":
			print ("ERROR when updating info from a site profiling")
			print("Incorrect task type, readed " + gridTaskType + " and should be \"benchmark\"")
			print ("	considering the execution as failed")
			gridTask.host.updateInfoAfterFailedProfiling()
			Session.add(gridTask)
			return
			
		if remoteHostName != gridTask.host.hostname:
			print ("ERROR when updating info from a site profiling")
			print("Incorrect host name, readed" + remoteHostName + " and should be " + gridTask.host.hostname)
			print ("	considering the execution as failed")
			gridTask.host.updateInfoAfterFailedProfiling()
			Session.add(gridTask)
			return
			
		totalActiveTime = InformationManager.readTotalActiveTime(gridTask.gwID)
		transferTime = totalActiveTime - waitingTime
		queueTime = InformationManager.readQueueTime(gridTask.gwID)
			
		if transferTime > 0:
			bandwidth = dataSize / transferTime
		else:
			bandwidth = -1
		#3.- suministrar esa info al host.
		gridTask.host.updateInfoFromSuccessFulExecution(whetstones, queueTime, bandwidth)
		
		
		#4.- eliminar archivos temporales
		try:
			#os.remove(execution_file)
			print ("IN application.py, I would be deleting " + execution_file)
			print ("Profiling file has been successfully deleted: " + execution_file)
		except:
			print ("Could not delete profiling file " + execution_file)

		gridTask.endDate = datetime.now()
Beispiel #49
0
def readRequirements(requirementsFile):
	

	myRequirementsFile = open(requirementsFile,'r')
	
	myname = mydesiredSamples = mymaxSupportedSamples =  \
	mymaxProfilingTime = mypreProcessScript = mypostProcessScript = myinputFiles = myoutputFiles = myworkingDirectory = None
	myRemoteMontera = myLrms = myInfrastructure = None		
		
	for line in myRequirementsFile.readlines():
		normalizedLine = line.lower().strip()
		
		if normalizedLine.startswith("#"):
			continue;
		if normalizedLine.count("name") > 0:
			myname = line.split("=")[1].strip()
			
		elif normalizedLine.count("num_samples") > 0:
			mydesiredSamples= int(line.split("=")[1].strip())
		
		elif normalizedLine.count("postprocess_script") > 0:
			mypostProcessScript = line.split("=")[1].strip()

		elif normalizedLine.count("preprocessScript") > 0:
			mypreProcessScript = line.split("=")[1].strip()
			
		elif normalizedLine.count("max_profiling_time") > 0:
			mymaxProfilingTime = line.split("=")[1].strip()
			
		elif normalizedLine.count("max_supported_samples") > 0:
			mymaxSupportedSamples = line.split("=")[1].strip()
			
			#los ficheros de entrada estan separados por comas
		elif normalizedLine.count("input_files") > 0:
			myinputFiles = line.split("=")[1].strip().replace(' ', '')
			
		elif normalizedLine.count("output_files") > 0:
			myoutputFiles = line.split("=")[1].strip().replace(' ', '')

		elif normalizedLine.count("working_directory") > 0:
			myworkingDirectory = line.split("=")[1].strip()		
			
		elif normalizedLine.count("schedulingalgorithm") > 0:
			mySchedulingAlgorithm = normalizedLine.split("=")[1].strip()
		
		elif normalizedLine.count("remoteMontera") > 0:
			myRemoteMontera = normalizedLine.split("=")[1].strip()			
						
		elif normalizedLine.count("infrastructure") > 0:
			myInfrastructure = normalizedLine.split("=")[1].strip()	
	
		elif normalizedLine.count("parametricjob") > 0:
			print("Parametric Job");
			mySpecificationFile = line.split("=")[1].strip()
			myAnalyzer = Analyzer(mySpecificationFile)
			mydesiredSamples = myAnalyzer.numTasks
			myRemoteMontera = "parametricRemoteMontera.sh" 
			
			
	
	myApplicationProfile = Session.query(AppProfile).filter(AppProfile.applicationName == myname).first()
	if myApplicationProfile == None:
		myApplicationProfile = AppProfile(myname)
				
	myApplication = Application(myname, mydesiredSamples, 
			maxSupportedSamples = mymaxSupportedSamples,
			maxProfilingTime = mymaxProfilingTime,
			preProcessScript = mypreProcessScript,
			postProcessScript = mypostProcessScript,
			inputFiles = myinputFiles,
			outputFiles = myoutputFiles,
			workingDirectory = myworkingDirectory, 
			profile = myApplicationProfile,
			schedulingAlgorithm = mySchedulingAlgorithm,
			remoteMontera = myRemoteMontera,
			infrastructure = myInfrastructure)
	
	
	return myApplication
Beispiel #50
0
from base import Session
from models import table_class_list

session = Session()


if __name__ == '__main__':
    tables = {index: table for index, table in enumerate(table_class_list)}

    while True:
        print('Filter table:', '\n')
        print('Tables list:', '\n')
        print(tables, '\n')
        user_response = input('Enter \'table #\' or \'x\' to exit: ')

        if user_response.lower() == 'x' or user_response == '':
            break
        else:
            # Get the Table Class
            table = tables.get(int(user_response), None)

            if table is not None:
                columns = {index: col.name
                           for index, col in enumerate(table.__table__.columns)
                           if col.name != 'id'}

                print(columns, '\n')
                user_response = input('Enter \'column #\' or \'c\' to cancel: ')
                if user_response.lower() == 'c' or user_response == '':
                    pass
                else:
Beispiel #51
0
from base import Session
from models import table_class_list
from list import list_all

session = Session()

if __name__ == '__main__':
    tables = {index: table for index, table in enumerate(table_class_list)}

    while True:
        print('Modify records:', '\n')
        print('Tables list:', '\n')
        print(tables, '\n')
        user_response = input('Enter \'table #\' or \'x\' to exit: ')

        if user_response.lower() == 'x' or user_response == '':
            break
        else:
            table = tables.get(int(user_response), None)

            if table is not None:
                # Retrieve list of columns
                columns = [col.name.replace('_', '') for col in table.__table__.columns if col.name != 'id']

                # List all records
                list_all(table)

                record_id = input('Enter \'record #\' or \'c\' to cancel...')
                if record_id.lower() != 'c' and record_id != '':

                    if input('Update record? y/n') == 'y' and record_id is not None:
	def load(self):	
		print ("LOAD - pilotInfrastructures")			
		#SITES
		pastHosts = Session.query(Host).filter(Host.type=="hosts").all()
		presentHosts = []
		
		#PILOTS
		pastPilots = Session.query(Pilot).all()
		print ("Deleting all past pilots from database")
		for pilot in pastPilots:
			base.Session.delete(pilot)
				
		presentPilots = []
	
		GWHosts = obtainGWResources()
		print ("Updating information")
		#load XML to memory and extract data from hosts
		try: 
			for resource in GWHosts:
				
				hostname = resource.getElementsByTagName("HOSTNAME")[0].firstChild.data.strip().lower() #TODO: remove "unicode" from TEXT
				try:	
					foundLrms = resource.getElementsByTagName("LRMS_NAME")[0].firstChild.data
				except:
					print ("Could not obtain resource LRMS for site " + hostname + ", skipping it")
					continue
				
				if foundLrms == "jobmanager-pilot":
					#nombre de pilot
					genericArgs = resource.getElementsByTagName("GENERIC_VAR_STR")
					for node in genericArgs:
						if node.attributes['NAME'].value =="PILOT_REAL_HOSTNAME":
							workerNode = node.attributes['VALUE'].value.strip().lower()
						if node.attributes['NAME'].value =="PILOT_REAL_RESOURCE":
							site = node.attributes['VALUE'].value.strip().lower()
						
					#numero de nodos libres. Esto en los pilots funciona bien, y es lo que se emplea para saber si está
					#activo o apagado
					try:	
						freeNodeCount = int(resource.getElementsByTagName("FREENODECOUNT")[0].firstChild.data)
					except:
						freeNodeCount = 0
					if not freeNodeCount > 0:
						continue;

					#cargamos el pilot en la base de datos:
					#primero buscamos el pilotResource, o lo creamos  si no existe
					#después creamos el pilot que emplee ese recurso
					
					pilotResource = base.Session.query(PilotResource).filter(PilotResource.site == site, PilotResource.workerNode == workerNode).first()
					
					if pilotResource == None:
						pilotResource = PilotResource(site, workerNode)
					
					pilot = Pilot(hostname, pilotResource=pilotResource)
					presentPilots.append(pilot)

				else:	#it is a site	
					for host in pastHosts:
						if host.hostname == hostname:
							presentHosts.append(host)
							break		
		
		except:	
			print("Error when parsing host information file, employing information from past executions")	
			self.hosts = pastHosts
			self.pilots = []
			return
			
		
		#the hosts that we will first employ are the resulting ones
		self.pilots = presentPilots
		self.hosts = presentHosts