def __init__(self,
base_uri=None,
username=None,
password=None,
headers=None,
additional_args=None,
timeout=g5k_configuration.get('api_timeout')):
""":param base_uri: server base uri. defaults to
``g5k_configuration.get('api_uri')``
:param username: username for the http connection. If None
(default), use default from
``g5k_configuration.get('api_username')``. If False, don't
use a username at all.
:param password: password for the http connection. If None
(default), get the password from a keyring (if available) or
interactively.
:param headers: http headers to use. If None (default),
default headers accepting json answer will be used.
:param additional_args: a dict of optional arguments (string
to string mappings) to pass at the end of the url of all
http requests.
:param timeout: timeout for the http connection.
"""
if not base_uri:
base_uri = g5k_configuration.get('api_uri')
self.base_uri = base_uri.rstrip("/")
if headers:
self.headers = headers
else:
self.headers = {'ACCEPT': 'application/json'}
self.additional_args = g5k_configuration["api_additional_args"]
if additional_args:
self.additional_args.update(additional_args)
if username:
self.username = username
else:
self.username = g5k_configuration.get('api_username')
self.password = password
if self.username and not self.password:
self.password = _get_api_password(self.username, self.base_uri)
self.timeout = timeout
def __init__(self, base_uri=None,
username=None, password=None,
headers=None, additional_args=None,
timeout=g5k_configuration.get('api_timeout')):
""":param base_uri: server base uri. defaults to
``g5k_configuration.get('api_uri')``
:param username: username for the http connection. If None
(default), use default from
``g5k_configuration.get('api_username')``. If False, don't
use a username at all.
:param password: password for the http connection. If None
(default), get the password from a keyring (if available) or
interactively.
:param headers: http headers to use. If None (default),
default headers accepting json answer will be used.
:param additional_args: a dict of optional arguments (string
to string mappings) to pass at the end of the url of all
http requests.
:param timeout: timeout for the http connection.
"""
if not base_uri:
base_uri = g5k_configuration.get('api_uri')
self.base_uri = base_uri.rstrip("/")
if headers:
self.headers = headers
else:
self.headers = {
'ACCEPT': 'application/json'
}
self.additional_args = g5k_configuration["api_additional_args"]
if additional_args:
self.additional_args.update(additional_args)
if username:
self.username = username
else:
self.username = g5k_configuration.get('api_username')
self.password = password
if self.username and not self.password:
self.password = _get_api_password(self.username, self.base_uri)
self.timeout = timeout
def get_frontend_host(frontend = None):
"""Given a frontend name, or None, and based on the global configuration, returns the frontend to connect to or None."""
if frontend == None:
frontend = get_default_frontend()
if g5k_configuration.get('no_ssh_for_local_frontend') == True and frontend == get_default_frontend():
frontend = None
if frontend:
frontend = Host(frontend)
return frontend
def get_frontend_host(frontend=None):
"""Given a frontend name, or None, and based on the global configuration, returns the frontend to connect to or None."""
if frontend == None:
frontend = get_default_frontend()
if g5k_configuration.get('no_ssh_for_local_frontend'
) == True and frontend == get_default_frontend():
frontend = None
if frontend:
frontend = Host(frontend)
return frontend
def _get_api_password_check_func(username, uri, password):
if g5k_configuration['api_verify_ssl_cert'] == False:
verify = False
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
else:
verify = True
response = requests.get(uri,
auth=(username, password),
verify=verify,
timeout=g5k_configuration.get('api_timeout'))
return (response.status_code in [200, 304])
def __site_charter_remaining(site, day, user = None):
try:
with G5kAutoPortForwarder(site,
'oardb.' + site + '.grid5000.fr',
g5k_configuration['oar_pgsql_ro_port']) as (host, port):
start, end = get_oar_day_start_end(day)
if not user:
user = g5k_configuration.get('api_username')
if not user:
user = os.environ['LOGNAME']
conn = psycopg2.connect(host = host, port = port,
user = g5k_configuration['oar_pgsql_ro_user'],
password = g5k_configuration['oar_pgsql_ro_password'],
database = g5k_configuration['oar_pgsql_ro_db'])
try:
logger.trace("getting jobs for user %s on %s for %s" % (user, site, day))
OOC_total_site_used = 0
OOC_site_quota = 0
for cluster in get_site_clusters(site):
total_cluster_used = 0
for j in _get_jobs(conn, cluster, user, start, end):
logger.trace("%s:%s - job: start %s, end %s, walltime %s, %s" % (
site, cluster, format_unixts(j[7]),
format_unixts(j[8]), format_seconds(j[6]), j))
if _job_intersect_charter_period(j):
cluster_used = (j[9] + j[10]) * j[6]
logger.trace("%s:%s job %i intersects charter -> uses %is of cluster quota" % (
site, cluster, j[0], cluster_used,))
total_cluster_used += cluster_used
#cluster_quota = cluster_num_cores(cluster) * 3600 * 2
cluster_quota = _cluster_num_available_cores(conn, cluster) * 3600 * 2
logger.trace("%s:%s total cluster used = %i (%s), cluster quota = %i (%s)" % (
site, cluster,
total_cluster_used, format_seconds(total_cluster_used),
cluster_quota, format_seconds(cluster_quota)))
threading.currentThread().remaining[cluster] = max(0, cluster_quota - total_cluster_used)
OOC_total_site_used += total_cluster_used
OOC_site_quota += cluster_quota
logger.trace("%s to compare with outofchart: total used = %i (%s), %i%% of site quota = %i (%s)" % (
site,
OOC_total_site_used, format_seconds(OOC_total_site_used),
int(float(OOC_total_site_used) / float(OOC_site_quota) * 100.0),
OOC_site_quota, format_seconds(OOC_site_quota)))
finally:
conn.close()
except Exception as e:
logger.warn("error connecting to oar database / getting planning from " + site)
logger.detail("exception:\n" + format_exc())
def __site_charter_remaining(site, day, user = None):
try:
with G5kAutoPortForwarder(site,
'oardb.' + site + '.grid5000.fr',
g5k_configuration['oar_pgsql_ro_port']) as (host, port):
start, end = get_oar_day_start_end(day)
if not user:
user = g5k_configuration.get('api_username')
if not user:
user = os.environ['LOGNAME']
conn = psycopg2.connect(host = host, port = port,
user = g5k_configuration['oar_pgsql_ro_user'],
password = g5k_configuration['oar_pgsql_ro_password'],
database = g5k_configuration['oar_pgsql_ro_db'])
try:
logger.trace("getting jobs for user %s on %s for %s" % (user, site, day))
OOC_total_site_used = 0
OOC_site_quota = 0
for cluster in get_site_clusters(site):
total_cluster_used = 0
for j in _get_jobs(conn, cluster, user, start, end):
logger.trace("%s:%s - job: start %s, end %s, walltime %s, %s" % (
site, cluster, format_unixts(j[7]),
format_unixts(j[8]), format_seconds(j[6]), j))
if _job_intersect_charter_period(j):
cluster_used = (j[9] + j[10]) * j[6]
logger.trace("%s:%s job %i intersects charter -> uses %is of cluster quota" % (
site, cluster, j[0], cluster_used,))
total_cluster_used += cluster_used
#cluster_quota = cluster_num_cores(cluster) * 3600 * 2
cluster_quota = _cluster_num_available_cores(conn, cluster) * 3600 * 2
logger.trace("%s:%s total cluster used = %i (%s), cluster quota = %i (%s)" % (
site, cluster,
total_cluster_used, format_seconds(total_cluster_used),
cluster_quota, format_seconds(cluster_quota)))
threading.currentThread().remaining[cluster] = max(0, cluster_quota - total_cluster_used)
OOC_total_site_used += total_cluster_used
OOC_site_quota += cluster_quota
logger.trace("%s to compare with outofchart: total used = %i (%s), %i%% of site quota = %i (%s)" % (
site,
OOC_total_site_used, format_seconds(OOC_total_site_used),
int(float(OOC_total_site_used) / float(OOC_site_quota) * 100.0),
OOC_site_quota, format_seconds(OOC_site_quota)))
finally:
conn.close()
except Exception as e:
logger.warn("error connecting to oar database / getting planning from " + site)
logger.detail("exception:\n" + format_exc())
def get_default_frontend():
"""Return the name of the default frontend."""
global __default_frontend, __default_frontend_cached #IGNORE:W0603
if not __default_frontend_cached:
__default_frontend_cached = True
if g5k_configuration.get("default_frontend"):
__default_frontend = g5k_configuration["default_frontend"]
else:
try:
localhost = socket.gethostname()
except socket.error:
localhost = ""
mo = re.search("^[^ \t\n\r\f\v\.]+\.([^ \t\n\r\f\v\.]+)\.grid5000.fr$", localhost)
if mo:
__default_frontend = mo.group(1)
else:
__default_frontend = None
return __default_frontend
def get_default_frontend():
"""Return the name of the default frontend."""
global __default_frontend, __default_frontend_cached #IGNORE:W0603
if not __default_frontend_cached:
__default_frontend_cached = True
if g5k_configuration.get("default_frontend"):
__default_frontend = g5k_configuration["default_frontend"]
else:
try:
localhost = socket.gethostname()
except socket.error:
localhost = ""
mo = re.search(
"^[^ \t\n\r\f\v\.]+\.([^ \t\n\r\f\v\.]+)\.grid5000.fr$",
localhost)
if mo:
__default_frontend = mo.group(1)
else:
__default_frontend = None
return __default_frontend