def compute_coorm_slots(planning, excluded_elements=None):
""" """
slots = []
limits = _slots_limits(planning)
for start in limits:
stop = 10**25
free_cores = {'grid5000': 0}
for site, site_planning in planning.items():
free_cores[site] = 0
for cluster, cluster_planning in site_planning.items():
free_cores[cluster] = 0
if cluster in get_g5k_clusters(queues=None):
for host, host_planning in cluster_planning.items():
for free_slot in host_planning['free']:
if free_slot[0] <= start and free_slot[0] < stop:
free_cores[cluster] += get_host_attributes(
host)['architecture']['nb_cores']
free_cores[site] += get_host_attributes(
host)['architecture']['nb_cores']
free_cores['grid5000'] += get_host_attributes(
host)['architecture']['nb_cores']
if free_slot[1] < stop:
stop = free_slot[1]
slots.append((start, stop, free_cores))
return slots
def get_hosts_jobs(hosts, walltime, out_of_chart=False):
"""Find the first slot when the hosts are available and return a
list of jobs_specs
:param hosts: list of hosts
:param walltime: duration of reservation
"""
hosts = map(lambda x: x.address if isinstance(x, Host) else x, hosts)
planning = get_planning(elements=hosts, out_of_chart=out_of_chart)
limits = _slots_limits(planning)
walltime = get_seconds(walltime)
for limit in limits:
all_host_free = True
for site_planning in planning.itervalues():
for cluster, cluster_planning in site_planning.iteritems():
if cluster in get_g5k_clusters():
for host_planning in cluster_planning.itervalues():
host_free = False
for free_slot in host_planning['free']:
if free_slot[0] <= limit and free_slot[
1] >= limit + walltime:
host_free = True
if not host_free:
all_host_free = False
if all_host_free:
startdate = limit
break
else:
logger.error('Unable to find a slot for %s', hosts)
return None
jobs_specs = []
for site in planning.keys():
site_hosts = map(get_host_longname,
filter(lambda h: get_host_site(h) == site, hosts))
sub_res = "{host in ('" + "','".join(site_hosts) + "')}/nodes=" + str(
len(site_hosts))
jobs_specs.append((OarSubmission(resources=sub_res,
reservation_date=startdate), site))
return jobs_specs
def get_hosts_jobs(hosts, walltime, out_of_chart=False):
"""Find the first slot when the hosts are available and return a
list of jobs_specs
:param hosts: list of hosts
:param walltime: duration of reservation
"""
hosts = map(lambda x: x.address if isinstance(x, Host) else x, hosts)
planning = get_planning(elements=hosts, out_of_chart=out_of_chart)
limits = _slots_limits(planning)
walltime = get_seconds(walltime)
for limit in limits:
all_host_free = True
for site_planning in planning.itervalues():
for cluster, cluster_planning in site_planning.iteritems():
if cluster in get_g5k_clusters():
for host_planning in cluster_planning.itervalues():
host_free = False
for free_slot in host_planning['free']:
if free_slot[0] <= limit and free_slot[1] >= limit + walltime:
host_free = True
if not host_free:
all_host_free = False
if all_host_free:
startdate = limit
break
else:
logger.error('Unable to find a slot for %s', hosts)
return None
jobs_specs = []
for site in planning.keys():
site_hosts = map(get_host_longname,
filter(lambda h: get_host_site(h) == site,
hosts))
sub_res = "{host in ('" + "','".join(site_hosts) + "')}/nodes=" + str(len(site_hosts))
jobs_specs.append((OarSubmission(resources=sub_res,
reservation_date=startdate), site))
return jobs_specs
def find_free_slot(slots, resources_wanted):
"""Return the first slot (a tuple start date, end date, resources) with enough resources
:param slots: list of slots returned by ``compute_slots``
:param resources_wanted: a dict describing the wanted ressources
``{'grid5000': 50, 'lyon': 20, 'stremi': 10 }``"""
# We need to add the clusters nodes to the total nodes of a site
real_wanted = resources_wanted.copy()
for cluster, n_nodes in resources_wanted.items():
if cluster in get_g5k_clusters(queues=None):
site = get_cluster_site(cluster)
if site in resources_wanted:
real_wanted[site] += n_nodes
for slot in slots:
vlan_free = True
if 'kavlan' in resources_wanted:
if isinstance(slot[2]['kavlan'], int):
if slot[2]['kavlan'] == 0:
vlan_free = False
elif isinstance(slot[2]['kavlan'], list):
if len(slot[2]['kavlan']) == 0:
vlan_free = False
slot_ok = True
for element, n_nodes in slot[2].items():
if element in real_wanted and real_wanted[element] > n_nodes \
and real_wanted != 'kavlan':
slot_ok = False
if slot_ok and vlan_free:
if 'kavlan' in resources_wanted:
resources_wanted['kavlan'] = slot[2]['kavlan']
return slot
return None, None, None
def get_vms_slot(vms, elements, slots, excluded_elements=None):
"""Return a slot with enough RAM and CPU """
chosen_slot = None
mem = vms[0]['mem']
cpu = vms[0]['n_cpu']
req_ram = sum([vm['mem'] for vm in vms])
req_cpu = sum([vm['n_cpu'] for vm in vms]) / 3
logger.debug('RAM %s CPU %s', req_ram, req_cpu)
for element in excluded_elements:
if element in get_g5k_sites():
excluded_elements += [
cluster for cluster in get_site_clusters(element)
if cluster not in excluded_elements
]
if 'grid5000' in elements:
clusters = [
cluster for cluster in get_g5k_clusters()
if cluster not in excluded_elements
and get_cluster_site not in excluded_elements
]
else:
clusters = [
element for element in elements if element in get_g5k_clusters()
and element not in excluded_elements
]
for element in elements:
if element in get_g5k_sites():
clusters += [
cluster for cluster in get_site_clusters(element)
if cluster not in clusters
and cluster not in excluded_elements
]
for slot in slots:
hosts = []
for element in slot[2]:
if str(element) in clusters:
n_hosts = slot[2][element]
for i in range(n_hosts):
hosts.append(Host(str(element + '-1.' + \
get_cluster_site(element) + '.grid5000.fr')))
attr = get_CPU_RAM_FLOPS(hosts)['TOTAL']
if attr['CPU'] > req_cpu and attr['RAM'] > req_ram:
chosen_slot = slot
break
del hosts[:]
if chosen_slot is None:
return None, None
resources_needed = {}
resources_available = chosen_slot[2]
logger.debug('resources available' + pformat(resources_available))
iter_clusters = cycle(clusters)
while req_ram > 0 or req_cpu > 0:
cluster = iter_clusters.next()
if resources_available[cluster] == 0:
clusters.remove(cluster)
iter_clusters = cycle(clusters)
else:
host = cluster + '-1'
attr = get_CPU_RAM_FLOPS([host])
resources_available[cluster] -= 1
req_ram -= float(attr[host]['RAM'] / mem) * mem
req_cpu -= float(attr[host]['CPU'] / cpu) * cpu
if cluster not in resources_needed:
resources_needed[cluster] = 0
resources_needed[cluster] += 1
if 'kavlan' in elements:
resources_needed['kavlan'] = 1
logger.debug('resources needed' + pformat(resources_needed))
return chosen_slot[0], distribute_hosts(chosen_slot[2], resources_needed,
excluded_elements)
def get_vms_slot(vms, elements, slots, excluded_elements=None):
"""Return a slot with enough RAM and CPU """
chosen_slot = None
mem = vms[0]['mem']
cpu = vms[0]['n_cpu']
req_ram = sum([vm['mem'] for vm in vms])
req_cpu = sum([vm['n_cpu'] for vm in vms]) / 3
logger.debug('RAM %s CPU %s', req_ram, req_cpu)
for element in excluded_elements:
if element in get_g5k_sites():
excluded_elements += [cluster for cluster
in get_site_clusters(element)
if cluster not in excluded_elements]
if 'grid5000' in elements:
clusters = [cluster for cluster in get_g5k_clusters()
if cluster not in excluded_elements
and get_cluster_site not in excluded_elements]
else:
clusters = [element for element in elements
if element in get_g5k_clusters()
and element not in excluded_elements]
for element in elements:
if element in get_g5k_sites():
clusters += [cluster
for cluster in get_site_clusters(element)
if cluster not in clusters
and cluster not in excluded_elements]
for slot in slots:
hosts = []
for element in slot[2]:
if str(element) in clusters:
n_hosts = slot[2][element]
for i in range(n_hosts):
hosts.append(Host(str(element + '-1.' + \
get_cluster_site(element) + '.grid5000.fr')))
attr = get_CPU_RAM_FLOPS(hosts)['TOTAL']
if attr['CPU'] > req_cpu and attr['RAM'] > req_ram:
chosen_slot = slot
break
del hosts[:]
if chosen_slot is None:
return None, None
resources_needed = {}
resources_available = chosen_slot[2]
logger.debug('resources available' + pformat(resources_available))
iter_clusters = cycle(clusters)
while req_ram > 0 or req_cpu > 0:
cluster = iter_clusters.next()
if resources_available[cluster] == 0:
clusters.remove(cluster)
iter_clusters = cycle(clusters)
else:
host = cluster + '-1'
attr = get_CPU_RAM_FLOPS([host])
resources_available[cluster] -= 1
req_ram -= float(attr[host]['RAM'] / mem) * mem
req_cpu -= float(attr[host]['CPU'] / cpu) * cpu
if cluster not in resources_needed:
resources_needed[cluster] = 0
resources_needed[cluster] += 1
if 'kavlan' in elements:
resources_needed['kavlan'] = 1
logger.debug('resources needed' + pformat(resources_needed))
return chosen_slot[0], distribute_hosts(chosen_slot[2], resources_needed,
excluded_elements)
def get_planning(elements=['grid5000'],
vlan=False,
subnet=False,
storage=False,
out_of_chart=False,
starttime=None,
endtime=None,
ignore_besteffort=True,
queues='default'):
"""Retrieve the planning of the elements (site, cluster) and others resources.
Element planning structure is ``{'busy': [(123456,123457), ... ], 'free': [(123457,123460), ... ]}.``
:param elements: a list of Grid'5000 elements ('grid5000', <site>, <cluster>)
:param vlan: a boolean to ask for KaVLAN computation
:param subnet: a boolean to ask for subnets computation
:param storage: a boolean to ask for sorage computation
:param out_of_chart: if True, consider that days outside weekends are busy
:param starttime: start of time period for which to compute the planning, defaults to now + 1 minute
:param endtime: end of time period for which to compute the planning, defaults to 4 weeks from now
:param ignore_besteffort: True by default, to consider the resources with besteffort jobs as available
:param queues: list of oar queues for which to get the planning
Return a dict whose keys are sites, whose values are dict whose keys
are cluster, subnets, kavlan or storage,
whose values are planning dicts, whose keys are hosts, subnet address range,
vlan number or chunk id planning respectively.
"""
if not starttime:
starttime = int(time() + timedelta_to_seconds(timedelta(minutes=1)))
starttime = int(get_unixts(starttime))
if not endtime:
endtime = int(starttime +
timedelta_to_seconds(timedelta(weeks=4, minutes=1)))
endtime = int(get_unixts(endtime))
if 'grid5000' in elements:
sites = elements = get_g5k_sites()
else:
sites = list(
set([site for site in elements if site in get_g5k_sites()] + [
get_cluster_site(cluster) for cluster in elements
if cluster in get_g5k_clusters(queues=queues)
] + [
get_host_site(host)
for host in elements if host in get_g5k_hosts()
or get_host_shortname(host) in get_g5k_hosts()
]))
if len(sites) == 0:
logger.error('Wrong elements given: %s' % (elements, ))
return None
planning = {}
for site in sites:
planning[site] = {}
for cluster in get_site_clusters(site, queues=queues):
planning[site][cluster] = {}
for site in sites:
if vlan:
planning[site].update({'vlans': {}})
if subnet:
planning[site].update({'subnets': {}})
if storage:
planning[site].update({'storage': {}})
if _retrieve_method == 'API':
_get_planning_API(planning, ignore_besteffort)
elif _retrieve_method == 'PostgreSQL':
_get_planning_PGSQL(planning, ignore_besteffort)
if out_of_chart:
_add_charter_to_planning(planning, starttime, endtime)
for site_pl in planning.values():
for res_pl in site_pl.values():
for el_planning in res_pl.values():
el_planning['busy'].sort()
_merge_el_planning(el_planning['busy'])
_trunc_el_planning(el_planning['busy'], starttime, endtime)
_fill_el_planning_free(el_planning, starttime, endtime)
# cleaning
real_planning = deepcopy(planning)
for site, site_pl in planning.items():
for cl, cl_pl in site_pl.items():
if cl in ['vlans']:
continue
keep_cluster = False
for h in cl_pl:
if not (get_host_site(h) in elements
or get_host_cluster(h) in elements
or get_host_shortname(h) in elements or h in elements):
del real_planning[site][cl][h]
else:
keep_cluster = True
if not keep_cluster:
del real_planning[site][cl]
return real_planning
def distribute_hosts(resources_available,
resources_wanted,
excluded_elements=None,
ratio=None):
""" Distribute the resources on the different sites and cluster
:param resources_available: a dict defining the resources available
:param resources_wanted: a dict defining the resources available you really want
:param excluded_elements: a list of elements that won't be used
:param ratio: if not None (the default), a float between 0 and 1,
to actually only use a fraction of the resources."""
if excluded_elements == None: excluded_elements = []
resources = {}
#Defining the cluster you want
clusters_wanted = {}
for element, n_nodes in resources_wanted.items():
if element in get_g5k_clusters(queues=None):
clusters_wanted[element] = n_nodes
for cluster, n_nodes in clusters_wanted.items():
nodes = n_nodes if n_nodes > 0 else resources_available[cluster]
resources_available[get_cluster_site(cluster)] -= nodes
resources[cluster] = nodes
# Blacklisting clusters
for element in excluded_elements:
if element in get_g5k_clusters(
queues=None) and element in resources_available:
resources_available['grid5000'] -= resources_available[element]
resources_available[get_cluster_site(
element)] -= resources_available[element]
resources_available[element] = 0
#Defining the sites you want
sites_wanted = {}
for element, n_nodes in resources_wanted.items():
if element in get_g5k_sites() and element not in excluded_elements:
sites_wanted[element] = n_nodes
for site, n_nodes in sites_wanted.items():
resources[site] = n_nodes if n_nodes > 0 else resources_available[site]
# Blacklisting sites
for element in excluded_elements:
if element in get_g5k_sites() and element in resources_available:
resources_available['grid5000'] -= resources_available[element]
resources_available[element] = 0
#Distributing hosts on grid5000 elements
logger.debug(pformat(resources_wanted))
if 'grid5000' in resources_wanted:
g5k_nodes = resources_wanted['grid5000'] if resources_wanted[
'grid5000'] > 0 else resources_available['grid5000']
total_nodes = 0
sites = [
element for element in resources_available
if element in get_g5k_sites()
]
iter_sites = cycle(sites)
while total_nodes < g5k_nodes:
site = next(iter_sites)
if resources_available[site] == 0:
sites.remove(site)
iter_sites = cycle(sites)
else:
resources_available[site] -= 1
if site in resources:
resources[site] += 1
else:
resources[site] = 1
total_nodes += 1
logger.debug(pformat(resources))
if 'kavlan' in resources_wanted:
resources['kavlan'] = resources_available['kavlan']
# apply optional ratio
if ratio != None:
resources.update(
(x, int(floor(y * ratio))) for x, y in resources.items())
return resources
def get_jobs_specs(resources, excluded_elements=None, name=None):
""" Generate the several job specifications from the dict of resources and
the blacklisted elements
:param resources: a dict, whose keys are Grid'5000 element and values the
corresponding number of n_nodes
:param excluded_elements: a list of elements that won't be used
:param name: the name of the jobs that will be given
"""
jobs_specs = []
if excluded_elements == None:
excluded_elements = []
# Creating the list of sites used
sites = []
real_resources = resources.copy()
for resource in resources:
if resource in get_g5k_sites() and resource not in sites:
sites.append(resource)
if resource in get_g5k_clusters(queues=None):
if resource not in excluded_elements:
site = get_cluster_site(resource)
if site not in sites:
sites.append(site)
if site not in real_resources:
real_resources[site] = 0
# Checking if we need a Kavlan, a KaVLAN global or none
get_kavlan = 'kavlan' in resources
if get_kavlan:
kavlan = 'kavlan'
n_sites = 0
for resource in real_resources:
if resource in sites:
n_sites += 1
if n_sites > 1:
kavlan += '-global'
break
blacklisted_hosts = {}
for element in excluded_elements:
if element not in get_g5k_clusters(queues=None) + get_g5k_sites():
site = get_host_site(element)
if not 'site' in blacklisted_hosts:
blacklisted_hosts[site] = [element]
else:
blacklisted_hosts[site].append(element)
for site in sites:
sub_resources = ''
# Adding a KaVLAN if needed
if get_kavlan:
if not 'global' in kavlan:
sub_resources = "{type='" + kavlan + "'}/vlan=1+"
get_kavlan = False
elif site in resources['kavlan']:
sub_resources = "{type='" + kavlan + "'}/vlan=1+"
get_kavlan = False
base_sql = '{'
end_sql = '}/'
# Creating blacklist SQL string for hosts
host_blacklist = False
str_hosts = ''
if site in blacklisted_hosts and len(blacklisted_hosts[site]) > 0:
str_hosts = ''.join([
"host not in ('" + get_host_longname(host) + "') and "
for host in blacklisted_hosts[site]
])
host_blacklist = True
#Adding the clusters blacklist
str_clusters = str_hosts if host_blacklist else ''
cl_blacklist = False
clusters_nodes = 0
for cluster in get_site_clusters(site, queues=None):
if cluster in resources and resources[cluster] > 0:
if str_hosts == '':
sub_resources += "{cluster='" + cluster + "'}"
else:
sub_resources += base_sql + str_hosts + "cluster='" + \
cluster + "'" + end_sql
sub_resources += "/nodes=" + str(resources[cluster]) + '+'
clusters_nodes += resources[cluster]
if cluster in excluded_elements:
str_clusters += "cluster not in ('" + cluster + "') and "
cl_blacklist = True
# Generating the site blacklist string from host and cluster blacklist
str_site = ''
if host_blacklist or cl_blacklist:
str_site += base_sql
if not cl_blacklist:
str_site += str_hosts[:-4]
else:
str_site += str_clusters[:-4]
str_site = str_site + end_sql
if real_resources[site] > 0:
sub_resources += str_site + "nodes=" + str(real_resources[site]) +\
'+'
if sub_resources != '':
jobs_specs.append((OarSubmission(resources=sub_resources[:-1],
name=name), site))
return jobs_specs
def compute_slots(planning, walltime, excluded_elements=None):
"""Compute the slots limits and find the number of available nodes for
each elements and for the given walltime.
Return the list of slots where a slot is ``[ start, stop, freehosts ]`` and
freehosts is a dict of Grid'5000 element with number of nodes available
``{'grid5000': 40, 'lyon': 20, 'reims': 10, 'stremi': 10 }``.
WARNING: slots does not includes subnets
:param planning: a dict of the resources planning, returned by ``get_planning``
:param walltime: a duration in a format supported by get_seconds where the resources
are available
:param excluded_elements: list of elements that will not be included in the slots
computation
"""
slots = []
walltime = get_seconds(walltime)
if excluded_elements is not None:
_remove_excluded(planning, excluded_elements)
limits = _slots_limits(planning)
# Checking if we need to compile vlans planning
kavlan = False
kavlan_global = False
if len(planning) > 0:
if 'vlans' in next(iter(planning.values())):
if len(planning) > 1:
kavlan_global = True
else:
kavlan = True
for limit in limits:
log = ''
free_elements = {'grid5000': 0}
if kavlan_global:
free_vlans_global = []
for site, site_planning in planning.items():
free_elements[site] = 0
for cluster, cluster_planning in site_planning.items():
if cluster in get_g5k_clusters(queues=None):
free_elements[cluster] = 0
for host, host_planning in cluster_planning.items():
host_free = False
for free_slot in host_planning['free']:
if free_slot[0] <= limit and free_slot[
1] >= limit + walltime:
host_free = True
if host_free:
free_elements['grid5000'] += 1
free_elements[site] += 1
free_elements[cluster] += 1
log += ', ' + host
if kavlan:
free_vlans = 0
for vlan, vlan_planning in site_planning['vlans'].items():
if int(vlan.split('-')[1]) < 10:
kavlan_free = False
for free_slot in vlan_planning['free']:
if free_slot[0] <= limit and free_slot[
1] >= limit + walltime:
kavlan_free = True
if kavlan_free:
free_vlans += 1
free_elements['kavlan'] = free_vlans
elif kavlan_global:
for vlan, vlan_planning in site_planning['vlans'].items():
if int(vlan.split('-')[1]) > 10:
kavlan_global_free = False
for free_slot in vlan_planning['free']:
if free_slot[0] <= limit and free_slot[
1] >= limit + walltime:
kavlan_global_free = True
if kavlan_global_free:
free_vlans_global.append(site)
free_elements['kavlan'] = free_vlans_global
## MISSING OTHER RESOURCES COMPUTATION
logger.debug(log)
slots.append([limit, limit + walltime, free_elements])
slots.sort(key=itemgetter(0))
return slots
def draw_slots(slots, colors=None, show=False, save=True, outfile=None):
"""Draw the number of nodes available for the clusters (requires Matplotlib >= 1.2.0)
:param slots: a list of slot, as returned by ``compute_slots``
:param colors: a dict to define element coloring ``{'element': (255., 122., 122.)}``
:param show: display the slots versus time
:param save: save the plot to outfile
:param outfile: specify the output file"""
startstamp = slots[0][0]
endstamp = slots[-1][1]
if outfile is None:
outfile = 'slots_' + format_date(startstamp)
logger.info('Saving slots diagram to %s', style.emph(outfile))
if colors is None:
colors = _set_colors()
xfmt = MD.DateFormatter('%d %b, %H:%M ')
if endstamp - startstamp <= timedelta_to_seconds(timedelta(days=7)):
x_major_locator = MD.HourLocator(byhour=[9, 19])
elif endstamp - startstamp <= timedelta_to_seconds(timedelta(days=17)):
x_major_locator = MD.HourLocator(byhour=[9])
else:
x_major_locator = MD.AutoDateLocator()
max_nodes = {}
total_nodes = 0
slot_limits = []
total_list = []
i_slot = 0
for slot in slots:
slot_limits.append(slot[0])
if i_slot + 1 < len(slots):
slot_limits.append(slots[i_slot + 1][0])
i_slot += 1
for element, n_nodes in slot[2].items():
if element in get_g5k_clusters(queues=None):
if not element in max_nodes:
max_nodes[element] = []
max_nodes[element].append(n_nodes)
max_nodes[element].append(n_nodes)
if element == 'grid5000':
total_list.append(n_nodes)
total_list.append(n_nodes)
if n_nodes > total_nodes:
total_nodes = n_nodes
slot_limits.append(endstamp)
slot_limits.sort()
dates = [unixts_to_datetime(ts) for ts in slot_limits]
datenums = MD.date2num(dates)
fig = PLT.figure(figsize=(15, 10), dpi=80)
ax = PLT.subplot(111)
ax.xaxis_date()
box = ax.get_position()
ax.set_position([box.x0 - 0.07, box.y0, box.width, box.height])
ax.set_xlim(unixts_to_datetime(startstamp), unixts_to_datetime(endstamp))
ax.set_xlabel('Time')
ax.set_ylabel('Nodes available')
ax.set_ylim(0, total_nodes * 1.1)
ax.axhline(y=total_nodes,
color='#000000',
linestyle='-',
linewidth=2,
label='ABSOLUTE MAXIMUM')
ax.yaxis.grid(color='gray', linestyle='dashed')
ax.xaxis.set_major_formatter(xfmt)
ax.xaxis.set_major_locator(x_major_locator)
PLT.xticks(rotation=15)
max_nodes_list = []
p_legend = []
p_rects = []
p_colors = []
for key, value in sorted(max_nodes.items()):
if key != 'grid5000':
max_nodes_list.append(value)
p_legend.append(key)
p_rects.append(PLT.Rectangle((0, 0), 1, 1, fc=colors[key]))
p_colors.append(colors[key])
plots = PLT.stackplot(datenums, max_nodes_list, colors=p_colors)
PLT.legend(p_rects,
p_legend,
loc='center right',
ncol=1,
shadow=True,
bbox_to_anchor=(1.2, 0.5))
if show:
PLT.show()
if save:
logger.debug('Saving file %s ...', outfile)
PLT.savefig(outfile, dpi=300)
walltime = "3:00:00"
sites = ['lyon', 'rennes', 'nancy', 'grenoble', 'nantes']
source_code = "/home/lpouilloux/src/executionware_backend/"
default_connection_params['user'] = '******'
logger.info("Looking for a running job")
job = get_job_by_name(job_name, sites)
if not job[0]:
planning = get_planning(sites)
blacklisted = ['talc', 'mbi']
slots = compute_slots(planning, walltime, excluded_elements=blacklisted)
wanted = {'grid5000': 1}
start_date, end_date, resources = find_first_slot(slots, wanted)
for c in filter(lambda x: x in get_g5k_clusters(), resources.keys()):
if resources[c] > 1:
wanted = {c: 1}
break
jobs_specs = get_jobs_specs(wanted, name=job_name)
for sub, frontend in jobs_specs:
sub.walltime = walltime
sub.job_type = "deploy"
job = oarsub(jobs_specs)[0]
nodes = get_oar_job_nodes(job[0], job[1])
logger.info('Deploying host %s', nodes[0].address)
deployed, undeployed = deploy(Deployment(nodes, env_name="jessie-x64-base"))
execware_host = list(deployed)[0]
logger.info('Installing required packages %s', style.emph(packages))
walltime = "3:00:00"
sites = ['lyon', 'rennes', 'nancy', 'grenoble', 'nantes']
source_code = "/home/lpouilloux/src/executionware_backend/"
default_connection_params['user'] = '******'
logger.info("Looking for a running job")
job = get_job_by_name(job_name, sites)
if not job[0]:
planning = get_planning(sites)
blacklisted = ['talc', 'mbi']
slots = compute_slots(planning, walltime, excluded_elements=blacklisted)
wanted = {'grid5000': 1}
start_date, end_date, resources = find_first_slot(slots, wanted)
for c in filter(lambda x: x in get_g5k_clusters(), resources.keys()):
if resources[c] > 1:
wanted = {c: 1}
break
jobs_specs = get_jobs_specs(wanted, name=job_name)
for sub, frontend in jobs_specs:
sub.walltime = walltime
sub.job_type = "deploy"
job = oarsub(jobs_specs)[0]
nodes = get_oar_job_nodes(job[0], job[1])
logger.info('Deploying host %s', nodes[0].address)
deployed, undeployed = deploy(Deployment(nodes,
env_name="jessie-x64-base"))
execware_host = list(deployed)[0]