class Host(models.Model):
"""
Host model - stores all host information for a cluster.
Relationships:
| account
| allocation
Attributes:
| name: Primary key.
| binaries: dict of label:path pairs for vasp binaries.
| check_queue: Path to showq command
| checked_time: datetime object for the last time the queue was
| checked.
| hostname: Full host name.
| ip_address: Full ip address.
| nodes: Total number of nodes.
| ppn: Number of processors per node.
| running: dict of PBS_ID:state pairs.
| sub_script: Path to qsub command
| sub_text: Path to queue file template.
| utilization: Number of active cores (based on showq).
| walltime: Maximum walltime on the machine.
| state: State code. 1=Up, 0=Full (auto-resets to 1 when jobs are
| collected), -1=Down.
"""
name = models.CharField(max_length=63, primary_key=True)
ip_address = models.IPAddressField(null=True)
hostname = models.CharField(max_length=255)
binaries = DictField()
ppn = models.IntegerField(default=8)
nodes = models.IntegerField(default=30)
walltime = models.IntegerField(default=3600 * 24)
sub_script = models.CharField(max_length=120)
sub_text = models.TextField(default='/usr/local/bin/qsub')
check_queue = models.CharField(max_length=180,
default='/usr/local/maui/bin/showq')
checked_time = models.DateTimeField(default=datetime.min)
running = DictField()
utilization = models.IntegerField(default=0)
state = models.IntegerField(default=1)
class Meta:
app_label = 'qmpy'
db_table = 'hosts'
def __str__(self):
return self.name
@staticmethod
def create():
"""
Classmethod to create a Host model. Script will ask you questions about
the host to add, and will return the created Host.
"""
host = {}
host['name'] = raw_input('Hostname:')
if Host.objects.filter(name=host['name']).exists():
print 'Host by that name already exists!'
exit(-1)
host['ip_address'] = raw_input('IP Address:')
if Host.objects.filter(ip_address=host['ip_address']).exists():
print 'Host at that address already exists!'
exit(-1)
host['ppn'] = raw_input('Processors per node:')
host['nodes'] = raw_input('Max nodes to run on:')
host['sub_script'] = raw_input('Command to submit a script '
'(e.g. /usr/local/bin/qsub):')
host['check_queue'] = raw_input('Command for showq (e.g.'
'/usr/local/maui/bin/showq):')
host['sub_text'] = raw_input('Path to qfile template:')
h = Host(**host)
h.save()
@classmethod
def get(cls, name):
try:
return cls.objects.get(name=name)
except cls.DoesNotExist:
return cls(name=name)
@property
def accounts(self):
return list(self.account_set.all())
@property
def jobs(self):
jobs = []
for acct in self.accounts:
jobs += list(acct.job_set.filter(state=1))
return jobs
@property
def active(self):
if self.state < 1:
return False
elif self.utilization > 5 * self.nodes * self.ppn:
return False
else:
return True
@property
def percent_utilization(self):
return 100. * float(self.utilization) / (self.nodes * self.ppn)
def get_utilization(self):
util = 0
for acct in self.account_set.all():
for job in acct.job_set.filter(state=1):
util += job.ncpus
self.utilization = util
return util
def get_project(self):
"""
Out of the active projects able to run on this host,
select one at random
Output:
Project, Active project able to run on this host
"""
proj = Project.objects.filter(allocations__host=self, state=1)
proj = proj.filter(task__state=0)
if proj.exists():
return random.choice(list(proj.distinct()))
def get_tasks(self, project=None):
tasks = queue.Task.objects.filter(state=0)
if project is None:
project = self.get_project()
if project is None:
return
tasks = tasks.filter(project_set=project)
tasks = tasks.filter(project_set__allocations__host=self)
tasks = tasks.filter(project_set__users__account__host=self)
return tasks.order_by('priority', 'id')
@property
def qfile(self):
return open(self.sub_text).read()
def get_binary(self, key):
return self.binaries[key]
def _try_login(self, timeout=5.0):
def _login():
self._tmp_acct = Allocation.get('b1004').get_account()
self._tmp_ssh = 'ssh {user}@{host} "{cmd}"'.format(
user=self._tmp_acct.user.username,
host=self._tmp_acct.host.ip_address,
cmd='whoami')
self._tmp_proc = subprocess.Popen(self._tmp_ssh,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = self._tmp_proc.communicate()
if stdout.strip() == self._tmp_acct.user.username:
print "quest is up"
self._tmp_thread = threading.Thread(target=_login)
self._tmp_thread.start()
self._tmp_thread.join(timeout)
if self._tmp_thread.is_alive():
print "unable login on quest"
self._tmp_proc.terminate()
self._tmp_thread.join()
return self._tmp_proc.returncode
def check_host(self):
"""Pings the host to see if it is online. Returns False if it is
offline."""
ret = subprocess.call("ping -c 1 -w 1 %s" % self.ip_address,
shell=True,
stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
if ret == 0:
self.state = 1
self.save()
write_resources()
return True
else:
"""Sometimes quest refuses to respond to ping requests. So, try
logging into it using an(y) account. Trying executing a command and
see if it is successful."""
if self.name == 'quest':
if self._try_login() == 0:
self.state = 1
self.save()
write_resources()
return True
self.state = -2
self.save()
return False
@property
def running_now(self):
if not self.state == 1:
return {}
if datetime.now() + timedelta(seconds=-60) > self.checked_time:
self.check_running()
return self.running
def check_running(self):
"""
Uses the hosts data and one of the associated accounts to check the PBS
queue on the Host. If it has been checked in the last 2 minutes, it
will return the previously returned result.
"""
self.checked_time = datetime.now()
if not self.state == 1:
self.running = {}
self.save()
return
account = random.choice(self.accounts)
raw_data = account.execute(self.check_queue)
running = {}
if not raw_data:
return
for line in raw_data.split('\n'):
if 'Active Jobs' in line:
continue
line = line.split()
if len(line) != 9:
continue
try:
# < Mohan
if 'Moab' in line[0]:
qid = int(line[0].strip().split('.')[1])
else:
qid = int(line[0])
running[qid] = {
'user': line[1],
'state': line[2],
'proc': int(line[3])
}
# Mohan >
except:
pass
self.running = running
self.save()
def get_running(self):
if self.running is not None:
return self.running
else:
return {}
def activate(self):
"""
Allow jobs to be run on this system. Remember to save() to enact change
"""
self.state = 1
def deactivate(self):
"""
Prevent new jobs from being started on this system.
Remember to save() changes
"""
self.state = -1
@property
def utilization_by_project(self):
utilization = defaultdict(int)
for job in self.jobs:
projects = job.task.project_set.all()
for p in projects:
utilization[str(p.name)] += float(job.ncpus) / len(projects)
if self.ppn * self.nodes > sum(utilization.values()):
utilization["Idle"] = self.ppn * self.nodes - sum(
utilization.values())
return utilization
@property
def utilization_json(self):
series = []
for k, v in self.utilization_by_project.items():
series.append({'data': v, 'label': k})
return json.dumps(series)
@property
def ncpus(self):
return self.ppn * self.nodes
class Element(models.Model):
"""
Core model for an element.
Relationships:
| :mod:`~qmpy.Atom` via atom_set
| :mod:`~qmpy.Species` via species_set
| :mod:`~qmpy.Structure` via structure_set
| :mod:`~qmpy.Entry` via entry_set
| :mod:`~qmpy.Composition` via composition_set
| :mod:`~qmpy.Calculation` via calculation_set
| :mod:`~qmpy.Potential` via potential_set
| :mod:`~qmpy.Hubbard` via hubbards
| :mod:`~qmpy.HubbardCorrection` via hubbardcorrection_set
| :mod:`~qmpy.ReferenceEnergy` via referenceenergy_set
Attributes:
| **Identification**
| z: atomic number
| name: full atomic name
| symbol: atomic symbol
| group: group in the periodic table
| period: period in the periodic table
|
| **Physical properties**
| mass: Atomic mass, in AMU (float)
| density: Density at STP, in g/cm^3 (float)
| volume: Atomic volume at STP, in A^3/atom (float)
| atomic_radii: in A (float)
| van_der_waals radii: in A (float)
| covalent_radii: in A (float)
| scattering_factors: A dictionary of scattering factor coeffs.
|
| **Thermodynamic properties**
| melt: melting point in K
| boil: boiling point in K
| specific_heat: C_p in J/K
|
| **Electronic properties**
| electronegativity: Pauling electronegativity
| ion_energy: First ionization energy. (eV)
| s_elec: # of s electrons
| p_elec: # of p electrons
| d_elec: # of d electrons
| f_elec: # of f electrons
|
| **Additional information**
| production: Annual tons of element produced.
| abundance: Amount in earths crust (ppm)
| radioactive: Are all isotopes unstable?
| HHI_P: Herfindahl-Hirschman Index for production.
| HHI_R: Herfindahl-Hirschman Index for reserve
Note:
HHI values from Gaultois, M. et al. Chem. Mater. 25, 2911-2920 (2013).
"""
### Identification
z = models.IntegerField()
name = models.CharField(max_length=20)
symbol = models.CharField(max_length=9, primary_key=True)
### Periodic table
group = models.IntegerField()
period = models.IntegerField()
### Phyical characteristics
mass = models.FloatField()
density = models.FloatField()
volume = models.FloatField()
atomic_radii = models.IntegerField()
van_der_waals_radii = models.IntegerField()
covalent_radii = models.IntegerField()
scattering_factors = DictField()
### Thermodynamics
melt = models.FloatField()
boil = models.FloatField()
specific_heat = models.FloatField()
### Electonic structure
electronegativity = models.FloatField()
first_ionization_energy = models.FloatField()
s_elec = models.IntegerField()
p_elec = models.IntegerField()
d_elec = models.IntegerField()
f_elec = models.IntegerField()
### misc
HHI_P = models.FloatField(default=0)
HHI_R = models.FloatField(default=0)
production = models.FloatField(default=0)
radioactive = models.BooleanField(default=False)
class Meta:
app_label = 'qmpy'
db_table = 'elements'
# builtins
def __str__(self):
return self.symbol
# accessor
@classmethod
def get(cls, value):
"""
Return an element object. Accepts symbols and atomic numbers, or a list
of symbols/atomic numbers.
Examples::
>>> Element.get('Fe')
>>> Element.get(26)
>>> Element.get(['Fe', 'O'])
"""
if isinstance(value, cls):
return value
elif isinstance(value, list):
return [cls.get(v) for v in value]
elif isinstance(value, int):
return cls.objects.get(z=value)
elif isinstance(value, str):
return cls.objects.get(symbol=value)
# methods
def species_distribution(self):
counts = {}
for s in self.species_set.all():
counts[s.ox] = s.structure_set.count()
return counts