def generate():
inp = open("pseudo.list")
# Generate structure
pseudo = OrderedDict()
str = inp.read()
pseudolist = str.split()
for name in pseudolist:
l = label(name)
if l not in pseudo.keys(): # key does not exist
pseudo[l] = []
pseudo[l].append(name)
out = open("pseudo.py", "w")
out.write(toString(pseudo))
inp.close()
out.close()
print "Done!"
def generate():
inp = open("pseudo.list")
# Generate structure
pseudo = OrderedDict()
str = inp.read()
pseudolist = str.split()
for name in pseudolist:
l = label(name)
if l not in pseudo.keys(): # key does not exist
pseudo[l] = []
pseudo[l].append(name)
out = open("pseudo.py", "w")
out.write(toString(pseudo))
inp.close()
out.close()
print "Done!"
class QEInput(object):
def __init__(self, filename=None, config=None, type='pw', parse=True):
"""
Initializes QEInput by passing either filename or config (not both)
parameters
filename: (str) -- Absolute or relative filename of file to be parsed
config: (str) -- Configuration text to be parsed
type: (str) -- Type of the simulation
"""
self.filename = filename
self.config = config
self._type = type.lower()
self.filters = []
self.header = None
self.namelists = OrderedDict()
self.cards = OrderedDict()
self.attach = None # Specific for 'matdyn', 'dynmat', etc.
self.qe = None # DEPRICATED
self.namelistRef = None
self.cardRef = None
self._read(filename, config, parse)
def parse(self):
"""
Parses the configuration file and stores the values in qe dictionary
"""
(self.header, self.namelists, self.cards, self.attach) = self.parser.parse()
def namelist(self, name):
"""
Returns namelist and adds to self.namelists. Return namelist if it exists.
name: (str) -- Name of the new namelist
"""
# If namelist with the name exists, then return it
name = name.lower()
if self.namelistExists(name):
return self.namelists[name]
return self._createNamelist(name)
def removeNamelist(self, name):
"""
Removes namelist if it exists or ignore otherwise
name: (str) -- Name of the existing namelist
"""
name = name.lower()
try:
del(self.namelists[name])
except KeyError: # parameter is not present
return
def setNamelist(self, namelist):
"""
Sets/replaces namelist for self.namelists. It overwrites namelist if it exists
already. So you this method with caution.
namelist: (object: Namelist) -- Namelist object
"""
if not namelist: # No namelist, just ignore it!
return
self.namelists[namelist.name()] = namelist
def namelistExists(self, name):
"""
Checks if namelist specified by name (lowered before) exists
name: (str) -- Name of the namelist
"""
return self._exists(name, self.namelists.keys())
def setCard(self, card):
"""
Adds card to self.cards
card: (object: Card) -- Card object
"""
if not card:
return
self.cards[card.name()] = card
def removeCard(self, name):
"""
Remove card if it exists or ignore otherwise
name: (str) -- Name of the existing card
"""
name = name.lower()
try:
del(self.cards[name])
except KeyError: # parameter is not present
return
# XXX: Make self.attach protected first!
# def attach(self):
# "Returns attachment"
# return self.attach
def addAttach(self, text):
"""
Sets attachment to text. If attachment is not None it still will be
overwritten
text: (str) -- Attachment text, usually is appended to the end of the
configuration file
"""
self.attach = text
def removeAttach(self):
"Sets attachment to None"
self.attach = None
def card(self, name):
"""
Returns card specified by name if exists or create a new one
"""
name = name.lower()
if self.cardExists(name): # If exists, return card
return self.cards[name]
return self._createCard(name)
def cardExists(self, name):
"Checks if card specified by name exists"
return self._exists(name, self.cards.keys())
def getObject(self, name, dict):
"""
Returns object specified by name
name: (str) -- Name that identifies an object through name() method
dict: (dict) -- Dictionary that stores objects as {name: object}
"""
if not name or not dict:
return None
for n in dict.values():
if n.name() == name:
return dict[name]
return None
def save(self, filename=None):
"""
Saves the QEInput structure to the configuration file
filename: (str) -- File name where the configuration is stored to
"""
default = "config.out"
if filename is None:
if self.filename is not None:
filename = self.filename
else:
filename = default
f = open(filename, "w")
f.write(self.toString())
f.close()
def type(self):
"Returns type of the configuration file"
return self._type
def applyFilter(self, filter, type=POSITIVE):
"""
Applies filter to the QEInput
filter: (object: Filter) -- Filter that applies changes to QEInput
"""
if not filter: # No filter, just ignore it
return None
filter.apply(self, type) # Apply filter to QEInput
self.filters.append(filter)
def toString(self):
"""
Dumps QEInput structure to string
"""
s = ''
if self.header: # Add header
s += "%s\n" % self.header
for name in self.namelistRef: # Add namelists
nl = self.getObject(name, self.namelists)
if nl is not None:
s += nl.toString()
for name in self.cardRef: # Add cards
c = self.getObject(name, self.cards)
if c is not None:
s += c.toString()
if self.attach: # Add attribute (e.g. for type='matdyn')
s += self.attach
return s
def structure(self):
"""
Returns basic atomic structure information as list of tuples.
Specific for 'pw' type
Example: [('Ni', '52.98', 'Ni.pbe-nd-rrkjus.UPF'), (...)]
"""
# Extract structure not from pw type input. Hard to do it from other types
if self._type != "pw":
return None
list = [] # list of structure
card = self.card("atomic_species")
for l in card.lines(): # Should have format: "<Label> <Mass> <Pseudo-Potential>"
l = l.strip()
if l == "": # Empty line
continue
list.append(l.split())
return list
def readFile(self, filename):
"""
Reads and parses configuration input from file
filename: (str) -- File name
"""
self.filename = filename
self._read(filename=filename)
def readString(self, config):
"""
Reads and parses configuration input from string
config: (str) -- Configuration string
"""
self.config = config
self._read(config=config)
def _read(self, filename=None, config=None, parse=True):
"Reads and parses configuration input specified by kwds parameters"
self.parser = QEParser(filename, config, self._type) #filename, config, type)
(self.namelistRef, self.cardRef) = self.parser.setReferences()
if parse and (filename or config):
QEInput.parse(self) # Avoid calling method parse() from subclass
self.qe = [self.header, self.namelists, self.cards, self.attach]
def _exists(self, name, list):
"Checks if lowered name is in the list"
if not name:
return False
name = name.lower()
if name in list:
return True
return False
def _createCard(self, name):
"""
Creates a new card
name: (str) -- Name of the card
"""
if not name in self.cardRef: # If not standard card, ignore it
return None
card = Card(name)
self.cards[name] = card
return card
def _createNamelist(self, name):
"""
Creates namelist specified by name
name: (str) -- Name of the namelist
"""
if not name in self.namelistRef: # If not standard card, ignore it
return None
# Otherwise create a new namelist
nl = Namelist(name)
self.namelists[name] = nl
return nl
# DEPRICATED: Use namelist() instead
def createNamelist(self, name):
return namelist(name)
# DEPRICATED: Use card() instead
def createCard(self, name):
return card(name)
# DEPRICATED: Use setNamelist() instead
def addNamelist(self, namelist):
self.setNamelist(namelist)
# DEPRICATED: Use setCard() instead
def addCard(self, card):
self.setCard(card)
class QEParser:
"""
Flexible Parser for Quantum Espresso (QE) configuration files.
It parses the file specified by filename or configuration string and stores
parameters in namelists, cards and attachment data
structures that later on can be used in parameters' manipulations
"""
def __init__(self, filename = None, configText = None, type = 'pw'):
"""
Parameters:
filename -- absolute or relative filename to be parsed
configText -- configuration text to be parsed
type -- type of the simulation
"""
self.header = None
self.namelists = OrderedDict() # Namelist dictionary
self.cards = OrderedDict() # Cards dictionary
self.attach = None
self.filename = filename
self.configText = configText
self.namelistRef = None
self.cardRef = None
self.type = type
def parse(self):
"""Parses string and returns namelists, cards, attachment and header"""
self.setReferences()
if self.configText is not None: # First try use configText
text = self.configText
elif self.filename is not None: # ... then from file
text = self._getText(self.filename)
else:
raise NameError('Dude, set config text or filename') # Compalain
self._parseHeader(text)
self._parseNamelists(text)
self._parseAttach(text)
self._parseCards(text)
return (self.header, self.namelists, self.cards, self.attach)
def toString(self):
"""Returns string of parsed values"""
s = ''
if self.header:
s += self.header
for n in self.namelists.keys():
s += self.namelists[n].toString()
for c in self.cards.keys():
s += self.cards[c].toString()
if self.attach:
s += self.attach
return s
def setReferences(self):
"""Sets reference names for namelists and cards for specified simulation type"""
input = "input%s" % self.type
module = _import("inputs.%s" % input)
self.namelistRef = getattr(module, "namelists")
self.cardRef = getattr(module, "cards")
return (self.namelistRef, self.cardRef)
def _parseHeader(self, text):
"""Cuts the first line if it header"""
start = self._namelistStart(text)
if start is not None and start == 0:
return # There is no header
lines = text.splitlines(True)
if lines:
self.header = lines[0]
def _parseNamelists(self, text):
"""Parses text and populates namelist dictionary"""
namelists = OrderedDict()
s1 = self._removeComments(text)
p2 = re.compile(NAMELIST)
matches = p2.findall(s1) # Finds all namelist blocks
for m in matches:
name = m[0].lower()
if name in self.namelistRef:
params = self._parseParams(m[1]) # Parse parameters from a namelist block
namelists[name.lower()] = params
self._convertNamelists(namelists)
def _removeComments(self, text):
"""Removes comments from the text"""
p = re.compile(COMMENT)
s = re.sub(p, '', text)
return self._removeEmptyLines(s)
def _convertNamelists(self, namelists):
"""Converts dictionary to Namelist"""
for name in namelists.keys():
nl = Namelist(name)
for p in namelists[name]:
nl.add(p[0], p[1])
self.namelists[name] = nl
def _parseParams(self, text):
"""Parses parameters"""
params = []
p = re.compile(EXPRESSION) # Match expression
matches = p.findall(text)
for m in matches:
pl = self._getParams(m) # Parameters list
params.append(pl)
return params
def _getParams(self, text):
""" Takes string like 'a = 2' and returns tuple ('a', 2) """
s = text.split('=')
for i in range(len(s)):
s[i] = s[i].strip()
param = s[0]
val = s[1]
# Assume that there are two values only: (variable, value) pair
assert len(s) == 2
return (param, val)
def _parseAttach(self, text):
"""Special case for simulations that have attachments"""
if self.type in ATTACHSIM:
self.attach = self._cutNamelistText(text)
def _parseCards(self, text):
"""Parses text and populates cards dictionary"""
if self.type in ATTACHSIM: # There should not be cards for simulations with attachment
return
s = self._cutNamelistText(text)
self._convertCards(self._getCards(self._rawlist(s) ))
def _cutNamelistText(self, text):
"""Cuts the namelist text"""
s = self._removeComments(text)
end = self._namelistEnd(s)
if end is not None:
s = s[end + 1:] # Suppose that cards and attachmet starts with new line
return s
def _rawlist(self, text):
"""Removes empty lines or lines with white spaces only"""
rawlist = []
s = text.splitlines(True)
for line in s:
stripped = line.strip()
if stripped != '':
rawlist.append(line) # rawlist contains both cardnames and card lines in order
return rawlist
def _removeEmptyLines(self, text):
"""Joins non empty lines or lines with white spaces only in the list to one string"""
return ''.join(self._rawlist(text))
def _getCards(self, rawlist):
cards = OrderedDict()
cardName = None
for l in rawlist:
isCardName = False
p = re.compile(CARD)
m = p.match(l)
if m is not None: # If card name matches
firstPart = m.group(1).lower()
secondPart = m.group(2).strip().lower() # Catch argument of the card
if firstPart in self.cardRef:
isCardName = True
cardName = firstPart
cards[cardName] = {}
if (secondPart != ''):
cards[cardName]["args"] = secondPart
else:
cards[cardName]["args"] = None
cards[cardName]["values"] = []
if cardName is not None and not isCardName:
cards[cardName]["values"].append(l)
return cards
def _convertCards(self, cards):
for cname in cards.keys():
c = Card(cname)
c.setArg(cards[cname]["args"])
for l in cards[cname]["values"]:
c.addLine(l)
self.cards[cname] = c
def _namelistStart(self, text):
"""Returns the start character position of the first namelist in the text"""
s = self._removeComments(text)
p = re.compile(NAMELIST)
matches = p.finditer(s) # Finds all namelist blocks
starts = []
for m in matches:
starts.append(m.start())
if len(starts):
return starts[0] # Get first value
return None
def _namelistEnd(self, text):
"""
Returns the end character position of the last namelist in the text
Notes:
- text should be clear from comments (filtered by _removeComments(text)),
otherwise the end character of the last namelist will be incorrect
"""
s = self._removeComments(text)
p = re.compile(NAMELIST)
matches = p.finditer(s) # Finds all namelist blocks
ends = []
for m in matches:
ends.append(m.end())
size = len(ends)
if size:
return ends[size-1] # Get last position value
return None
def _getText(self, filename):
try:
f = open(filename)
except IOError:
print "I/O error"
except:
import sys
print "Unexpected error:", sys.exc_info()[0]
raise
text = f.read()
f.close()
return text
class Namelist(Block):
def __init__(self, name):
"""
name: (str) -- Name of the namelist in lower case. Example: "control"
"""
self._name = name.lower() # keeps lower name
self._params = OrderedDict() # Replace dictionary by ordered dictionry
def get(self, param, quotes = True):
"""
Returns paramater value. If no parameter exists, return None.
When quotes=True quotes are not added to parameter's value.
param: (str) -- Parameter of the namelist
quotes: (bool) -- True - if add quotes '' to parameters value,
False - otherwise
Note: replaces param()
"""
if not self._paramExists(param):
return None
param = param.lower()
if quotes:
return self._params[param]
return self._unquote(self._params[param])
def set(self, param, val, quotes = False):
"""
Sets existing parameter to the specified value.
If no parameter exists, create one
param: (str) -- Parameter name
val: (str) -- Parameter value
quotes: (bool) -- Add quotes to the value or not
"""
param = param.lower()
if quotes:
val = self._quote(val)
self._params[param] = val
# TODO: Rename to params() in the future
def paramlist(self):
"""
Returns list of parameter names
"""
return self._params.keys()
def remove(self, param):
"""
Deletes parameter
param: (str) -- Name of the parameter
"""
if self._paramExists(param):
del(self._params[param])
def exists(self,param):
"""
Checks if parameter exists in the namelist
param: (str) -- Name of the parameter
"""
return self._paramExists(param)
def _quote(self, val):
"""
Quotes value with "'" quote mark
val: (str) -- Value to be quoted
"""
return "'" + val.strip('"').strip("'") + "'"
def _unquote(self, val):
"""
Removes quotes "'" (unquotes) on both sides of the string
val: (str) -- Value to be unquoted
"""
return val.strip('"').strip("'")
def toString(self, indent = 4, br = "\n"):
"""
Dumps namelist as a sting
indent: (int) -- Number of spaces in indent for parameters
br: (str) -- Separator between parameters
"""
ind = ""
for i in range(indent): # Populate indent
ind += " "
s = '&%s%s' % (self.name().upper(), br)
for p in self._params.keys():
s += '%s%s = %s,%s' % (ind, p, self._params[p], br)
s += "/%s" % br
return s
def _paramExists(self, param):
"""
Checks if parameter exists in self._params
param: (str) -- Name of the parameter
"""
try:
param = param.lower()
self._params[param]
return True
except KeyError: # parameter is not present
return False
# DEPRICATED METHODS:
# DEPRICATED: Use get() instead
def param(self, param, quotes = True):
return self.get(param, quotes)
# DEPRICATED: Use set() instead!
def add(self, param, val, quotes = False):
self.set(param, val, quotes)
# DEPRICATED: Use set() instead!
def editParam(self, param, val):
self.set(param, val)
# DEPRICATED: Use set() instead!
def addParam(self, param, val):
self.add(param, val)
# DEPRICATED: Use remove() instead!
def removeParam(self, param):
self.remove(param)
class QEInput(object):
"""Quantum Espresso configuration class. It can:
- Parse existing configuration file
- Add, Edit or Remove parameters from/to namelist or card
"""
# Either filename or config (not both) can be specified
def __init__(self, filename=None, config=None, type='pw'):
self.header = None
self.filename = filename
self.config = config
self.parser = QEParser(filename, config, type)
self.type = type
self.namelists = OrderedDict()
self.cards = OrderedDict()
self.attach = None # Specific for 'matdyn', 'dynmat'
self.namelistRef = None
self.cardRef = None
self.qe = [self.header, self.namelists, self.cards, self.attach]
def parse(self):
""" Parses the configuration file and stores the values in qe dictionary """
(self.header, self.namelists, self.cards,
self.attach) = self.parser.parse()
def createNamelist(self, name):
"""Creates namelist and adds to QEInput. """
nl = Namelist(name)
self.namelists[name] = nl
return nl
def addNamelist(self, namelist):
"""Adds namelist. """
self.namelists[namelist.name()] = namelist
def removeNamelist(self, name):
try:
del (self.namelists[name])
except KeyError: # parameter is not present
return
def namelist(self, name):
"Returns namelist specified by name if exists or create a new one"
if self.namelistExists(name): # If exists, return namelist
return self.namelists[name]
return self.createNamelist(name) # otherwise create a new one
def namelistExists(self, name):
"Checks if namelist specified by name exists"
return self._exists(name, self.namelists.keys())
def createCard(self, name):
"Creates card and adds to QEInput. "
card = Card(name)
self.cards[name] = card
return card
def addCard(self, card):
"Adds card"
self.cards[card.name()] = card
def removeCard(self, name):
try:
del (self.cards[name])
except KeyError: # parameter is not present
return
def attach(self):
"Returns attachment"
return self.attach
def addAttach(self, text):
"""
Sets attachment to some string.
If attachment is not None it still will be overwritten
"""
self.attach = text
def removeAttach(self):
"Sets attachment to None"
self.attach = None
def card(self, name):
"Returns card specified by name if exists or create a new one"
if self.cardExists(name): # If exists, return card
return self.cards[name]
return self.createCard(name)
def cardExists(self, name):
"Checks if card specified by name exists"
return self._exists(name, self.cards.keys())
def getObject(self, name, dict):
"""Returns object that corresponds to 'name'"""
for n in dict.values():
if n.name() == name:
return dict[name]
return None
def save(self, filename=None):
""" Saves the QEInput to the configuration file"""
default = "config.out"
if filename is None:
if self.filename is not None:
filename = self.filename
else:
filename = default
f = open(filename, "w")
f.write(self.toString())
f.close()
def type(self):
return self.type
def structure(self):
"""Returns basic structure information as list tuples
Example: [('Ni', '52.98', 'Ni.pbe-nd-rrkjus.UPF'), (...)]
"""
# Hard to extract structure not from pw type input
# TODO: Should also have "atomic_species" card
if self.type != "pw":
return None
list = [] # list of structure
card = self.card("atomic_species")
for l in card.lines(
): # Should have format: "<Label> <Mass> <Pseudo-Potential>"
l = l.strip()
if l == "": # Empty line
continue
list.append(l.split())
return list
def toString(self):
(self.namelistRef, self.cardRef) = self.parser.setReferences()
s = ''
if self.header: # Add header
s += "%s\n" % self.header
for name in self.namelistRef: # Add namelists
nl = self.getObject(name, self.namelists)
if nl is not None:
s += nl.toString()
for name in self.cardRef: # Add cards
c = self.getObject(name, self.cards)
if c is not None:
s += c.toString()
if self.attach: # Add attribute (e.g. for type='matdyn')
s += self.attach
return s
def _exists(self, name, list):
if name in list:
return True
return False
class Namelist():
"""Namelist class that corresponds to Namelist in QE config file"""
def __init__(self, name):
self.__name = name.lower() # keeps lower name
self.params = OrderedDict() # Replace dictionary by ordered dictionry
def name(self):
return self.__name
def setName(self, name):
self.__name = name.lower()
def param(self, param, quotes=True):
"""Returns value of parameter 'param'"""
if self.__paramExists(param):
if quotes:
return self.params[param]
else:
return self._unquote(self.params[param])
return self.params[param]
return None
def add(self, param, val, quotes=False):
"Adds parameter to the namelist"
#param = param.lower() # Should be lowered?
if quotes:
val = self._quote(val)
self.params[param] = val
def set(self, param, val, quotes=False):
# Replaces editParam() and addParam(). Merge with add()?
"Sets existing parameter to the specified value. If it doesn't exist, it just ignores it " ""
if self.__paramExists(param):
if quotes:
val = self._quote(val)
self.params[param] = val
def remove(self, param):
"""Deletes parameter"""
if self.__paramExists(param):
del (self.params[param])
def exists(self, param):
return self.__paramExists(param)
def _quote(self, val):
return "'" + val.strip('"').strip("'") + "'"
def _unquote(self, val):
return val.strip('"').strip("'")
def toString(self, indent=" ", br="\n"):
# Dump namelist
# Should I use space?
s = '&%s%s' % (self.name().upper(), br)
for p in self.params.keys():
s += '%s%s = %s,%s' % (indent, p, self.params[p], br)
s += "/%s" % br
return s
def __paramExists(self, param):
try:
param = param.lower()
self.params[param]
return True
except KeyError: # parameter is not present
return False
# Depricated methods:
# Depricated
def addParam(self, param, val):
self.add(param, val)
# Depricated
def editParam(self, param, val):
self.set(param, val)
# Depricated
def removeParam(self, param):
self.remove()
class Namelist():
"""Namelist class that corresponds to Namelist in QE config file"""
def __init__(self, name):
self.__name = name.lower() # keeps lower name
self.params = OrderedDict() # Replace dictionary by ordered dictionry
def name(self):
return self.__name
def setName(self, name):
self.__name = name.lower()
def param(self, param):
"""Returns value of parameter 'param'"""
if self.__paramExists(param):
return self.params[param]
def add(self, param, val):
# Replaces addParam() Add verification?
param = param.lower()
self.params[param] = val
def set(self, param, val):
# Replaces editParam() and addParam(). Merge with add()?
"""Edits parameter. If it doesn't exist, it just ignores it """
if self.__paramExists(param):
self.params[param] = val
def remove(self, param):
"""Deletes parameter"""
if self.__paramExists(param):
del (self.params[param])
def __paramExists(self, param):
try:
param = param.lower()
self.params[param]
return True
except KeyError: # parameter is not present
return False
def toString(self, indent=" ", br="\n"):
# Dump namelist
# Should I use space?
s = '&%s%s' % (self.name().upper(), br)
for p in self.params.keys():
s += '%s%s = %s,%s' % (indent, p, self.params[p], br)
s += "/%s" % br
return s
# Depricated methods:
# Depricated
def addParam(self, param, val):
self.add(param, val)
# Depricated
def editParam(self, param, val):
self.set(param, val)
# Depricated
def removeParam(self, param):
self.remove()
class QEParser:
"""
Flexible Parser for Quantum Espresso (QE) configuration files. It parses the file specified
by filename or configuration string and stores parameters in namelists, cards and
attachment (specific for matdyn) data structures that later on can be used in
parameters' manipulations
"""
def __init__(self, filename=None, configText=None, type='pw'):
self.namelists = OrderedDict()
self.cards = OrderedDict()
self.attach = None
self.filename = filename
self.configText = configText
self.namelistRef = None
self.cardRef = None
self.type = type
def parse(self):
self.getReferences()
if self.configText is not None: # First try use configText
text = self.configText
elif self.filename is not None: # ... then from file
text = self._getText(self.filename)
else:
raise NameError('Dude, set config text or filename') # Compalain
self._parseNamelists(text)
self._parseCards(text)
return (self.namelists, self.cards, self.attach)
def toString(self):
for n in self.namelists.keys():
print self.namelists[n].toString()
for c in self.cards.keys():
print self.cards[c].toString()
if self.attach:
print self.attach
def getReferences(self):
input = "input%s" % self.type
module = _import("inputs.%s" % input)
self.namelistRef = getattr(module, "namelists")
self.cardRef = getattr(module, "cards")
return (self.namelistRef, self.cardRef)
def _parseNamelists(self, text):
namelists = OrderedDict()
p = re.compile(COMMENT)
s1 = re.sub(p, '', text) # Remove comments
p2 = re.compile(NAMELIST)
matches = p2.findall(s1) # Finds all namelist blocks
for m in matches:
name = m[0].lower()
if name in self.namelistRef:
params = self._parseParams(
m[1]) # Parse parameters from a namelist block
namelists[name.lower()] = params
self._convertNamelists(namelists)
# Converts from dictionary to Namelist
def _convertNamelists(self, namelists):
for name in namelists.keys():
nl = Namelist(name)
for p in namelists[name]:
nl.add(p[0], p[1])
self.namelists[name] = nl
# Parses parameters
def _parseParams(self, text):
params = []
p = re.compile(EXPRESSION) # Match expression
matches = p.findall(text)
for m in matches:
pl = self._getParams(m) # Parameters list
params.append(pl)
return params
def _getParams(self, text):
""" Takes string like 'a = 2' and returns tuple ('a', 2) """
s = text.split('=')
for i in range(len(s)):
s[i] = s[i].strip()
param = s[0]
val = s[1]
# Assume that there are two values only: (variable, value) pair
assert len(s) == 2
return (param, val)
def _parseCards(self, text):
p = re.compile(COMMENT)
s1 = re.sub(p, '', text) # Remove comments
p2 = re.compile(NAMELIST)
s2 = re.sub(p2, '', s1) # Remove namelists
# Special case for 'matdyn'
if self.type == 'matdyn':
self.attach = s2.strip()
return
rawlist = []
p = re.compile(EMPTY_LINE)
s = s2.split('\n')
for line in s:
line = line.strip()
if line != '':
rawlist.append(
line
) # rawlist contains both cardnames and card lines in order
self._convertCards(self._getCards(rawlist))
def _getCards(self, rawlist):
cards = OrderedDict()
cardName = None
for l in rawlist:
isCardName = False
p = re.compile(CARD)
m = p.match(l)
if m is not None: # If card name matches
firstPart = m.group(1).lower()
secondPart = m.group(
2).strip().lower() # Catch argument of the card
if firstPart in self.cardRef:
isCardName = True
cardName = firstPart
cards[cardName] = {}
if (secondPart != ''):
cards[cardName]["args"] = secondPart
else:
cards[cardName]["args"] = None
cards[cardName]["values"] = []
if cardName is not None and not isCardName:
cards[cardName]["values"].append(l)
return cards
def _convertCards(self, cards):
for cname in cards.keys():
c = Card(cname)
c.setArg(cards[cname]["args"])
for l in cards[cname]["values"]:
c.addLine(l)
self.cards[cname] = c
def _getText(self, filename):
try:
f = open(filename)
except IOError:
print "I/O error"
except:
import sys
print "Unexpected error:", sys.exc_info()[0]
raise
text = f.read()
f.close()
return text
def callservice(conn, schemaname, servicename, querystring):
try:
t1 = datetime.datetime.now()
# log the request
logging.basicConfig(
filename='../../htdocs/mstmp/REST_Services_Log.log',
level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
)
# logging.info("REST REQUEST: " + web.ctx.home + web.ctx.path + web.ctx.query)
# PARSE THE STANDARD OPTIONAL INPUT PARAMETERS
# get the input parameters
params = getQueryStringParams(
querystring
) # the unquoting is to handle encoded parameters (like from extJS - 1,2,3 as a parameter becomes 1%2C2%2C3
# get the standard optional parameters from the url
format = params.setdefault('format', 'json')
fields = params.setdefault('fields', '').split(
","
) # fields will be passed as an array, e.g. iucn_species_id,wdpa_id
includemetadata = params.setdefault('includemetadata', 'true')
metadataName = params.setdefault('metadataname', 'metadata')
rootName = params.setdefault('rootname', 'records')
parseparams = params.setdefault('parseparams', 'true')
sortField = params.setdefault('sortfield', '')
decimalPlaceLimit = params.setdefault('dplimit', '2')
isHadoop = (
'true' if (servicename[-2:] == '_h') else 'false'
) # if the service is a call to a hadoop method then set a flag
# remove the standard optional parameters from the dictionary so we are left with just the parameters required for the function
del (params['format'], params['fields'], params['includemetadata'],
params['parseparams'], params['metadataname'], params['rootname'],
params['sortfield'], params['dplimit'])
if 'callback' in params.keys():
del (params['callback'])
# check if the service name is valid
if not (isValidServiceName(servicename)):
raise RESTServicesError('Invalid servicename')
# authorise with ecas if needed
# if requiresAuthentication(servicename):
# if isAuthenticated() == False:
# web.ctx.status = '401 Unauthorized'
# web.header("Content-Type", "text/html")
# return "<html><head></html><body><h1>Authentication required</h1></body></html>"
# if it is a Hadoop query then we need to run if first before we actually use the values to get the data from postgresql
if (isHadoop.lower() == 'true'):
hadoopData = runHadoopQuery(conn, servicename, params)
if hadoopData == '[]': hadoopData = '[-1]'
servicename = "_" + servicename # now call the postgresql function
params.clear()
params['species_ids'] = str(hadoopData)[1:-1]
# PARSE AND CONVERT THE DATA TYPES OF THE OTHER INPUT PARAMETERS
# get all the parameters for the function from postgresql
conn.cur.callproc('utils.dopa_rest_getparams', [servicename])
# get the function parameters as a string and split this into a list, e.g. wdpa_id integer, presence_id integer[] --> ['wdpa_id integer', ' presence_id integer[]']
functionparams = conn.cur.fetchone()
hasparams = True if functionparams[0] else False
if hasparams:
functionparams = functionparams[0].split(',')
# get the names of the function parameters which are array types
arrayparamnames = [
p.strip().split(" ")[0] for p in functionparams if '[' in p
]
# convert the array values into lists
for key in params.keys():
if key in arrayparamnames:
strlist = params[key].split(",")
isnum = isNumeric(strlist[0])
if isnum:
params[key] = [int(s) for s in strlist]
else:
params[key] = strlist
# get the full list of function parameter names
functionparamnames = [
p.strip().split(" ")[0] for p in functionparams
]
# check that all parameters are correct
invalidparamnames = [
n for n in params.keys() if n not in functionparamnames
]
if invalidparamnames and parseparams == 'true':
raise RESTServicesError('Invalid parameters: ' +
",".join(invalidparamnames))
# put the input parameters in the right order
params = OrderedDict([(n, params[n]) for n in functionparamnames
if n in params.keys()])
# GET THE SORT CLAUSE
if sortField != "":
sortClause = ' ORDER BY "' + sortField + '"'
else:
sortClause = ""
# GET THE FIELDS CLAUSE
if fields != ['']:
fieldsClause = ",".join(fields)
else:
fieldsClause = "*"
# RUN THE QUERY
if hasparams:
sql = "SELECT " + fieldsClause + " from " + schemaname + "." + servicename + "(" + ",".join(
[n + ":=%(" + n + ")s" for n in params]
) + ")" + sortClause + ";" # run the query using named parameters
conn.cur.execute(sql, params)
else:
sql = "SELECT * from " + schemaname + "." + servicename + "()" + sortClause + ";"
conn.cur.execute(sql)
rows = conn.cur.fetchall()
# PROCESS THE ROWS AND WRITE THEM BACK TO THE CLIENT
conn.cur.close()
t2 = datetime.datetime.now()
# METADATA SECTION OF RESPONSE
allfields = [d.name for d in conn.cur.description]
if (fields == ['']): fields = allfields
fieldcount = len(fields)
fieldsdict = [
dict([("name", d.name),
("type", gettypefromtypecode(d.type_code))])
for d in conn.cur.description if (d.name in fields)
]
if len(fieldsdict) != len(fields):
raise RESTServicesError('Invalid output fields')
metadatadict = OrderedDict([
("duration", str(t2 - t1)), ("error", None),
("idProperty", conn.cur.description[0].name),
("successProperty", 'success'), ("totalProperty", 'recordCount'),
("success", True), ("recordCount", int(conn.cur.rowcount)),
("root", rootName), ("fields", fieldsdict)
])
# RECORDS SECTION OF THE RESPONSE
# parse the float values and set the correct number of decimal places according to the decimalPlaceLimit variable - dont include lat/long fields as these must have more decimal places
floatColumns = [
i for i, d in enumerate(fieldsdict)
if d['type'] == 'float' and d['name'] not in ['lat', 'lng']
]
if len(floatColumns) > 0:
for floatColumn in floatColumns:
for row in rows:
if type(row[floatColumn]
) != NoneType: # check that the data is not null
row[floatColumn] = round(row[floatColumn],
int(decimalPlaceLimit))
# return the data
colsRequired = [allfields.index(field) for field in fields]
if format in ['json', 'array']:
if format == 'json':
recordsdict = [
OrderedDict([(allfields[col], row[col])
for col in range(fieldcount)
if (col in colsRequired)]) for row in rows
]
else:
recordsdict = [[
row[col] for col in range(fieldcount)
if (col in colsRequired)
] for row in rows]
json.encoder.FLOAT_REPR = lambda f: (
"%.14g" % f
) # this specifies how many decimal places are returned in the json with float values - currently set to 14 - good enough for returning lat/long coordinates
if (includemetadata.lower() == 'true'):
responsejson = json.dumps(dict([(metadataName, metadatadict),
(rootName, recordsdict)]),
indent=1,
cls=CustomJSONEncoder)
else:
responsejson = json.dumps(dict([(rootName, recordsdict)]),
indent=1,
cls=CustomJSONEncoder)
return getJsonResponse(responsejson)
elif format in ['xml', 'xmlverbose']:
root = etree.Element('results')
recordsnode = etree.Element(rootName)
recordsdicts = [
OrderedDict([
(allfields[col], str(row[col]).decode('utf-8'))
for col in range(fieldcount)
if (col in colsRequired) and str(row[col]) != 'None'
]) for row in rows
] #
if format == 'xml':
recordselements = [
etree.Element('record', element)
for element in recordsdicts
]
for recordelement in recordselements:
recordsnode.append(recordelement)
else:
for recordelement in recordsdicts:
record = etree.Element('record')
for (n, v) in recordelement.items():
el = etree.Element(n)
el.text = v
record.append(el)
recordsnode.append(record)
root.append(recordsnode)
web.header("Content-Type", "text/xml")
# web.header("Content-Type", "application/Excel") # doesnt work!
# web.header("Content-Disposition", "attachment; filename=test.xml")
return etree.tostring(root)
elif format == 'sms':
_twilio = twilio()
client = TwilioRestClient(
_twilio.twilio_account_sid,
_twilio.twilio_auth_token) # use the twilio api account
bodystr = 'Hi Andrew - test species data: '
bodystr = bodystr + str(rows[0])[:160 - len(bodystr)]
message = client.sms.messages.create(to="+393668084920",
from_="+19712647662",
body=bodystr) # my mobile
return message
elif format == 'email':
_amazon_ses = amazon_ses()
amazonSes = AmazonSES(
_amazon_ses.AccessKeyID, _amazon_ses.SecretAccessKey
) # use the amazon simple email service api account
message = EmailMessage()
message.subject = 'JRC REST Services Information Request'
message.bodyHtml = getResultsAsHTML(rows, fieldcount, colsRequired,
metadatadict)
result = amazonSes.sendEmail('*****@*****.**',
'*****@*****.**',
message) # to me
return result
elif format == 'html':
htmlData = getResultsAsHTML(rows, fieldcount, colsRequired,
metadatadict)
web.header("Content-Type", "text/html")
return "<html><head></head><body>" + htmlData + "</body></html>"
elif format == 'csv':
data = [[
row[col] for col in range(fieldcount) if (col in colsRequired)
] for row in rows]
colnames = ",".join([f["name"]
for f in metadatadict["fields"]]) + "\n"
output = colnames + "\n".join([
p for p in [
",".join(h) for h in [[getStringValue(col) for col in row]
for row in data]
]
])
filename = "dataDownload.csv" #hardcoded for now
f = open(r'../../htdocs/mstmp/' + filename, 'wb')
f.write(output)
f.close()
web.header("Content-Type", "text/plain")
web.header("Content-Disposition",
"attachment; filename=%s" % filename)
return output
elif format == 'pdf':
config = pdfkit.configuration(
wkhtmltopdf='/usr/local/bin/wkhtmltopdf')
web.header("Content-Type", "application/pdf")
htmlData = getResultsAsHTML(rows, fieldcount, colsRequired,
metadatadict)
return pdfkit.from_string(htmlData.decode('utf8'),
False,
configuration=config,
options={
'quiet': '',
'encoding': "UTF-8"
})
else:
raise RESTServicesError('Invalid response format: ' + format)
except (RESTServicesError, DataError, ProgrammingError,
exceptions.TypeError, IndexError, IntegrityError, AmazonError,
OperationalError) as e:
# web.webapi.internalerror() #returns a internal server error 500
t2 = datetime.datetime.now()
msg = "There was an error sending the email. Make sure that the email address has been verified in Amazon Simple Email Services" if type(
e) == AmazonError else e.message
logging.error(msg + "\n")
if type(e) == ProgrammingError:
if ("column" in e.message) & ("does not exist"
in e.message) & (sortField != ""):
msg = "Invalid sortfield parameter: " + sortField
return returnError(metadataName, rootName, t2 - t1, msg)
class Namelist(Block):
def __init__(self, name):
"""
name: (str) -- Name of the namelist in lower case. Example: "control"
"""
self._name = name.lower() # keeps lower name
self._params = OrderedDict() # Replace dictionary by ordered dictionry
def get(self, param, quotes=True):
"""
Returns paramater value. If no parameter exists, return None.
When quotes=True quotes are not added to parameter's value.
param: (str) -- Parameter of the namelist
quotes: (bool) -- True - if add quotes '' to parameters value,
False - otherwise
Note: replaces param()
"""
if not self._paramExists(param):
return None
param = param.lower()
if quotes:
return self._params[param]
return self._unquote(self._params[param])
def set(self, param, val, quotes=False):
"""
Sets existing parameter to the specified value.
If no parameter exists, create one
param: (str) -- Parameter name
val: (str) -- Parameter value
quotes: (bool) -- Add quotes to the value or not
"""
param = param.lower()
if quotes:
val = self._quote(val)
self._params[param] = val
# TODO: Rename to params() in the future
def paramlist(self):
"""
Returns list of parameter names
"""
return self._params.keys()
def remove(self, param):
"""
Deletes parameter
param: (str) -- Name of the parameter
"""
if self._paramExists(param):
del (self._params[param])
def exists(self, param):
"""
Checks if parameter exists in the namelist
param: (str) -- Name of the parameter
"""
return self._paramExists(param)
def _quote(self, val):
"""
Quotes value with "'" quote mark
val: (str) -- Value to be quoted
"""
return "'" + val.strip('"').strip("'") + "'"
def _unquote(self, val):
"""
Removes quotes "'" (unquotes) on both sides of the string
val: (str) -- Value to be unquoted
"""
return val.strip('"').strip("'")
def toString(self, indent=4, br="\n"):
"""
Dumps namelist as a sting
indent: (int) -- Number of spaces in indent for parameters
br: (str) -- Separator between parameters
"""
ind = ""
for i in range(indent): # Populate indent
ind += " "
s = '&%s%s' % (self.name().upper(), br)
for p in self._params.keys():
s += '%s%s = %s,%s' % (ind, p, self._params[p], br)
s += "/%s" % br
return s
def _paramExists(self, param):
"""
Checks if parameter exists in self._params
param: (str) -- Name of the parameter
"""
try:
param = param.lower()
self._params[param]
return True
except KeyError: # parameter is not present
return False
# DEPRICATED METHODS:
# DEPRICATED: Use get() instead
def param(self, param, quotes=True):
return self.get(param, quotes)
# DEPRICATED: Use set() instead!
def add(self, param, val, quotes=False):
self.set(param, val, quotes)
# DEPRICATED: Use set() instead!
def editParam(self, param, val):
self.set(param, val)
# DEPRICATED: Use set() instead!
def addParam(self, param, val):
self.add(param, val)
# DEPRICATED: Use remove() instead!
def removeParam(self, param):
self.remove(param)
class Namelist():
"""Namelist class that corresponds to Namelist in QE config file"""
def __init__(self, name):
self.__name = name.lower() # keeps lower name
self.params = OrderedDict() # Replace dictionary by ordered dictionry
def name(self):
return self.__name
def setName(self, name):
self.__name = name.lower()
def param(self, param, quotes = True):
"""Returns value of parameter 'param'"""
if self.__paramExists(param):
if quotes:
return self.params[param]
else:
return self._unquote(self.params[param])
return self.params[param]
return None
def add(self, param, val, quotes = False):
# Replaces addParam() Add verification?
param = param.lower()
if quotes:
val = self._quote(val)
self.params[param] = val
def set(self, param, val, quotes = False):
# Replaces editParam() and addParam(). Merge with add()?
"""Edits parameter. If it doesn't exist, it just ignores it """
if self.__paramExists(param):
if quotes:
val = self._quote(val)
self.params[param] = val
def remove(self, param):
"""Deletes parameter"""
if self.__paramExists(param):
del(self.params[param])
def exists(self,param):
return self.__paramExists(param)
def _quote(self, val):
return "'" + val.strip('"').strip("'") + "'"
def _unquote(self, val):
return val.strip('"').strip("'")
def toString(self, indent=" ", br="\n"):
# Dump namelist
# Should I use space?
s = '&%s%s' % (self.name().upper(), br)
for p in self.params.keys():
s += '%s%s = %s,%s' % (indent, p, self.params[p], br)
s += "/%s" % br
return s
def __paramExists(self, param):
try:
param = param.lower()
self.params[param]
return True
except KeyError: # parameter is not present
return False
# Depricated methods:
# Depricated
def addParam(self, param, val):
self.add(param, val)
# Depricated
def editParam(self, param, val):
self.set(param, val)
# Depricated
def removeParam(self, param):
self.remove()
class QEInput(object):
def __init__(self, filename=None, config=None, type='pw', parse=True):
"""
Initializes QEInput by passing either filename or config (not both)
parameters
filename: (str) -- Absolute or relative filename of file to be parsed
config: (str) -- Configuration text to be parsed
type: (str) -- Type of the simulation
"""
self.filename = filename
self.config = config
self._type = type.lower()
self.filters = []
self.header = None
self.namelists = OrderedDict()
self.cards = OrderedDict()
self.attach = None # Specific for 'matdyn', 'dynmat', etc.
self.qe = None # DEPRICATED
self.namelistRef = None
self.cardRef = None
self._read(filename, config, parse)
def parse(self):
"""
Parses the configuration file and stores the values in qe dictionary
"""
(self.header, self.namelists, self.cards, self.attach) = self.parser.parse()
def namelist(self, name):
"""
Returns namelist and adds to self.namelists. Return namelist if it exists.
name: (str) -- Name of the new namelist
"""
# If namelist with the name exists, then return it
name = name.lower()
if self.namelistExists(name):
return self.namelists[name]
return self._createNamelist(name)
def removeNamelist(self, name):
"""
Removes namelist if it exists or ignore otherwise
name: (str) -- Name of the existing namelist
"""
name = name.lower()
try:
del(self.namelists[name])
except KeyError: # parameter is not present
return
def setNamelist(self, namelist):
"""
Sets/replaces namelist for self.namelists. It overwrites namelist if it exists
already. So you this method with caution.
namelist: (object: Namelist) -- Namelist object
"""
if not namelist: # No namelist, just ignore it!
return
self.namelists[namelist.name()] = namelist
def namelistExists(self, name):
"""
Checks if namelist specified by name (lowered before) exists
name: (str) -- Name of the namelist
"""
return self._exists(name, self.namelists.keys())
def setCard(self, card):
"""
Adds card to self.cards
card: (object: Card) -- Card object
"""
if not card:
return
self.cards[card.name()] = card
def removeCard(self, name):
"""
Remove card if it exists or ignore otherwise
name: (str) -- Name of the existing card
"""
name = name.lower()
try:
del(self.cards[name])
except KeyError: # parameter is not present
return
# XXX: Make self.attach protected first!
# def attach(self):
# "Returns attachment"
# return self.attach
def addAttach(self, text):
"""
Sets attachment to text. If attachment is not None it still will be
overwritten
text: (str) -- Attachment text, usually is appended to the end of the
configuration file
"""
self.attach = text
def removeAttach(self):
"Sets attachment to None"
self.attach = None
def card(self, name):
"""
Returns card specified by name if exists or create a new one
"""
name = name.lower()
if self.cardExists(name): # If exists, return card
return self.cards[name]
return self._createCard(name)
def cardExists(self, name):
"Checks if card specified by name exists"
return self._exists(name, self.cards.keys())
def getObject(self, name, dict):
"""
Returns object specified by name
name: (str) -- Name that identifies an object through name() method
dict: (dict) -- Dictionary that stores objects as {name: object}
"""
if not name or not dict:
return None
for n in dict.values():
if n.name() == name:
return dict[name]
return None
def save(self, filename=None):
"""
Saves the QEInput structure to the configuration file
filename: (str) -- File name where the configuration is stored to
"""
default = "config.out"
if filename is None:
if self.filename is not None:
filename = self.filename
else:
filename = default
f = open(filename, "w")
f.write(self.toString())
f.close()
def type(self):
"Returns type of the configuration file"
return self._type
def applyFilter(self, filter, type=POSITIVE):
"""
Applies filter to the QEInput
filter: (object: Filter) -- Filter that applies changes to QEInput
"""
if not filter: # No filter, just ignore it
return None
filter.apply(self, type) # Apply filter to QEInput
self.filters.append(filter)
def toString(self):
"""
Dumps QEInput structure to string
"""
s = ''
if self.header: # Add header
s += "%s\n" % self.header
for name in self.namelistRef: # Add namelists
nl = self.getObject(name, self.namelists)
if nl is not None:
s += nl.toString()
for name in self.cardRef: # Add cards
c = self.getObject(name, self.cards)
if c is not None:
s += c.toString()
if self.attach: # Add attribute (e.g. for type='matdyn')
s += self.attach
return s
def structure(self):
"""
Returns basic atomic structure information as list of tuples.
Specific for 'pw' type
Example: [('Ni', '52.98', 'Ni.pbe-nd-rrkjus.UPF'), (...)]
"""
# Extract structure not from pw type input. Hard to do it from other types
if self._type != "pw":
return None
list = [] # list of structure
card = self.card("atomic_species")
for l in card.lines(): # Should have format: "<Label> <Mass> <Pseudo-Potential>"
l = l.strip()
if l == "": # Empty line
continue
list.append(l.split())
return list
def readFile(self, filename):
"""
Reads and parses configuration input from file
filename: (str) -- File name
"""
self.filename = filename
self._read(filename=filename)
def readString(self, config):
"""
Reads and parses configuration input from string
config: (str) -- Configuration string
"""
self.config = config
self._read(config=config)
def _read(self, filename=None, config=None, parse=True):
"Reads and parses configuration input specified by kwds parameters"
self.parser = QEParser(filename, config, self._type) #filename, config, type)
(self.namelistRef, self.cardRef) = self.parser.setReferences()
if parse and (filename or config):
QEInput.parse(self) # Avoid calling method parse() from subclass
self.qe = [self.header, self.namelists, self.cards, self.attach]
def _exists(self, name, list):
"Checks if lowered name is in the list"
if not name:
return False
name = name.lower()
if name in list:
return True
return False
def _createCard(self, name):
"""
Creates a new card
name: (str) -- Name of the card
"""
if not name in self.cardRef: # If not standard card, ignore it
return None
card = Card(name)
self.cards[name] = card
return card
def _createNamelist(self, name):
"""
Creates namelist specified by name
name: (str) -- Name of the namelist
"""
if not name in self.namelistRef: # If not standard card, ignore it
return None
# Otherwise create a new namelist
nl = Namelist(name)
self.namelists[name] = nl
return nl
# DEPRICATED: Use namelist() instead
def createNamelist(self, name):
return namelist(name)
# DEPRICATED: Use card() instead
def createCard(self, name):
return card(name)
# DEPRICATED: Use setNamelist() instead
def addNamelist(self, namelist):
self.setNamelist(namelist)
# DEPRICATED: Use setCard() instead
def addCard(self, card):
self.setCard(card)