def delete_file(filename):
"""Delete a Parser staged or ready file from the server.
Uses the request referrer value to determine which page initiates
the request. The deletion will occur within context of the referring
page. For example, delete request on the Uploads page will remove
a file from the staging directory.
**Args:**
*filename*: name of the file to delete
"""
referrer = flask.request.referrer
log.debug('File deletion referrer: "%s"' % referrer)
parsed_referrer_url = urlparse.urlparse(referrer)
log.debug('File deletion referrer path: "%s"' %
parsed_referrer_url.path)
delete_path = baip_parser_ui.app.config['STAGING_DIR']
route = 'upload'
if parsed_referrer_url.path == '/parser/download':
delete_path = baip_parser_ui.app.config['READY_DIR']
route = 'download'
delete_path = os.path.join(delete_path, filename)
log.info('Attempting file delete: "%s"' % delete_path)
remove_files(delete_path)
return flask.redirect(flask.url_for(route))
def _extract_xpath_remove(tree):
"""Generate XPath expressions relating to section removal.
*tree* is a :mod:`lxml.etree._ElementTree` object parses from
the source XML-based configuration file
**Args:**
*tree*: :mod:`lxml.etree._ElementTree` structure
**Returns:**
list of tuples representing the *remove* token
and the corresponding xpath expression. For example::
[('r', '//table'), (...), ...]
Specific to section removal is the ``r`` (*remove*) token
that defines the action to perform on the section
"""
xpath_expressions = []
for section_remover in tree.xpath('//Doc/Section/sectionRemover'):
start_section = section_remover.xpath('../startSection')
end_section = section_remover.xpath('../endSection')
xpath = None
if (len(end_section) == 0
or start_section[0].text == end_section[0].text):
xpath = '//{0}'.format(start_section[0].text)
log.debug('Generated sectionRemove xpath: "%s"' % xpath)
xpath_expressions.append(('d', xpath))
return xpath_expressions
def allowed_file(filename, extensions):
"""Check if *filename* has an extension that is acceptable for upload.
**Args:**
*filename*: the textual representation of the file to upload
*extensions*: list of accepted filename extension. For example::
['html', 'htm', ...]
**Returns:**
Boolean ``True`` if extension is allowed. ``False`` otherwise.
"""
is_file_allowed = False
extension = os.path.splitext(filename)[1].strip('.')
log.debug('File name|valid extensions: "%s"|%s' % (filename, extensions))
if extension in extensions:
is_file_allowed = True
log.info('File "%s" has accepted extension?: %s' %
(filename, is_file_allowed))
return is_file_allowed
def allowed_file(filename, extensions):
"""Check if *filename* has an extension that is acceptable for upload.
**Args:**
*filename*: the textual representation of the file to upload
*extensions*: list of accepted filename extension. For example::
['html', 'htm', ...]
**Returns:**
Boolean ``True`` if extension is allowed. ``False`` otherwise.
"""
is_file_allowed = False
extension = os.path.splitext(filename)[1].strip('.')
log.debug('File name|valid extensions: "%s"|%s' %
(filename, extensions))
if extension in extensions:
is_file_allowed = True
log.info('File "%s" has accepted extension?: %s' %
(filename, is_file_allowed))
return is_file_allowed
def delete_file(filename):
"""Delete a Munger staged or ready file from the server.
Uses the request referrer value to determine which page initiates
the request. The deletion will occur within context of the referring
page. For example, delete request on the Uploads page will remove
a file from the staging directory.
**Args:**
*filename*: name of the file to delete
"""
referrer = flask.request.referrer
log.debug('File deletion referrer: "%s"' % referrer)
parsed_referrer_url = urlparse.urlparse(referrer)
log.debug('File deletion referrer path: "%s"' % parsed_referrer_url.path)
delete_path = baip_munger_ui.app.config['STAGING_DIR']
route = 'upload'
if parsed_referrer_url.path == '/munger/download':
delete_path = baip_munger_ui.app.config['READY_DIR']
route = 'download'
delete_path = os.path.join(delete_path, filename)
log.info('Attempting file delete: "%s"' % delete_path)
remove_files(delete_path)
return flask.redirect(flask.url_for(route))
def replace_tag(self, xpath, new_tag, new_tag_attributes=None):
"""Replace element tag from *xpath* expression search to
*new_tag*.
**Args:**
*xpath*: standard XPath expression used to query against *html*
*new_tag*: new element tag name to replace
*new_tag_attributes*: list of tuples representing attributes
name|value pairs to add to the new tag
"""
log.info('Replace element tag XPath: "%s"' % xpath)
for tag in self.root.xpath(xpath):
log.debug('Replacing element tag "%s" with "%s"' %
(tag.tag, new_tag))
new_element = lxml.etree.Element(new_tag)
new_element.text = tag.text_content()
if new_tag_attributes is not None and len(new_tag_attributes):
for new_tag_attribute in new_tag_attributes:
name, value = new_tag_attribute
if value is None:
value = str()
new_element.attrib[name] = value
else:
log.debug('Copying over existing attributes: "%s"' %
tag.attrib)
for key, value in tag.attrib.iteritems():
new_element.attrib[key] = value
tag.getparent().replace(tag, new_element)
def update_attr(element, attribute, value, old_value):
log.debug('Updating attr "%s" from tag "%s" with "%s"' %
(attribute, element.tag, value))
if old_value is not None:
if element.attrib[attribute] == old_value:
element.attrib[attribute] = value
else:
element.attrib[attribute] = value
def check_args(self, script_name, command=None):
"""Verify that the daemon arguments are as expected.
Sets the controller command to (for example stop, start or status)
unless :attr:`command` is predefined.
Attempts to make a sane assesment of options against the given
:attr:`command`.
**Args**:
*script_name*: name of the executing script
**Kwargs**:
*command*: name of the executing script
**Raises**:
``SystemExit`` (program exit) if one argument is not provided
on the command line (unless the :attr:`command` is predefined.
"""
(options, args) = self.parser.parse_args()
if options.dry is not None:
set_console()
cmd = command
if command is None:
cmd = (args.pop(0))
if (cmd not in self.supported_commands):
self.parser.error('command "%s" not supported' % cmd)
if (cmd != 'start' and
(options.dry or options.batch)):
self.parser.error('invalid option(s) with command "%s"' %
cmd)
if len(args):
self.parser.error("unknown arguments")
if cmd == 'status':
set_console()
if options.verbose == 0:
set_log_level('INFO')
else:
log.debug('Logging verbosity set to "DEBUG" level')
self.command = cmd
self.script_name = script_name
self.options = options
self.args = args
if self.command == 'start':
self.dry = (self.options.dry is not None)
self.batch = (self.options.batch is not None)
def parse(path='.'):
"""Parser parse.
"""
status = False
if flask.request.method == 'POST':
config_file = baip_parser_ui.app.config['PARSER_CONF_FILE']
conf = baip_parser.ParserConfig(config_file)
conf.parse_config()
in_dir = baip_parser_ui.app.config['STAGING_DIR']
files_to_process = get_directory_files_list(in_dir)
results = []
for file_to_process in files_to_process:
log.info('Processing file: %s' % file_to_process)
parser = baip_parser.Parser()
parser.open(file_to_process)
log.debug('xxx: %s' % conf.cells_to_extract)
parser.cells_to_extract = conf.cells_to_extract
parser.skip_sheets = conf.skip_sheets
results.append(parser.parse_sheets())
if len(results):
parserd = baip_parser.ParserDaemon(pidfile=None, conf=conf)
tmp_file = parserd.dump(results)
ready_dir = baip_parser_ui.app.config['READY_DIR']
target_file = os.path.join(ready_dir,
os.path.basename(tmp_file))
log.info('Moving "%s" to target "%s"' % (tmp_file, target_file))
shutil.move(tmp_file, target_file)
status = True
remove_files(files_to_process)
enabled = False
if get_directory_files_list(baip_parser_ui.app.config['STAGING_DIR']):
enabled = True
kwargs = {
'path': path,
'template': 'dashboard/parse.html',
'template_context': {
'enabled': enabled,
'status': status,
},
'endpoint': '.parse',
}
return baip_parser_ui.staging_index.render_autoindex(**kwargs)
def _validate(self):
"""Validator method called during object initialisation.
..note::
When starting a new process, the PID file will be relative
to '/' once the process is forked.
**Raises:**
:mod:`daemoniser.DaemonError` if *pidfile* contains
invalid content.
"""
if not os.path.isabs(self.pidfile):
log.debug('PID file "%s" is relative -- make absolute' %
self.pidfile)
self.pidfile = os.path.join(os.sep, self.pidfile)
log.debug('PID file now "%s"' % self.pidfile)
# Check if the PID file exists.
if os.path.exists(self.pidfile):
# PID file exists, so the process may be active.
# Set the current process PID.
log.debug('PID file "%s" exists' % self.pidfile)
try:
self.pid = int(file(self.pidfile, 'r').read().strip())
log.debug('Stored PID is: %d' % self.pid)
except ValueError as error:
raise DaemonError('Error reading PID file: %s' % error)
def _parse_strip_chars(xpath, section):
"""Parse ``sectionStripChars`` config items.
"""
config_items = []
for action in section.xpath('sectionStripChars'):
conf_item = {'xpath': xpath}
chars = action.xpath('stripChars/text()')
log.debug('sectionStripChars value: "%s"' % chars)
if len(chars):
conf_item['chars'] = chars[0]
config_items.append(conf_item)
return config_items
def _parse_insert_tag(xpath, section):
"""Parse ``sectionInsertTag`` config items.
"""
config_items = []
for action in section.xpath('sectionInsertTag'):
conf_item = {'xpath': xpath}
new_tag = action.xpath('newTag/text()')
log.debug('sectionInsertTag value: "%s"' % new_tag)
if len(new_tag):
conf_item['new_tag'] = new_tag[0]
config_items.append(conf_item)
return config_items
def munge(path='.'):
"""Munger munge.
"""
status = False
if flask.request.method == 'POST':
in_dir = baip_munger_ui.app.config['STAGING_DIR']
log.debug('Munging files: %s' % get_directory_files_list(in_dir))
ready_dir = baip_munger_ui.app.config['READY_DIR']
for html_file in get_directory_files_list(in_dir):
target_file = os.path.join(ready_dir,
os.path.basename(html_file))
munger = baip_munger.Munger()
status = munger.munge(baip_munger_ui.app.config['MUNGER_ACTIONS'],
html_file,
target_file)
if status:
remove_files(html_file)
else:
break
enabled = False
if (baip_munger_ui.app.config['MUNGER_ACTIONS'] is not None and
get_directory_files_list(baip_munger_ui.app.config['STAGING_DIR'])):
enabled = True
kwargs = {
'path': path,
'template': 'dashboard/munge.html',
'template_context': {
'enabled': enabled,
'status': status,
},
'endpoint': '.munge',
}
return baip_munger_ui.staging_index.render_autoindex(**kwargs)
def munge(path='.'):
"""Munger munge.
"""
status = False
if flask.request.method == 'POST':
in_dir = baip_munger_ui.app.config['STAGING_DIR']
log.debug('Munging files: %s' % get_directory_files_list(in_dir))
ready_dir = baip_munger_ui.app.config['READY_DIR']
for html_file in get_directory_files_list(in_dir):
target_file = os.path.join(ready_dir, os.path.basename(html_file))
munger = baip_munger.Munger()
status = munger.munge(baip_munger_ui.app.config['MUNGER_ACTIONS'],
html_file, target_file)
if status:
remove_files(html_file)
else:
break
enabled = False
if (baip_munger_ui.app.config['MUNGER_ACTIONS'] is not None
and get_directory_files_list(
baip_munger_ui.app.config['STAGING_DIR'])):
enabled = True
kwargs = {
'path': path,
'template': 'dashboard/munge.html',
'template_context': {
'enabled': enabled,
'status': status,
},
'endpoint': '.munge',
}
return baip_munger_ui.staging_index.render_autoindex(**kwargs)
def stop(self):
"""Stop the daemon.
Will run a series of checks around the existence of a PID file
before attempting to terminate the daemon.
**Returns:**
boolean::
``True`` -- success
``False`` -- failure
"""
stop_status = False
if self.pid:
# OK to terminate.
log.debug('Stopping daemon process with PID: %s' % self.pid)
try:
os.kill(int(self.pid), signal.SIGTERM)
except OSError as error:
log.error('PID "%s" stop: "%s"' % (self.pid, error))
if error.errno == 3:
# For a daemon process, remove the PID file.
if self.pidfile is not None:
log.warn('Removing PID file "%s"' % self.pidfile)
remove_files(self.pidfile)
else:
stop_status = True
self.pid = None
elif self.pid is None:
# PID or PID file does not exist.
log.warn('Stopping process but unable to find PID')
else:
# Should not happen, but ...
log.warn('PID file exists with invalid value: "%s"' %
str(self.pid))
return stop_status
def remove_section(html, xpath, root_tag):
"""Remove a section from *html* based on the *xpath* expression.
If *root_tag* is a nested ancestor of the *xpath* expression match
then the section removal will stem from this level.
**Args:**
*html*: HTML document as a string
*xpath*: standard XPath expression used to query against *html*
*root_tag*: tag name of the ancestor element to remove if
a match/matches are produced by *xpath*
**Returns:**
the resultant HTML document as a string
"""
root = lxml.html.fromstring(html)
log.debug('Section removal XPath: "%s"' % xpath)
for element in root.xpath(xpath):
log.debug('Removing element tag: "%s"' % element.tag)
if element.tag == root_tag:
element.getparent().remove(element)
for ancestor in element.iterancestors():
log.debug('Removing ancestor tag: "%s"' % ancestor.tag)
ancestor.getparent().remove(ancestor)
if ancestor.tag == root_tag:
break
return lxml.html.tostring(root)
def _parse_add_attributes(xpath, section):
"""Parse ``sectionAddAttribute`` config items.
"""
config_items = []
for action in section.xpath('sectionAddAttribute'):
conf_item = {'xpath': xpath}
attr = action.xpath('attributeName/text()')
value = action.xpath('attributeValue/text()')
log.debug('sectionAddAttribute attr|value: "%s|%s"' %
(attr, value))
if len(attr):
conf_item['attribute'] = attr[0]
if len(value):
conf_item['value'] = value[0]
conf_item['add'] = True
config_items.append(conf_item)
return config_items
def _start_daemon(self):
"""Start the daemon process.
..note::
Daemon will only start if :attr:`daemonise.pidfile` exists
(not ``None``).
The :meth:`daemonise.start` method checks for an existing PID before
preparing the daemon environment. Finally, it will initiate
the daemon process run sequence.
**Returns:**
boolean::
``True`` -- success
``False`` -- failure
**Raises:**
:mod:`daemoniser.DaemonError`, if:
* PID file has not been defined
* PID file is not writable
"""
start_status = False
# If we have got this far, check that we have a valid PID file.
if self.pidfile is None:
raise DaemonError('PID file has not been defined')
log.debug('checking daemon status with PID: %s' % self.pid)
if self.pid is not None:
msg = ('PID file "%s" exists. Daemon may be running?\n' %
self.pidfile)
log.warn(msg)
else:
# Check if PID file is writable to save hassles later on.
log.debug('No PID file -- creating handle')
try:
log.debug('starting daemon')
self.daemonize()
self._start(self.exit_event)
start_status = True
except IOError as error:
err_msg = 'Cannot write to PID file: IOError "%s"' % error
raise DaemonError(err_msg)
return start_status
def strip_char(self, xpath, chars):
"""Strip *chars* from *xpath* expression search.
**Args:**
*xpath*: standard XPath expression used to query against *html*
*chars*: characters to strip from the element tag text
"""
log.info('Strip chars XPath expression: "%s"' % xpath)
for tag in self.root.xpath(xpath):
for child_tag in tag.iter():
if child_tag.text is not None:
log.debug('Stipping "%s" from tag "%s" text: "%s"' %
(chars, child_tag.tag, child_tag.text))
child_tag.text = child_tag.text.strip(chars)
log.debug('Resultant text: "%s"' % child_tag.text)
if child_tag.tail is not None:
log.debug('Stipping tail text: "%s" from "%s"' %
(chars, child_tag.tail))
child_tag.tail = child_tag.tail.strip(chars)
log.debug('Resultant tail text: "%s"' % child_tag.tail)
def _parse_replace_tag(xpath, section):
"""Parse ``sectionReplaceTag`` config items.
"""
config_items = []
for action in section.xpath('sectionReplaceTag'):
conf_item = {'xpath': xpath}
new_tag = action.xpath('newTag/text()')
log.debug('sectionReplaceTag value: "%s"' % new_tag)
# Check for attributes.
new_tag_attr = 'newTagAttribute'
new_attrs = action.xpath(new_tag_attr)
new_tag_attributes = []
for new_attr in new_attrs:
log.debug('newTagAttribute element found')
names = new_attr.xpath('attributeName/text()')
attr_name = None
if len(names):
attr_name = names[0]
values = new_attr.xpath('attributeValue/text()')
attr_value = None
if len(values):
attr_value = values[0]
if attr_name is not None:
new_tag_attributes.append((attr_name, attr_value))
log.debug('sectionReplaceTag attribute "%s"' %
(new_tag_attributes[-1], ))
if len(new_tag):
conf_item['new_tag'] = new_tag[0]
conf_item['new_tag_attributes'] = new_tag_attributes
config_items.append(conf_item)
return config_items
def daemonize(self):
"""Prepare the daemon environment.
Does the UNIX double-fork magic, see Stevens' "Advanced
Programming in the UNIX Environment" for details
(ISBN 0201563177).
The second child PID is written out to the PID file. This acts
as a persistent state check, where no file implies that the
daemon process is idle.
The daemon process environment acts as a container for your
process logic. As such, once the daemon process ends it will
remove its associated PID file automatically.
"""
log.debug('Attempting first fork ...')
try:
pid = os.fork()
# Exit first parent.
if pid > 0:
if self.term_parent:
sys.exit(0)
else:
return
except OSError as error:
sys.stderr.write("Fork #1 failed: %d (%s)\n" % (error.errno,
error.strerror))
sys.exit(1)
# Decouple from parent environment.
os.chdir("/")
os.setsid()
os.umask(0)
# Do second fork.
try:
pid = os.fork()
# Exit from second parent.
if pid > 0:
if self.term_parent:
sys.exit(0)
else:
return
except OSError as error:
sys.stderr.write("fork #2 failed: %d (%s)\n" % (error.errno,
error.strerror))
sys.exit(1)
# Close all file descriptors except from non-console logging
# handlers.
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = MAXFD
filenos = []
for handler in log.handlers:
if (hasattr(handler, 'stream') and
hasattr(handler.stream, 'fileno') and
handler.stream.fileno() > 2):
filenos.append(handler.stream.fileno())
for fd in range(0, maxfd):
try:
if fd not in filenos:
os.close(fd)
except OSError:
pass
# Redirect stdin, stdout, stderr to null
os.open(os.devnull, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
# Write out to pidfile.
child_pid = str(os.getpid())
log.debug('PID of child process: %s' % child_pid)
file(self.pidfile, 'w+').write("%s\n" % child_pid)
# Remove the PID file when the process terminates.
atexit.register(self._delpid)
def insert_tag(self, xpath, new_tag):
"""Insert *new_tag* element tag from *xpath* expression search.
Workflow is:
* identify elements from XPath expression
* group same parent/sequential elements
* construct new HTML element based on *new_tag*
* insert into :att:`root` :mod:`lxml.html` tree
**Args:**
*xpath*: standard XPath expression used to query against
*html*
*new_tag*: new element tag name to replace as a string.
Method will convert to a :mod:`lxml.etree.Element`
"""
def build_xml(new_tag, tags_to_extend):
new_element = lxml.etree.Element(new_tag)
new_element.extend(tags_to_extend)
xml = lxml.etree.XML(lxml.etree.tostring(new_element))
return xml
def child_xml_insert(start_index, node_count, parent_element, xml):
insert_index = start_index - node_count + 1
log.info('Child element insert "%s ..." at index: %d' %
(lxml.html.tostring(xml), insert_index))
parent_element.insert(insert_index, xml)
log.info('Insert element tag XPath: "%s"' % xpath)
tags = self.root.xpath(xpath)
current_parent = None
prev_index = None
tags_to_extend = []
for tag in tags:
parent = tag.getparent()
index = parent.index(tag)
log.debug('Index (current): %d' % index)
if current_parent is None:
current_parent = parent
log.debug('Set current parent (initial) %s:"%s"' %
(current_parent, current_parent.tag))
log.debug('Extending tag (parent): %s' %
(lxml.html.tostring(tag)))
tags_to_extend.append(tag)
prev_index = index
log.debug('Previous index (parent): %d' % prev_index)
continue
if parent == current_parent:
if index == (prev_index + 1):
log.debug('Extending tag: %s' % (lxml.html.tostring(tag)))
tags_to_extend.append(tag)
prev_index = index
log.debug('Previous index (parent match): %d' % prev_index)
continue
else:
log.debug('Sequential index interrupted: inserting')
else:
log.debug('Parent change: inserting')
xml = build_xml(new_tag, tags_to_extend)
if parent != current_parent:
current_parent.insert(prev_index - 1, xml)
current_parent = parent
log.debug('Set current parent %s:"%s"' %
(current_parent, current_parent.tag))
else:
child_xml_insert(prev_index, len(tags_to_extend),
current_parent, xml)
# Reset our control variables.
prev_index = parent.index(tag)
log.debug('New index after insert: %d' % prev_index)
del tags_to_extend[:]
log.debug('Extending tag (pass through): %s' %
(lxml.html.tostring(tag)))
tags_to_extend.append(tag)
# Insert the laggards (if any).
if len(tags_to_extend):
xml = build_xml(new_tag, tags_to_extend)
child_xml_insert(prev_index, len(tags_to_extend), current_parent,
xml)
def update_element_attribute(self,
xpath,
attribute,
value=None,
old_value=None,
add=False):
"""Update element *attribute* from *xpath* expression search.
If *value* is ``None`` then the attribute will be deleted.
Otherwise, the existing attribute value will be replaced with
the string contained within *value*.
If the *attribute* is not part of the tag definition and
*add* is set to ``True`` then the attribute will be added
**Args:**
*xpath*: standard XPath expression used to query against *html*
*attribute*: element attribute name to action
*value*: if not ``None``, string to replace *attribute* value
*old_value*: if not ``None``, old string to match *attribute*
value against before a replacement occurs
*add*: boolean flag which if set, will add the attribute
if it already not part of the tag definition
"""
def update_attr(element, attribute, value, old_value):
log.debug('Updating attr "%s" from tag "%s" with "%s"' %
(attribute, element.tag, value))
if old_value is not None:
if element.attrib[attribute] == old_value:
element.attrib[attribute] = value
else:
element.attrib[attribute] = value
def recursive_update_attr(element, attribute, value, old_value):
if element.attrib.get(attribute) is not None:
update_attr(element, attribute, value, old_value)
# Now, perform the check recursively for each parent.
parent = element.getparent()
while parent is not None:
if parent.attrib.get(attribute) is not None:
update_attr(parent, attribute, value, old_value)
parent = parent.getparent()
log.debug('Update attribute XPath: "%s"' % xpath)
for tag in self.root.xpath(xpath):
if value is None:
if add:
log.debug('Adding attr "%s" from tag "%s"' %
(attribute, tag.tag))
tag.attrib[attribute] = str()
elif tag.attrib.get(attribute):
log.debug('Removing attr "%s" from tag "%s"' %
(attribute, tag.tag))
tag.attrib.pop(attribute)
else:
if add:
log.debug('Adding attr "%s" from tag "%s" with "%s"' %
(attribute, tag.tag, value))
tag.attrib[attribute] = value
# else tag.attrib.get(attribute) is not None:
else:
recursive_update_attr(tag, attribute, value, old_value)
def _delpid(self):
"""Simple wrapper method around file deletion.
"""
log.debug('Removing PID file at "%s"' % self.pidfile)
os.remove(self.pidfile)