def render(self, req, parent): buf = self.buf if isinstance(buf, (bytes, bytearray)): buf = buf.decode() parent.text = etree.CDATA(buf)
def task_to_element(task) -> etree.Element:
"""Serialize task into XML Element."""
element = etree.Element('task')
element.set('id', task.get_id())
element.set('status', task.get_status())
element.set('uuid', task.get_uuid())
element.set('recurring', str(task.get_recurring()))
tags = etree.SubElement(element, 'tags')
for t in task.get_tags():
tag_tag = etree.SubElement(tags, 'tag')
tag_tag.text = str(t.tid)
title = etree.SubElement(element, 'title')
title.text = task.get_title()
dates = etree.SubElement(element, 'dates')
added_date = etree.SubElement(dates, 'added')
added_date.text = task.get_added_date().isoformat()
modified_date = etree.SubElement(dates, 'modified')
modified_date.text = Date(task.get_modified()).xml_str()
done_date = etree.SubElement(dates, 'done')
done_date.text = task.get_closed_date().xml_str()
due_date = task.get_due_date()
due_tag = 'fuzzyDue' if due_date.is_fuzzy() else 'due'
due = etree.SubElement(dates, due_tag)
due.text = due_date.xml_str()
start_date = task.get_start_date()
start_tag = 'fuzzyStart' if start_date.is_fuzzy() else 'start'
start = etree.SubElement(dates, start_tag)
start.text = start_date.xml_str()
recurring = etree.SubElement(element, 'recurring')
recurring.set('enabled', str(task.recurring).lower())
recurring_term = etree.SubElement(recurring, 'term')
recurring_term.text = str(task.get_recurring_term())
subtasks = etree.SubElement(element, 'subtasks')
for subtask_id in task.get_children():
sub = etree.SubElement(subtasks, 'sub')
sub.text = subtask_id
content = etree.SubElement(element, 'content')
text = task.get_text()
# Poor man's encoding.
# CDATA's only poison is this combination of characters.
text = text.replace(']]>', ']]>')
content.text = etree.CDATA(text)
return element
def test_query(dict, query, reference, name):
global failures
global SERVER_DIED
print "{0:100}".format('Dictionary: ' + dict + ' Name: ' + name +
": "),
sys.stdout.flush()
report_testcase = et.Element("testcase", attrib={"name": name})
reference_file = os.path.join(args.reference, reference) + '.reference'
stdout_file = os.path.join(args.reference, reference) + '.stdout'
stderr_file = os.path.join(args.reference, reference) + '.stderr'
command = '{ch} --port {port} --query "{query}" > {stdout_file} 2> {stderr_file}'.format(
ch=args.client,
port=args.port,
query=query,
stdout_file=stdout_file,
stderr_file=stderr_file)
proc = Popen(command, shell=True)
start_time = datetime.now()
while (datetime.now() - start_time
).total_seconds() < args.timeout and proc.poll() is None:
sleep(0)
if proc.returncode is None:
try:
proc.kill()
except OSError as e:
if e.errno != ESRCH:
raise
failure = et.Element("failure", attrib={"message": "Timeout"})
report_testcase.append(failure)
failures = failures + 1
print("{0} - Timeout!".format(MSG_FAIL))
else:
stdout = open(stdout_file,
'r').read() if os.path.exists(stdout_file) else ''
stdout = unicode(stdout, errors='replace', encoding='utf-8')
stderr = open(stderr_file,
'r').read() if os.path.exists(stderr_file) else ''
stderr = unicode(stderr, errors='replace', encoding='utf-8')
if proc.returncode != 0:
failure = et.Element("failure",
attrib={
"message":
"return code {}".format(
proc.returncode)
})
report_testcase.append(failure)
stdout_element = et.Element("system-out")
stdout_element.text = et.CDATA(stdout)
report_testcase.append(stdout_element)
failures = failures + 1
print("{0} - return code {1}".format(MSG_FAIL,
proc.returncode))
if stderr:
stderr_element = et.Element("system-err")
stderr_element.text = et.CDATA(stderr)
report_testcase.append(stderr_element)
print(stderr)
if 'Connection refused' in stderr or 'Attempt to read after eof' in stderr:
SERVER_DIED = True
elif stderr:
failure = et.Element("failure",
attrib={"message": "having stderror"})
report_testcase.append(failure)
stderr_element = et.Element("system-err")
stderr_element.text = et.CDATA(stderr)
report_testcase.append(stderr_element)
failures = failures + 1
print("{0} - having stderror:\n{1}".format(
MSG_FAIL, stderr.encode('utf-8')))
elif 'Exception' in stdout:
failure = et.Element("error",
attrib={"message": "having exception"})
report_testcase.append(failure)
stdout_element = et.Element("system-out")
stdout_element.text = et.CDATA(stdout)
report_testcase.append(stdout_element)
failures = failures + 1
print("{0} - having exception:\n{1}".format(
MSG_FAIL, stdout.encode('utf-8')))
elif not os.path.isfile(reference_file):
skipped = et.Element("skipped",
attrib={"message": "no reference file"})
report_testcase.append(skipped)
print("{0} - no reference file".format(MSG_UNKNOWN))
else:
(diff, _) = Popen(['diff', reference_file, stdout_file],
stdout=PIPE).communicate()
if diff:
failure = et.Element(
"failure",
attrib={"message": "result differs with reference"})
report_testcase.append(failure)
stdout_element = et.Element("system-out")
stdout_element.text = et.CDATA(diff)
report_testcase.append(stdout_element)
failures = failures + 1
print("{0} - result differs with reference:\n{1}".format(
MSG_FAIL, diff))
else:
print(MSG_OK)
if os.path.exists(stdout_file):
os.remove(stdout_file)
if os.path.exists(stderr_file):
os.remove(stderr_file)
dump_report(args.output, dict, name, report_testcase)
def dump_scan_template(profile_id, dump_path=None):
"""
:param profile_id:
:param dump_path:
:return:
"""
template_style_version = '1.0.0'
if not dump_path:
dump_path = '{0}'.format(settings.UPGRADE_ROOT)
dump_plugin_whitelist = os.path.join(dump_path, 'plugins', 'whitelist')
dump_plugin_blacklist = os.path.join(dump_path, 'plugins', 'blacklist')
make_dir([dump_path, dump_plugin_whitelist, dump_plugin_blacklist])
if profile_id:
profile = get_profile_by_id(profile_id)
if profile.config:
profile_config = ast.literal_eval(profile.config)
else:
profile_config = None
data = ET.Element('template')
data.set('version', template_style_version)
conf = ET.SubElement(data, 'config')
conf.set('sync', 'True')
conf.set('enable', 'True')
name = ET.SubElement(conf, 'name')
name.text = profile.name
version = ET.SubElement(conf, 'version')
version.text = str(profile.revision)
exclude_dir = ET.SubElement(conf, 'exclude_dir')
exclude_dir.text = profile.exclude_dir
exclude_ext = ET.SubElement(conf, 'exclude_ext')
exclude_ext.text = profile.exclude_ext
exclude_file = ET.SubElement(conf, 'exclude_file')
exclude_file.text = profile.exclude_file
task_timeout = ET.SubElement(conf, 'task_timeout')
task_timeout.text = str(profile.task_timeout)
engines = ET.SubElement(data, 'engines')
for engine in profile.engines.all():
engine_node = ET.SubElement(engines, 'engine')
engine_node.set('name', engine.name)
# parameters
parameters = ET.SubElement(engine_node, 'parameters')
engine_config = ast.literal_eval(engine.config)
for k, v in engine_config.items():
item = ET.SubElement(parameters, 'item')
item.set("name", k)
item.text = str(v)
if profile_config and engine.module_name in profile_config:
for k1, v2 in profile_config[engine.module_name].items():
item = ET.SubElement(parameters, 'item')
item.set("name", k1)
item.text = str(v2)
# rule
if engine.module_name == 'seecode_scanner.lib.engines.rulescanner':
component = ET.SubElement(engine_node, 'component')
# component
for item in profile.tactics.filter(rule_match_type=4,
engine__id=engine.id):
item_node = ET.SubElement(component, 'item')
item_node.set("id", str(item.id))
item_node_name = ET.SubElement(item_node, 'name')
item_node_name.text = item.name
item_node_key = ET.SubElement(item_node, 'key')
item_node_key.text = item.key
item_node_revision = ET.SubElement(item_node, 'revision')
item_node_revision.text = str(item.revision)
item_node_risk = ET.SubElement(item_node, 'risk')
item_node_risk.set("id", str(item.risk))
item_node_risk.text = get_risk_name(item.risk)
item_node_category = ET.SubElement(item_node, 'category')
item_node_category.text = get_tactic_type(item.type)
item_node_match_type = ET.SubElement(
item_node, 'match_type')
if item.rule_value == '1':
item_node_match_type.text = 'groupId'
else:
item_node_match_type.text = 'name'
item_node_match_content = ET.SubElement(
item_node, 'match_content')
item_node_match_content.text = ET.CDATA(
item.component_name)
item_node_match_regex = ET.SubElement(
item_node, 'match_regex')
item_node_match_regex.text = ET.CDATA(str(item.rule_regex))
item_node_match_regex.set("flag", item.rule_regex_flag)
# whitelist
whitelist = ET.SubElement(engine_node, 'whitelist')
for item in profile.tactics.filter(
nature_type=1, attribution_type=1,
engine__id=engine.id).exclude(rule_match_type=4):
item_node = ET.SubElement(whitelist, 'item')
item_node.set("id", str(item.id))
item_node_name = ET.SubElement(item_node, 'name')
item_node_name.text = item.name
item_node_key = ET.SubElement(item_node, 'key')
item_node_key.text = item.key
item_node_revision = ET.SubElement(item_node, 'revision')
item_node_revision.text = str(item.revision)
item_node_risk = ET.SubElement(item_node, 'risk')
item_node_risk.set("id", str(item.risk))
item_node_risk.text = get_risk_name(item.risk)
item_node_category = ET.SubElement(item_node, 'category')
item_node_category.text = get_tactic_type(item.type)
item_node_category.set("id", str(item.type))
item_node_match_type = ET.SubElement(
item_node, 'match_type')
if item.rule_match_type == 1:
item_node_match_type.text = 'dir'
elif item.rule_match_type == 2:
item_node_match_type.text = 'file'
elif item.rule_match_type == 3:
item_node_match_type.text = 'content'
item_node_match_ext = ET.SubElement(
item_node, 'match_ext')
item_node_match_ext.text = item.rule_value
item_node_match_regex = ET.SubElement(
item_node, 'match_regex')
item_node_match_regex.text = str(item.rule_regex)
item_node_match_regex.set("flag", item.rule_regex_flag)
# blacklist
blacklist = ET.SubElement(engine_node, 'blacklist')
for item in profile.tactics.filter(
nature_type=2, attribution_type=1,
engine__id=engine.id).exclude(rule_match_type=4):
item_node = ET.SubElement(blacklist, 'item')
item_node.set("id", str(item.id))
item_node_name = ET.SubElement(item_node, 'name')
item_node_name.text = item.name
item_node_key = ET.SubElement(item_node, 'key')
item_node_key.text = item.key
item_node_revision = ET.SubElement(item_node, 'revision')
item_node_revision.text = str(item.revision)
item_node_risk = ET.SubElement(item_node, 'risk')
item_node_risk.set("id", str(item.risk))
item_node_risk.text = get_risk_name(item.risk)
item_node_category = ET.SubElement(item_node, 'category')
item_node_category.text = get_tactic_type(item.type)
item_node_category.set("id", str(item.type))
item_node_match_type = ET.SubElement(
item_node, 'match_type')
if item.rule_match_type == 1:
item_node_match_type.text = 'dir'
elif item.rule_match_type == 2:
item_node_match_type.text = 'file'
elif item.rule_match_type == 3:
item_node_match_type.text = 'content'
item_node_match_ext = ET.SubElement(
item_node, 'match_ext')
item_node_match_ext.text = item.rule_value
item_node_match_regex = ET.SubElement(
item_node, 'match_regex')
item_node_match_regex.text = str(item.rule_regex)
item_node_match_regex.set("flag", item.rule_regex_flag)
# plugin
if engine.module_name == 'seecode_scanner.lib.engines.pluginscanner':
# whitelist
whitelist = ET.SubElement(engine_node, 'whitelist')
for item in profile.tactics.filter(nature_type=1,
attribution_type=2,
engine__id=engine.id):
item_node = ET.SubElement(whitelist, 'item')
item_node.set("id", str(item.id))
item_node_name = ET.SubElement(item_node, 'name')
item_node_name.text = item.name
item_node_key = ET.SubElement(item_node, 'key')
item_node_key.text = item.key
item_node_revision = ET.SubElement(item_node, 'revision')
item_node_revision.text = str(item.revision)
item_node_risk = ET.SubElement(item_node, 'risk')
item_node_risk.set("id", str(item.risk))
item_node_risk.text = get_risk_name(item.risk)
item_node_category = ET.SubElement(item_node, 'category')
item_node_category.text = get_tactic_type(item.type)
item_node_category.set("id", str(item.type))
item_node_module = ET.SubElement(item_node, 'module')
item_node_module.text = item.plugin_module_name
item_node_script = ET.SubElement(item_node, 'script')
item_node_script.text = 'plugins/whitelist/{0}.py'.format(
item.plugin_name)
with open(
os.path.join(dump_plugin_whitelist,
'{0}.py'.format(item.plugin_name)),
'w') as fp:
fp.write(item.plugin_content)
# blacklist
blacklist = ET.SubElement(engine_node, 'blacklist')
for item in profile.tactics.filter(nature_type=2,
attribution_type=2,
engine__id=engine.id):
item_node = ET.SubElement(blacklist, 'item')
item_node.set("id", str(item.id))
item_node_name = ET.SubElement(item_node, 'name')
item_node_name.text = item.name
item_node_key = ET.SubElement(item_node, 'key')
item_node_key.text = item.key
item_node_revision = ET.SubElement(item_node, 'revision')
item_node_revision.text = str(item.revision)
item_node_risk = ET.SubElement(item_node, 'risk')
item_node_risk.set("id", str(item.risk))
item_node_risk.text = get_risk_name(item.risk)
item_node_category = ET.SubElement(item_node, 'category')
item_node_category.text = get_tactic_type(item.type)
item_node_category.set("id", str(item.type))
item_node_module = ET.SubElement(item_node, 'module')
item_node_module.text = item.plugin_module_name
item_node_script = ET.SubElement(item_node, 'script')
item_node_script.text = 'plugins/blacklist/{0}.py'.format(
item.plugin_name)
with open(
os.path.join(dump_plugin_blacklist,
'{0}.py'.format(item.plugin_name)),
'w') as fp:
fp.write(item.plugin_content)
mydata = ET.tostring(data, encoding='utf-8')
file_name = os.path.join(dump_path, '{0}.xml'.format(profile.name))
with open(file_name, "wb") as fp:
fp.write(
b"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n"
)
fp.write(mydata)
def atom_entry(self, extensions=True):
'''Create an ATOM entry and return it.'''
entry = etree.Element('entry')
if not (self.__atom_id and self.__atom_title and self.__atom_updated):
raise ValueError('Required fields not set')
id = etree.SubElement(entry, 'id')
id.text = self.__atom_id
title = etree.SubElement(entry, 'title')
title.text = self.__atom_title
updated = etree.SubElement(entry, 'updated')
updated.text = self.__atom_updated.isoformat()
# An entry must contain an alternate link if there is no content element.
if not self.__atom_content:
if not True in [ l.get('rel') == 'alternate' \
for l in self.__atom_link or [] ]:
raise ValueError('Entry must contain an alternate link or ' +
'a content element.')
# Add author elements
for a in self.__atom_author or []:
# Atom requires a name. Skip elements without.
if not a.get('name'):
continue
author = etree.SubElement(entry, 'author')
name = etree.SubElement(author, 'name')
name.text = a.get('name')
if a.get('email'):
email = etree.SubElement(author, 'email')
email.text = a.get('email')
if a.get('uri'):
email = etree.SubElement(author, 'url')
email.text = a.get('uri')
if self.__atom_content:
content = etree.SubElement(entry, 'content')
type = self.__atom_content.get('type')
if self.__atom_content.get('src'):
content.attrib['src'] = self.__atom_content['src']
elif self.__atom_content.get('content'):
# Surround xhtml with a div tag, parse it and embed it
if type == 'xhtml':
content.append(etree.fromstring('''<div
xmlns="http://www.w3.org/1999/xhtml">%s</div>''' % \
self.__atom_content.get('content')))
elif type == 'CDATA':
content.text = etree.CDATA(self.__atom_content)
# Emed the text in escaped form
elif not type or type.startswith('text') or type == 'html':
content.text = self.__atom_content.get('content')
# Parse XML and embed it
elif type.endswith('/xml') or type.endswith('+xml'):
content.append(
etree.fromstring(self.__atom_content['content']))
# Everything else should be included base64 encoded
else:
raise ValueError(
'base64 encoded content is not supported at the moment.'
+ 'If you are interested , please file a bug report.')
# Add type description of the content
if type:
content.attrib['type'] = type
for l in self.__atom_link or []:
link = etree.SubElement(entry, 'link', href=l['href'])
if l.get('rel'):
link.attrib['rel'] = l['rel']
if l.get('type'):
link.attrib['type'] = l['type']
if l.get('hreflang'):
link.attrib['hreflang'] = l['hreflang']
if l.get('title'):
link.attrib['title'] = l['title']
if l.get('length'):
link.attrib['length'] = l['length']
if self.__atom_summary:
summary = etree.SubElement(entry, 'summary')
summary.text = self.__atom_summary
for c in self.__atom_category or []:
cat = etree.SubElement(feed, 'category', term=c['term'])
if c.get('schema'):
cat.attrib['schema'] = c['schema']
if c.get('label'):
cat.attrib['label'] = c['label']
# Add author elements
for c in self.__atom_contributor or []:
# Atom requires a name. Skip elements without.
if not c.get('name'):
continue
contrib = etree.SubElement(feed, 'contributor')
name = etree.SubElement(contrib, 'name')
name.text = c.get('name')
if c.get('email'):
email = etree.SubElement(contrib, 'email')
email.text = c.get('email')
if c.get('uri'):
email = etree.SubElement(contrib, 'url')
email.text = c.get('uri')
if self.__atom_published:
published = etree.SubElement(entry, 'published')
published.text = self.__atom_published.isoformat()
if self.__atom_rights:
rights = etree.SubElement(feed, 'rights')
rights.text = self.__atom_rights
if extensions:
for ext in self.__extensions.values() or []:
if ext.get('atom'):
ext['inst'].extend_atom(entry)
return entry
def customer_xml():
"""
Generates an XML file suitable for Customer usage
"""
from lxml import etree
location_attribute = '{%s}noNameSpaceSchemaLocation' % "http://www.w3.org/2001/XMLSchema-instance"
kvasir_results_xml = etree.Element('KvasirResults', attrib={ location_attribute: 'kvasir.xsd', })
summary_xml = etree.SubElement(kvasir_results_xml, 'summary')
customer = etree.SubElement(summary_xml, 'customer')
customer.text = settings.customer or 'CUSTOMER NAME'
assessment = etree.SubElement(summary_xml, 'assessment')
assessment.set('type', settings.assessment_type)
start_date = etree.SubElement(assessment, 'start-date')
start_date.text = settings.start_date or 'START DATE'
end_date = etree.SubElement(assessment, 'end-date')
end_date.text = settings.end_date or 'END DATE'
hosts_xml = etree.SubElement(kvasir_results_xml, 'hosts')
os_xml = etree.SubElement(kvasir_results_xml, 'os_records')
vulns_xml = etree.SubElement(kvasir_results_xml, 'vulns')
# this is a little hack to ensure a record is either blank or None
# use it as "if variable not in notin:"
notin = [ None, '' ]
unknown_cpeid_counter = 0
# go through each host, adding the os, services and vulns accordingly
query = create_hostfilter_query(session.hostfilter)
for host_rec in db(query).select():
host_xml = etree.SubElement(hosts_xml, 'host')
host_xml.set('ipaddr', host_rec.f_ipaddr)
host_xml.set('assetgroup', host_rec.f_asset_group)
if host_rec.f_macaddr:
host_xml.set('macaddr', host_rec.f_macaddr)
if host_rec.f_hostname:
host_xml.set('hostname', host_rec.f_hostname.decode('utf-8'))
if host_rec.f_netbios_name:
host_xml.set('netbios', host_rec.f_netbios_name.decode('utf-8'))
# build the os information using the highest certainty record
highest = (0, None)
for os_rec in db(db.t_host_os_refs.f_hosts_id == host_rec.id).select():
if os_rec.f_certainty > highest[0]:
highest = (os_rec.f_certainty, os_rec)
if highest[0] > 0:
# add os element to the host
record = highest[1]
os = etree.SubElement(host_xml, 'os')
os.set('certainty', str(highest[0]))
if record.f_class not in notin:
os.set('class', record.f_class)
if record.f_family not in notin:
os.set('family', record.f_family)
# since some os records may not have a cpe id we'll mask them with
# using their title, replacing spaces with underscores
t_os_rec = db.t_os[record.f_os_id]
if t_os_rec.f_cpename in notin:
cpeid = t_os_rec.f_title.replace(' ', '_')
else:
cpeid = t_os_rec.f_cpename
os.set('id', cpeid)
# if the id isn't in os_records, add it
if len(os_xml.findall('.//os[@id="%s"]' % (os.get('id', None)))) < 1:
os_info_xml = etree.SubElement(os_xml, 'os')
os_rec = db.t_os[highest[1].f_os_id]
os_info_xml.set('id', cpeid)
os_info_xml.set('title', os_rec.f_title)
if os_rec.f_vendor not in notin:
vendor = etree.SubElement(os_info_xml, 'vendor')
vendor.text = os_rec.f_vendor
if os_rec.f_product not in notin:
product = etree.SubElement(os_info_xml, 'product')
product.text = os_rec.f_product
if os_rec.f_version not in notin:
version = etree.SubElement(os_info_xml, 'version')
version.text = os_rec.f_version
if os_rec.f_update not in notin:
update = etree.SubElement(os_info_xml, 'update')
update.text = os_rec.f_update
if os_rec.f_edition not in notin:
edition = etree.SubElement(os_info_xml, 'edition')
edition.text = os_rec.f_edition
if os_rec.f_language not in notin:
language = etree.SubElement(os_info_xml, 'language')
language.text = os_rec.f_language
# snmp strings
snmp_recs = db(db.t_snmp.f_hosts_id == host_rec.id).select()
if len(snmp_recs) > 0:
snmp_top_xml = etree.SubElement(hosts_xml, 'snmps')
for record in snmp_recs:
snmp_xml = etree.SubElement(snmp_top_xml, 'snmp')
if record.f_community not in notin:
snmp_xml.set('community', record.f_community.decode('utf-8'))
snmp_xml.set('version', record.f_version)
snmp_xml.set('access', record.f_access)
# netbios information
netb_record = db(db.t_netbios.f_hosts_id == host_rec.id).select().first() or None
if netb_record:
netbios_xml = etree.SubElement(hosts_xml, 'netbios')
if netb_record.f_type not in notin:
netbios_xml.set('type', netb_record.f_type)
if netb_record.f_domain not in notin:
netbios_xml.set('domain', netb_record.f_domain.decode('utf-8'))
if netb_record.f_lockout_limit not in notin:
netbios_xml.set('lockout_limit', str(netb_record.f_lockout_limit))
if netb_record.f_lockout_duration not in notin:
netbios_xml.set('lockout_duration', str(netb_record.f_lockout_duration))
if netb_record.f_advertised_names is not None:
adv_names_xml = etree.SubElement(netbios_xml, 'advertised_names')
for name in netb_record.f_advertised_names:
name_xml = etree.SubElement(adv_names_xml, 'name')
name.text = name.decode('utf-8')
# build the services and vulnerabilities
services_xml = etree.SubElement(host_xml, 'services')
for svc_rec in db(db.t_services.f_hosts_id == host_rec.id).select():
service_xml = etree.SubElement(services_xml, 'service')
service_xml.set('proto', svc_rec.f_proto)
service_xml.set('number', svc_rec.f_number)
if svc_rec.f_name not in notin:
name = etree.SubElement(service_xml, 'name')
name.text = svc_rec.f_name.decode('utf-8')
if svc_rec.f_banner not in notin:
banner = etree.SubElement(service_xml, 'banner')
banner.text = svc_rec.f_banner.decode('utf-8')
# service configuration records
svc_info_recs = db(db.t_service_info.f_services_id == svc_rec.id).select()
if len(svc_info_recs) > 0:
config_xml = etree.SubElement(service_xml, 'configuration')
for info_rec in svc_info_recs:
rec_xml = etree.SubElement(config_xml, 'config')
if info_rec.f_name not in notin:
rec_xml.set('name', info_rec.f_name)
if info_rec.f_text not in notin:
rec_xml.text = info_rec.f_text.decode('utf-8')
# vulnerabilities
svc_vuln_recs = db(db.t_service_vulns.f_services_id == svc_rec.id).select()
if len(svc_vuln_recs) > 0:
svc_vulns_xml = etree.SubElement(service_xml, 'vulns')
for vuln_rec in svc_vuln_recs:
vuln_xml = etree.SubElement(svc_vulns_xml, 'vuln')
vuln_xml.set('status', vuln_rec.f_status)
vuln_xml.set('id', db.t_vulndata[vuln_rec.f_vulndata_id].f_vulnid)
proof = etree.SubElement(vuln_xml, 'proof')
proof.text = etree.CDATA(unicode(MARKMIN(vuln_rec.f_proof).xml(), 'utf-8'))
# search for the nexpose id in vulns_xml
if len(vuln_xml.findall('.//vuln[@id="%s"]' % vuln_xml.get('id', None))) < 1:
new_vuln_xml = etree.SubElement(vulns_xml, 'vuln')
vulndata = db.t_vulndata[vuln_rec.f_vulndata_id]
new_vuln_xml.set('id', vulndata.f_vulnid)
new_vuln_xml.set('title', vulndata.f_title)
new_vuln_xml.set('severity', str(vulndata.f_severity))
new_vuln_xml.set('pci_sev', str(vulndata.f_pci_sev))
new_vuln_xml.set('cvss_score', str(vulndata.f_cvss_score))
new_vuln_xml.set('cvss_metric', cvss_metrics(vulndata))
description = etree.SubElement(new_vuln_xml, 'description')
description.text = etree.CDATA(unicode(MARKMIN(vulndata.f_description).xml(), 'utf-8'))
solution = etree.SubElement(new_vuln_xml, 'solution')
solution.text = etree.CDATA(unicode(MARKMIN(vulndata.f_solution).xml(), 'utf-8'))
# find vulnerability references and add them
vuln_refs = db(db.t_vuln_references.f_vulndata_id == vulndata.id).select()
if len(vuln_refs) > 0:
refs_xml = etree.SubElement(new_vuln_xml, 'references')
for ref_rec in vuln_refs:
record = db.t_vuln_refs[ref_rec.f_vuln_ref_id]
ref_xml = etree.SubElement(refs_xml, 'reference')
ref_xml.set('source', record.f_source)
ref_xml.text = record.f_text.decode('utf-8')
# accounts
accounts = db(db.t_accounts.f_services_id == svc_rec.id).select()
if len(accounts) > 0:
accounts_xml = etree.SubElement(service_xml, 'accounts')
for acct_rec in accounts:
acct_xml = etree.SubElement(accounts_xml, 'account')
if acct_rec.f_username not in notin:
elem = etree.SubElement(acct_xml, 'username')
elem.text = acct_rec.f_username.decode('utf-8')
if acct_rec.f_fullname not in notin:
elem = etree.SubElement(acct_xml, 'fullname')
elem.text = acct_rec.f_fullname.decode('utf-8')
if acct_rec.f_password not in notin:
elem = etree.SubElement(acct_xml, 'password')
elem.text = acct_rec.f_password.decode('utf-8')
if acct_rec.f_hash1 not in notin:
elem = etree.SubElement(acct_xml, 'hash1')
elem.text = acct_rec.f_hash1
if acct_rec.f_hash1_type not in notin:
elem = etree.SubElement(acct_xml, 'hash1_type')
elem.text = acct_rec.f_hash1_type
if acct_rec.f_hash2 not in notin:
elem = etree.SubElement(acct_xml, 'hash2')
elem.text = acct_rec.f_hash2
if acct_rec.f_hash2_type not in notin:
elem = etree.SubElement(acct_xml, 'hash2_type')
elem.text = acct_rec.f_hash2_type
if acct_rec.f_uid not in notin:
elem = etree.SubElement(acct_xml, 'uid')
elem.text = acct_rec.f_uid
if acct_rec.f_gid not in notin:
elem = etree.SubElement(acct_xml, 'gid')
elem.text = acct_rec.f_gid
if acct_rec.f_level not in notin:
elem = etree.SubElement(acct_xml, 'level')
elem.text = acct_rec.f_level
if acct_rec.f_domain not in notin:
elem = etree.SubElement(acct_xml, 'domain')
elem.text = acct_rec.f_domain.decode('utf-8')
if acct_rec.f_description not in notin:
elem = etree.SubElement(acct_xml, 'description')
elem.text = acct_rec.f_description.decode('utf-8')
result = etree.tostring(kvasir_results_xml, pretty_print=True, encoding=unicode)
return result
def setRungContent(self, Content):
assert type(Content) == str
self.rungContent.text = etree.CDATA(Content)
<br/> \
Camera Make: {make}; \
Camera Model: {model}; \
Lens Model: {lens_model}; \
<br/> \
Program: {exposure_program}; \
<br/> \
Aperture: {aperture}; \
Focal Length: {focal_length}; \
Shutter: {shutter_speed}; \
ISO: {iso} \
<br/><br/><img src='{thumbnail_url}'/><br/>Image ID: {img_id}"
print(f"DESCRIPTION: {item_description_str}")
#item_description_str = "del<br/><img x='del'/>"
item_description.text = ET.CDATA(item_description_str)
except:
print(f"ERROR: ID: {img_id}")
rss_str = ET.tostring(rss, encoding='unicode')
print(rss_str)
tmp = minidom.parseString(rss_str)
pretty_xml = tmp.toprettyxml(indent=" ", encoding="utf-8")
f = open(output_file, "wb")
f.write(pretty_xml)
f.close()
# IGNORE ----
# my_xml = ET.Element("my_name")
def evaluate_population(pop, record_history=False):
seed = pop.seed
# clear old .vxd robot files from the data directory
sub.call("rm data{}/*.vxd".format(seed), shell=True)
# remove old sim output.xml if we are saving new stats
if not record_history:
sub.call("rm output{}.xml".format(seed), shell=True)
num_evaluated_this_gen = 0
for n, ind in enumerate(pop):
# don't evaluate if invalid
if not ind.phenotype.is_valid():
for rank, goal in pop.objective_dict.items():
if goal["name"] != "age":
setattr(ind, goal["name"], goal["worst_value"])
print "Skipping invalid individual"
# otherwise create a vxd
else:
num_evaluated_this_gen += 1
pop.total_evaluations += 1
(x, y, z) = ind.genotype.orig_size_xyz
root = etree.Element("VXD") # new vxd root
if record_history:
sub.call("rm a{0}_gen{1}.hist".format(seed, pop.gen), shell=True)
history = etree.SubElement(root, "RecordHistory")
history.set('replace', 'VXA.Simulator.RecordHistory')
etree.SubElement(history, "RecordStepSize").text = '100'
structure = etree.SubElement(root, "Structure")
structure.set('replace', 'VXA.VXC.Structure')
structure.set('Compression', 'ASCII_READABLE')
etree.SubElement(structure, "X_Voxels").text = str(x)
etree.SubElement(structure, "Y_Voxels").text = str(y)
etree.SubElement(structure, "Z_Voxels").text = str(z)
for name, details in ind.genotype.to_phenotype_mapping.items():
state = details["state"]
flattened_state = state.reshape(z, x*y)
data = etree.SubElement(structure, name)
for i in range(flattened_state.shape[0]):
layer = etree.SubElement(data, "Layer")
if name == "Data":
str_layer = "".join([str(c) for c in flattened_state[i]])
else:
str_layer = "".join([str(c)+", " for c in flattened_state[i]])
layer.text = etree.CDATA(str_layer)
# hacky code to make sure the muscles actuate in counter phase; need to implement in voxelyze #
if pop.material_wide_phase_offset:
for name, details in ind.genotype.to_phenotype_mapping.items():
state = details["state"]
flattened_state = state.reshape(z, x * y)
if name == "Data":
mat_phase = np.zeros((z, x * y), dtype=np.float16)
mat_phase[flattened_state == 3] = 0
mat_phase[flattened_state == 4] = 0.5
data = etree.SubElement(structure, "PhaseOffset")
for i in range(mat_phase.shape[0]):
layer = etree.SubElement(data, "Layer")
str_layer = "".join([str(c) + ", " for c in mat_phase[i]])
layer.text = etree.CDATA(str_layer)
# end hack #
# md5 so we don't eval the same vxd more than once
m = hashlib.md5()
m.update(etree.tostring(root))
ind.md5 = m.hexdigest()
# don't evaluate if identical phenotype has already been evaluated
if ind.md5 in pop.already_evaluated:
for rank, goal in pop.objective_dict.items():
if goal["tag"] is not None:
setattr(ind, goal["name"], pop.already_evaluated[ind.md5][rank])
print "Age {0} individual already evaluated: cached fitness is {1}".format(ind.age, ind.fitness)
else:
# save the vxd to data folder
with open('data'+str(seed)+'/bot_{:04d}.vxd'.format(ind.id), 'wb') as vxd:
vxd.write(etree.tostring(root))
# ok let's finally evaluate all the robots in the data directory
if record_history: # just save history, don't assign fitness
print "Recording the history of the run champ"
sub.call("./Voxelyze3 -i data{0} > a{0}_gen{1}.hist".format(seed, pop.gen), shell=True)
else: # normally, we will just want to update fitness and not save the trajectory of every voxel
print "GENERATION {}".format(pop.gen)
print "Launching {0} voxelyze calls, out of {1} individuals".format(num_evaluated_this_gen, len(pop))
while True:
try:
sub.call("./Voxelyze3 -i data{0} -o output{1}.xml".format(seed, seed), shell=True)
# sub.call waits for the process to return
# after it does, we collect the results output by the simulator
root = etree.parse("output{}.xml".format(seed)).getroot()
break
except IOError:
print "Uh oh, there was an IOError! I'll re-simulate this batch again..."
pass
except IndexError:
print "Uh oh, there was an IndexError! I'll re-simulate this batch again..."
pass
for ind in pop:
if ind.phenotype.is_valid() and ind.md5 not in pop.already_evaluated:
# init_x = float(root.findall("detail/bot_{:04d}/Init/x".format(ind.id))[0].text) / 0.001
# init_y = float(root.findall("detail/bot_{:04d}/Init/y".format(ind.id))[0].text) / 0.001
# init_z = float(root.findall("detail/bot_{:04d}/Init/z".format(ind.id))[0].text) / 0.001
# final_x = float(root.findall("detail/bot_{:04d}/CoM/x".format(ind.id))[0].text) / 0.001
# final_y = float(root.findall("detail/bot_{:04d}/CoM/y".format(ind.id))[0].text) / 0.001
# final_z = float(root.findall("detail/bot_{:04d}/CoM/z".format(ind.id))[0].text) / 0.001
# dist2 = (final_x-init_x)**2 + (final_y-init_y)**2 - (final_z-init_z)**2
dist2 = float(root.findall("detail/bot_{:04d}/fitness_score".format(ind.id))[0].text)
ind.fitness = pop.objective_dict[0]["meta_func"](dist2, ind)
print "Assigning ind {0} fitness {1}".format(ind.id, ind.fitness)
pop.already_evaluated[ind.md5] = [getattr(ind, details["name"])
for rank, details in
pop.objective_dict.items()]
def _getSyncMLData(self, syncml_response, min_gid, max_gid):
"""
Compare data from source with data stored in signature from previous
synchronization. If there is any change, add command into the syncml
message
syncml_response : SyncML message to fill with command
min_gid = the lower limit for browsing data
max_gid = the upper limit for browsing data
"""
syncml_logger.info("getSyncMLData, min %s - max %r" % (min_gid, max_gid,))
conduit = self.getConduit()
portal = self.getPortalObject()
traverse = portal.restrictedTraverse
# Check deletion now ?
if portal.portal_preferences.getPreferredCheckDeleteAtEnd() is False:
raise NotImplementedError
object_list = self.z_get_syncml_path_list(
min_gid=min_gid,
max_gid=max_gid,
path=self.getSearchableSourcePath())
syncml_logger.info("getSyncMLData, object list is %s" % ([x.path for x in object_list]))
alert_code = self.getSyncmlAlertCode()
sync_all = alert_code in ("refresh_from_client_only", "slow_sync")
# XXX Quick & dirty hack to prevent signature creation, this must be defined
# on pub/sub instead
create_signature = alert_code != "refresh_from_client_only"
if not len(object_list) and (min_gid or max_gid):
raise ValueError("No object retrieved althoud min/max gid (%s/%s) is provided"
% (min_gid, max_gid))
more_data = False
for result in object_list:
# XXX We need a way to stop the loop when we reach a given packet size
document_path = result.path
gid = result.gid
document_data = result.data
signature = self.getSignatureFromGid(gid)
if signature:
syncml_logger.info("signature is %s = %s" %(signature.getRelativeUrl(),
signature.getValidationState()))
if not document_data:
raise ValueError("No data for %s / %s" %(gid, document_path))
# For the case it was never synchronized, we have to send everything
if not signature or sync_all:
# Either it is the first time we get this object
# either the synchronization process required
# to send every data again as if it was never done before
if create_signature:
if not signature:
signature = self.newContent(portal_type='SyncML Signature',
id=gid,
reference=document_path,
temporary_data=document_data)
syncml_logger.info("Created a signature %s for gid = %s, path %s"
% (signature.getPath(), gid, document_path))
more_data = self._generateSyncCommand(
action=ADD_ACTION,
signature=signature,
data_diff=document_data,
document_data=document_data,
gid=gid,
conduit=conduit,
syncml_response=syncml_response)
elif signature.hasPartialData():
# Case of partially sent data
# XXX Cutting must be managed by conduit
# Here it is too specific to XML data
xml_string = signature.getFirstPdataChunk(MAX_LEN)
if signature.hasPartialData():
more_data = True
# We need to convert XML to a CDATA type to prevent collision
# with syncml's XML
document_data = etree.CDATA(xml_string.decode('utf-8'))
syncml_logger.info("adding partial sync command for %s" %(gid,))
syncml_response.addSyncCommand(
sync_command=signature.getPartialAction(),
gid=gid,
data=document_data,
more_data=more_data,
media_type=conduit.getContentType())
if not more_data:
syncml_logger.info("signature %s is syncing from partial"
% (signature.getRelativeUrl(),))
elif signature.getValidationState() in ('no_conflict',
'conflict_resolved_with_merge'):
# We don't have synchronized this object yet but it has a signature
if signature.getValidationState() == 'conflict_resolved_with_merge':
# XXX Why putting confirmation message here
# Server can get confirmation of sync although it has not yet
# send its data modification to the client
# This must be checked against specifications
# Right now, this message will tell the other side to apply the
# diff without checking conflicts
# We then send the modifications
syncml_response.addConfirmationMessage(
source_ref=gid,
sync_code='conflict_resolved_with_merge',
command='Replace')
syncml_logger.info("\tMD5 is %s for %s" %((signature.checkMD5(document_data)),
signature.getReference()))
if not signature.checkMD5(document_data):
# MD5 checksum tell there is a modification of the object
# XXX this diff generation must managed by the conduit
# we just need to have conduit.generateDocumentDiff(new_data, former_data)
if conduit.getContentType() != 'text/xml':
# If there is no xml, we re-send the whole object
data_diff = document_data
else:
# Compute the diff
new_document = conduit.replaceIdFromXML(document_data, 'gid', gid)
previous_document = conduit.replaceIdFromXML(signature.getData(),
'gid', gid)
data_diff = conduit.generateDiff(new_data=new_document,
former_data=previous_document)
if not data_diff:
# MD5 Checksum can detect changes like <lang/> != <lang></lang>
# but Diff generator will return no diff for it
# in this case, no need to send diff
syncml_logger.info("\tFake diff, signature %s is synchronized"
% (signature.getRelativeUrl(),))
continue
# Reindex modified document
syncml_logger.info("\tGot a diff for %s : %s" %(gid, data_diff))
more_data = self._generateSyncCommand(
action=REPLACE_ACTION,
signature=signature,
data_diff=data_diff,
document_data=document_data,
gid=gid,
conduit=conduit,
syncml_response=syncml_response)
elif signature.getValidationState() == \
'conflict_resolved_with_client_command_winning':
# We have decided to apply the update
# XXX previous_xml will be getXML instead of getTempXML because
# some modification was already made and the update
# may not apply correctly
xml_update = signature.getPartialData()
previous_xml_with_gid = conduit.replaceIdFromXML(signature.getData(),
'gid', gid,
as_string=False)
conduit.updateNode(xml=xml_update, object=traverse(document_path),
previous_xml=previous_xml_with_gid, force=True,
gid=gid,
signature=signature,
domain=self)
syncml_response.addConfirmationMessage(
target_ref=gid,
sync_code='conflict_resolved_with_client_command_winning',
command='Replace')
signature.synchronize()
syncml_logger.debug("signature %s is synchronized"
% (signature.getRelativeUrl(),))
if more_data:
syncml_logger.info("Splitting document")
break
syncml_logger.info("_getSyncMLData end with more_data %s"
% (more_data,))
return not more_data
def gen_ncclient_script(username, payload):
"""
Generate Netconf / Restconf RPC
"""
payload = payload.replace('<metadata>', '')
payload = payload.replace('</metadata>', '')
_, device, fmt, lock, rpc = Adapter.parse_request(payload)
if fmt == 'xpath' and rpc == '':
rpc = Adapter.gen_rpc(username, payload)
if rpc is None:
logging.error('gen_script: Invalid RPC Generated')
return None
parser = NetconfParser(rpc)
op = parser.get_operation()
data = ET.tostring(parser.get_data(), pretty_print=True)
datastore = parser.get_datastore()
# setup template args
args = dict()
args['data'] = data.strip()
args['datastore'] = datastore
args['host'] = device.get('host', '')
args['port'] = device.get('port', '830')
args['user'] = device.get('user', '')
args['passwd'] = device.get('passwd', '')
args['platform'] = device.get('platform', '')
if not args['host']:
args['host'] = '<address>'
if not args['user']:
args['user'] = '******'
if not args['passwd']:
args['passwd'] = '<password>'
if not args['platform']:
args['platform'] = 'csr'
if op == 'get':
args['nccall'] = 'm.get(payload).xml'
elif op == 'get-config':
args[
'nccall'] = "m.get_config(source='%s', filter=payload).xml" % datastore
elif op == 'edit-config':
e_opt = parser.get_error_option()
if e_opt is None or e_opt == '':
args[
'nccall'] = "m.edit_config(target='%s', config=payload).xml" % datastore
else:
args[
'nccall'] = "m.edit_config(target='%s', error_option='%s', config=payload).xml" % (
datastore, e_opt)
args['lock'] = lock
else:
args['nccall'] = "m.dispatch(ET.fromstring(payload)).xml"
# generate script
rendered = render_to_string('pyscript.py', args)
script = ET.Element('script')
script.text = ET.CDATA(rendered)
return script
def generate_junit_xml(junit_xml_path, tag_expression, scenario_ids, features):
from lxml import etree
end_states = {State.PASSED, State.FAILED}
total_duration = sum(
(f.duration() for f in features if f.state in end_states), timedelta())
testsuites_element = etree.Element(
"testsuites",
OrderedDict([("name", "radish"),
("time", "{:.3f}".format(total_duration.total_seconds()))
]))
for feature in (f for f in features
if f.has_to_run(tag_expression, scenario_ids)):
testsuite_states = {
"failures": 0,
"errors": 0,
"skipped": 0,
"tests": 0
}
for rule in feature.rules:
# generate test suites stats
for scenario in (s for s in rule.scenarios
if s.has_to_run(tag_expression, scenario_ids)):
testsuite_states["tests"] += 1
if scenario.state in [
State.UNTESTED, State.PENDING, State.SKIPPED
]:
testsuite_states["skipped"] += 1
if scenario.state is State.FAILED:
testsuite_states["failures"] += 1
testsuite_element = etree.Element(
"testsuite",
OrderedDict([
("name", feature.short_description),
("tests", str(testsuite_states["tests"])),
("skipped", str(testsuite_states["skipped"])),
("failures", str(testsuite_states["failures"])),
("errors", str(testsuite_states["errors"])),
("time",
"{:.3f}".format(feature.duration().total_seconds()))
]))
for scenario in (s for s in rule.scenarios
if s.has_to_run(tag_expression, scenario_ids)):
testcase_element = etree.Element(
"testcase",
OrderedDict([
("classname", feature.short_description),
("name", scenario.short_description),
("time",
"{:.3f}".format(scenario.duration().total_seconds())),
]))
if scenario.state in [
State.UNTESTED, State.PENDING, State.SKIPPED
]:
skipped_element = etree.Element("skipped")
testcase_element.append(skipped_element)
if scenario.state is State.FAILED:
steps_text = []
steps = scenario.steps
if scenario.background:
steps = scenario.background.steps + steps
for step in steps:
step_line = "{} {}".format(step.keyword, step.text)
steps_text.append(step_line)
if step.state is State.FAILED:
failure_element = etree.Element(
"failure",
OrderedDict([
("type", step.failure_report.name),
("message", step_line),
]))
failure_element.text = etree.CDATA(
"{}\n\n{}".format(
"\n".join(steps_text),
_strip_ansi(step.failure_report.traceback),
))
testcase_element.append(failure_element)
testsuite_element.append(testcase_element)
testsuites_element.append(testsuite_element)
with open(junit_xml_path, "wb+") as f:
content = etree.tostring(
testsuites_element,
pretty_print=True,
xml_declaration=True,
encoding="utf-8",
)
f.write(content)
def request_authorization(doc, access_key, empresa, tipo_comprobante,
documento):
messages = []
m = ""
client = Client(SriService.get_active_ws()[1])
result = client.service.autorizacionComprobante(access_key)
print "El resultado es ", result
ruta_actual = os.path.join(os.path.dirname(__file__))
ahora = datetime.datetime.now()
year = str(ahora.year)
if ahora.month < 10:
month = '0' + str(ahora.month)
else:
month = str(ahora.month)
if tipo_comprobante == 'out_invoice':
tipo = 'fact_'
if tipo_comprobante == 'in_withholding':
tipo = 'c_r'
if tipo_comprobante == 'out_credit_note':
tipo = 'n_c'
if tipo_comprobante == 'out_debit_note':
tipo = 'n_d'
if tipo_comprobante == 'out_shipment':
tipo = 'g_r'
nuevaruta = NUEVA_RUTA + '/comprobantes/' + empresa + '/' + year + '/' + month
if result.autorizaciones:
autorizacion = result.autorizaciones[0][0]
if autorizacion.estado == 'AUTORIZADO':
num = str(autorizacion.numeroAutorizacion)
ruc = num[10:23]
est = num[24:27]
emi = num[27:30]
sec = num[30:39]
numero = ruc + '_' + est + '-' + emi + '-' + sec
ruta_db = os.getcwd(
) + '/comprobantes/' + empresa + '/' + year + '/' + month + '/' + tipo + numero
autorizacion_xml = etree.Element('autorizacion')
etree.SubElement(autorizacion_xml,
'estado_sri').text = autorizacion.estado
etree.SubElement(
autorizacion_xml,
'numeroAutorizacion').text = autorizacion.numeroAutorizacion
#etree.SubElement(autorizacion_xml, 'ambiente').text = 'PRODUCCION'
etree.SubElement(
autorizacion_xml, 'ambiente'
).text = 'PRUEBAS' #autorizacion.ambiente.replace("Ó","O") #Nodux autorizacion.ambiente
etree.SubElement(autorizacion_xml,
'comprobante').text = etree.CDATA(
autorizacion.comprobante)
autorizacion_xml = etree.tostring(autorizacion_xml,
encoding='utf8',
method='xml')
messages = " ".join(messages)
auth = autorizacion.estado
if not os.path.exists(nuevaruta):
os.makedirs(nuevaruta)
return autorizacion_xml, False, 'AUTORIZADO', ruta_db, numero, num
else:
identificador = result.autorizaciones[0][0].mensajes[0][
0].identificador #cls.replace_character(str(result.autorizaciones[0][0].mensajes[0][0].identificador))
mensaje = result.autorizaciones[0][0].mensajes[0][
0].mensaje #cls.replace_character(str(result.autorizaciones[0][0].mensajes[0][0].mensaje))
informacion = result.autorizaciones[0][0].mensajes[0][
0].informacionAdicional #cls.replace_character(str(result.autorizaciones[0][0].mensajes[0][0].informacionAdicional))
tipo = result.autorizaciones[0][0].mensajes[0][
0].tipo #cls.replace_character(str(result.autorizaciones[0][0].mensajes[0][0].tipo))
mensaje = 'Tipo: ' + tipo + '\nIdentificador: ' + identificador + '\nMensaje: ' + mensaje + '\nInformacion Adicional: ' + informacion
num = str(access_key)
ruc = num[10:23]
est = num[24:27]
emi = num[27:30]
sec = num[30:39]
numero = ruc + '_' + est + '-' + emi + '-' + sec
ruta_db = os.getcwd(
) + '/comprobantes/' + empresa + '/' + year + '/' + month + '/' + tipo + numero
autorizacion_xml = etree.Element('autorizacion')
etree.SubElement(autorizacion_xml,
'estado_sri').text = 'NO AUTORIZADO'
etree.SubElement(autorizacion_xml, 'numeroAutorizacion').text = num
#etree.SubElement(autorizacion_xml, 'ambiente').text = 'PRODUCCION'
etree.SubElement(
autorizacion_xml, 'ambiente'
).text = 'PRUEBAS' #autorizacion.ambiente.replace("Ó","O") #Nodux autorizacion.ambiente
etree.SubElement(autorizacion_xml,
'comprobante').text = etree.CDATA(documento)
autorizacion_xml = etree.tostring(autorizacion_xml,
encoding='utf8',
method='xml')
return mensaje, False, 'NO AUTORIZADO', ruta_db, numero, num
def transform(self, pretty_print=True, **kwargs):
"""change the self.html and return it with CSS turned into style
attributes.
"""
if hasattr(self.html, "getroottree"):
# skip the next bit
root = self.html.getroottree()
page = root
tree = root
else:
if self.method == 'xml':
parser = etree.XMLParser(ns_clean=False,
resolve_entities=False)
else:
parser = etree.HTMLParser()
stripped = self.html.strip()
tree = etree.fromstring(stripped, parser).getroottree()
page = tree.getroot()
# lxml inserts a doctype if none exists, so only include it in
# the root if it was in the original html.
root = tree if stripped.startswith(tree.docinfo.doctype) else page
assert page is not None
if self.disable_leftover_css:
head = None
else:
head = get_or_create_head(tree)
#
# style selectors
#
rules = []
index = 0
for element in CSSSelector('style,link[rel~=stylesheet]')(page):
# If we have a media attribute whose value is anything other than
# 'all' or 'screen', ignore the ruleset.
media = element.attrib.get('media')
if media and media not in ('all', 'screen'):
continue
data_attribute = element.attrib.get(self.attribute_name)
if data_attribute:
if data_attribute == 'ignore':
del element.attrib[self.attribute_name]
continue
else:
warnings.warn('Unrecognized %s attribute (%r)' % (
self.attribute_name,
data_attribute,
))
is_style = element.tag == 'style'
if is_style:
css_body = element.text
else:
href = element.attrib.get('href')
css_body = self._load_external(href)
these_rules, these_leftover = self._parse_style_rules(
css_body, index)
index += 1
rules.extend(these_rules)
parent_of_element = element.getparent()
if these_leftover or self.keep_style_tags:
if is_style:
style = element
else:
style = etree.Element('style')
style.attrib['type'] = 'text/css'
if self.keep_style_tags:
style.text = css_body
else:
style.text = self._css_rules_to_string(these_leftover)
if self.method == 'xml':
style.text = etree.CDATA(style.text)
if not is_style:
element.addprevious(style)
parent_of_element.remove(element)
elif not self.keep_style_tags or not is_style:
parent_of_element.remove(element)
# external style files
if self.external_styles:
for stylefile in self.external_styles:
css_body = self._load_external(stylefile)
self._process_css_text(css_body, index, rules, head)
index += 1
# css text
if self.css_text:
for css_body in self.css_text:
self._process_css_text(css_body, index, rules, head)
index += 1
# rules is a tuple of (specificity, selector, styles), where
# specificity is a tuple ordered such that more specific
# rules sort larger.
rules.sort(key=operator.itemgetter(0))
# collecting all elements that we need to apply rules on
# id is unique for the lifetime of the object
# and lxml should give us the same everytime during this run
# item id -> {item: item, classes: [], style: []}
elements = {}
for _, selector, style in rules:
new_selector = selector
class_ = ''
if ':' in selector:
new_selector, class_ = re.split(':', selector, 1)
class_ = ':%s' % class_
# Keep filter-type selectors untouched.
if class_ in FILTER_PSEUDOSELECTORS:
class_ = ''
else:
selector = new_selector
sel = CSSSelector(selector)
items = sel(page)
if len(items):
# same so process it first
processed_style = csstext_to_pairs(style)
for item in items:
item_id = id(item)
if item_id not in elements:
elements[item_id] = {
'item': item,
'classes': [],
'style': [],
}
elements[item_id]['style'].append(processed_style)
elements[item_id]['classes'].append(class_)
# Now apply inline style
# merge style only once for each element
# crucial when you have a lot of pseudo/classes
# and a long list of elements
for _, element in elements.items():
final_style = merge_styles(
element['item'].attrib.get('style', ''),
element['style'],
element['classes'],
remove_unset_properties=self.remove_unset_properties,
)
if final_style:
# final style could be empty string because of
# remove_unset_properties
element['item'].attrib['style'] = final_style
self._style_to_basic_html_attributes(element['item'],
final_style,
force=True)
if self.remove_classes:
# now we can delete all 'class' attributes
for item in page.xpath('//@class'):
parent = item.getparent()
del parent.attrib['class']
# Capitalize Margin properties
# To fix weird outlook bug
# https://www.emailonacid.com/blog/article/email-development/outlook.com-does-support-margins
if self.capitalize_float_margin:
for item in page.xpath('//@style'):
mangled = capitalize_float_margin(item)
item.getparent().attrib['style'] = mangled
# Add align attributes to images if they have a CSS float value of
# right or left. Outlook (both on desktop and on the web) are bad at
# understanding floats, but they do understand the HTML align attrib.
if self.align_floating_images:
for item in page.xpath('//img[@style]'):
image_css = cssutils.parseStyle(item.attrib['style'])
if image_css.float == 'right':
item.attrib['align'] = 'right'
elif image_css.float == 'left':
item.attrib['align'] = 'left'
#
# URLs
#
if self.base_url:
if not urlparse(self.base_url).scheme:
raise ValueError('Base URL must have a scheme')
for attr in ('href', 'src'):
for item in page.xpath("//@%s" % attr):
parent = item.getparent()
url = parent.attrib[attr]
if (attr == 'href' and self.preserve_internal_links
and url.startswith('#')):
continue
if (attr == 'src' and self.preserve_inline_attachments
and url.startswith('cid:')):
continue
if attr == 'href' and url.startswith('tel:'):
continue
parent.attrib[attr] = urljoin(self.base_url, url)
if hasattr(self.html, "getroottree"):
return root
else:
kwargs.setdefault('method', self.method)
kwargs.setdefault('pretty_print', pretty_print)
kwargs.setdefault('encoding', 'utf-8') # As Ken Thompson intended
out = etree.tostring(root, **kwargs).decode(kwargs['encoding'])
if self.method == 'xml':
out = _cdata_regex.sub(
lambda m: '/*<![CDATA[*/%s/*]]>*/' % m.group(1), out)
if self.strip_important:
out = _importants.sub('', out)
return out
def convert_task(task: et.Element, ds: datastore) -> Optional[et.Element]:
"""Convert old task XML into the new format."""
tid = task.attrib['id']
real_task = ds.task_factory(tid)
if task is None:
return
# Get the old task properties
# TIDs were stored as UUID, but sometimes they were not present
tid = task.get('uuid') or real_task.get_uuid() or tid_cache[tid]
status = task.get('status')
title = task.find('title').text
content = task.find('content')
try:
done_date = task.find('donedate').text
except AttributeError:
done_date = None
try:
due_date = task.find('duedate').text
except AttributeError:
due_date = None
try:
modified = task.find('modified').text
except AttributeError:
modified = None
try:
added = task.find('added').text
except AttributeError:
added = None
try:
start = task.find('startdate').text
except AttributeError:
start = None
# Build the new task
new_task = et.Element('task')
new_task.set('status', status)
new_task.set('id', tid)
new_title = et.SubElement(new_task, 'title')
new_title.text = title
tags = et.SubElement(new_task, 'tags')
for tag_name in task.get('tags').split(','):
if tag_name:
tag_id = tags_cache[tag_name]
task_tag = et.SubElement(tags, 'tag')
task_tag.text = tag_id
dates = et.SubElement(new_task, 'dates')
new_added = et.SubElement(dates, 'added')
new_modified = et.SubElement(dates, 'modified')
if added:
added = str(Date(added))
else:
added = date.today().isoformat()
new_added.text = added
if modified:
modified = modified[:10]
modified = str(Date(modified))
else:
modified = date.today().isoformat()
new_modified.text = modified
if done_date:
new_done = et.SubElement(dates, 'done')
new_done.text = str(Date(done_date))
if start:
start = Date(start)
if start.is_fuzzy():
new_start = et.SubElement(dates, 'fuzzyStart')
else:
new_start = et.SubElement(dates, 'start')
new_start.text = str(start)
if due_date:
due_date = Date(due_date)
if due_date.is_fuzzy():
new_due = et.SubElement(dates, 'fuzzyDue')
else:
new_due = et.SubElement(dates, 'due')
new_due.text = str(due_date)
recurring = et.SubElement(new_task, 'recurring')
recurring.set('enabled', 'false')
subtasks = et.SubElement(new_task, 'subtasks')
for sub in task.findall('subtask'):
new_sub = et.SubElement(subtasks, 'sub')
new_sub.text = tid_cache[sub.text]
new_content = et.SubElement(new_task, 'content')
if content is not None:
new_content.text = et.CDATA(convert_content(content.text))
else:
new_content.text = et.CDATA('')
return new_task
def genele(parent, name, attrs, item):
ele = etree.SubElement(parent, name, attrs)
if 'lxml' in sys.modules:
ele.text = (etree.CDATA(srepr(item)) if not (isinstance(item, bool) or isinstance(item, int)) else repr(item))
else:
ele.text = srepr(item)
def genKMLPoint(self, event=None):
# makes a KML marker for either a deployment, current, or final location. If
# an argument is passed it is assumed to
# be the current/final location, and if they are missing, the deployment location is used.
# event : an optional drifterdat postion dictionary object.
# set-up either deployment or current or final by toggling off of event and status
dtime = self.Ddeployment['startstr']
if event == None:
lat = self.Ddeployment['latitude']
lon = self.Ddeployment['longitude']
time = dtime
keyword = 'Deployment'
if self.Ddeployment['tag'].find('IceTracker'):
# Ice Trackers not moving much and full-sized markers block current location pins
icon = 'http://maps.google.com/mapfiles/kml/paddle/grn-blank_maps.png'
else:
icon = 'http://maps.google.com/mapfiles/kml/shapes/open-diamond.png'
else:
lat = str(event['latitude'])
lon = str(event['longitude'])
time = str(event['timestamp'])
# Mark currently functioning drifters with green thumbtacks, and failed
# drifters with red dots.
if self.Ddeployment['status'] == 'OK':
keyword = 'Current'
icon = 'http://maps.google.com/mapfiles/kml/paddle/ltblu-blank_maps.png'
else:
keyword = 'Final'
icon = 'http://maps.google.com/mapfiles/kml/pal4/icon49.png'
# set common variables to be inserted into KML
contacts = self.Ddeployment['contacts']
name = keyword + ' Location of Drifter ' + self.Ddeployment['drifter']
cstr = lon + ',' + lat + ',5'
snippet = keyword + ' location of the drifter'
# prep the description
destxt = '\n\
<b>Area:</b> ' + self.Ddeployment['area'] + '<br \>\n\
<b>Purpose:</b> ' + self.Ddeployment[
'purpose'] + '<br \>\n\
<b>Time Deployed:</b> ' + dtime + ' <br \>\n\
<b>Contact:</b> <br \>\n\
' + contacts + '<br \>\n\
<p>\n'
if keyword is not 'Deployment':
destxt = destxt + '\
<b>' + keyword + ' Latitude:</b> ' + lat + ' <br \>\n\
<b>' + keyword + ' Longitude:</b> ' + lon + ' <br \>\n\
<b>Time of Last Fix:</b> ' + time + ' <br \>\n\
'
description = etree.Element('description')
description.text = etree.CDATA(destxt)
# build the KML
leaf = E.Placemark(E.name(name), E.snippet(snippet), description,
E.styleUrl('#drifter_info'),
E.Style(E.IconStyle(E.Icon(E.href(icon)))),
E.Point(E.coordinates(cstr)))
return leaf
def rss_entry(self, extensions=True):
"""Create a RSS item and return it."""
entry = etree.Element('item')
if self.__rss_itemUrl:
entry.attrib['{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about'] = \
self.__rss_itemUrl
if not (self.__rss_title or self.__rss_description
or self.__rss_content):
raise ValueError('Required fields not set')
if self.__rss_title:
title = etree.SubElement(entry, 'title')
title.text = self.__rss_title
if self.__rss_link:
link = etree.SubElement(entry, 'link')
link.text = self.__rss_link
if self.__rss_seeAlso:
seeAlso = etree.SubElement(
entry, '{http://www.w3.org/2000/01/rdf-schema#}seeAlso')
seeAlso.attrib['{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource'] = \
self.__rss_seeAlso
if self.__rss_description and self.__rss_content:
description = etree.SubElement(entry, 'description')
description.text = self.__rss_description
XMLNS_CONTENT = 'http://purl.org/rss/1.0/modules/content/'
content = etree.SubElement(entry, '{%s}encoded' % XMLNS_CONTENT)
content.text = etree.CDATA(self.__rss_content['content']) \
if self.__rss_content.get('type', '') == 'CDATA' \
else self.__rss_content['content']
elif self.__rss_description:
description = etree.SubElement(entry, 'description')
description.text = self.__rss_description
elif self.__rss_content:
description = etree.SubElement(entry, 'description')
description.text = etree.CDATA(self.__rss_content['content']) \
if self.__rss_content.get('type', '') == 'CDATA' \
else self.__rss_content['content']
for a in self.__rss_author or []:
author = etree.SubElement(entry, 'author')
author.text = a
if self.__rss_guid.get('guid'):
guid = etree.SubElement(entry, 'guid')
guid.text = self.__rss_guid['guid']
permaLink = str(self.__rss_guid.get('permalink', False)).lower()
guid.attrib['isPermaLink'] = permaLink
for cat in self.__rss_category or []:
category = etree.SubElement(entry, 'category')
category.text = cat['value']
if cat.get('domain'):
category.attrib['domain'] = cat['domain']
if self.__rss_comments:
comments = etree.SubElement(entry, 'comments')
comments.text = self.__rss_comments
if self.__rss_enclosure:
enclosure = etree.SubElement(entry, 'enclosure')
enclosure.attrib['url'] = self.__rss_enclosure['url']
enclosure.attrib['length'] = self.__rss_enclosure['length']
enclosure.attrib['type'] = self.__rss_enclosure['type']
if self.__rss_pubDate:
pubDate = etree.SubElement(entry, 'pubDate')
pubDate.text = formatRFC2822(self.__rss_pubDate)
if self.__rss_source:
source = etree.SubElement(entry,
'source',
url=self.__rss_source['url'])
source.text = self.__rss_source['title']
if extensions:
for ext in self.__extensions.values() or []:
if ext.get('rss'):
ext['inst'].extend_rss(entry)
return entry
def setup_for_event_testing(ssh_client, db, listener_info, providers):
# FIX THE ENV ERROR IF PRESENT
if ssh_client.run_command("ruby -v")[0] != 0:
success = ssh_client.run_command("echo 'source /etc/default/evm' >> .bashrc")[0] == 0
assert success, "Issuing the patch command was unsuccessful"
# Verify it works
assert ssh_client.run_command("ruby -v")[0] == 0, "Patch failed"
# IMPORT AUTOMATE NAMESPACE
qe_automate_namespace_xml = "qe_event_handler.xml"
qe_automate_namespace_script = "qe_event_handler.rb"
local_automate_script = local(__file__)\
.new(basename="../data/%s" % qe_automate_namespace_script)\
.strpath
local_automate_file = local(__file__)\
.new(basename="../data/%s" % qe_automate_namespace_xml)\
.strpath
tmp_automate_file = "/tmp/%s" % qe_automate_namespace_xml
# Change the information
with open(local_automate_file, "r") as input_xml, \
open(tmp_automate_file, "w") as output_xml:
tree = etree.parse(input_xml)
root = tree.getroot()
def set_text(xpath, text):
field = root.xpath(xpath)
assert len(field) == 1
field[0].text = text
set_text("//MiqAeSchema/MiqAeField[@name='url']",
re.sub(r"^http://([^/]+)/?$", "\\1", listener_info.host))
set_text("//MiqAeSchema/MiqAeField[@name='port']", str(listener_info.port))
# Put the custom script from an external file
with open(local_automate_script, "r") as script:
set_text("//MiqAeMethod[@name='relay_events']",
etree.CDATA(script.read()))
et = etree.ElementTree(root)
et.write(output_xml)
# copy xml file to appliance
# but before that, let's check whether it's there because we may have already applied this file
if ssh_client.run_command("ls /root/%s" % qe_automate_namespace_xml)[0] != 0:
ssh_client.put_file(tmp_automate_file, '/root/')
# We have to convert it first for new version
convert_cmd = version.pick({
"default": None,
"5.3.0.0":
"evm:automate:convert DOMAIN=Default FILE=/root/{} ZIP_FILE=/root/{}.zip".format(
qe_automate_namespace_xml, qe_automate_namespace_xml),
})
if convert_cmd is not None:
logger.info("Converting namespace for use on newer appliance...")
return_code, stdout = ssh_client.run_rake_command(convert_cmd)
if return_code != 0:
logger.error("Namespace conversion was unsuccessful")
logger.error(stdout)
# We didn't successfully do that so remove the file to know
# that it's needed to do it again when run again
ssh_client.run_command("rm -f /root/%s*" % qe_automate_namespace_xml)
raise AutomateImportError(stdout)
# run rake cmd on appliance to import automate namespace
rake_cmd = version.pick({
"default": "evm:automate:import FILE=/root/{}".format(qe_automate_namespace_xml),
"5.3.0.0":
"evm:automate:import ZIP_FILE=/root/{}.zip DOMAIN=Default OVERWRITE=true "
"PREVIEW=false".format(qe_automate_namespace_xml),
})
logger.info("Importing the QE Automation namespace ...")
return_code, stdout = ssh_client.run_rake_command(rake_cmd)
if return_code != 0:
logger.error("Namespace import was unsuccessful")
logger.error(stdout)
# We didn't successfully do that so remove the file to know
# that it's needed to do it again when run again
ssh_client.run_command("rm -f /root/%s*" % qe_automate_namespace_xml)
raise AutomateImportError(stdout)
# CREATE AUTOMATE INSTANCE HOOK
if db is None or db.session.query(db['miq_ae_instances'].name)\
.filter(db['miq_ae_instances'].name == "RelayEvents").count() == 0:
# Check presence
instance = Instance(
name="RelayEvents",
display_name="RelayEvents",
description="relationship hook to link to custom QE events relay namespace",
values={
"rel2": {
"value": "/QE/Automation/APIMethods/relay_events?event=$evm.object['event']"
}
},
cls=Class(name="Automation Requests (Request)", namespace=Namespace("System"))
)
instance.create()
# IMPORT POLICIES
policy_yaml = "profile_relay_events.yaml"
policy_path = local(__file__).new(basename="../data/%s" % policy_yaml)
if not is_imported("Automate event policies"):
import_file(policy_path.strpath)
# ASSIGN POLICY PROFILES
for provider in providers:
prov_obj = get_from_config(provider)
prov_obj.assign_policy_profiles("Automate event policies")
flash.assert_no_errors()
def atom_entry(self, extensions=True):
"""Create an ATOM entry and return it."""
entry = etree.Element('entry')
if not (self.__atom_id and self.__atom_title and self.__atom_updated):
raise ValueError('Required fields not set')
id = etree.SubElement(entry, 'id')
id.text = self.__atom_id
title = etree.SubElement(entry, 'title')
title.text = self.__atom_title
updated = etree.SubElement(entry, 'updated')
updated.text = self.__atom_updated.isoformat()
# An entry must contain an alternate link if there is no content
# element.
if not self.__atom_content:
links = self.__atom_link or []
if not [l for l in links if l.get('rel') == 'alternate']:
raise ValueError('Entry must contain an alternate link or ' +
'a content element.')
XMLELEMENTS_NS = 'http://www.w3.org/XML/1998/namespace'
# Add author elements
for a in self.__atom_author or []:
# Atom requires a name. Skip elements without.
if not a.get('name'):
continue
author = etree.SubElement(entry, 'author')
if a.get('lang'):
author.set('{%s}lang' % XMLELEMENTS_NS, a.get('lang'))
name = etree.SubElement(author, 'name')
name.text = a.get('name')
if a.get('email'):
email = etree.SubElement(author, 'email')
email.text = a.get('email')
if a.get('uri'):
uri = etree.SubElement(author, 'uri')
uri.text = a.get('uri')
if self.__atom_content:
content = etree.SubElement(entry, 'content')
type = self.__atom_content.get('type')
lang = self.__atom_content.get('lang')
if self.__atom_content.get('src'):
content.attrib['src'] = self.__atom_content['src']
elif self.__atom_content.get('content'):
# Surround xhtml with a div tag, parse it and embed it
if type == 'xhtml':
content.append(
etree.fromstring(
'<div xmlns="http://www.w3.org/1999/xhtml">' +
self.__atom_content.get('content') + '</div>'))
elif type == 'CDATA':
content.text = etree.CDATA(
self.__atom_content.get('content'))
# Emed the text in escaped form
elif not type or type.startswith('text') or type == 'html':
content.text = self.__atom_content.get('content')
# Parse XML and embed it
elif type.endswith('/xml') or type.endswith('+xml'):
content.append(
etree.fromstring(self.__atom_content['content']))
# Everything else should be included base64 encoded
else:
raise ValueError('base64 encoded content is not ' +
'supported at the moment. Pull requests' +
' adding support are welcome.')
# Add type description of the content
if type:
content.attrib['type'] = type
if lang:
content.set('{%s}lang' % XMLELEMENTS_NS, lang)
for l in self.__atom_link or []:
link = etree.SubElement(entry, 'link', href=l['href'])
if l.get('rel'):
link.attrib['rel'] = l['rel']
if l.get('type'):
link.attrib['type'] = l['type']
if l.get('hreflang'):
link.attrib['hreflang'] = l['hreflang']
if l.get('title'):
link.attrib['title'] = l['title']
if l.get('length'):
link.attrib['length'] = l['length']
if self.__atom_summary:
summary = etree.SubElement(entry, 'summary')
summary.text = self.__atom_summary
for c in self.__atom_category or []:
cat = etree.SubElement(entry, 'category', term=c['term'])
if c.get('scheme'):
cat.attrib['scheme'] = c['scheme']
if c.get('label'):
cat.attrib['label'] = c['label']
# Add author elements
for c in self.__atom_contributor or []:
# Atom requires a name. Skip elements without.
if not c.get('name'):
continue
contrib = etree.SubElement(entry, 'contributor')
name = etree.SubElement(contrib, 'name')
name.text = c.get('name')
if c.get('email'):
email = etree.SubElement(contrib, 'email')
email.text = c.get('email')
if c.get('uri'):
uri = etree.SubElement(contrib, 'uri')
uri.text = c.get('uri')
if self.__atom_published:
published = etree.SubElement(entry, 'published')
published.text = self.__atom_published.isoformat()
if self.__atom_rights:
rights = etree.SubElement(entry, 'rights')
rights.text = self.__atom_rights
if self.__atom_source:
source = etree.SubElement(entry, 'source')
if self.__atom_source.get('title'):
source_title = etree.SubElement(source, 'title')
source_title.text = self.__atom_source['title']
if self.__atom_source.get('link'):
etree.SubElement(source,
'link',
href=self.__atom_source['link'])
if extensions:
for ext in self.__extensions.values() or []:
if ext.get('atom'):
ext['inst'].extend_atom(entry)
return entry
def setComment(self, Comment):
assert type(Comment) == str
if self.Comment == None:
self.Comment = etree.SubElement(self.root, 'Comment')
self.root.append(self.Comment)
self.Comment.text = etree.CDATA(Comment)
def __renderHistory(self, pager):
#<comment>
# <![CDATA[<blockTable style="historyhdrtablestyle" rowHeights="24" colWidths="42,30,25,45,38">
# <tr>
# <td><para alignment="LEFT" fontName="Helvetica-Bold" fontSize="9">Date (UTC)</para></td>
# <td><para alignment="RIGHT" fontName="Helvetica-Bold" fontSize="9">Dist. (km)</para></td>
# <td><para alignment="CENTER" fontName="Helvetica-Bold" fontSize="9">Mag.</para></td>
# <td><para alignment="CENTER" fontName="Helvetica-Bold" fontSize="9">Max MMI(#)</para></td>
# <td><para alignment="RIGHT" fontName="Helvetica-Bold" fontSize="9">Shaking Deaths</para></td>
# </tr>
# </blockTable>
# <blockTable style="historytablestyle" rowHeights="12,12,12" colWidths="42,30,25,45,38">
# <tr>
# <td><para alignment="LEFT" fontName="Helvetica" fontSize="9">1993-03-06</para></td>
# <td><para alignment="RIGHT" fontName="Helvetica" fontSize="9">238</para></td>
# <td><para alignment="CENTER" fontName="Helvetica" fontSize="9">6.6</para></td>
# <td background="#7aff93"><para alignment="CENTER" fontName="Helvetica" fontSize="9">V(7k)</para></td>
# <td><para alignment="RIGHT" fontName="Helvetica" fontSize="9">0</para></td>
# </blockTable><para style="commentstyle"></para>]]> </comment>
if not any(self._pagerdict['historical_earthquakes']):
return pager
table_dict = {
'style': 'historyhdrtablestyle',
'rowHeights': '24',
'colWidths': '42,30,25,45,38'
}
header_tag = etree.Element('blockTable', attrib=table_dict)
hdr_alignments = ['LEFT', 'RIGHT', 'CENTER', 'CENTER', 'RIGHT']
hdr_fonts = ['Helvetica-Bold'] * 5
hdr_sizes = ['9'] * 5
hdr_data = [
'Date (UTC)', 'Dist. (km)', 'Mag.', 'Max MMI(#)', 'Shaking Deaths'
]
row1_tag = etree.SubElement(header_tag, 'tr')
for i in range(0, 5):
align = hdr_alignments[i]
font = hdr_fonts[i]
size = hdr_sizes[i]
hdr_text = hdr_data[i]
pdict = {'alignment': align, 'fontName': font, 'fontSize': font}
cell_tag = etree.SubElement(row1_tag, 'td')
para_tag = etree.SubElement(row1_tag,
'para',
attrib=pdict,
text=hdr_text)
bdict = {
'style': 'historytablestyle',
'rowHeights': '12,12,12',
'colWidths': '42,30,25,45,38'
}
block_tag = etree.Element('blockTable', attrib=bdict)
for event in self._pagerdict['historical_earthquakes']:
rom_maxmmi = dec_to_roman(event['MaxMMI'])
nmmi_str = pop_round_short(event['MaxMMI'], usemillion=True)
deaths = event['ShakingDeaths']
if np.isnan(deaths):
deaths = 'NA'
else:
deaths = '%i' % deaths
content = [
event['Time'][0:10],
'%i' % (event['Distance']),
'%.1f' % (event['Magnitude']),
'%s(%s)' % (rom_maxmmi, nmmi_str),
'%s' % (deaths)
]
row_tag = etree.SubElement(block_tag, 'tr')
for i in range(0, 5):
align = hdr_alignments[i]
font = hdr_fonts[i]
size = hdr_sizes[i]
td_tag = etree.SubElement(row_tag, 'td')
pdict = {
'alignment': align,
'fontName': font,
'fontSize': size
}
if i == 3:
pdict['background'] = event['Color']
para_tag = etree.SubElement(td_tag,
'para',
attrib=pdict,
text=content[i])
para_tag = etree.SubElement(header_tag,
'para',
attrib={'style': 'commentstyle'})
history_text = etree.tostring(header_tag, pretty_print=True)
comment_tag = etree.SubElement(pager, 'comment')
comment_tag.text = etree.CDATA(history_text)
return pager
def toxml(self, from_, to_, funcflag='0', pretty_print=False):
import time
from lxml import etree
r = etree.Element('xml')
tousername = etree.SubElement(r, 'ToUserName')
tousername.text = etree.CDATA(to_)
fromusername = etree.SubElement(r, 'FromUserName')
fromusername.text = etree.CDATA(from_)
createtime = etree.SubElement(r, 'CreateTime')
createtime.text = etree.CDATA(str(int(time.time())))
msgtype = etree.SubElement(r, 'MsgType')
if self.rule_type == 'text':
msgtype.text = etree.CDATA('text')
content = etree.SubElement(r, 'Content')
content.text = etree.CDATA(self.content)
if self.rule_type == 'news':
msgtype.text = etree.CDATA('news')
content = etree.SubElement(r, 'Content')
content.text = etree.CDATA('')
articlecount = etree.SubElement(r, 'ArticleCount')
articlecount.text = etree.CDATA(str(self.article_set.count()))
articles = etree.SubElement(r, 'Articles')
for a in self.article_set.all():
item = etree.SubElement(articles, 'item')
title = etree.SubElement(item, 'Title')
title.text = etree.CDATA(a.title)
desp = etree.SubElement(item, 'Description')
desp.text = etree.CDATA(a.desp)
picurl = etree.SubElement(item, 'PicUrl')
picurl.text = etree.CDATA(a.imgurl())
url = etree.SubElement(item, 'Url')
url.text = etree.CDATA(a.url)
if self.rule_type == 'music':
m = self.music_set.all()[0]
msgtype.text = etree.CDATA('music')
music = etree.SubElement(r, 'Music')
title = etree.SubElement(music, 'Title')
title.text = etree.CDATA(m.title)
desp = etree.SubElement(music, 'Description')
desp.text = etree.CDATA(m.desp)
musicurl = etree.SubElement(music, 'MusicUrl')
musicurl.text = etree.CDATA(m.mfileurl())
hqmusicurl = etree.SubElement(music, 'HQMusicUrl')
hqmusicurl.text = etree.CDATA(m.hqmfileurl())
funcflag_ = etree.SubElement(r, 'FuncFlag')
funcflag_.text = etree.CDATA(funcflag)
xml = etree.tostring(r, encoding='utf-8', pretty_print=pretty_print)
return xml
def getXml(self):
# xml_dict = Dict.toRecursiveDict(self._correios_log)
correios_log = etree.Element('correioslog')
tipo_arquivo = etree.Element('tipo_arquivo')
versao_arquivo = etree.Element('versao_arquivo')
correios_log.append(tipo_arquivo)
correios_log.append(versao_arquivo)
# remetente = etree.Element('remetente')
# objeto_postal = etree.Element('objeto_postal')
tipo_arquivo.text = unicode(self._correios_log['tipo_arquivo'])
versao_arquivo.text = unicode(self._correios_log['versao_arquivo'])
# plp node
etree_plp = etree.Element('plp')
plpobj = self._correios_log['plp']
etree_id_plp = etree.Element('id_plp')
etree_valor_global = etree.Element('valor_global')
etree_mcu_unidade_postagem = etree.Element('mcu_unidade_postagem')
etree_nome_unidade_postagem = etree.Element('nome_unidade_postagem')
etree_cartao_postagem = etree.Element('cartao_postagem')
etree_id_plp.text = unicode(plpobj.id_plp)
etree_valor_global.text = unicode(plpobj.valor_global)
etree_mcu_unidade_postagem.text = unicode(plpobj.mcu_unidade_postagem)
etree_nome_unidade_postagem.text = unicode(plpobj.nome_unidade_postagem)
etree_cartao_postagem.text = unicode(plpobj.cartao_postagem)
etree_plp.append(etree_id_plp)
etree_plp.append(etree_valor_global)
etree_plp.append(etree_mcu_unidade_postagem)
etree_plp.append(etree_nome_unidade_postagem)
etree_plp.append(etree_cartao_postagem)
correios_log.append(etree_plp)
# remetente node
etree_remetente = etree.Element('remetente')
etree_numero_contrato = etree.Element('numero_contrato')
etree_numero_diretoria = etree.Element('numero_diretoria')
etree_codigo_administrativo = etree.Element('codigo_administrativo')
etree_nome_remetente = etree.Element('nome_remetente')
etree_logradouro_remetente = etree.Element('logradouro_remetente')
etree_numero_remetente = etree.Element('numero_remetente')
etree_complemento_remetente = etree.Element('complemento_remetente')
etree_bairro_remetente = etree.Element('bairro_remetente')
etree_cep_remetente = etree.Element('cep_remetente')
etree_cidade_remetente = etree.Element('cidade_remetente')
etree_uf_remetente = etree.Element('uf_remetente')
etree_telefone_remetente = etree.Element('telefone_remetente')
etree_fax_remetente = etree.Element('fax_remetente')
etree_email_remetente = etree.Element('email_remetente')
remetenteobj = self._correios_log['remetente']
etree_numero_contrato.text = unicode(remetenteobj.numero_contrato)
etree_numero_diretoria.text = unicode(remetenteobj.numero_diretoria)
etree_codigo_administrativo.text = unicode(remetenteobj.codigo_administrativo)
etree_nome_remetente.text = etree.CDATA(unicode(remetenteobj.nome_remetente)) if remetenteobj.nome_remetente != '' else None
etree_logradouro_remetente.text = etree.CDATA(unicode(remetenteobj.logradouro_remetente)) if remetenteobj.logradouro_remetente != '' else None
etree_numero_remetente.text = etree.CDATA(unicode(remetenteobj.numero_remetente)) if remetenteobj.numero_remetente != '' else None
etree_complemento_remetente.text = etree.CDATA(unicode(remetenteobj.complemento_remetente)) if remetenteobj.complemento_remetente != '' else None
etree_bairro_remetente.text = etree.CDATA(unicode(remetenteobj.bairro_remetente)) if remetenteobj.bairro_remetente != '' else None
etree_cep_remetente.text = etree.CDATA(unicode(remetenteobj.cep_remetente)) if remetenteobj.cep_remetente != '' else None
etree_cidade_remetente.text = etree.CDATA(unicode(remetenteobj.cidade_remetente)) if remetenteobj.cidade_remetente != '' else None
etree_uf_remetente.text = etree.CDATA(unicode(remetenteobj.uf_remetente)) if remetenteobj.uf_remetente != '' else None
etree_telefone_remetente.text = etree.CDATA(unicode(remetenteobj.telefone_remetente)) if remetenteobj.telefone_remetente != '' else None
etree_fax_remetente.text = etree.CDATA(unicode(remetenteobj.fax_remetente)) if remetenteobj.fax_remetente != '' else None
etree_email_remetente.text = etree.CDATA(unicode(remetenteobj.email_remetente)) if remetenteobj.email_remetente != '' else None
etree_remetente.append(etree_numero_contrato)
etree_remetente.append(etree_numero_diretoria)
etree_remetente.append(etree_codigo_administrativo)
etree_remetente.append(etree_nome_remetente)
etree_remetente.append(etree_logradouro_remetente)
etree_remetente.append(etree_numero_remetente)
etree_remetente.append(etree_complemento_remetente)
etree_remetente.append(etree_bairro_remetente)
etree_remetente.append(etree_cep_remetente)
etree_remetente.append(etree_cidade_remetente)
etree_remetente.append(etree_uf_remetente)
etree_remetente.append(etree_telefone_remetente)
etree_remetente.append(etree_fax_remetente)
etree_remetente.append(etree_email_remetente)
correios_log.append(etree_remetente)
forma_pagamento = etree.Element('forma_pagamento')
correios_log.append(forma_pagamento)
# objeto_postal node list
for objeto_postal_obj in self._correios_log['objeto_postal']:
etree_objeto_postal = etree.Element('objeto_postal')
etree_codigo_objeto_cliente = etree.Element('codigo_objeto_cliente')
etree_data_postagem_sara = etree.Element('data_postagem_sara')
etree_status_processamento = etree.Element('status_processamento')
etree_numero_comprovante_postagem = etree.Element('numero_comprovante_postagem')
etree_numero_etiqueta = etree.Element('numero_etiqueta')
etree_codigo_servico_postagem = etree.Element('codigo_servico_postagem')
etree_cubagem = etree.Element('cubagem')
etree_peso = etree.Element('peso')
etree_rt1 = etree.Element('rt1')
etree_rt2 = etree.Element('rt2')
etree_destinatario = etree.Element('destinatario')
etree_nacional = etree.Element('nacional')
etree_servico_adicional = etree.Element('servico_adicional')
etree_dimensao_objeto = etree.Element('dimensao_objeto')
etree_valor_cobrado = etree.Element('valor_cobrado')
etree_status_processamento.text = unicode('0')
etree_numero_etiqueta.text = unicode(objeto_postal_obj.numero_etiqueta)
etree_codigo_servico_postagem.text = unicode(objeto_postal_obj.codigo_servico_postagem)
etree_cubagem.text = unicode(objeto_postal_obj.cubagem)
etree_peso.text = unicode(objeto_postal_obj.peso)
etree_rt1.text = etree.CDATA(unicode(objeto_postal_obj.rt1)) if objeto_postal_obj.rt1 != '' else None
etree_rt2.text = etree.CDATA(unicode(objeto_postal_obj.rt2)) if objeto_postal_obj.rt2 != '' else None
# objeto_postal.destinatario node
destinatario_obj = objeto_postal_obj.destinatario
etree_nome_destinatario = etree.Element('nome_destinatario')
etree_telefone_destinatario = etree.Element('telefone_destinatario')
etree_celular_destinatario = etree.Element('celular_destinatario')
etree_email_destinatario = etree.Element('email_destinatario')
etree_logradouro_destinatario = etree.Element('logradouro_destinatario')
etree_complemento_destinatario = etree.Element('complemento_destinatario')
etree_numero_end_destinatario = etree.Element('numero_end_destinatario')
etree_nome_destinatario.text = etree.CDATA(unicode(destinatario_obj.nome_destinatario)) if destinatario_obj.nome_destinatario != '' else ''
etree_telefone_destinatario.text = etree.CDATA(unicode(destinatario_obj.telefone_destinatario)) if destinatario_obj.telefone_destinatario != '' else ''
etree_celular_destinatario.text = etree.CDATA(unicode(destinatario_obj.celular_destinatario)) if destinatario_obj.celular_destinatario != '' else ''
etree_email_destinatario.text = etree.CDATA(unicode(destinatario_obj.email_destinatario)) if destinatario_obj.email_destinatario != '' else ''
etree_logradouro_destinatario.text = etree.CDATA(unicode(destinatario_obj.logradouro_destinatario)) if destinatario_obj.logradouro_destinatario != '' else ''
etree_complemento_destinatario.text = etree.CDATA(unicode(destinatario_obj.complemento_destinatario)) if destinatario_obj.complemento_destinatario != '' else ''
etree_numero_end_destinatario.text = etree.CDATA(unicode(destinatario_obj.numero_end_destinatario)) if destinatario_obj.numero_end_destinatario != '' else ''
etree_destinatario.append(etree_nome_destinatario)
etree_destinatario.append(etree_telefone_destinatario)
etree_destinatario.append(etree_celular_destinatario)
etree_destinatario.append(etree_email_destinatario)
etree_destinatario.append(etree_logradouro_destinatario)
etree_destinatario.append(etree_complemento_destinatario)
etree_destinatario.append(etree_numero_end_destinatario)
# objeto_postal.nacional node
nacional_obj = objeto_postal_obj.nacional
etree_bairro_destinatario = etree.Element('bairro_destinatario')
etree_cidade_destinatario = etree.Element('cidade_destinatario')
etree_uf_destinatario = etree.Element('uf_destinatario')
etree_cep_destinatario = etree.Element('cep_destinatario')
etree_numero_nota_fiscal = etree.Element('numero_nota_fiscal')
etree_descricao_objeto = etree.Element('descricao_objeto')
etree_valor_a_cobrar = etree.Element('valor_a_cobrar')
etree_natureza_nota_fiscal = etree.Element('natureza_nota_fiscal')
etree_codigo_usuario_postal = etree.Element('codigo_usuario_postal')
etree_centro_custo_cliente = etree.Element('centro_custo_cliente')
etree_serie_nota_fiscal = etree.Element('serie_nota_fiscal')
etree_valor_nota_fiscal = etree.Element('valor_nota_fiscal')
etree_bairro_destinatario.text = etree.CDATA(unicode(nacional_obj.bairro_destinatario)) if nacional_obj.bairro_destinatario != '' else ''
etree_cidade_destinatario.text = etree.CDATA(unicode(nacional_obj.cidade_destinatario)) if nacional_obj.cidade_destinatario != '' else ''
etree_uf_destinatario.text = etree.CDATA(unicode(nacional_obj.uf_destinatario)) if nacional_obj.uf_destinatario != '' else ''
etree_cep_destinatario.text = etree.CDATA(unicode(nacional_obj.cep_destinatario)) if nacional_obj.cep_destinatario != '' else ''
etree_numero_nota_fiscal.text = unicode(nacional_obj.numero_nota_fiscal) if nacional_obj.numero_nota_fiscal != '' else ''
etree_descricao_objeto.text = etree.CDATA(unicode(nacional_obj.descricao_objeto)) if nacional_obj.descricao_objeto != '' else ''
etree_valor_a_cobrar.text = unicode(nacional_obj.valor_a_cobrar) if nacional_obj.valor_a_cobrar != '' else ''
etree_natureza_nota_fiscal.text = etree.CDATA(unicode(nacional_obj.natureza_nota_fiscal)) if nacional_obj.natureza_nota_fiscal != '' else ''
etree_codigo_usuario_postal.text = etree.CDATA(unicode(nacional_obj.codigo_usuario_postal)) if nacional_obj.codigo_usuario_postal != '' else ''
etree_centro_custo_cliente.text = etree.CDATA(unicode(nacional_obj.centro_custo_cliente)) if nacional_obj.centro_custo_cliente != '' else ''
etree_serie_nota_fiscal.text = etree.CDATA(unicode(nacional_obj.serie_nota_fiscal)) if nacional_obj.serie_nota_fiscal != '' else ''
etree_valor_nota_fiscal.text = unicode(nacional_obj.valor_nota_fiscal) if nacional_obj.valor_nota_fiscal !='' else ''
etree_nacional.append(etree_bairro_destinatario)
etree_nacional.append(etree_cidade_destinatario)
etree_nacional.append(etree_uf_destinatario)
etree_nacional.append(etree_cep_destinatario)
etree_nacional.append(etree_codigo_usuario_postal)
etree_nacional.append(etree_centro_custo_cliente)
etree_nacional.append(etree_numero_nota_fiscal)
etree_nacional.append(etree_serie_nota_fiscal)
etree_nacional.append(etree_valor_nota_fiscal)
etree_nacional.append(etree_natureza_nota_fiscal)
etree_nacional.append(etree_descricao_objeto)
etree_nacional.append(etree_valor_a_cobrar)
# objeto_postal.servico_adicional node
servico_adicional_obj = objeto_postal_obj.servico_adicional
for servico_adicional in servico_adicional_obj.codigo_servico_adicional:
codigo_servico_adicional = etree.Element('codigo_servico_adicional')
codigo_servico_adicional.text = unicode(servico_adicional)
etree_servico_adicional.append(codigo_servico_adicional)
etree_valor_declarado = etree.Element('valor_declarado')
etree_valor_declarado.text = unicode(servico_adicional_obj.valor_declarado)
etree_servico_adicional.append(etree_valor_declarado)
# objeto_postal.dimensao_objeto node
etree_tipo_objeto = etree.Element('tipo_objeto')
etree_dimensao_altura = etree.Element('dimensao_altura')
etree_dimensao_largura = etree.Element('dimensao_largura')
etree_dimensao_comprimento = etree.Element('dimensao_comprimento')
etree_dimensao_diametro = etree.Element('dimensao_diametro')
etree_tipo_objeto.text = unicode(objeto_postal_obj.dimensao_objeto.tipo_objeto)
etree_dimensao_altura.text = unicode(objeto_postal_obj.dimensao_objeto.dimensao_altura) if objeto_postal_obj.dimensao_objeto.dimensao_altura != '' else ''
etree_dimensao_largura.text = unicode(objeto_postal_obj.dimensao_objeto.dimensao_largura) if objeto_postal_obj.dimensao_objeto.dimensao_largura != '' else ''
etree_dimensao_comprimento.text = unicode(objeto_postal_obj.dimensao_objeto.dimensao_comprimento) if objeto_postal_obj.dimensao_objeto.dimensao_comprimento != '' else ''
etree_dimensao_diametro.text = unicode(objeto_postal_obj.dimensao_objeto.dimensao_diametro) if objeto_postal_obj.dimensao_objeto.dimensao_diametro != '' else ''
etree_dimensao_objeto.append(etree_tipo_objeto)
etree_dimensao_objeto.append(etree_dimensao_altura)
etree_dimensao_objeto.append(etree_dimensao_largura)
etree_dimensao_objeto.append(etree_dimensao_comprimento)
etree_dimensao_objeto.append(etree_dimensao_diametro)
etree_objeto_postal.append(etree_numero_etiqueta)
etree_objeto_postal.append(etree_codigo_objeto_cliente)
etree_objeto_postal.append(etree_codigo_servico_postagem)
etree_objeto_postal.append(etree_cubagem)
etree_objeto_postal.append(etree_peso)
etree_objeto_postal.append(etree_rt1)
etree_objeto_postal.append(etree_rt2)
etree_objeto_postal.append(etree_destinatario)
etree_objeto_postal.append(etree_nacional)
etree_objeto_postal.append(etree_servico_adicional)
etree_objeto_postal.append(etree_dimensao_objeto)
etree_objeto_postal.append(etree_data_postagem_sara)
etree_objeto_postal.append(etree_status_processamento)
etree_objeto_postal.append(etree_numero_comprovante_postagem)
etree_objeto_postal.append(etree_valor_cobrado)
correios_log.append(etree_objeto_postal)
# correios_log = parseDict(xml_dict, 'correioslog')
soap_package = etree.tostring(correios_log, encoding='ISO-8859-1')
return soap_package.replace('\n', '')
def recipe_to_krecipes_xml(recipe, author=None):
"""
Export recipe to KRecipes XML string
"""
sauthor = u''
if not empty(recipe.author):
sauthor += '%s@' % recipe.author
if author is None:
sauthor += 'Cookboob'
else:
sauthor += author
header = u'<?xml version="1.0" encoding="UTF-8" ?>\n'
initial_xml = '''\
<krecipes version='2.0-beta2' lang='fr' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:noNamespaceSchemaLocation='krecipes.xsd'>
<krecipes-recipe id='1'>
</krecipes-recipe>
</krecipes>'''
doc = ET.fromstring(initial_xml)
xrecipe = doc.find('krecipes-recipe')
desc = ET.SubElement(xrecipe, 'krecipes-description')
title = ET.SubElement(desc, 'title')
title.text = recipe.title
authors = ET.SubElement(desc, 'author')
authors.text = sauthor
eyield = ET.SubElement(desc, 'yield')
if not empty(recipe.nb_person):
amount = ET.SubElement(eyield, 'amount')
if len(recipe.nb_person) == 1:
amount.text = '%s' % recipe.nb_person[0]
else:
mini = ET.SubElement(amount, 'min')
mini.text = u'%s' % recipe.nb_person[0]
maxi = ET.SubElement(amount, 'max')
maxi.text = u'%s' % recipe.nb_person[1]
etype = ET.SubElement(eyield, 'type')
etype.text = 'persons'
if not empty(recipe.preparation_time):
preptime = ET.SubElement(desc, 'preparation-time')
preptime.text = '%02d:%02d' % (recipe.preparation_time / 60, recipe.preparation_time % 60)
if recipe.picture and recipe.picture.url:
data = requests.get(recipe.picture.url).content
datab64 = base64.encodestring(data)[:-1]
pictures = ET.SubElement(desc, 'pictures')
pic = ET.SubElement(pictures, 'pic', {'format': 'JPEG', 'id': '1'})
pic.text = ET.CDATA(datab64)
if not empty(recipe.ingredients):
ings = ET.SubElement(xrecipe, 'krecipes-ingredients')
pat = re.compile('^[0-9,.]*')
for i in recipe.ingredients:
sname = u'%s' % i
samount = ''
sunit = ''
first_nums = pat.match(i).group()
if first_nums != '':
samount = first_nums
sname = i.lstrip('0123456789 ')
ing = ET.SubElement(ings, 'ingredient')
am = ET.SubElement(ing, 'amount')
am.text = samount
unit = ET.SubElement(ing, 'unit')
unit.text = sunit
name = ET.SubElement(ing, 'name')
name.text = sname
if not empty(recipe.instructions):
instructions = ET.SubElement(xrecipe, 'krecipes-instructions')
instructions.text = recipe.instructions
if not empty(recipe.comments):
ratings = ET.SubElement(xrecipe, 'krecipes-ratings')
for c in recipe.comments:
rating = ET.SubElement(ratings, 'rating')
if c.author:
rater = ET.SubElement(rating, 'rater')
rater.text = c.author
if c.text:
com = ET.SubElement(rating, 'comment')
com.text = c.text
crits = ET.SubElement(rating, 'criterion')
if c.rate:
crit = ET.SubElement(crits, 'criteria')
critname = ET.SubElement(crit, 'name')
critname.text = 'Overall'
critstars = ET.SubElement(crit, 'stars')
critstars.text = c.rate.split('/')[0]
return header + ET.tostring(doc, encoding='UTF-8', pretty_print=True).decode('utf-8')
def set_text(self, content):
"""Sets the style content text as a CDATA section"""
self.text = etree.CDATA(str(content))
def product(edition, facet=None):
ebooks = facet.filter_model("Ebook", edition.ebooks.filter(
active=True)) if facet else edition.ebooks.filter(active=True)
ebooks = ebooks.order_by('-created')
# Just because an edition satisfies 2 facets with multiple ebooks doesn't mean that there is a single ebook satisfies both facets
if not ebooks.exists():
return None
work = edition.work
product_node = etree.Element("Product")
product_node.append(
text_node("RecordReference",
"it.unglue.work.%s.%s" % (work.id, edition.id)))
product_node.append(text_node("NotificationType", "03")) # final
ident_node = etree.SubElement(product_node, "ProductIdentifier")
ident_node.append(text_node("ProductIDType", "01")) #proprietary
ident_node.append(text_node("IDTypeName",
"unglue.it edition id")) #proprietary
ident_node.append(text_node("IDValue", unicode(edition.id)))
# wrong isbn better than no isbn
isbn = edition.isbn_13 if edition.isbn_13 else edition.work.first_isbn_13()
if isbn:
ident_node = etree.SubElement(product_node, "ProductIdentifier")
ident_node.append(text_node("ProductIDType", "03")) #proprietary
ident_node.append(text_node("IDValue", isbn))
# Descriptive Detail Block
descriptive_node = etree.SubElement(product_node, "DescriptiveDetail")
descriptive_node.append(text_node("ProductComposition",
"00")) # single item
descriptive_node.append(text_node("ProductForm", "ED")) # download
ebook = None
latest_ebooks = []
ebook_formats = []
for ebook in ebooks:
if ebook.format not in ebook_formats:
ebook_formats.append(ebook.format)
latest_ebooks.append(ebook)
if ebook.format == 'epub':
descriptive_node.append(text_node("ProductFormDetail", "E101"))
elif ebook.format == 'pdf':
descriptive_node.append(text_node("ProductFormDetail", "E107"))
elif ebook.format == 'mobi':
descriptive_node.append(text_node("ProductFormDetail", "E116"))
if ebook.rights:
license_node = etree.SubElement(descriptive_node, "EpubLicense")
license_node.append(text_node("EpubLicenseName", ebook.rights))
lic_expr_node = etree.SubElement(license_node, "EpubLicenseExpression")
lic_expr_node.append(text_node("EpubLicenseExpressionType",
'01')) #human readable
lic_expr_node.append(
text_node("EpubLicenseExpressionLink",
ccinfo(ebook.rights).url))
title_node = etree.SubElement(descriptive_node, "TitleDetail")
title_node.append(text_node("TitleType", '01')) #distinctive title
title_el = etree.SubElement(title_node, "TitleElement")
title_el.append(text_node("TitleElementLevel", '01'))
title_el.append(text_node("TitleText", edition.title))
contrib_i = 0
for contrib in edition.relators.all():
contrib_i += 1
contrib_node = etree.SubElement(descriptive_node, "Contributor")
contrib_node.append(text_node("SequenceNumber", unicode(contrib_i)))
contrib_node.append(
text_node("ContributorRole",
relator_contrib.get(contrib.relation.code, "")))
contrib_node.append(text_node("PersonName", contrib.author.name))
contrib_node.append(
text_node("PersonNameInverted", contrib.author.last_name_first))
(lang, locale) = (edition.work.language, None)
if '_' in lang:
(lang, locale) = lang.split('_')
if len(lang) == 2:
lang = iso639.get(lang, None)
if lang:
lang_node = etree.SubElement(descriptive_node, "Language")
lang_node.append(text_node("LanguageRole", "01"))
lang_node.append(text_node("LanguageCode", lang))
if locale:
lang_node.append(text_node("CountryCode", locale))
for subject in work.subjects.all():
subj_node = etree.SubElement(descriptive_node, "Subject")
if subject.authority == 'lcsh':
subj_node.append(text_node("SubjectSchemeIdentifier", "04"))
subj_node.append(text_node("SubjectHeadingText", subject.name))
elif subject.authority == 'lcc':
subj_node.append(text_node("SubjectSchemeIdentifier", "03"))
subj_node.append(text_node("SubjectCode", subject.name))
elif subject.authority == 'bisacsh':
subj_node.append(text_node("SubjectSchemeIdentifier", "10"))
subj_node.append(text_node("SubjectCode",
bisac.code(subject.name)))
subj_node.append(text_node("SubjectHeadingText", subject.name))
else:
subj_node.append(text_node("SubjectSchemeIdentifier", "20"))
subj_node.append(text_node("SubjectHeadingText", subject.name))
# audience range composite
if work.age_level:
range_match = re.search(r'(\d?\d?)-(\d?\d?)', work.age_level)
if range_match:
audience_range_node = etree.SubElement(descriptive_node,
"AudienceRange")
audience_range_node.append(
text_node("AudienceRangeQualifier",
"17")) #Interest age, years
if range_match.group(1):
audience_range_node.append(
text_node("AudienceRangePrecision", "03")) #from
audience_range_node.append(
text_node("AudienceRangeValue", range_match.group(1)))
if range_match.group(2):
audience_range_node.append(
text_node("AudienceRangePrecision", "04")) #from
audience_range_node.append(
text_node("AudienceRangeValue", range_match.group(2)))
# Collateral Detail Block
coll_node = etree.SubElement(product_node, "CollateralDetail")
desc_node = etree.SubElement(coll_node, "TextContent")
desc_node.append(text_node("TextType", '03')) # description
desc_node.append(text_node("ContentAudience", '00')) #unrestricted
desc = (
work.description if work.description else ''
) + '<br /><br />Listed by <a href="https://unglue.it/work/%s/">Unglue.it</a>.' % work.id
try:
content = etree.XML("<div>" + desc + "</div>")
content_node = etree.SubElement(desc_node,
"Text",
attrib={"textformat": "05"}) #xhtml
content_node.append(content)
except etree.XMLSyntaxError:
content_node = etree.SubElement(desc_node,
"Text",
attrib={"textformat": "02"}) #html
content_node.text = etree.CDATA(desc)
supp_node = etree.SubElement(coll_node, "SupportingResource")
supp_node.append(text_node("ResourceContentType", '01')) #front cover
supp_node.append(text_node("ContentAudience", '00')) #unrestricted
supp_node.append(text_node("ResourceMode", '03')) #image
cover_node = etree.SubElement(supp_node, "ResourceVersion")
cover_node.append(text_node("ResourceForm", '01')) #linkable
coverfeat_node = etree.SubElement(cover_node, "ResourceVersionFeature")
coverfeat_node.append(text_node("ResourceVersionFeatureType",
'01')) #image format
coverfeat_node.append(text_node("FeatureValue", 'D502')) #jpeg
cover_node.append(
text_node("ResourceLink", edition.cover_image_thumbnail())) #link
# Publishing Detail Block
pubdetail_node = etree.SubElement(product_node, "PublishingDetail")
if edition.publisher_name:
pub_node = etree.SubElement(pubdetail_node, "Publisher")
pub_node.append(text_node("PublishingRole", '01')) #publisher
pub_node.append(text_node("PublisherName",
edition.publisher_name.name))
pubdetail_node.append(text_node("PublishingStatus", '00')) #unspecified
#consumers really want a pub date
publication_date = edition.publication_date if edition.publication_date else edition.work.earliest_publication_date
if publication_date:
pubdate_node = etree.SubElement(pubdetail_node, "PublishingDate")
pubdate_node.append(text_node("PublishingDateRole",
'01')) #nominal pub date
pubdate_node.append(
text_node("Date", publication_date.replace('-', '')))
# Product Supply Block
supply_node = etree.SubElement(product_node, "ProductSupply")
market_node = etree.SubElement(supply_node, "Market")
terr_node = etree.SubElement(market_node, "Territory")
terr_node.append(text_node("RegionsIncluded", 'WORLD'))
supply_detail_node = etree.SubElement(supply_node, "SupplyDetail")
supplier_node = etree.SubElement(supply_detail_node, "Supplier")
supplier_node.append(text_node("SupplierRole",
'11')) #non-exclusive distributer
supplier_node.append(text_node("SupplierName",
'Unglue.it')) #non-exclusive distributer
for ebook in latest_ebooks:
website_node = etree.SubElement(supplier_node, "Website")
website_node.append(text_node("WebsiteRole", '29')) #full content
website_node.append(
text_node("WebsiteDescription",
'%s file download' % ebook.format,
attrib={'textformat': '06'})) #full content
website_node.append(text_node("WebsiteLink",
ebook.download_url)) #full content
supply_detail_node.append(text_node("ProductAvailability",
'20')) #Available
price_node = etree.SubElement(supply_detail_node, "Price")
price_node.append(text_node("PriceType", '01')) #retail excluding tax
price_node.append(text_node("PriceAmount", '0.00')) #retail excluding tax
price_node.append(text_node("CurrencyCode", 'USD')) #retail excluding tax
return product_node
url = enclosure.attrib["url"]
sig = enclosure.attrib.get(sparkle("dsaSignature"))
version = enclosure.attrib[sparkle("version")]
enclosure.set("url", f"{LUNAR_SITE}/download/{version}")
if key_path and not sig:
dmg = appcast_path.with_name(f"Lunar-{version}.dmg")
enclosure.set(sparkle("dsaSignature"), get_signature(dmg))
release_notes_file = release_notes / f"{version}.md"
if description is None and release_notes_file.exists():
changelog = html.fromstring(
markdown_path(str(release_notes_file), extras=["header-ids"]))
description = E.description(
etree.CDATA(
html.tostring(H.div(CHANGELOG_STYLE, changelog),
encoding="unicode").replace("\n", "")))
item.append(description)
for delta in item.findall(sparkle("deltas")):
for enclosure in delta.findall("enclosure"):
new_version = enclosure.attrib[sparkle("version")]
old_version = enclosure.attrib[sparkle("deltaFrom")]
sig = enclosure.attrib.get(sparkle("dsaSignature"))
enclosure.set("url",
f"{LUNAR_SITE}/delta/{new_version}/{old_version}")
if key_path and not sig:
enclosure.set(
sparkle("dsaSignature"),
get_signature(
def transform(self, pretty_print=True):
"""change the self.html and return it with CSS turned into style
attributes.
"""
if etree is None:
return self.html
parser = etree.HTMLParser()
stripped = self.html.strip()
tree = etree.fromstring(stripped, parser).getroottree()
page = tree.getroot()
# lxml inserts a doctype if none exists, so only include it in
# the root if it was in the original html.
root = tree if stripped.startswith(tree.docinfo.doctype) else page
if page is None:
print repr(self.html)
raise PremailerError("Could not parse the html")
assert page is not None
##
## style selectors
##
rules = []
index = 0
for element in CSSSelector('style,link[rel~=stylesheet]')(page):
# If we have a media attribute whose value is anything other than
# 'screen', ignore the ruleset.
media = element.attrib.get('media')
if media and media != 'screen':
continue
is_style = element.tag == 'style'
if is_style:
css_body = element.text
else:
href = element.attrib.get('href')
if not href:
continue
css_body = self._load_external(href)
these_rules, these_leftover = self._parse_style_rules(
css_body, index)
index += 1
rules.extend(these_rules)
parent_of_element = element.getparent()
if these_leftover:
if is_style:
style = element
else:
style = etree.Element('style')
style.attrib['type'] = 'text/css'
style.text = '\n'.join([
'%s {%s}' % (k, make_important(v))
for (k, v) in these_leftover
])
if self.method == 'xml':
style.text = etree.CDATA(style.text)
if not is_style:
element.addprevious(style)
parent_of_element.remove(element)
elif not self.keep_style_tags or not is_style:
parent_of_element.remove(element)
if self.external_styles:
for stylefile in self.external_styles:
css_body = self._load_external(stylefile)
these_rules, these_leftover = self._parse_style_rules(
css_body, index)
index += 1
rules.extend(these_rules)
# rules is a tuple of (specificity, selector, styles), where specificity is a tuple
# ordered such that more specific rules sort larger.
rules.sort(key=operator.itemgetter(0))
first_time = []
first_time_styles = []
for __, selector, style in rules:
new_selector = selector
class_ = ''
if ':' in selector:
new_selector, class_ = re.split(':', selector, 1)
class_ = ':%s' % class_
# Keep filter-type selectors untouched.
if class_ in FILTER_PSEUDOSELECTORS:
class_ = ''
else:
selector = new_selector
sel = CSSSelector(selector)
for item in sel(page):
old_style = item.attrib.get('style', '')
if not item in first_time:
new_style = merge_styles(old_style, style, class_)
first_time.append(item)
first_time_styles.append((item, old_style))
else:
new_style = merge_styles(old_style, style, class_)
item.attrib['style'] = new_style
self._style_to_basic_html_attributes(item,
new_style,
force=True)
# Re-apply initial inline styles.
for item, inline_style in first_time_styles:
old_style = item.attrib.get('style', '')
if not inline_style:
continue
new_style = merge_styles(old_style, inline_style, class_)
item.attrib['style'] = new_style
self._style_to_basic_html_attributes(item, new_style, force=True)
if self.remove_classes:
# now we can delete all 'class' attributes
for item in page.xpath('//@class'):
parent = item.getparent()
del parent.attrib['class']
##
## URLs
##
if self.base_url:
for attr in ('href', 'src'):
for item in page.xpath("//@%s" % attr):
parent = item.getparent()
if attr == 'href' and self.preserve_internal_links \
and parent.attrib[attr].startswith('#'):
continue
if not self.base_url.endswith('/'):
self.base_url += '/'
parent.attrib[attr] = urlparse.urljoin(
self.base_url, parent.attrib[attr].strip('/'))
out = etree.tostring(root,
method=self.method,
pretty_print=pretty_print)
if self.method == 'xml':
out = _cdata_regex.sub(
lambda m: '/*<![CDATA[*/%s/*]]>*/' % m.group(1), out)
if self.strip_important:
out = _importants.sub('', out)
return out
def handle(self, *args, **options):
category_dict = {
4: "Товары для компьютера",
3: "Игры, приставки и программы",
1: "Ноутбуки",
2: "Настольные компьютеры",
16: "Настольные компьютеры",
31: "Настольные компьютеры",
9: "Настольные компьютеры",
5: "Товары для компьютера",
20: "Товары для компьютера",
43: "Товары для компьютера",
23: "Товары для компьютера",
24: "Товары для компьютера",
38: "Товары для компьютера",
39: "Товары для компьютера",
41: "Товары для компьютера",
44: "Товары для компьютера",
6: "Товары для компьютера",
10: "Товары для компьютера",
11: "Товары для компьютера",
21: "Товары для компьютера",
12: "Ноутбуки",
15: "Планшеты и электронные книги",
14: "Телефоны",
13: "Аудио и видео",
42: "Оргтехника и расходники",
8: "Фототехника",
}
sub_category_dict = {
20: "Оперативная память",
43: "Оперативная память",
4: "Мониторы",
3: "Игровые приставки",
24: "Процессоры",
38: "Материнские платы",
39: "Блоки питания",
41: "Корпусы",
44: "Блоки питания",
6: "Видеокарты",
10: "Жёсткие диски",
11: "Жёсткие диски",
21: "Жёсткие диски",
15: "Планшеты",
42: "МФУ, копиры и сканеры",
13: "Телевизоры и проекторы",
8: "Зеркальные фотоаппараты",
}
comp_list = [
'Acer', 'Apple', 'ASUS', 'Compaq', 'Dell', 'Fujitsu', 'HP',
'Lenovo', 'MSI', 'Microsoft', 'Samsung', 'Sony', 'Toshiba',
'Packard Bell'
]
product = products.objects.filter(on_avito=True)
tel_list = [
'Acer', 'Alcatel', 'ASUS', 'BlackBerry', 'BQ', 'DEXP', 'Explay',
'Fly', 'Highscreen', 'HTC', 'Huawei', 'iPhone', 'Lenovo', 'LG',
'Meizu', 'Micromax', 'Microsoft', 'Motorola', 'MTS', 'Nokia',
'Panasonic', 'Philips', 'Prestigio', 'Samsung', 'Siemens',
'SkyLink', 'Sony', 'teXet', 'Vertu', 'Xiaomi', 'ZTE'
]
# Создание корневого элемента html
page = etree.Element('Ads', formatVersion="3", target="Avito.ru")
for i in product:
# Добавление двух дочерних элементов - <head> и <body>
headElt = etree.SubElement(page, 'Ad')
# bodyElt = etree.SubElement(page, 'body')
# Пример: добавление элемента <title>Your page title here</title>
title = etree.SubElement(headElt, 'Id')
title.text = str(i.id)
title = etree.SubElement(headElt, 'DateBegin')
# title.text = "2015-11-27"
title = etree.SubElement(headElt, 'DateEnd')
# title.text = "2015-11-27"
title = etree.SubElement(headElt, 'AdStatus')
title.text = "Free"
title = etree.SubElement(headElt, 'AllowEmail')
title.text = "Да"
title = etree.SubElement(headElt, 'ManagerName')
title.text = "Александр"
title = etree.SubElement(headElt, 'ContactPhone')
title.text = "83832770000"
title = etree.SubElement(headElt, 'Address')
title.text = "Новосибирск, проспект Карла Маркса, 14"
title = etree.SubElement(headElt, 'Category')
z = category_dict.get(i.category_type.id)
if z:
title.text = str(z)
title = etree.SubElement(headElt, 'GoodsType')
f = sub_category_dict.get(i.category_type.id)
if f:
title.text = str(f)
else:
if i.category_type.id == 12 or i.category_type.id == 1:
title.text = "Другой"
if i.category_type.id == 14:
if title.text not in tel_list:
title.text = "Другие марки"
g = i.category_type.id
if g == 1:
z = AtributeProduct.objects.filter(
prod=i, atributevalue__atribute__id=6)
if z:
for j in z:
title.text = j.atributevalue.name
if i.category_type.id == 12 or i.category_type.id == 1:
if title.text not in comp_list:
title.text = "Другой"
if i.category_type.id == 14:
if title.text not in tel_list:
title.text = "Другие марки"
else:
title.text = "Другой"
title = etree.SubElement(headElt, 'Title')
title.text = str(i.short_descriptions)
title = etree.SubElement(headElt, 'Description')
attrib = AtributeProduct.objects.filter(prod=i)
desk = i.descriptions
# pprint.pprint()
go = ""
tempz = desk.split('\n')
for k in tempz:
go += "<p>" + k + "</p>"
# for k in attrib:
# # d = i.atributevalue.atribute "," i.atributevalue.name
# # i.atributevalue.atribute
# # i.atributevalue.name
# hot = ""
# if k.atributevalue.atribute.id == 4 or k.atributevalue.atribute.id == 3:
# hot += "GB"
# if k.atributevalue.atribute.id == 2:
# hot += "Mb"
# desk += "<p>"+ str(k.atributevalue.atribute)+ " - " + str(k.atributevalue.name) +" " + hot+"</p>"
teting = "<p>30 дней на полную проверку ноутбука и год сервисного обслуживания от магазина.</p><p>Кредит, безнал.</p><p>Если вас заинтересовал этот товар, позвоните ꟷ Будем рады Вам помочь.</p><p>Мы находимся по адресу: Проспект Карла Маркса 14, магазин CompX. Работаем 7 дней в неделю с 10.00 до 20.00</p>"
new_str = "<p>------------------------------------------------------------------------------------</p><p>Гарантия: до 3 мес,1 год сервисного обслуживания.</p><p>------------------------------------------------------------------------------------</p><p>Возможно приобрести в рассрочку или кредит!</p><p>------------------------------------------------------------------------------------</p><p>Принимаем в зачет Вашу старую технику (ноутбуки, компьютеры)</p><p>------------------------------------------------------------------------------------</p><p>Возможна доставка!</p><p>------------------------------------------------------------------------------------</p><p>Мы находимся по адресу: Проспект Карла Маркса, 14.</p><p>Часы работы: без обеда и выходных с 10:00 до 20:00. Широкий ассортимент подержанных ноутбуков и компьютеров.</p>"
title.text = etree.CDATA(go + new_str)
title = etree.SubElement(headElt, 'Price')
title.text = str(i.output_price)
titleIm = etree.SubElement(headElt, 'Images')
if i.image:
title = etree.SubElement(
titleIm,
'Image',
url="http://compx.tlab-nsk.ru/static/media/" +
str(i.image))
dop_image = SecondImage.objects.filter(prod=i)
for j in dop_image:
title = etree.SubElement(
titleIm,
'Image',
url="http://compx.tlab-nsk.ru/static/media/" +
str(j.image))
# Создание и сохранение документа
doc = etree.ElementTree(page)
outFile = open(
'/home/c/ch44828/compx/public_html/app/static/media/exportavito.xml',
'wb')
#outFile = open('exportavito.xml', 'wb')
doc.write(outFile, encoding="UTF-8")
self.stdout.write("OK")
