def generate_settings_accessor(in_file: str, out_header: str, out_src: str,
mvvm_files: list):
node_tree = SettingsNode("")
# parse primary file
xml = parse(in_file)
root = xml.getroot()
includes = add_tree_normal(node_tree, root)
# additional mvvm files
for file in mvvm_files:
xml = parse(file)
add_tree_mvvm(node_tree, xml)
# print a flat key list
flatten(node_tree)
# create the actual header
with open(out_header, "w") as file:
prefix = root.attrib["prefix"] if "prefix" in root.attrib else ""
create_settings_file_header(file, root.attrib["name"], node_tree,
includes, prefix)
with open(out_src, "w") as file:
backend = root.attrib[
"backend"] if "backend" in root.attrib else "QSettingsAccessor"
create_settings_file_source(file, os.path.basename(out_header),
root.attrib["name"], node_tree, backend)
def test_diff(self):
logging.debug("Starting diff tests:\n")
subdir = 'diff_tests'
#Go through the 'expected' files as the driver.
test_files = []
for item in os.listdir(os.path.join(TEST_DIRECTORY, subdir)):
if 'expected.' in item:
test_files.append(item)
failed_tests = []
for test in test_files:
actual_file = test.replace('expected', 'actual')
actual_file = os.path.join(os.path.join(TEST_DIRECTORY, subdir),
actual_file)
expected_file = os.path.join(os.path.join(TEST_DIRECTORY, subdir),
test)
logging.debug("Testing: " + test)
actual_xml = parse(actual_file).getroot()
expected_xml = parse(expected_file).getroot()
compare_sql = False
compare_tuples = False
if 'expected.sql' in test:
compare_sql = True
if 'expected.tuples' in test:
compare_tuples = True
if 'expected.both' in test:
compare_tuples = True
compare_sql = True
test_config = TdvtTestConfig(compare_sql, compare_tuples)
results = tdvt_core.TestResult(test_config=test_config)
results.add_test_results(actual_xml, actual_file)
expected_output = tdvt_core.TestResult(test_config=test_config)
expected_output.add_test_results(expected_xml, '')
num_diffs, diff_string = tdvt_core.diff_test_results(
results, expected_output)
results.set_best_matching_expected_output(expected_output,
expected_file, 0, [0])
if results.all_passed() and 'shouldfail' not in test:
logging.debug("Test passed: " + test)
self.assertTrue(True)
elif not results.all_passed() and 'shouldfail' in test:
logging.debug("Test passed: " + test)
self.assertTrue(True)
else:
logging.debug("Test failed: " + test)
failed_tests.append(test)
self.assertTrue(False)
logging.debug("\n")
if failed_tests:
logging.debug("All failed tests:\n")
for item in failed_tests:
logging.debug("Failed -- " + item)
logging.debug("Ending diff tests:\n")
return len(test_files), len(failed_tests)
def main(self, url, login, password, path):
try:
self.client = wordpress_xmlrpc.Client(url, login, password)
except wordpress_xmlrpc.exceptions.ServerConnectionError as exc:
if '301 Moved Permanently' in exc.args[0]:
print("301 redirect, trying HTTPS protocol.")
ssl_url = url.replace('http://', 'https://')
self.client = wordpress_xmlrpc.Client(ssl_url, login, password)
# This is just to make sure that the credentials are OK before we jump
# to XML parsing.
self.client.call(wordpress_xmlrpc.methods.users.GetUsers())
# Parse the XML. Give 2 seconds for parsing to prevent abuse.
signal.alarm(PARSE_TIMEOUT)
if path.endswith('.gz'):
target = gzip.open(path)
else:
target = path
self.tree = parse(target)
signal.alarm(0)
entries = self.tree.findall('.//entry')
for n, entry in enumerate(entries, 1):
print("%d/%d" % (n, len(entries)))
self.handle_post(entry)
print("Done. You can now change wordpress password.")
def _format(self, data: AnyStr) -> Union[List, Dict]:
"""Transform raw data from server into python native type."""
data = parse(StringIO(data))
xml_data = []
child_tag = list(data.getroot().iter())[1]
if child_tag.tag == ResponseDataConst.STATUS:
child_tag_element = list(child_tag.iter())[1]
if child_tag_element.tag == ResponseDataConst.ERROR:
# error message does not follow
# same structure as data message.
# trying to follow the same error message
# structure from json error response.
attribs = child_tag_element.attrib
xml_data = {
MiscConst.ERRORS: [{
ResponseConst.CODE:
attribs[ResponseDataConst.CODE],
ResponseConst.DETAIL:
attribs[ResponseDataConst.DESCRIPTION]
}]
}
else:
xml_data = [child.attrib for child in list(data.iter())[1:]]
return xml_data
def __init__(self, input_filename, template_model=None, savings_dir=None, verbosity=0, tmax=None):
self.input_filename = input_filename
self.hdul = fits.open(input_filename)
self.verbosity = verbosity
if self.verbosity > 0:
self.hdul.info()
if tmax is not None and tmax < 1:
raise Exception('Need a non-zero positive tmax')
self.tmax = tmax
self.model_filename = template_model
self.model_tree = None
if self.model_filename:
self.model_tree = parse(self.model_filename)
self.savings_dir = '.'
if savings_dir:
if os.path.isabs(savings_dir):
self.savings_dir = savings_dir
else:
self.savings_dir = os.path.join(self.savings_dir, savings_dir)
try:
os.makedirs(self.savings_dir)
if self.verbosity > 0:
print("Created the data dir: {}".format(self.savings_dir), file=sys.stderr)
except FileExistsError as e:
if self.verbosity > 1:
print("The data dir already exists", file=sys.stderr)
def _getroot(self) -> str:
r"""获取Xml节点
:Usage:
_getroot()
"""
return parse(self.filename).getroot()
def _load(self):
try:
data = BytesIO(self._pdf.Root.Metadata.get_stream_buffer())
except AttributeError:
self._create_xmp()
else:
self._xmp = parse(data)
def _get_requirements_pom_xml(path: str) -> list:
"""
Get list of requirements from Maven project.
Files supported are pom.xml
:param path: Project path
"""
reqs = []
namespaces = {'xmlns': 'http://maven.apache.org/POM/4.0.0'}
for full_path in full_paths_in_dir(path):
if not full_path.endswith('pom.xml'):
continue
tree = parse(full_path)
root = tree.getroot()
deps = root.findall(".//xmlns:dependency", namespaces=namespaces)
for dep in deps:
artifact_id = dep.find("xmlns:artifactId", namespaces=namespaces)
version = dep.find("xmlns:version", namespaces=namespaces)
if version is not None:
if version.text.startswith('$'):
reqs.append((full_path, artifact_id.text, None))
else:
reqs.append((full_path, artifact_id.text, version.text))
else:
reqs.append((full_path, artifact_id.text, None))
return reqs
def loadFromFile(self,filename):
xmlf=parse(filename)
reports=xmlf.getroot()
if reports.get('lib').upper()=='QTRPT':
self.Pages=[ReportPage(pg) for pg in reports]
else:
self.Pages=[]
def add_signature(fp):
if not HMAC_KEY:
return fp
new_archive_fp = BytesIO()
with ZipFile(new_archive_fp, "w", compression=ZIP_DEFLATED) as new_archive:
with ZipFile(fp, "r") as archive:
for file_name in archive.namelist():
if file_name != "docProps/custom.xml":
with archive.open(file_name, "r") as f:
new_archive.writestr(file_name, f.read())
signature = compute_hmac(archive, HMAC_KEY)
with archive.open("docProps/custom.xml", "r") as f:
xml = parse(f)
hmac_prop_value = extract_signature_node_from_xml(xml)
if hmac_prop_value is not None:
hmac_prop_value.text = signature
custom_props_fp = BytesIO()
xml.write(custom_props_fp, xml_declaration=True, encoding="UTF-8")
custom_props_fp.seek(0)
new_archive.writestr("docProps/custom.xml", custom_props_fp.read())
return new_archive_fp
def warn_file_specific_rules_tdr(path_to_file: Path):
xml_tree = parse(str(path_to_file))
root = xml_tree.getroot()
attribute_list = root.find(
'.//connection-normalizer/required-attributes/attribute-list')
if not attribute_list:
return
authentication_attr_exists = False
server_attr_exists = False
for attr in attribute_list.iter('attr'):
if attr.text == 'authentication':
authentication_attr_exists = True
if attr.text == 'server':
server_attr_exists = True
if not authentication_attr_exists:
logger.warning(
"Warning: 'authentication' attribute is missing from "
"<connection-normalizer>/<required-attributes>/<attribute-list> in "
+ str(path_to_file) + ".")
if not server_attr_exists:
logger.warning(
"Warning: 'server' attribute is missing from "
"<connection-normalizer>/<required-attributes>/<attribute-list> in "
+ str(path_to_file) + ".")
def access_request_handler():
access_details = readAccessReq() # Reading the Access Request
try:
tree = parse('scanner_details.xml') # Reading the Scanner Details
root = tree.getroot()
# Access request handler for scanners
# Read Nessus scanner details
scanner = root.find('nessus')
execute_nessus = scanner.get('enabled')
if execute_nessus == '1':
# print(scanner)
if scanner.find('host').text is None or scanner.find('username').text is None or scanner.find('host').text is None:
xml_error("Nessus data missing in scanner_details.xml")
print("Nessus" + " host@:" + scanner.find('host').text)
# print(scanner.find('username').text)
usr_passwd = input("Please enter your password for " + " Nessus" + ": ")
nessus_details = {'uname': scanner.find('username').text, 'passwd': usr_passwd, 'host': scanner.find('host').text}
# Scanner task calls from here
Utilities.printLog("Executing Nessus tasks")
nessusObj = nes.Nessus(nessus_details) # Create Nessus scanner class obj
msg = nessusObj.handleAccessReq(access_details, nessus_details) # Login | Add User | Logout
# Read Nexpose scanner details
scanner = root.find('nexpose')
execute_nexpose = scanner.get('enabled')
if execute_nexpose == '1':
# print(scanner)
if scanner.find('host').text is None or scanner.find('username').text is None or scanner.find('host').text is None:
xml_error("Nexpose data missing in scanner_details.xml")
print("Nexpose" + " host@:" + scanner.find('host').text)
# print(scanner.find('username').text)
usr_passwd = input("Please enter your password for " + " Nexpose" + ": ")
nexpose_details = {'uname': scanner.find('username').text, 'passwd': usr_passwd, 'host': scanner.find('host').text}
# Scanner task calls from here
Utilities.printLog("Executing Nexpose tasks")
nexposeObj = nex.Nexpose(nexpose_details) # Create Nexpose scanner class obj
msg += "\n"+nexposeObj.handleAccessReq(access_details, nexpose_details) # Login | SaveSite | Add User | Logout
# Read Qualys scanner details
scanner = root.find('qualys')
execute_qualys = scanner.get('enabled')
if execute_qualys == '1':
# print(scanner)
if scanner.find('host').text is None or scanner.find('username').text is None or scanner.find('host').text is None:
xml_error("Qualys data missing in scanner_details.xml")
print("Qualys" + " host@:" + scanner.find('host').text)
# print(scanner.find('username').text)
usr_passwd = input("Please enter your password for " + " Qualys" + ": ")
qualys_details = {'uname': scanner.find('username').text, 'passwd': usr_passwd, 'host': scanner.find('host').text}
# Scanner task calls from here
Utilities.printLog("Executing Qualys tasks")
qualysObj = qua.Qualys(qualys_details) # Create Qualys scanner class obj
qualysObj.handleAccessReq(access_details, qualys_details) # Login | Add Asset | Add Asset Grp | Add User
msg +="\nQualys\nDetails send to email."
Utilities.write_to_file(msg)
except Exception as e:
Utilities.printException("In fun access_request_handler():"+ str(e))
def readAccessReq():
# read access request from XML
try:
ip = ""
usrlst = []
tree = parse('access_request.xml')
root = tree.getroot()
for child in root.findall('user'):
uname = child.find('uname').text
name = child.find('name').text
email = child.find('email').text
if uname is None or name is None or email is None:
xml_error("Data missing in access_request.xml")
usrlst.append(uname + ',' + name + ',' + email)
asst_det = root.find('asset_details')
site_det = root.find('site')
site_name = site_det.get('name')
site_desc = site_det.get('desc')
for ipchild in asst_det.findall('ip'):
if ipchild.text is None:
xml_error("IP missing in access_request.xml")
ip = ip + "," + ipchild.text
# print(ip)
ip = ip.strip(',')
access_req = {'userList': usrlst, 'ip': ip, 'site_name': site_name, 'site_desc': site_desc}
# print(access_req)
return access_req
except Exception as e:
Utilities.printException("Error with access_request.xml."+ str(e))
def get_proposal_code(proposal_zip: Union[str, BinaryIO]) -> Optional[str]:
"""Extract the proposal code from a proposal file."""
if not zipfile.is_zipfile(proposal_zip):
raise ValueError("The file supplied is not a zip file")
try:
archive = ZipFile(proposal_zip, "r")
file = archive.open("Proposal.xml")
except KeyError:
raise KeyError("The zip file contains no file Proposal.xml.") from None
tree = parse(file)
proposal = tree.getroot()
if "code" not in proposal.attrib:
raise ValueError("No proposal code supplied in the file Proposal.xml.")
_, _, tag = proposal.tag.rpartition("}") # ignore namespace
if tag != "Proposal":
raise ValueError(
"The root element in the file Proposal.xml is not called Proposal"
)
proposal_code = proposal.attrib["code"]
if proposal_code.startswith("2"):
return proposal_code
elif proposal_code.startswith("Unsubmitted-") or proposal_code == "":
return None
else:
raise ValueError(f"Invalid proposal code: {proposal_code}.")
def validate_file_specific_rules(file_to_test: ConnectorFile,
path_to_file: Path,
xml_violations_buffer: List[str]) -> bool:
if file_to_test.file_type == 'connection-fields':
xml_tree = parse(str(path_to_file))
root = xml_tree.getroot()
for child in root.iter('field'):
if 'name' in child.attrib:
field_name = child.attrib['name']
if not (field_name in PLATFORM_FIELD_NAMES
or field_name.startswith(VENDOR_FIELD_NAME_PREFIX)):
xml_violations_buffer.append(
"Element 'field', attribute 'name'='" + field_name +
"' not an allowed value. See 'Connection Field Platform Integration' section of documentation for allowed values."
)
return False
if 'category' in child.attrib:
category = child.attrib['category']
optional = child.attrib.get('optional', 'true') == 'true'
default_present = child.attrib.get('default-value', '') != ''
if category == 'advanced' and not optional and not default_present:
xml_violations_buffer.append(
"Element 'field', attribute 'name'='" + field_name +
"': Required fields in the Advanced category must be assigned a default value."
)
return False
return True
def readAccessReq():
# read access request from XML
try:
ip = ""
usrlst = []
tree = parse('access_request.xml')
root = tree.getroot()
for child in root.findall('user'):
uname = child.find('uname').text
name = child.find('name').text
email = child.find('email').text
if uname is None or name is None or email is None:
xml_error("Data missing in access_request.xml")
usrlst.append(uname + ',' + name + ',' + email)
asst_det = root.find('asset_details')
site_det = root.find('site')
site_name = site_det.get('name')
site_desc = site_det.get('desc')
for ipchild in asst_det.findall('ip'):
if ipchild.text is None:
xml_error("IP missing in access_request.xml")
ip = ip + "," + ipchild.text
# print(ip)
ip = ip.strip(',')
access_req = {
'userList': usrlst,
'ip': ip,
'site_name': site_name,
'site_desc': site_desc
}
# print(access_req)
return access_req
except Exception as e:
Utilities.printException("Error with access_request.xml." + str(e))
def main(self, url, login, password, path):
if not url.endswith('xmlrpc.php'):
print("[EN] WARNING: URL doesn't end with xmlrpc.php. Perhaps you "
"forgot to put it here?\n"
"[PL] OSTREZENIE: URL nie konczy sie na xmlrpc.php. To dosc "
"nietypowe - moze to pomylka?\n")
# This is just to make sure that the credentials are OK before we jump
# to XML parsing.
self.client = self._login(url, login, password)
# Parse the XML. Give 2 seconds for parsing to prevent abuse.
signal.alarm(PARSE_TIMEOUT)
if path.endswith('.gz'):
target = gzip.open(path)
else:
target = path
self.tree = parse(target)
signal.alarm(0)
entries = self.tree.findall('.//entry')
for n, entry in enumerate(entries, 1):
print("%d/%d" % (n, len(entries)))
self.handle_post(entry)
print("Done. You can now change wordpress password.")
def validate_file_specific_rules(file_to_test: ConnectorFile,
path_to_file: Path,
xml_violations_buffer: List[str]) -> bool:
if file_to_test.file_type == 'connection-fields':
field_names = set()
xml_tree = parse(str(path_to_file))
root = xml_tree.getroot()
for child in root.iter('field'):
if 'name' in child.attrib:
field_name = child.attrib['name']
if not (field_name in PLATFORM_FIELD_NAMES
or field_name.startswith(VENDOR_FIELD_NAME_PREFIX)):
xml_violations_buffer.append(
"Element 'field', attribute 'name'='" + field_name +
"' not an allowed value. See 'Connection Field Platform Integration' section of documentation for allowed values."
)
return False
if field_name in field_names:
xml_violations_buffer.append(
"A field with the field name = " + field_name +
" already exists. Cannot have multiple fields with the same name."
)
return False
if field_name == 'instanceurl':
used_for_oauth = False
for conditions in child.iter("conditions"):
for condition in conditions.iter("condition"):
if 'field' in condition.attrib:
if condition.attrib[
'field'] == 'authentication':
if 'value' in condition.attrib:
if condition.attrib[
'value'] == 'oauth':
used_for_oauth = True
if not used_for_oauth:
xml_violations_buffer.append(
"Element 'field', attribute 'name'='instanceurl' can only be used conditional on field "
+
"'authentication' with 'value'='oauth'. See 'Connection Field Platform Integration' section "
+ "of documentation for more information.")
return False
field_names.add(field_name)
if 'category' in child.attrib:
category = child.attrib['category']
optional = child.attrib.get('optional', 'true') == 'true'
default_present = child.attrib.get('default-value', '') != ''
if category == 'advanced' and not optional and not default_present:
xml_violations_buffer.append(
"Element 'field', attribute 'name'='" + field_name +
"': Required fields in the Advanced category must be assigned a default value."
)
return False
return True
def _read_gpx(self):
"""Parses an xml file containing GPS data
Data points are assumed to be in a Geographic coordinate system
Returns a pandas DataFrame containing the parsed GPS data if parse was
successful and None otherwise.
"""
# Parsing XML file
elements = parse(self.source)
points = []
for element in elements.iter():
if element.tag.find("trkpt") == -1:
continue
latitude = element.attrib.get("lat", None)
longitude = element.attrib.get("lon", None)
elevation = element[0].text
if (not latitude or
not longitude or
not elevation):
raise ValueError("Unexpected tag for lat/lon/ele.")
try:
latitude = pd.to_numeric(latitude)
longitude = pd.to_numeric(longitude)
elevation = pd.to_numeric(elevation)
utm = UtmCoordinate.create_from_geographic(
latitude,
longitude,
elevation)
entry = (latitude,
longitude,
utm.elevation,
utm.northing,
utm.easting)
points.append(entry)
except Exception as exception:
raise exception
if len(points) == 0:
raise(ValueError("Unable to find valid points within the provided GPX file."))
# Generate DataFrame
columns = ['latitude',
'longitude',
'elevation',
'northing',
'easting']
data = pd.DataFrame.from_records(points,
columns=columns)
# Generate x, y, z
data['x'] = data['easting'] - data['easting'].min()
data['y'] = data['northing'] - data['northing'].min()
data['z'] = data['elevation'] - data['elevation'].min()
selection = ['x', 'y', 'z', 'elevation']
return data[selection]
def _from_xml_oem(cls, file_path):
parts = parse(str(file_path)).getroot()
header = components.HeaderSection._from_xml(parts)
segments = [
components.EphemerisSegment._from_xml(part, header.version)
for part in parts[1]
]
return cls(header, segments)
def __init__(self, saveFile, read_data=False):
self.saveFile = saveFile
if read_data == False:
root = parse(saveFile).getroot()
else:
root = ElementTree.fromstring(saveFile)
self.root = root
def __init__(self, saveFile, read_data=False):
self.saveFile = saveFile
if read_data == False:
root = parse(saveFile).getroot()
else:
root = ElementTree.fromstring(saveFile)
self.root = root
def parse_entities(file_path="CorePlusUAWithModern.xml"):
tree = parse(file_path)
root = tree.getroot()
for child in root.findall('item'):
try:
parse_and_store_item(child)
except ItemAlreadyExists:
print("Small dirty place for testing")
def scan_root(file):
"""
Function returns the root element for tree of given nessus file with scan results.
:param file: given nessus file
:return: root element for this tree.
"""
scan_file_parsed = parse(file)
root = scan_file_parsed.getroot()
return root
def get_xml():
"""Load and parse safely xml."""
try:
return parse("theoryexamhe_data.xml")
except Exception as e:
print(e)
return None
def get_report_data(xmlfile, file_path_list=[]):
"""Convert xml file to dict
:param xmlfile: xml file to parse
:param file_path_list: Full file path for any manipulation
"""
issues = []
metrics = {}
file_ref = {}
if not file_path_list:
file_path_list = []
et = parse(xmlfile)
root = et.getroot()
for child in root:
issue = {}
if child.tag.lower() == "BugInstance".lower():
issue = child.attrib
if "priority" in child.attrib:
priority = child.attrib["priority"]
if priority in PRIORITY_MAP:
issue["issue_severity"] = PRIORITY_MAP.get(
priority, priority)
if "cweid" in child.attrib and child.attrib["cweid"]:
issue["test_id"] = "CWE-" + child.attrib["cweid"]
elif "type" in child.attrib and child.attrib["type"]:
issue["test_id"] = child.attrib["type"]
for ele in child.iter():
if ele.tag.lower() == "ShortMessage".lower():
issue["title"] = ele.text
if ele.tag.lower() == "LongMessage".lower():
issue["description"] = ele.text
if ele.tag.lower() == "Message".lower():
issue["description"] = issue[
"description"] + " \n" + ele.text
if ele.tag.lower() == "SourceLine".lower() and (
ele.attrib.get("synthetic") == "true"
or ele.attrib.get("primary") == "true"):
issue["line"] = ele.attrib["start"]
fname = ele.attrib["sourcepath"]
if fname in file_ref:
fname = file_ref[fname]
else:
# FIXME: This logic is too slow.
# Tools like find-sec-bugs are not reliably reporting the full path
# so such a lookup is required
for tf in file_path_list:
if tf.endswith(fname):
file_ref[fname] = tf
fname = tf
break
issue["filename"] = fname
issues.append(issue)
if child.tag.lower() == "FindBugsSummary".lower():
metrics = {"summary": child.attrib}
return issues, metrics
def parse_heavym(self):
tree = parse(self.heavym_path)
root = tree.getroot()
groups_root = root.find('sequence').find('groups')
groups = root.findall("./sequence/groups/group")
self.next_group_id = max(list(map(lambda x: int(x.get('id')),
groups))) + 1
faces = root.findall(".//groups/group/faces/face")
self.next_face_id = max(list(map(lambda x: int(x.get('id')),
faces))) + 1
def warn_file_specific_rules_dialect(path_to_file: Path):
xml_tree = parse(str(path_to_file))
root = xml_tree.getroot()
if 'base' in root.attrib and root.attrib['base'] == 'DefaultSQLDialect':
logger.warning(
'Warning: DefaultSQLDialect is not a recommended base to inherit from, '
'please see the documentation for current best practices: '
'https://tableau.github.io/connector-plugin-sdk/docs/design#choose-a-dialect'
)
def create_qm_files(self):
with tempfile.TemporaryDirectory() as tmpdir:
self.tmpdir = tmpdir
self.fname = os.path.splitext(os.path.basename(self.in_file))[0]
xml_in = parse(self.in_file)
root = xml_in.getroot()
depends = root.find("dependencies")
for locale in self.collect_locales(depends.get("root")):
self.create_qm_metafile(copy.deepcopy(xml_in), locale)
def parseXML(xml):
tree = parse(xml)
root = tree.getroot()
urls = []
for item in root.findall('./channel/item/guid'):
urls.append(item.text)
return urls
def main():
for f, file in enumerate(files[0:1]):
ns = "{http://www.w3.org/2001/XMLSchema-instance}"
locations = parse(file).getroot().find('locations').findall('GameLocation')
assert locations[1].attrib[ns+'type'] == 'Farm'
farm = locations[1]
print(str(f)+' of '+str(len(files)-1))
structure.update(moveRecursivelyOverXml(farm))
# print structure
return structure
def run_diff(test_config, diff):
root_directory = get_root_dir()
allowed_test_path = os.path.join(root_directory, diff)
test_path_base = os.path.split(allowed_test_path)[0]
test_name = os.path.split(allowed_test_path)[1]
actual, actual_diff, setup, expected_files, next_path = get_test_file_paths(
test_path_base, test_name, test_config.output_dir)
logging.debug('actual_path: ' + actual)
diff_count_map = {}
for f in expected_files:
logging.debug('expected_path: ' + f)
if os.path.isfile(f) and os.path.isfile(actual):
logging.debug("Diffing " + actual + " and " + f)
actual_xml = None
expected_xml = None
try:
actual_xml = parse(actual).getroot()
except ParseError as e:
logging.debug("Exception parsing actual file: " + actual +
" exception: " + str(e))
continue
try:
expected_xml = parse(f).getroot()
except ParseError as e:
logging.debug("Exception parsing expected file: " + f +
" exception: " + str(e))
continue
result = TestResult(test_config=test_config)
result.add_test_results(actual_xml, actual)
expected_output = TestResult(test_config=test_config)
expected_output.add_test_results(expected_xml, '')
num_diffs, diff_string = result.diff_test_results(expected_output)
logging.debug(diff_string)
diff_count_map[f] = sum(num_diffs)
for t in diff_count_map:
logging.debug(t + ' Number of differences: ' + str(diff_count_map[t]))
return 0
def q1(path: str = "news.rss") -> None:
tree = parse(path)
with open("description.txt", "w") as df, open("output.txt", "w") as of:
for elem in tree.iter(tag="description"):
elem_text = re.sub(r"<.+?>", "", elem.text)
news_story = []
text = elem_text.replace(" ", "")
if text:
df.write(text + "\n")
news_story += jieba.lcut(text)
of.write(" ".join(news_story) + "\n")
def main():
for f, file in enumerate(files[0:1]):
ns = "{http://www.w3.org/2001/XMLSchema-instance}"
locations = parse(file).getroot().find("locations").findall(
"GameLocation")
assert locations[1].attrib[ns + "type"] == "Farm"
farm = locations[1]
print(str(f) + " of " + str(len(files) - 1))
structure.update(moveRecursivelyOverXml(farm))
# print structure
return structure
def validate_file_specific_rules_tdr(file_to_test: ConnectorFile,
path_to_file: Path,
xml_violations_buffer: List[str],
properties: ConnectorProperties) -> bool:
xml_tree = parse(str(path_to_file))
root = xml_tree.getroot()
attribute_list = root.find(
'.//connection-normalizer/required-attributes/attribute-list')
# The connection resolver appears after the dialog elements in the manifest's xml, so we know
# USES_TCD is accurate here
if not attribute_list and properties.uses_tcd:
xml_violations_buffer.append(
"Connectors using a .tcd file cannot use inferred connection resolver,"
"must manually populate required-attributes/attributes-list in " +
str(path_to_file) + ".")
return False
# Check that all the connection-fields attributes are in the required attributes
if properties.connection_fields and attribute_list:
attributes = []
for attr in attribute_list.iter():
attributes.append(attr.text)
if len(attributes) > 0 and properties.connection_fields:
for field in properties.connection_fields:
if field == 'instanceurl':
continue
if field not in attributes:
xml_violations_buffer.append(
"Attribute '" + field +
"' in connection-fields but not in required-attributes list."
)
return False
if len(
attributes
) > 0 and properties.connection_metadata_database and not properties.uses_tcd:
if 'dbname' not in attributes:
logger.warning(
"Warning: connection-metadata 'database' enabled but 'dbname' is not in required-attributes list. Consider adding it if the value is used in connection-builder or connection-properties scripts"
)
properties_builder = root.find('.//connection-properties')
if not properties_builder and properties.is_jdbc:
xml_violations_buffer.append(
"Connectors using a 'jdbc' superclass must declare a <connection-properties> element in "
+ str(path_to_file) + ".")
return False
return True
def create_dataset_from_zip(context, data_dict):
upload = data_dict.get('upload')
private = data_dict.get('private', True)
map_package = upload.file
tempdir = tempfile.mkdtemp('-mapactionzip')
metadata_paths = []
file_paths = []
with zipfile.ZipFile(map_package, 'r') as z:
z.extractall(tempdir)
for f in z.namelist():
full_path = os.path.join(tempdir, f)
if f.endswith('.xml'):
metadata_paths.append(full_path)
else:
file_paths.append(full_path)
assert len(metadata_paths) == 1
metadata_file = metadata_paths[0]
et = parse(metadata_file)
dataset_dict = {}
owner_org = data_dict.get('owner_org')
if owner_org:
dataset_dict['owner_org'] = owner_org
else:
private = False
dataset_dict['title'] = join_lines(et.find('.//mapdata/title').text)
map_id = et.find('.//mapdata/ref').text
operation_id = et.find('.//mapdata/operationID').text
dataset_dict['name'] = slugify('%s %s' % (operation_id, map_id))
dataset_dict['notes'] = join_lines(et.find('.//mapdata/summary').text)
dataset_dict['private'] = private
dataset_dict['extras'] = [
{'key': k, 'value': v} for (k, v) in
metadataimporter.map_metadata_to_ckan_extras(et).items()
]
dataset = toolkit.get_action('package_create')(context, dataset_dict)
for resource_file in file_paths:
resource = {
'package_id': dataset['id'],
'path': resource_file,
}
_create_and_upload_local_resource(context, resource)
return dataset
def __init__(self):
self._configuration_file_ = "robobackup-configuration.xml"
# parse robobackup-configuration.xml
# check root element
try:
path = os.path.join(os.path.dirname(\
os.path.relpath(__file__)), \
self._configuration_file_)
etree = parse(path)
except: # pylint: disable=bare-except
logbook.critical(_("Error parsing ") + \
self._configuration_file_)
self._root_ = etree.getroot()
if self._root_.tag != "backup":
logbook.critical(self._configuration_file_ + \
_(" does not have <backup> as root-element"))
def getFarmInfo(saveFileLocation, read_data=False):
sprite = namedtuple('Sprite', ['name', 'x', 'y', 'w', 'h', 'index', 'type', 'growth', 'flipped', 'orientation'])
ns = "{http://www.w3.org/2001/XMLSchema-instance}"
farm = {}
if read_data is False:
root = parse(saveFileLocation).getroot()
else:
root = ElementTree.fromstring(saveFileLocation)
# Farm Objects
locations = root.find('locations').findall("GameLocation")
s = []
for item in locations[1].find('objects').iter("item"):
f = False
obj = item.find('value').find('Object')
name = obj.find('name').text
x = int(item.find('key').find('Vector2').find('X').text)
y = int(item.find('key').find('Vector2').find('Y').text)
i = int(obj.find('parentSheetIndex').text)
t = obj.find('type').text
a = False
if obj.find('flipped').text == 'true':
f = True
if 'Fence' in name or name == 'Gate':
t = int(obj.find('whichType').text)
a = False
if name == 'Gate':
a = True
name = 'Fence'
else:
name = 'Object'
s.append(sprite(name, x, y, 0, 0, i, t, a, f, obj.find('name').text))
d = {k[0]: [a for a in s if a[0] == k[0]] for k in s}
try:
farm['Fences'] = checkSurrounding(d['Fence'])
except Exception as e:
pass
farm['objects'] = [a for a in s if a.name != 'Fence']
# Terrain Features
tf = []
crops = []
for item in locations[1].find('terrainFeatures').iter('item'):
s = None
loc = None
f = False
name = item.find('value').find('TerrainFeature').get(ns+'type')
if name == 'Tree':
t = int(item.find('value').find('TerrainFeature').find('treeType').text)
s = int(item.find('value').find('TerrainFeature').find('growthStage').text)
if item.find('value').find('TerrainFeature').find('flipped').text == 'true': f= True
if name == 'Flooring':
t = int(item.find('value').find('TerrainFeature').find('whichFloor').text)
s = int(item.find('value').find('TerrainFeature').find('whichView').text)
if name == "HoeDirt":
crop = item.find('value').find('TerrainFeature').find('crop')
if crop is not None:
crop_x = int(item.find('key').find('Vector2').find('X').text)
crop_y = int(item.find('key').find('Vector2').find('Y').text)
crop_phase = int(crop.find('currentPhase').text)
crop_location = int(crop.find('rowInSpriteSheet').text)
if crop_location in [26, 27, 28, 29, 31]:
r = int(crop.find('tintColor').find('R').text)
g = int(crop.find('tintColor').find('G').text)
b = int(crop.find('tintColor').find('B').text)
days = int(crop.find('dayOfCurrentPhase').text)
o = ((r, g, b), days)
else:
o = None
crop_flip = False
if crop.find('flip').text == 'true':
crop_flip = True
crop_dead = False
if crop.find('dead').text == 'true':
crop_dead = True
crops.append(sprite('HoeDirtCrop', crop_x, crop_y, 1, 1, crop_dead, crop_location, crop_phase, crop_flip, o))
if name == "FruitTree":
t = int(item.find('value').find('TerrainFeature').find('treeType').text)
s = int(item.find('value').find('TerrainFeature').find('growthStage').text)
if item.find('value').find('TerrainFeature').find('flipped').text == 'true': f= True
if name == "Grass":
t = int(item.find('value').find('TerrainFeature').find('grassType').text)
s = int(item.find('value').find('TerrainFeature').find('numberOfWeeds').text)
loc = int(item.find('value').find('TerrainFeature').find('grassSourceOffset').text)
x = int(item.find('key').find('Vector2').find('X').text)
y = int(item.find('key').find('Vector2').find('Y').text)
tf.append(sprite(name, x, y, 1, 1, loc, t, s, f, None))
d = {k[0]: [a for a in tf if a[0] == k[0]] for k in tf}
excludes = ['Flooring', 'HoeDirt', 'Crop']
farm['terrainFeatures'] = [a for a in tf if a.name not in excludes]
farm['Crops'] = crops
try:
farm['Flooring'] = checkSurrounding(d['Flooring'])
except Exception as e:
pass
try:
farm['HoeDirt'] = (checkSurrounding(d['HoeDirt']))
except:
pass
# Resource Clumps
s = []
for item in locations[1].find('resourceClumps').iter('ResourceClump'):
name = item.get(ns+'type')
if name is None:
name = 'ResourceClump'
t = int(item.find('parentSheetIndex').text)
x = int(item.find('tile').find('X').text)
y = int(item.find('tile').find('Y').text)
w = int(item.find('width').text)
h = int(item.find('height').text)
s.append(sprite(name, x, y, w, h, None, t, None, None, None))
farm['resourceClumps'] = s
s = []
for item in locations[1].find('buildings').iter('Building'):
name = 'Building'
x = int(item.find('tileX').text)
y = int(item.find('tileY').text)
w = int(item.find('tilesWide').text)
h = int(item.find('tilesHigh').text)
t = item.find('buildingType').text
s.append(sprite(name, x, y, w, h, None, t, None, None, None))
farm['buildings'] = s
house = sprite('House',
58, 14, 10, 6,
int(root.find('player').find('houseUpgradeLevel').text),
None,
None,
None,
None)
hasGreenhouse = False
for location in locations:
if location.find('name').text == "CommunityCenter":
cats = location.find('areasComplete').findall('boolean')
if cats[0].text == 'true':
hasGreenhouse = True
# Check for letter to confirm player has unlocked greenhouse, thanks /u/BumbleBHE
for letter in root.find('player').find('mailReceived').iter('string'):
if letter.text == "ccPantry":
hasGreenhouse = True
if hasGreenhouse:
greenHouse = sprite('Greenhouse',
25, 12, 0, 6, 1,
None, None, None, None)
else:
greenHouse = sprite('Greenhouse',
25, 12, 0, 6, 0,
None, None, None, None)
farm['misc'] = [house, greenHouse]
return farm
import sys
from defusedxml.ElementTree import fromstring, parse
if len(sys.argv) > 1:
fn = sys.argv[1]
else:
fn = input('Filename: ')
try:
root = parse(fn).getroot()
print(root)
except FileNotFoundError:
try:
root = parse(fn+'.xml').getroot()
print(root)
except FileNotFoundError:
print('File '+fn+'not found!')
def lineReceived(self, line):
"As soon as any data is received, write it back."
data_as_xml = parse(line)
print("DEBUG: line is {0}".format(line))
self.sendLine(line)
def parse_nmap_xml(nmap_xml):
"""Build the global list of variables to be called throughout this script"""
global v4_addr, mac_addr, v6_addr, os_cpe, ports_info, mac_vendor, host_name, cpe, ex_info, \
svc_nse_script_id, svc_nse_script_output, product_type, product_vendor, product_name, product_version,\
product_update, product_edition, product_language, os_product, svc_cpe_product_type, svc_cpe_product_name,\
svc_cpe_product_version, svc_cpe_product_update, svc_cpe_product_edition, svc_cpe_product_language,\
protocol, portid, service_name, ex_info, service_product, host_nse_script_id, host_nse_script_output,\
inventory_svcs, add_inventory_svcs, inventory_svcs_id_list, svc_product, add_svc_product
"""Connect to the database"""
Session = modules.db_connect.connect()
session = Session()
"""Parse the nmap xml files from this directory, and build the tree"""
tree = parse(nmap_xml)
root = tree.getroot()
if root.tag == 'nmaprun':
"""Find all the hosts in the nmap scan"""
for host in root.findall('host'):
"""Set variables"""
v4_addr = None
v6_addr = None
mac_addr = None
mac_vendor = None
ostype = None
os_cpe = None
ports_info = []
host_name = None
cpe = 'None'
product_type = None
product_vendor = None
product_name = None
product_version = None
product_update = None
product_edition = None
product_language = None
os_product = 'None'
svc_cpe_product_type = 'None'
svc_cpe_product_vendor = None
svc_cpe_product_name = None
svc_cpe_product_version = None
svc_cpe_product_update = None
svc_cpe_product_edition = None
svc_cpe_product_language = None
protocol = None
portid = None
service_name = None
ex_info = None
service_product = None
host_nse_script_id = None
host_nse_script_output = None
svc_nse_script_id = None
svc_nse_script_output = None
svc_nse_script_id_a = []
svc_nse_script_output_a = []
host_nse_script_id_a = []
host_nse_script_output_a = []
"""Get the hosts state, and find all addresses"""
state = host[0].get('state')
if state == 'up':
addresses = host.findall('address')
for address in addresses:
if address.attrib.get('addrtype') == 'ipv4':
v4_addr = address.attrib.get('addr')
if address.attrib.get('addrtype') == 'mac':
mac_addr = address.attrib.get('addr')
mac_vendor = address.attrib.get('vendor')
if address.attrib.get('addrtype') == 'ipv6':
v6_addr = address.attrib.get('addr')
"""Get the hostname"""
host_info = host.find('hostnames')
hostname = host_info.find('hostname')
try:
host_name = hostname.get('name')
except AttributeError:
"""Nothing found"""
"""Get OS Info"""
os_elm = host.find('os')
try:
osmatch_elm = os_elm.find('osmatch')
osclass_elm = osmatch_elm.findall('osclass')
ostype = osclass_elm[0].get('type')
os_cpe = osclass_elm[0][0].text
except:
"""Nothing found"""
"""Get Host NSE Script Info"""
try:
find_host_nse_scripts = host.find('hostscript')
for host_nse_script in find_host_nse_scripts:
host_nse_script_id = host_nse_script.get('id')
host_nse_script_output = host_nse_script.get('output')
"""Build a dictionary of the NSE script names and output"""
host_nse_script_id_a.append(host_nse_script_id)
host_nse_script_output_a.append(host_nse_script_output)
except TypeError:
"""Nothing found"""
current_state = state
name = host_name
ipv4 = v4_addr
ipv6 = v6_addr
mac = mac_addr
m_vendor = mac_vendor
current_os = os_cpe
"""Clean the slate"""
stmt = (update(InventoryHost).where(InventoryHost.ipv4_addr == ipv4)
.values(macaddr=None,
ipv6_addr=None,
host_name=None,
mac_vendor_id=None,
state='down',
product_id=None))
try:
session.execute(stmt)
session.commit()
except sqlalchemy.exc.IntegrityError:
session.rollback()
"""Find Vendor for MAC address"""
add_mac_vendor = MACVendor(name=m_vendor)
try:
session.add(add_mac_vendor)
session.commit()
select_mac = session.query(MACVendor).filter_by(name=m_vendor).first()
except sqlalchemy.exc.IntegrityError:
session.rollback()
select_mac = session.query(MACVendor).filter_by(name=m_vendor).first()
"""Build product info for host OS"""
try:
os_product = current_os.split(':')
except AttributeError:
"""Nothing Found"""
try:
product_type = os_product[1]
except IndexError:
"""Nothing Found"""
try:
product_vendor = os_product[2]
except IndexError:
"""Nothing Found"""
try:
product_name = os_product[3]
except IndexError:
"""Nothing Found"""
try:
product_version = os_product[4]
except IndexError:
"""Nothing Found"""
try:
product_update = os_product[5]
except IndexError:
"""Nothing Found"""
try:
product_edition = os_product[6]
except IndexError:
"""Nothing Found"""
try:
product_language = os_product[6]
except IndexError:
"""Nothing Found"""
"""Add the OS Vendor to the database"""
add_prod_vendor = Vendor(name=product_vendor)
try:
session.add(add_prod_vendor)
session.commit()
"""Get vendor.id for product info"""
select_prod_vendor = session.query(Vendor).filter_by(name=product_vendor).first()
except sqlalchemy.exc.IntegrityError:
"""Vendor exists"""
session.rollback()
"""Get vendor.id for product info"""
select_prod_vendor = session.query(Vendor).filter_by(name=product_vendor).first()
"""Get the OS product.id if it exists"""
os_product = session.query(Product).filter_by(product_type=product_type.replace('/', ''),
vendor_id=select_prod_vendor.id,
name=product_name,
version=product_version,
product_update=product_update,
edition=product_edition,
language=product_language).first()
"""Add the OS product to the database"""
add_product = Product(product_type=product_type.replace('/', ''),
vendor_id=select_prod_vendor.id,
name=product_name,
version=product_version,
product_update=product_update,
edition=product_edition,
language=product_language)
"""If the OS product does not exist, add it"""
if os_product is None:
try:
session.add(add_product)
session.commit()
"""Get the OS product.id"""
os_product = session.query(Product).filter_by(product_type=product_type.replace('/', ''),
vendor_id=select_prod_vendor.id,
name=product_name,
version=product_version,
product_update=product_update,
edition=product_edition,
language=product_language).first()
except sqlalchemy.exc.IntegrityError:
"""This should not happen, but if it does.."""
session.rollback()
"""Add host info to database"""
add_inventory_host = InventoryHost(ipv4_addr=ipv4,
ipv6_addr=ipv6,
macaddr=mac,
host_type=ostype,
host_name=name,
mac_vendor_id=select_mac.id,
state=current_state,
product_id=os_product.id)
try:
session.add(add_inventory_host)
session.commit()
except sqlalchemy.exc.IntegrityError:
"""I exist, update me instead"""
session.rollback()
stmt = (update(InventoryHost).where(InventoryHost.ipv4_addr == ipv4)
.values(macaddr=mac,
ipv6_addr=ipv6,
host_name=name,
host_type=ostype,
mac_vendor_id=select_mac.id,
state=current_state,
product_id=os_product.id))
session.execute(stmt)
session.commit()
"""Get the inventory_hosts.id for this host"""
inventory_hosts = session.query(InventoryHost).filter_by(ipv4_addr=ipv4).first()
"""Clean out the old HostNseScript"""
session.query(HostNseScript).filter(HostNseScript.host_id == inventory_hosts.id).delete()
session.commit()
"""Add the new HostNseScripts"""
host_nse_script_keys = host_nse_script_id_a
host_nse_script_values = host_nse_script_output_a
host_nse_scripts_dict = dict(zip(host_nse_script_keys, host_nse_script_values))
for k, v in host_nse_scripts_dict.items():
add_nse_script_op = HostNseScript(host_id=inventory_hosts.id,
name=k,
output=v)
session.add(add_nse_script_op)
session.commit()
"""Find all port Info"""
port_info = host.findall('ports')
"""Clean out the old Inventory_svcs"""
session.query(InventorySvc).filter(InventorySvc.host_id == inventory_hosts.id).delete()
session.commit()
for ports in port_info:
inventory_svcs_id_list = []
p = ports.findall('port')
for each_port in p:
protocol = each_port.get('protocol')
portid = each_port.get('portid')
service_info = each_port.find('service')
findall_cpe = service_info.findall('cpe')
try:
cpe = findall_cpe[0].text
except IndexError:
"""Nothing Found"""
try:
ex_info = service_info.get('extrainfo')
except IndexError:
"""Nothing Found"""
"""Get the NSE script info"""
findall_svc_nse_scripts = each_port.findall('script')
try:
svc_nse_scripts = findall_svc_nse_scripts
for svc_nse_script in svc_nse_scripts:
svc_nse_script_id = svc_nse_script.get('id')
svc_nse_script_output = svc_nse_script.get('output')
"""Build a dictionary of the NSE script names and output"""
svc_nse_script_id_a.append(svc_nse_script_id)
svc_nse_script_output_a.append(svc_nse_script_output)
except IndexError:
"""Nothing Found"""
service_name = service_info.get('name')
service_product = service_info.get('product')
"""Build product info for SVC CPE"""
try:
svc_cpe_product = cpe.split(':')
except AttributeError:
"""Nothing Found"""
try:
svc_cpe_product_type = svc_cpe_product[1]
except IndexError:
"""Nothing Found"""
try:
svc_cpe_product_vendor = svc_cpe_product[2]
except IndexError:
"""Nothing Found"""
try:
svc_cpe_product_name = svc_cpe_product[3]
except IndexError:
"""Nothing Found"""
try:
svc_cpe_product_version = svc_cpe_product[4]
except IndexError:
"""Nothing Found"""
try:
svc_cpe_product_update = svc_cpe_product[5]
except IndexError:
"""Nothing Found"""
try:
svc_cpe_product_edition = svc_cpe_product[6]
except IndexError:
"""Nothing Found"""
try:
svc_cpe_product_language = svc_cpe_product[6]
except IndexError:
"""Nothing Found"""
"""Add SVC and CPE info to database"""
if 'cpe:' in cpe:
"""Add CPE Vendor info to the database"""
add_svc_cpe_prod_vendor = Vendor(name=svc_cpe_product_vendor)
try:
session.add(add_svc_cpe_prod_vendor)
session.commit()
"""Get the CPE vendor.id"""
select_cpe_prod_vendor = session.query(Vendor).filter_by(name=svc_cpe_product_vendor).first()
except sqlalchemy.exc.IntegrityError:
"""You must already exist, just get the CPE vendor.id"""
session.rollback()
select_cpe_prod_vendor = session.query(Vendor).filter_by(name=svc_cpe_product_vendor).first()
finally:
"""What ever you do, make sure you get the CPE vendor.id!"""
select_cpe_prod_vendor = session.query(Vendor).filter_by(name=svc_cpe_product_vendor).first()
"""Add SVC CPE Product info to database"""
try:
"""Get the products.id"""
svc_product = session.query(Product)\
.filter_by(product_type=svc_cpe_product_type.replace('/', ''),
vendor_id=select_cpe_prod_vendor.id,
name=svc_cpe_product_name,
version=svc_cpe_product_version,
product_update=svc_cpe_product_update,
edition=svc_cpe_product_edition,
language=svc_cpe_product_language).first()
except AttributeError:
"""Nothing Found"""
try:
"""Add the new Product to the database"""
add_svc_product = Product(product_type=svc_cpe_product_type.replace('/', ''),
vendor_id=select_cpe_prod_vendor.id,
name=svc_cpe_product_name,
version=svc_cpe_product_version,
product_update=svc_cpe_product_update,
edition=svc_cpe_product_edition,
language=svc_cpe_product_language)
except AttributeError:
"""Nothing Found"""
"""If the product does not exist, add it"""
if svc_product is None:
try:
session.add(add_svc_product)
session.commit()
"""Get the products.id of the Product you just added"""
svc_product = session.query(Product)\
.filter_by(product_type=svc_cpe_product_type.replace('/', ''),
vendor_id=select_cpe_prod_vendor.id,
name=svc_cpe_product_name,
version=svc_cpe_product_version,
product_update=svc_cpe_product_update,
edition=svc_cpe_product_edition,
language=svc_cpe_product_language).first()
except sqlalchemy.exc.IntegrityError:
"""Then you must already exist."""
session.rollback()
"""Add the new inventory_svc"""
add_inventory_svcs = InventorySvc(host_id=inventory_hosts.id,
protocol=protocol,
portid=portid,
name=service_name,
svc_product=service_product,
product_id=svc_product.id,
extra_info=ex_info)
try:
session.add(add_inventory_svcs)
session.commit()
"""Get the inventory_svcs.id"""
svc = session.query(InventorySvc).filter_by(host_id=inventory_hosts.id,
protocol=protocol,
portid=portid,
name=service_name,
svc_product=service_product,
product_id=svc_product.id,
extra_info=ex_info).first()
"""Add the new SvcNseScripts"""
svc_nse_script_keys = svc_nse_script_id_a
svc_nse_script_values = svc_nse_script_output_a
"""Put the two NSE id and output lists together as a dictionary"""
svc_nse_scripts_dict = dict(zip(svc_nse_script_keys, svc_nse_script_values))
for k, v in svc_nse_scripts_dict.items():
add_nse_script_op = SvcNseScript(svc_id=svc.id,
name=k,
output=v)
session.add(add_nse_script_op)
session.commit()
except sqlalchemy.exc.IntegrityError:
"""Then I must exist, but I can't..."""
session.rollback()
"""Clear variables so loop back is clean"""
v4_addr = None
v6_addr = None
mac_addr = None
mac_vendor = None
ostype = None
os_cpe = None
ports_info = []
host_name = None
cpe = 'None'
product_type = None
product_vendor = None
product_name = None
product_version = None
product_update = None
product_edition = None
product_language = None
os_product = 'None'
svc_cpe_product_type = 'None'
svc_cpe_product_vendor = None
svc_cpe_product_name = None
svc_cpe_product_version = None
svc_cpe_product_update = None
svc_cpe_product_edition = None
svc_cpe_product_language = None
protocol = None
portid = None
service_name = None
ex_info = None
service_product = None
host_nse_script_id = None
host_nse_script_output = None
svc_nse_script_id = None
svc_nse_script_output = None
svc_nse_script_id_a = []
svc_nse_script_output_a = []
host_nse_script_id_a = []
host_nse_script_output_a = []
"""Add SVC only info to database"""
else:
add_inventory_svcs = InventorySvc(host_id=inventory_hosts.id,
protocol=protocol,
portid=portid,
name=service_name,
svc_product=service_product,
extra_info=ex_info)
try:
session.add(add_inventory_svcs)
session.commit()
"""Get the inventory_svcs.id"""
inventory_svcs = session.query(InventorySvc).filter_by(host_id=inventory_hosts.id,
protocol=protocol,
portid=portid,
name=service_name,
svc_product=service_product,
extra_info=ex_info).first()
except sqlalchemy.exc.IntegrityError:
"""Then I must exist, but I can't..."""
session.rollback()
session.commit()
"""Add the new SvcNseScripts"""
svc_nse_script_keys = svc_nse_script_id_a
svc_nse_script_values = svc_nse_script_output_a
svc_nse_scripts_dict = dict(zip(svc_nse_script_keys, svc_nse_script_values))
for k, v in svc_nse_scripts_dict.items():
add_nse_script_op = SvcNseScript(svc_id=inventory_svcs.id,
name=k,
output=v)
session.add(add_nse_script_op)
session.commit()
"""Clear variables so loop back is clean"""
v4_addr = None
v6_addr = None
mac_addr = None
mac_vendor = None
ostype = None
os_cpe = None
ports_info = []
host_name = None
cpe = 'None'
product_type = None
product_vendor = None
product_name = None
product_version = None
product_update = None
product_edition = None
product_language = None
os_product = 'None'
svc_cpe_product_type = 'None'
svc_cpe_product_vendor = None
svc_cpe_product_name = None
svc_cpe_product_version = None
svc_cpe_product_update = None
svc_cpe_product_edition = None
svc_cpe_product_language = None
protocol = None
portid = None
service_name = None
ex_info = None
service_product = None
host_nse_script_id = None
host_nse_script_output = None
svc_nse_script_id = None
svc_nse_script_output = None
svc_nse_script_id_a = []
svc_nse_script_output_a = []
host_nse_script_id_a = []
host_nse_script_output_a = []
else:
print('This XML file is not supported!')
exit()
session.close()
try:
from defusedxml.ElementTree import parse
except ImportError:
print("defusedxml not found, downgrading to builtin XML parsing library.")
from xml.etree.ElementTree import parse
if sys.argv[1] is None:
raise SystemExit("need a file to convert")
if not os.path.exists(sys.argv[1]):
raise SystemExit("File {} does not exist".format(sys.argv[1]))
# keep file name, to use for outputs
name = os.path.splitext(sys.argv[1])[0]
# parse file, extract hosts, map by open port found
et = parse(sys.argv[1])
et.findall("host")
xhosts = et.findall("host")
portmap = ddict(list)
for xhost in xhosts:
_hostaddr = xhost.getchildren()[0].items()[1][1]
_port = xhost.getchildren()[1].getchildren()[0].items()[1][1]
portmap[_port].append(_hostaddr)
# dump to files corresponding to each port name
for port, hosts in portmap.iteritems():
outname = "{}-port{}.list".format(name, port)
with open(outname, "w") as ofd:
for host in hosts:
ofd.write("{}\n".format(host))
print("wrote {}".format(outname))
def decode(self, data, charset=None, mimetype=None): try: return parse(data) except ParseError as exc: raise DecodeError(u'Could not decode as %s: %s' % (self.mimetype, exc,))
def setUp(self):
et = parse(custom_helpers.get_test_xml())
self.extras_dict = metadataimporter.map_metadata_to_ckan_extras(et)
self.assertTrue(len(self.extras_dict) > 0)
from defusedxml.ElementTree import parse
from collections import Counter
import sys
sys.path.append("..")
from models import User_Music
from app import db
parsed_file = parse("itunes.xml")
root = parsed_file.getroot()
listings = root.findall('dict/dict/dict')
def parse_iTunes(listings):
parsed = []
for listing in listings:
for index, child in enumerate(listing):
if child.text == "Artist":
parsed.append(listing[index + 1].text)
return parsed
parsed_results = parse_iTunes(listings)
counted_results = Counter(parsed_results)
sorted_results = sorted(counted_results.items(), key=lambda x:x[1])
sorted_results.reverse()
def add_user_music_to_database(results):
for result in results:
band = User_Music(artistName = result[0], count = result[1], userId = "6acebe51-1c90-4dd8-828f-d1c13d52f743")
def loadNotes(self):
self._preProcessFile()
from defusedxml.ElementTree import parse
tree = parse(self.tempname)
root = tree.getroot()
for part in root.findall('part'):
repCount = 0
#Reading the part:
for measure in part.findall('measure'):
self._lastAcc = dict()
self._accidental = self._getAccidentalOfFirstNote(measure.find('note'))
#Reading one Measure
data = []
rep = measure.find('attributes/measure-style/measure-repeat')
if rep is not None and rep.attrib['type'].lower() == 'start':
repCount = 1
else:
if rep is not None and rep.attrib['type'].lower() == 'stop':
c = int(rep.text)
block = []
repCount = repCount - 1
for i in range(len(self.measures) - repCount, len(self.measures)):
block.append(self.measures[i])
self.measures = self.measures + block
repCount = 0
if repCount > 0:
repCount = repCount + 1
continue
# Here we load the notes
for inthebox in measure.findall('DoletSibelius-Unexported-box'):
#reading all the BOX REPEATS:
rep = inthebox.find('direction/direction-type/words')
if rep is not None:
rep = int(rep.text)
minLine = float("Inf")
maxLine = 0
extraRep = 0
extraLine = 0
for i in range(0, rep):
if i == 1:
extraRep = 1 / (rep + 1)
extraLine = extraRep / (maxLine - minLine + 1)
for note in inthebox.findall('note'):
#Notes in the box
new = self._readNote(note)
if new is None:
continue
if i == 0:
if new['lineNumber'] < minLine:
minLine = new['lineNumber']
if new['lineNumber'] > maxLine:
maxLine = new['lineNumber']
else:
new['lineNumber'] = self._calcLineNumber(new['lineNumber'],
i, [minLine, maxLine, extraRep, extraLine])
data.append(new)
for note in measure.findall('note'):
#regular notes:
new = self._readNote(note)
if new is None:
continue
data.append(new)
if len(data) > 0:
self.measures.append(data)
del data
def parse_xml(self, fh=None, etree=None, resources=None,
capability=None, sitemapindex=None):
"""Parse XML Sitemap and add to resources object.
Reads from fh or etree and adds resources to a resorces object
(which must support the add method). Returns the resources object.
Also sets self.resources_created to be the number of resources created.
We adopt a very lax approach here. The parsing is properly namespace
aware but we search just for the elements wanted and leave everything
else alone.
This method will read either sitemap or sitemapindex documents. Behavior
depends on the sitemapindex parameter:
- None - will read either
- False - SitemapIndexError exception if sitemapindex detected
- True - SitemapIndexError exception if sitemap detected
Will set self.parsed_index based on whether a sitemap or sitemapindex
document was read:
- False - sitemap
- True - sitemapindex
"""
if (resources is None):
resources = ResourceContainer()
if (fh is not None):
etree = parse(fh)
elif (etree is None):
raise ValueError("Neither fh or etree set")
# check root element: urlset (for sitemap), sitemapindex or bad
root_tag = etree.getroot().tag
resource_tag = None # will be <url> or <sitemap> depending on type
self.parsed_index = None
if (root_tag == '{' + SITEMAP_NS + "}urlset"):
self.parsed_index = False
if (sitemapindex is not None and sitemapindex):
raise SitemapIndexError(
"Got sitemap when expecting sitemapindex", etree)
resource_tag = '{' + SITEMAP_NS + "}url"
elif (root_tag == '{' + SITEMAP_NS + "}sitemapindex"):
self.parsed_index = True
if (sitemapindex is not None and not sitemapindex):
raise SitemapIndexError(
"Got sitemapindex when expecting sitemap", etree)
resource_tag = '{' + SITEMAP_NS + "}sitemap"
else:
raise SitemapParseError(
"XML is not sitemap or sitemapindex (root element is <%s>)" %
root_tag)
# have what we expect, read it
in_preamble = True
self.resources_created = 0
seen_top_level_md = False
for e in etree.getroot().getchildren():
# look for <rs:md> and <rs:ln>, first <url> ends
# then look for resources in <url> blocks
if (e.tag == resource_tag):
in_preamble = False # any later rs:md or rs:ln is error
r = self.resource_from_etree(e, self.resource_class)
try:
resources.add(r)
except SitemapDupeError:
self.logger.warning(
"dupe of: %s (lastmod=%s)" %
(r.uri, r.lastmod))
self.resources_created += 1
elif (e.tag == "{" + RS_NS + "}md"):
if (in_preamble):
if (seen_top_level_md):
raise SitemapParseError(
"Multiple <rs:md> at top level of sitemap")
else:
resources.md = self.md_from_etree(e, 'preamble')
seen_top_level_md = True
else:
raise SitemapParseError(
"Found <rs:md> after first <url> in sitemap")
elif (e.tag == "{" + RS_NS + "}ln"):
if (in_preamble):
resources.ln.append(self.ln_from_etree(e, 'preamble'))
else:
raise SitemapParseError(
"Found <rs:ln> after first <url> in sitemap")
else:
# element we don't recognize, ignore
pass
# check that we read to right capability document
if (capability is not None):
if ('capability' not in resources.md):
if (capability == 'resourcelist'):
self.logger.warning(
'No capability specified in sitemap, assuming resourcelist')
resources.md['capability'] = 'resourcelist'
else:
raise SitemapParseError("Expected to read a %s document, but no capability specified in sitemap" %
(capability))
if (resources.md['capability'] != capability):
raise SitemapParseError("Expected to read a %s document, got %s" %
(capability, resources.md['capability']))
# return the resource container object
return(resources)