def allCitiesAndEventsXML(): # Query to get data of interest city = session.query(City).all() # Declare root node of XML top = Element('allEvents') comment = Comment('XML Response with all cities and events') top.append(comment) # Loop through query responses and format as XML for c in city: event = SubElement(top, 'event') child = SubElement(event, 'id') child.text = str(c.id) child = SubElement(event, 'city') child.text = c.name child = SubElement(event, 'state') child.text = c.state eventInfo = SubElement(event, 'eventInfo') # Add new node for Events for e in c.events: en = SubElement(eventInfo, 'event_name') en.text = e.name child = SubElement(en, 'description') child.text = e.description child = SubElement(en, 'event_date') child.text = str(e.event_date) child = SubElement(en, 'event_url') child.text = e.event_url child = SubElement(en, 'user_id') child.text = str(e.user_id) print tostring(top) return app.response_class(tostring(top), mimetype='application/xml')
def _enable_wms_caching(layer, cache_secs, disable=False): '''This should go into gsconfig''' layer.resource.fetch() # ensure we fetch stuff fresh or we'll overwrite layer.catalog._cache.pop(layer.resource.href, None) dom = layer.resource.dom metadata = dom.find('metadata') if metadata is None: metadata = _el('metadata') dom.append(metadata) def set_entry(k, v): entries = metadata.findall("entry") entries = [ e for e in entries if e.attrib.get('key', None) == k] entry = entries[0] if entries else None if v: if entry is None: entry = _el('entry', v, key=k) metadata.append(entry) else: entry.text = str(v) elif entry is not None: metadata.remove(entry) set_entry('cacheAgeMax', cache_secs) set_entry('cachingEnabled', 'false' if disable or cache_secs is 0 else 'true') print tostring(dom) headers, resp = layer.resource.catalog.http.request( layer.resource.href, 'PUT', tostring(dom), {'content-type' : 'text/xml'}) if headers.status != 200: raise Exception('error enabling wms caching: %s' % resp)
def qInequalities_template(): '''Solve inequalities. e.g. 2-3x <= 8.''' leftside_section1 = randint(-100,100) leftside_section2 = randint(-100,100) left_side = leftside_section1 + leftside_section2 right_side = randint(-100,100) equality_type = randint(0,3) #<, <=, >, >= question = None x = Symbol('x', real=True) #For 4U Maths use complex=True for ImaginaryNum question_str = "Solve " if equality_type == 0: question = Lt(leftside_section1 + leftside_section2*x, right_side) elif equality_type == 1: question = Le(leftside_section1 + leftside_section2*x, right_side) elif equality_type == 2: question = Gt(leftside_section1 + leftside_section2*x, right_side) elif equality_type == 3: question = Ge(leftside_section1 + leftside_section2*x, right_side) question_str += tostring(am.parse(str(question))) steps = [] if leftside_section1 < 0: steps.append('Move by +' + str(leftside_section1*-1) + ' to both ' \ +'sides') else: steps.append('Move by -' + str(leftside_section1) + ' to both ' \ +'sides') steps.append('Divide left and right side by ' + str(leftside_section2)) answer = [] answer.append(steps) answer.append(tostring(am.parse(str(solve(question, x))))) return question_str, answer
def test_xml(): the_tree = etree.parse('test.xml') for test in the_tree.getiterator('test'): name = test.get('name') if not name: raise ValueError('test element does not have a name attribute') the_string = '' for string in test.getiterator('string'): the_string += string.text test_xml = ascii_to_math_tree(the_string) indent(test_xml) test_xml_string = tostring(test_xml).encode('utf8') standard_test_xml_string = xml_copy_tree(test_xml_string) match = None for result in test.getiterator('result'): math_tree = result[0] indent(math_tree) xml_string = tostring(math_tree) standard_result_xml_string = xml_copy_tree(xml_string) if standard_test_xml_string == standard_result_xml_string: match = True break if not match: sys.stderr.write('Error for test "%s"\n' % (name)) sys.stderr.write(standard_test_xml_string.encode("utf8")) sys.stderr.write('\ndoes not match\n') sys.stderr.write(standard_result_xml_string.encode("utf8"))
def update(self): ResourceInfo.update(self) doc = self.metadata title = doc.find("title") abstract = doc.find("description") projection = doc.find("srs") enabled = doc.find("enabled") native_format = doc.find("nativeFormat") default_interpolation_method = doc.find("defaultInterpolationMethod") request_srs = doc.find("requestSRS/string") response_srs = doc.find("responseSRS/string") if title is None: print self.href print tostring(doc) self.title = title.text if title is not None else None self.abstract = abstract.text if abstract is not None else None self.keywords = [(kw.text or None) for kw in doc.findall("keywords/string")] self.native_bbox = bbox(doc.find("nativeBoundingBox")) self.latlon_bbox = bbox(doc.find("latLonBoundingBox")) self.projection = projection.text if projection is not None else None self.enabled = enabled.text == "true" if enabled is not None else False self.extra_config = dict((entry.attrib['key'], entry.text) for entry in doc.findall("metadata/entry")) self.dimensions = [coverage_dimension(d) for d in doc.findall("dimensions/coverageDimension")] self.native_format = native_format.text if native_format is not None else None self.grid = None # TODO: i guess this merits a little class of its own self.supported_formats = [format.text for format in doc.findall("supportedFormats/string")] self.default_interpolation_method = default_interpolation_method.text if default_interpolation_method is not None else None self.interpolation_methods = [method.text for method in doc.findall("interpolationMethods/string")] self.request_srs = request_srs.text if request_srs is not None else None self.response_srs = response_srs.text if response_srs is not None else None self.metadata_links = [md_link(n) for n in self.metadata.findall("metadataLinks/metadataLink")]
def deploy_product(self, ip, product_name, product_version, attributes_string): url = "%s/%s/%s/%s" % (self.sdc_url, "vdc", self.vdc, "productInstance") headers = {'X-Auth-Token': self.token, 'Tenant-Id': self.vdc, 'Accept': "application/json", 'Content-Type': "application/xml"} productrequest = ProductRequest(self.keystone_url, self.sdc_url, self.tenant, self.user, self.password) productrequest.get_product_info(product_name) attributes = self.__process_attributes(attributes_string) product_release = ProductReleaseDto(product_name, product_version) productInstanceDto = ProductInstanceDto(ip, product_release, attributes) payload = productInstanceDto.to_xml() print url print headers print tostring(payload) response = http.post(url, headers, tostring(payload)) ## Si la respuesta es la adecuada, creo el diccionario de los datos en JSON. if response.status != 200: print 'error to add the product sdc ' + str(response.status) sys.exit(1) else: http.processTask(headers, json.loads(response.read()))
def verify(self, manager, uri, response, respdata, args): #@UnusedVariable # Get arguments files = args.get("filepath", []) # status code must be 200, 207 if response.status not in (200,207): return False, " HTTP Status Code Wrong: %d" % (response.status,) # look for response data if not respdata: return False, " No response body" # look for one file if len(files) != 1: return False, " No file to compare response to" # read in all data from specified file fd = open( files[0], "r" ) try: try: data = fd.read() finally: fd.close() except: data = None if data is None: return False, " Could not read data file" data = manager.server_info.subs(data) result = True if data != respdata: data = data.replace("\n", "\r\n") if data != respdata: # If we have an iCalendar file, then unwrap data and do compare if files[0].endswith(".ics"): data = data.replace("\r\n ", "") respdata = respdata.replace("\r\n ", "") if data != respdata: result = False elif files[0].endswith(".xml"): try: respdata = tostring(ElementTree(file=StringIO(respdata)).getroot()) except Exception: return False, " Could not parse XML response: %s" %(respdata,) try: data = tostring(ElementTree(file=StringIO(data)).getroot()) except Exception: return False, " Could not parse XML data: %s" %(data,) if data != respdata: result = False else: result = False if result: return True, "" else: error_diff = "\n".join([line for line in unified_diff(data.split("\n"), respdata.split("\n"))]) return False, " Response data does not exactly match file data %s" % (error_diff,)
def brandsCatalogXML(brand_id): if 'username' not in login_session: response = make_response(json.dumps("You are not authorized, Login is required"), 401) response.headers['Content-Type'] = 'application/json' return response else: brand = session.query(Brand).filter_by(id=brand_id).one() items = session.query(Item).filter_by(brand_id=brand_id).all() root = Element('html') for i in items: body = SubElement(root, 'body') ID = SubElement(body, 'ID') ID.text = str(i.id) name = SubElement(body, 'name') name.text = i.name price = SubElement(body, 'price') price.text = i.price description = SubElement(body, 'description') description.text = i.description image = SubElement(body, 'image') image.text = i.image print tostring(root) return app.response_class(tostring(root), mimetype='application/xml')
def sign_file(xml_file, root_id, assert_id, key_file, cert_file): """sign *xml_file* with *key_file* and include content of *cert_file*. *xml_file* can be a file, a filename string or an HTTP/FTP url. *key_file* contains the PEM encoded private key. It must be a filename string. *cert_file* contains a PEM encoded certificate (corresponding to *key_file*), included as `X509Data` in the dynamically created `Signature` template. """ # template aware infrastructure from dm.xmlsec.binding.tmpl import parse, Element, SubElement, \ fromstring, XML from dm.xmlsec.binding.tmpl import Signature doc = parse(xml_file) xmlsec.addIDs(doc.getroot(),['ID']) assertion = doc.findall('saml:Assertion', {"saml": "urn:oasis:names:tc:SAML:2.0:assertion"})[0] assertion_xml = sign_xml(tostring(assertion), assert_id, key_file, cert_file) assertion_doc = fromstring(assertion_xml) doc.getroot().remove(doc.findall("saml:Assertion", {"saml": "urn:oasis:names:tc:SAML:2.0:assertion"})[0]) doc.getroot().insert(0, assertion_doc) return sign_xml(tostring(doc), root_id, key_file, cert_file)
def _serverParseXML_QueryRQ(self, e_root, e_header, e_body): # # start to parse a query class # e_query = e_root.find(".//fiap:query", namespaces=_NSMAP) if e_query == None: self.emsg = "query is not specified. [%s]" % tostring(e_root) return None # # check uuid. # XXX it's not used. # uuid = e_query.get("id") # # check the type attribute. # either storage or stream is valid. # type = e_query.get("type") if type == None: self.emsg = "type is not specified. [%s]" % tostring(e_query) return None elif type == "storage": return self._serverParseXML_FETCH(e_root, e_query) elif type == "stream": return self._serverParseXML_TRAP(e_root, e_query) else: self.emsg = "invalid type is specified. [%s]" % type return None
def getItemsXML(expedition_id, category_id): """ Endpoint to return an XML List of all items associated with a certain expedition and category :param expedition_id: :param category_id: """ items = session.query(Item).filter_by( expedition_id=expedition_id, category_id=category_id).all() root = Element('allItems') comment = Comment('XML Endpoint Listing ' 'all Item for a specific Category and Expedition') root.append(comment) for i in items: ex = SubElement(root, 'expedition') ex.text = i.expedition.title category_name = SubElement(ex, 'category_name') category_description = SubElement(category_name, 'category_description') category_picture = SubElement(category_name, 'category_picture') category_name.text = i.category.name category_description.text = i.category.description category_picture.text = i.category.picture item_name = SubElement(category_name, 'item_name') item_decription = SubElement(item_name, 'item_description') item_picture = SubElement(item_name, 'item_picture') item_name.text = i.name item_decription.text = i.description item_picture.text = i.picture print tostring(root) return app.response_class(tostring(root), mimetype='application/xml')
def test_package_serialization(self): items = [ self.request_delivery_factory.factory_item( ware_key=uuid.uuid1().get_hex()[:10], cost=Decimal(450.0 * 30), payment=Decimal(450.0), weight=500, weight_brutto=600, amount=4, comment=u'Комментарий на русском', link=u'http://shop.ru/item/44' ), self.request_delivery_factory.factory_item( ware_key=uuid.uuid1().get_hex()[:10], cost=Decimal(250.0 * 30), payment=Decimal(250.0), weight=500, weight_brutto=600, amount=4, comment=u'Комментарий на русском', link=u'http://shop.ru/item/42' ) ] package = self.request_delivery_factory.factory_package( number=uuid.uuid1().hex[:10], weight=3000, items=items ) self.assertIsInstance(package, PackageRequestObject) tostring(package.to_xml_element(u'Package'), encoding='UTF-8').replace("'", "\"")
def codeToXML(self, code): self.code = code #root = 0 f=tempfile.NamedTemporaryFile(suffix='.cpp', delete=False) #code='//this is the content\n' f.write(code) f.close() #print f.name #fpath='./test.c' #g=open(f.name) #print '\n'.join(g.readlines()) #g.close() try: p1 = Popen([os.path.dirname(os.path.realpath(__file__))+"/../bin/src2srcml", f.name], stdout=PIPE) except: print 'unable to open src2srcml binary' exit(-1) #print p1.stdout.read() #content='<unit>\n'+'\n'.join(p1.stdout.read().split('\n')[2:]) content='\n'.join(p1.stdout.read().split('\n')[1:]) content=content.replace('xmlns=','namsp=') #content=content.replace('xmlns:cpp=','namsp=') #content=p1.stdout.read() if DEBUG: print content #os.remove(f.name) root = ET.fromstring(content) if DEBUG: print tostring(root) #del root.attrib["xmlns"] self.root = root return root
def srcml_code2xml(code): srcml_sample = srcML() if DEBUG: print 'HERE:' tostring(srcml_sample.codeToXML(code)) print '2HERE:' return srcml_sample.codeToXML(code)
def orgHandler(node): """ node is the start node of an Organization orHandler parses the xml info for this organizaton gathered info is stored in django module database """ assert node is not None o = getEntity(Organization, node.attrib.get('ID')) o.name = node.attrib.get('Name') for attr in node: if attr.tag == 'Crises': for elem in attr: o.crises.add(getEntity(Crisis, elem.attrib.get('ID'))) if attr.tag == 'People': for elem in attr: o.people.add(getEntity(Person, elem.attrib.get('ID'))) if attr.tag == 'Kind': o.kind = attr.text if attr.tag == 'Location': o.location = attr.text if attr.tag == 'History': o.history += ''.join([v for v in [tostring(li).strip() for li in attr] if v not in o.history]) if attr.tag == 'ContactInfo': o.contact += ''.join([v for v in [tostring(li).strip() for li in attr] if v not in o.contact]) if attr.tag == 'Common': comHandler(attr, o) assert o is not None o.save()
def qSignificantFigures_template(): '''Significant figures. e.g. Evaluate cuberoot(651/3) to 5 sig fig.''' sig_fig = randint(2,5) root_val = randint(2,5) numerator = randint(1,1000) denom = randint(1,1000) val = 'root%s(%s/(%s*pi))' % (root_val, numerator, denom) question = 'Evaluate ' + tostring(am.parse(val)) + ' to %s' % (sig_fig) question += ' Significant Figures.' steps = [] inside_root = (numerator/(denom*pi)).evalf() val = root(inside_root, root_val) steps.append('This has to be done with a calculator.') steps.append('Do the inside calucation first: ' + tostring(am.parse('%s/(%s*pi)'%(numerator,denom))) + tostring(am.parse('*')) + str(am.parse('pi'))) steps.append('This should give ' + tostring(am.parse(str(inside_root)))) steps.append('Then you need to square root the answer.') steps.append('Use either [1/y] key or similar on calculator and press ' + str(root_val)) steps.append('Please refer to your calculator manual if in doubt.') steps.append('Then look for %s significant figures.' % sig_fig) steps.append('Note: First non-zero digit is 1st signifiant figure,' + \ ' going from left to right. Each digit after that is a' + \ ' significant figure.') answer = [] answer.append(steps) answer.append(round(val, sig_fig-int(floor(log10(val)))-1)) return question, answer
def createAndWriteXML(upload_id, writeToDB): completeVase = Element('vase') for i in range(len(finalRight)): pointpair = SubElement(completeVase, 'pointpair') left = SubElement(pointpair, 'left') right = SubElement(pointpair, 'right') t = SubElement(pointpair, 't') xRight = SubElement(right, 'x') xLeft = SubElement(left, 'x') yRight = SubElement(right, 'y') yLeft = SubElement(left, 'y') t.text = "0" xRight.text = str(finalRight[i][0]) xLeft.text = str(finalLeft[i][0]) yRight.text = str(finalRight[i][1]) yLeft.text = str(finalLeft[i][1]) if writeToDB == 'True': writeToDB(upload_id, tostring(completeVase)) #print "Content-type: text/html" #print print tostring(completeVase)
def handle_noargs(self, **options): """ Poll flickr for photos and create save an onject to the database for each that isn't currently represented. Todo: keys and ids from settings """ for account in settings.DJANGR_ACCOUNTS: flickr = flickrapi.FlickrAPI(settings.DJANGR_APIKEY) photos_el = flickr.photos_search(user_id=account['user_id']) for photo_el in photos_el.findall('photos/photo'): id = int(photo_el.get('id')) secret = photo_el.get('secret') # Reference existing photo or create new try: photo = Photo.objects.get(id=id) print 'Updating photo %s' % id except Photo.DoesNotExist: print 'Creating a new photo %s' % id photo = Photo() photo.id = id photo.active = True photo.secret = secret # Request more information on the photo info_el = flickr.photos_getinfo( user_id=account['user_id'], photo_id=id, secret=secret) photoinfo_el = info_el.find('photo') photo.infoxml = tostring(photoinfo_el) # Parse xml photo.username = photoinfo_el.find('owner').get('username') photo.dateuploaded = datetime.fromtimestamp(float( photoinfo_el.get('dateuploaded'))) datetaken = photoinfo_el.find('dates').get('taken', '') photo.date = datetime.strptime(datetaken, '%Y-%m-%d %H:%M:%S') photo.description = photoinfo_el.findtext('description') photo.title = photoinfo_el.findtext('title') photo.farm = int(photoinfo_el.get('farm')) photo.server = int(photoinfo_el.get('server')) photo.xml = tostring(photo_el) photo.owner = photo_el.get('owner') # Parse optional properties, always delete from object when # none exist - they may be newly removed location = photoinfo_el.find('location') if location: photo.latitude = location.get('latitude') photo.longitude = location.get('longitude') else: photo.longitude = None photo.latitude = None photo.save()
def video_notification(request): """Receive video completion notifications from encoding.com A URL pointing to this view should be sent to encoding.com with transcode requests so that the hit this page and trigger EncodedVideo model updates """ logger = logging.getLogger('vod_aws.views.video_notification') # Only handle POST requests, otherwise just show a blank page if request.method == 'POST': try: result = fromstring(request.POST['xml']) except KeyError: logger.error('request POST contained no XML data') return HttpResponseBadRequest() except ExpatError, e: logger.error('request POST xml parse error: %s' % (e.message)) return HttpResponseBadRequest() # At this point, we should have XML that looks like this: #======================================================================= # <result> # <mediaid>[MediaID]</mediaid> # <source>[SourceFile]</source> # <status>[MediaStatus]</status> # <description>[ ErrorDescription]</description> <!-- Only in case of Status = Error --> # <format> # <output>[OutputFormat]</output> # <destination>[DestFile]</destination> <!-- Only in case of Status = Finished --> # <status>[TaskStatus]</status> # <description>[ErrorDescription]</description> <!-- Only in case of Status = Error --> # <suggestion>[ErrorSuggestion]</suggestion> <!-- Only in case of Status = Error --> # </format> # <format> # ... # </format> # </result> # # Note the lowercase 'mediaid', instead of 'MediaID' as it is elsewhere #======================================================================= # TODO: Currently this only acts on videos for which we've been notified. # Doing a batch "GetStatus" on the MediaID we've been told about as well as # old jobs that (for whatever reason) we don't know about would be a good idea # status can be "Error" or "Finished" status = result.findtext('status') if status == 'Finished': mediaid = result.findtext('mediaid') logger.info('Encoding.com job with MediaID %s is finished.' % mediaid) vod_aws.tasks.process_video_notification.delay(mediaid) elif status == 'Error': # Error message is result.find('description').text # Log the XML to get more debugging from encoding.com logger.error('Encoding.com reported an error: "%s". Enable DEBUG logging for full XML.' % result.findtext('description')) logger.debug(tostring(result)) else: logger.error('Unknown status %s from encoding.com. Enable DEBUG logging for full XML.' % status) logger.debug(tostring(result))
def test_himpact3(self): root = fromstring(himpact3) model = createHumanImpact(root) elem = Element('test') buildHumanImpact(elem, model) rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def test_eimpact3(self): root = fromstring(eimpact3) model = createEconomicImpact(root) elem = Element('test') buildEconomicImpact(elem, model) rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def test_impact2(self) : root = fromstring(impact2) model = createImpact(root) elem = Element('test') buildImpact(elem, model) rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def test_crisisInfo3(self): root = fromstring(cInfo3) model = createCrisisInfo(root) elem = Element('test') buildCrisisInfo(elem, model) rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def test_location3(self): root = fromstring(location3) model = createLocationInfo(root) elem = Element('test') buildLocation(elem, model) rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def test_link1(self): root = fromstring(p_imageLink) model = createLink('primaryImage', root) elem = Element('primaryImage') buildLink(elem, model.site, model.title, model.url, model.description) rootstring = tostring(root) elemstring = tostring(elem) self.assert_(rootstring == elemstring)
def test_link5(self): root = fromstring(extLink1) model = createLink('ext', root) elem = Element('ext') buildLink(elem, model.site, model.title, model.url, model.description) rootstring = tostring(root) elemstring = tostring(elem) self.assert_(rootstring == elemstring)
def test_crisis4(self): root = fromstring(crisis4) model = createCrisis(root) elem = Element('crisis') buildCrisisPage(elem, model) rootstring = tostring(root) elemstring = tostring(elem) self.assert_(rootstring == elemstring)
def test_external_links4(self): root = fromstring(refs4) model = createRefLinks(root) elem = Element('test') buildExternalRefs(elem, model) rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def test_date2(self): root = fromstring(cDate2) model = createDate(root) elem = Element('test') buildDate(elem, model, 'time') rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def test_address3(self): root = fromstring(address3) model = createFullAddress(root) elem = Element('test') buildAddress(elem, model) rootstring = tostring(root) elemstring = tostring(elem[0]) self.assert_(rootstring == elemstring)
def main(): # For py2exe builds freeze_support() # Handle SIGINT to terminate processes signal.signal(signal.SIGINT, sigint_handler) start_time = time() #--PLUGINS INITIALIZATION-- sslyze_plugins = PluginsFinder() available_plugins = sslyze_plugins.get_plugins() available_commands = sslyze_plugins.get_commands() # Create the command line parser and the list of available options sslyze_parser = CommandLineParser(available_plugins, PROJECT_VERSION) try: # Parse the command line (command_list, target_list, shared_settings) = sslyze_parser.parse_command_line() except CommandLineParsingError as e: print e.get_error_msg() return if not shared_settings['quiet'] and shared_settings['xml_file'] != '-': print '\n\n\n' + _format_title('Available plugins') print '' for plugin in available_plugins: print ' ' + plugin.__name__ print '\n\n' #--PROCESSES INITIALIZATION-- # Three processes per target from MIN_PROCESSES up to MAX_PROCESSES nb_processes = max(MIN_PROCESSES, min(MAX_PROCESSES, len(target_list) * 3)) if command_list.https_tunnel: nb_processes = 1 # Let's not kill the proxy task_queue = JoinableQueue() # Processes get tasks from task_queue and result_queue = JoinableQueue( ) # put the result of each task in result_queue # Spawn a pool of processes, and pass them the queues for _ in xrange(nb_processes): priority_queue = JoinableQueue() # Each process gets a priority queue p = WorkerProcess(priority_queue, task_queue, result_queue, available_commands, \ shared_settings) p.start() process_list.append( (p, priority_queue)) # Keep track of each process and priority_queue #--TESTING SECTION-- # Figure out which hosts are up and fill the task queue with work to do if not shared_settings['quiet'] and shared_settings['xml_file'] != '-': print _format_title('Checking host(s) availability') targets_OK = [] targets_ERR = [] # Each server gets assigned a priority queue for aggressive commands # so that they're never run in parallel against this single server cycle_priority_queues = cycle(process_list) target_results = ServersConnectivityTester.test_server_list( target_list, shared_settings) for target in target_results: if target is None: break # None is a sentinel here # Send tasks to worker processes targets_OK.append(target) (_, current_priority_queue) = cycle_priority_queues.next() for command in available_commands: if getattr(command_list, command): args = command_list.__dict__[command] if command in sslyze_plugins.get_aggressive_commands(): # Aggressive commands should not be run in parallel against # a given server so we use the priority queues to prevent this current_priority_queue.put((target, command, args)) else: # Normal commands get put in the standard/shared queue task_queue.put((target, command, args)) for exception in target_results: targets_ERR.append(exception) if not shared_settings['quiet'] and shared_settings['xml_file'] != '-': print ServersConnectivityTester.get_printable_result( targets_OK, targets_ERR) print '\n\n' # Put a 'None' sentinel in the queue to let the each process know when every # task has been completed for (proc, priority_queue) in process_list: task_queue.put(None) # One sentinel in the task_queue per proc priority_queue.put(None) # One sentinel in each priority_queue # Keep track of how many tasks have to be performed for each target task_num = 0 for command in available_commands: if getattr(command_list, command): task_num += 1 # --REPORTING SECTION-- processes_running = nb_processes # XML output xml_output_list = [] # Each host has a list of results result_dict = {} for target in targets_OK: result_dict[target] = [] # If all processes have stopped, all the work is done while processes_running: result = result_queue.get() if result is None: # Getting None means that one process was done processes_running -= 1 else: # Getting an actual result (target, command, plugin_result) = result result_dict[target].append((command, plugin_result)) if len(result_dict[target]) == task_num: # Done with this target # Print the results and update the xml doc if shared_settings['xml_file']: xml_output_list.append( _format_xml_target_result(target, result_dict[target])) if not shared_settings[ 'quiet'] and shared_settings['xml_file'] != '-': print _format_txt_target_result( target, result_dict[target]) else: print _format_txt_target_result(target, result_dict[target]) result_queue.task_done() # --TERMINATE-- # Make sure all the processes had time to terminate task_queue.join() result_queue.join() #[process.join() for process in process_list] # Causes interpreter shutdown errors exec_time = time() - start_time # Output XML doc to a file if needed if shared_settings['xml_file']: result_xml_attr = { 'httpsTunnel': str(shared_settings['https_tunnel_host']), 'totalScanTime': str(exec_time), 'defaultTimeout': str(shared_settings['timeout']), 'startTLS': str(shared_settings['starttls']) } result_xml = Element('results', attrib=result_xml_attr) # Sort results in alphabetical order to make the XML files (somewhat) diff-able xml_output_list.sort(key=lambda xml_elem: xml_elem.attrib['host']) for xml_element in xml_output_list: result_xml.append(xml_element) xml_final_doc = Element('document', title="SSLyze Scan Results", SSLyzeVersion=PROJECT_VERSION, SSLyzeWeb=PROJECT_URL) # Add the list of invalid targets xml_final_doc.append( ServersConnectivityTester.get_xml_result(targets_ERR)) # Add the output of the plugins xml_final_doc.append(result_xml) # Remove characters that are illegal for XML # https://lsimons.wordpress.com/2011/03/17/stripping-illegal-characters-out-of-xml-in-python/ xml_final_string = tostring(xml_final_doc, encoding='UTF-8') illegal_xml_chars_RE = re.compile( u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]') xml_sanitized_final_string = illegal_xml_chars_RE.sub( '', xml_final_string) # Hack: Prettify the XML file so it's (somewhat) diff-able xml_final_pretty = minidom.parseString( xml_sanitized_final_string).toprettyxml(indent=" ", encoding="utf-8") if shared_settings['xml_file'] == '-': # Print XML output to the console if needed print xml_final_pretty else: # Otherwise save the XML output to the console with open(shared_settings['xml_file'], 'w') as xml_file: xml_file.write(xml_final_pretty) if not shared_settings['quiet'] and shared_settings['xml_file'] != '-': print _format_title('Scan Completed in {0:.2f} s'.format(exec_time))
def tostring(self): return tostring(self.toxml())
def __str__(self): return tostring(self.xml(), encoding='utf-8')
def create_annotation(xml_file, from_fasst): """Create annotations by importing from FASST sleep scoring file. Parameters ---------- xml_file : path to xml file annotation file that will be created from_fasst : path to FASST file .mat file containing the scores Returns ------- instance of Annotations TODO ---- Merge create_annotation and create_empty_annotations """ xml_file = Path(xml_file) try: mat = loadmat(str(from_fasst), variable_names='D', struct_as_record=False, squeeze_me=True) except ValueError: raise UnrecognizedFormat( str(from_fasst) + ' does not look like a FASST .mat file') D = mat['D'] info = D.other.info score = D.other.CRC.score microsecond, second = modf(info.hour[2]) start_time = datetime(*info.date, int(info.hour[0]), int(info.hour[1]), int(second), int(microsecond * 1e6)) first_sec = score[3, 0][0] last_sec = score[0, 0].shape[0] * score[2, 0] root = Element('annotations') root.set('version', VERSION) info = SubElement(root, 'dataset') x = SubElement(info, 'filename') x.text = D.other.info.fname x = SubElement(info, 'path') # not to be relied on x.text = D.other.info.fname x = SubElement(info, 'start_time') x.text = start_time.isoformat() x = SubElement(info, 'first_second') x.text = str(int(first_sec)) x = SubElement(info, 'last_second') x.text = str(int(last_sec)) xml = parseString(tostring(root)) with xml_file.open('w') as f: f.write(xml.toxml()) annot = Annotations(xml_file) n_raters = score.shape[1] for i_rater in range(n_raters): rater_name = score[1, i_rater] epoch_length = int(score[2, i_rater]) annot.add_rater(rater_name, epoch_length=epoch_length) for epoch_start, epoch in enumerate(score[0, i_rater]): if isnan(epoch): continue annot.set_stage_for_epoch(epoch_start * epoch_length, FASST_STAGE_KEY[int(epoch)], save=False) annot.save() return annot
def to_svg(self, file=None, canvas_shape=None): """Convert the current layer state to an SVG. Parameters ---------- file : path-like object, optional An object representing a file system path. A path-like object is either a str or bytes object representing a path, or an object implementing the `os.PathLike` protocol. If passed the svg will be written to this file canvas_shape : 4-tuple, optional View box of SVG canvas to be generated specified as `min-x`, `min-y`, `width` and `height`. If not specified, calculated from the last two dimensions of the layer. Returns ---------- svg : string SVG representation of the layer. """ if canvas_shape is None: min_shape = [r[0] for r in self.dims.range[-2:]] max_shape = [r[1] for r in self.dims.range[-2:]] shape = np.subtract(max_shape, min_shape) else: shape = canvas_shape[2:] min_shape = canvas_shape[:2] props = { 'xmlns': 'http://www.w3.org/2000/svg', 'xmlns:xlink': 'http://www.w3.org/1999/xlink', } xml = Element( 'svg', height=f'{shape[0]}', width=f'{shape[1]}', version='1.1', **props, ) transform = f'translate({-min_shape[1]} {-min_shape[0]})' xml_transform = Element('g', transform=transform) xml_list = self.to_xml_list() for x in xml_list: xml_transform.append(x) xml.append(xml_transform) svg = ('<?xml version=\"1.0\" standalone=\"no\"?>\n' + '<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n' + '\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n' + tostring(xml, encoding='unicode', method='xml')) if file: # Save svg to file with open(file, 'w') as f: f.write(svg) return svg
def main(): # Load maintainer mapping (imported from dist-git). # TODO: check that mapping exist / error. print("Loading maintainer mapping...") with open(SCM_MAINTAINER_MAPPING) as raw: maintainer_mapping = json.load(raw) # Group databases files. databases = {} db_pattern = re.compile('^(fedora|epel)-([\w|-]+)_(primary|filelists|other).sqlite$') for db in os.listdir(DBS_DIR): if (not db_pattern.match(db)): sys.exit("Invalid object in {}: {}".format(DBS_DIR, db)) (product, branch, db_type) = db_pattern.findall(db)[0] release_branch = "{}-{}".format(product, branch) if release_branch in databases: databases[release_branch][db_type] = db else: databases[release_branch] = { db_type: db } # Build internal package metadata structure / cache. packages = {} srpm_pattern = re.compile("^(.+)-.+-.+.src.rpm$") changelog_mail_pattern = re.compile("<(.+@.+)>") release_branch_pattern = re.compile("^([fedora|epel]+-[\w|\d]+)-?([a-z|-]+)?$") for release_branch in databases.keys(): print("> Processing database files for {}.".format(release_branch)) for db_type in ["primary", "filelists", "other"]: if db_type not in databases[release_branch]: sys.exit("No {} database for {}.".format(db_type, release_branch)) (_, primary) = open_db(databases[release_branch]["primary"]) (_, filelist) = open_db(databases[release_branch]["filelists"]) (_, other) = open_db(databases[release_branch]["other"]) for raw in primary.execute('SELECT * FROM packages'): pkg = packages.get(raw["name"]) revision = "{}-{}".format(raw["version"], raw["release"]) first_pkg_encounter = False # Register unknown packages. if pkg == None: pkg = Package(raw["name"]) packages[pkg.name] = pkg first_pkg_encounter = True # Override package metadata with rawhide (= lastest) values. if first_pkg_encounter or release_branch == "rawhide": pkg.summary = raw["summary"] pkg.description = raw["description"] pkg.upstream = raw["url"] pkg.license = raw["rpm_license"] pkg.maintainers = maintainer_mapping["rpms"].get(pkg.name, []) # Handle subpackage specific case. (srpm_name) = srpm_pattern.findall(raw["rpm_sourcerpm"])[0] if pkg.name != srpm_name: pkg.subpackage_of = srpm_name # XXX: we do not resolve files and changelog here because storing # them in the packages hash would require multiple GBs of RAM # (roughly 1GB per repository). # Always register branch-specific metadata. (release, branch) = release_branch_pattern.findall(release_branch)[0] if branch == "": branch = "base" pkg.set_release(release, raw["pkgKey"], branch, revision) # Set license and maintainers for subpackages. We have to wait for all # packages to have been processed since subpackage might have been # processed before its parent. print(">> Handling subpackages...") for pkg in packages.values(): if pkg.subpackage_of != None: parent = packages.get(pkg.subpackage_of) if parent != None: parent.subpackages += [pkg.name] pkg.maintainers = packages[pkg.subpackage_of].maintainers print(">>> {} packages have been extracted.".format(len(packages))) print("Sending data to Solr index...") pkg_xml = Element('add') pkg_count = 0 max_pkg_count = len(packages) # Submit packages to Solr index, 500 at a time. for pkg in packages.values(): pkg_el = Element('doc') pkg_el_name = Element('field', { "name": "id" }) pkg_el_name.text = pkg.name pkg_el.append(pkg_el_name) pkg_el_desc = Element('field', { "name": "description" }) pkg_el_desc.text = pkg.description pkg_el.append(pkg_el_desc) pkg_xml.append(pkg_el) pkg_count += 1 if (pkg_count % 500 == 0 or pkg_count == max_pkg_count): requests.post(f"{SOLR_URL}solr/{SOLR_CORE}/update?commit={str(pkg_count == max_pkg_count).lower()}", data=tostring(pkg_xml), headers={'Content-Type': 'application/xml'}) pkg_xml.clear() # print("Submitted {}/{} packages.".format(pkg_count, max_pkg_count)) print("DONE.") print("> {} packages submitted to solr.".format(len(packages)))
#! /usr/bin/env python #-*- coding:utf-8 -*- # version : Python 2.7.13 from xml.etree.ElementTree import Element from xml.etree.ElementTree import tostring def dict_to_xml(tag, d): elem = Element(tag) for key, val in d.items(): child = Element(key) child.text = str(val) elem.append(child) return elem d1 = {'id': 1, 'name': 'test1'} e = dict_to_xml('student', d1) print tostring(e)
def __str__(self): if self.prettyprint: self.indent(self.xml) s = tostring(self.xml).decode(self.encoding) return s
# -*- coding: utf-8 -*-import itertoolsimport osimport plistlibimport unicodedataimport sys from xml.etree.ElementTree import Element, SubElement, tostring """You should run your script via /bin/bash with all escape options ticked.The command line should bepython yourscript.py "{query}" arg2 arg3 ..."""UNESCAPE_CHARACTERS = u""" ;()""" _MAX_RESULTS_DEFAULT = 9 preferences = plistlib.readPlist('info.plist')bundleid = preferences['bundleid'] class Item(object): @classmethod def unicode(cls, value): try: items = iter(value.items()) except AttributeError: return unicode(value) else: return dict(map(unicode, item) for item in items) def __init__(self, attributes, title, subtitle, icon=None): self.attributes = attributes self.title = title self.subtitle = subtitle self.icon = icon def __str__(self): return tostring(self.xml()).decode('utf-8') def xml(self): item = Element(u'item', self.unicode(self.attributes)) for attribute in (u'title', u'subtitle', u'icon'): value = getattr(self, attribute) if value is None: continue if len(value) == 2 and isinstance(value[1], dict): (value, attributes) = value else: attributes = {} SubElement(item, attribute, self.unicode(attributes)).text = self.unicode(value) return item def args(characters=None): return tuple(unescape(decode(arg), characters) for arg in sys.argv[1:]) def config(): return _create('config') def decode(s): return unicodedata.normalize('NFD', s.decode('utf-8')) def uid(uid): return u'-'.join(map(str, (bundleid, uid))) def unescape(query, characters=None): for character in (UNESCAPE_CHARACTERS if (characters is None) else characters): query = query.replace('\\%s' % character, character) return query def work(volatile): path = { True: '~/Library/Caches/com.runningwithcrayons.Alfred-2/Workflow Data', False: '~/Library/Application Support/Alfred 2/Workflow Data' }[bool(volatile)] return _create(os.path.join(os.path.expanduser(path), bundleid)) def write(text): sys.stdout.write(text) def xml(items, maxresults=_MAX_RESULTS_DEFAULT): root = Element('items') for item in itertools.islice(items, maxresults): root.append(item.xml()) return tostring(root, encoding='utf-8') def _create(path): if not os.path.isdir(path): os.mkdir(path) if not os.access(path, os.W_OK): raise IOError('No write access: %s' % path) return path
def extract_proxy_info(tr_el): """ Yields an (ip:port, proxy type, anon level) tuple from the provided table row element. :type tr_el: xml.etree.ElementTree.Element :rtype: (unicode, unicode, unicode) """ port = None kind = None level = None for td_num, td_el in enumerate(tr_el): if not td_el.text: continue if re.match(r'(?m)\s*\d+\s*', td_el.text): if port is not None: raise Exception('Cannot cope with multiple ports: port=%s text=<<%s>>' % (port, td_el.text)) port = int(re.sub(r'\s*', '', td_el.text)) elif re.match(r'HTTPS?|SOCKS[45]', td_el.text): kind = td_el.text elif 7 == td_num: # re.match(r'(?i)low|high') level = td_el.text if port is None: raise ValueError('Expected port, found None') # we have to grab the style's parent because ElementTree # doesn't permit .parentNode and "dom" doesn't have .findall :-/ #: :type: list[xml.etree.ElementTree.Element] style_parent_nl = tr_el.findall('.//*[style]') if not style_parent_nl: LOG.error('Expected "style" parent, found None in %s', tostring(tr_el)) return if 1 != len(style_parent_nl): LOG.warning('Expected only one style parent, found %d of them', len(style_parent_nl)) style_parent = style_parent_nl[0] #: :type: xml.etree.ElementTree.Element sty = style_parent.find('.//style') if sty is None: LOG.warning('Expected "style" child, found None: %s', tostring(style_parent)) return bad_classes = find_display_none_classes(sty.text) LOG.debug('style[%s] -= %s', sty.text, bad_classes) #: :type: list[unicode] parts = [] for ip_el in style_parent: LOG.debug('IP_EL=((%s))', tostring(ip_el)) if 'style' == ip_el.tag: # don't continue or you'll eat the .tail text # which very well could contain a number or dot pass elif 'class' in ip_el.attrib: sp_class = ip_el.attrib['class'] css = [] if sp_class: css = sp_class.split(' ') ok = reduce(lambda a, b: a and b, [x not in bad_classes for x in css], True) if ok: LOG.debug('#class(%s)=%s', css, ip_el.text) parts.append(ip_el.text) elif 'style' in ip_el.attrib: st = ip_el.attrib['style'] if 'x' not in find_display_none_classes('.x{%s}' % st): LOG.debug('#style(%s)=%s', st, ip_el.text) parts.append(ip_el.text) else: if ip_el.text: LOG.warning('??=<<%s>>' % ip_el.text) if ip_el.tail: parts.append(ip_el.tail) if not parts: LOG.warning('Your TD contained no IP parts') return ip_addr = u''.join(parts) if not re.match(r'\d+\.\d+\.\d+\.\d+', ip_addr): raise ValueError('That does not appear to be an IP: %s' % ip_addr) ip_port = '%s:%d' % (ip_addr, port) LOG.debug('ip=%s', ip_port) return ip_port, kind, level
def _dump_tree(self, filename='/Users/rsenseman/Desktop/dump.xml'): with open(filename, 'w') as f: f.write(tostring(self.tree.getroot(), encoding='utf8').decode('utf8')) return None
#random number function def rn1(): random.seed(int(datetime.now().strftime("%f")) * 1000) rn1 = random.random() * 1000000000 return int(rn1) #ID input from user def getID(): ID = str(input("\nEnter ID: \n")) ID = ID.upper() return "ftp://*****:*****@10.10.70.200/UMS/VOD/" + ID + "/" + ID + ".xml" #WSDL URL client = Client('http://10.226.187.29:5954/bme/services/AMSService?WSDL') #RAW SOAP message check messagexml = client.create_message(client.service, 'notify', notifyRequest={ 'MD5CcheckSum': "", 'NO': rn1(), 'URL': getID() }) xml1 = tostring(messagexml, encoding="unicode") print(xml1) #POST SOAP message #print(client.service.notify(notifyRequest={'MD5CcheckSum':"", 'NO':rn1(), 'URL':getID()"}))
'SSID_end': header.find('SSID')+4, 'BSSID_beg': header.find('BSSID'), 'BSSID_end': header.find('RSSI')-1, 'RSSI_beg': header.find('RSSI'), 'RSSI_end': header.find('CHANNEL')-1, 'SECURITY_beg': header.find('SECURITY') } output_results = [] for item in cmd_results: output_result = {} output_result['SSID'] = item[0:formatter['SSID_end']].strip() output_result['BSSID'] = item[formatter['BSSID_beg']:formatter['BSSID_end']].strip() output_result['RSSI'] = item[formatter['RSSI_beg']:formatter['RSSI_end']].strip() output_result['SECURITY'] = item[formatter['SECURITY_beg']:].strip() output_results.append(output_result) #Generate XML items = Element('items') for result in output_results: item = SubElement(items, 'item', {'autocomplete':result['SSID'], 'uid':result['BSSID'], 'arg':result['SSID']}) title = SubElement(item, 'title') title.text = result['SSID'] subtitle = SubElement(item, 'subtitle') subtitle.text = 'RSSI:'+result['RSSI']+'\t\t\t\t'+'Security:'+result['SECURITY'] icon = SubElement(item, 'icon') icon.text = 'icon.png' print tostring(items)
def print_xml(self): ''' print string version of the elements ''' print tostring(self._xml_root)
def dict_to_xml(tag, d): ''' Turn a simple dict of key/value pairs into XML ''' elem = Element(tag) for key, val in d.items(): child = Element(key) child.text = str(val) elem.append(child) return elem # An example s = {'name': 'GOOG', 'shares': 100, 'price': 490.1} e = dict_to_xml('stock', s) print(e) print() print(tostring(e)) print() e.set('_id', '1234') print(tostring(e)) print() print(escape('<spam>')) print(unescape(_))
def _read_option(section_name, opt): """ Reads an option (inside a section) and returns the name and the value. """ opt_name = opt.tag.lower() if section_name == 'open-scap': if opt.attrib: opt_value = {} for a in opt.attrib: opt_value[a] = opt.attrib[a] # profiles profiles_list = [] for profiles in opt.iter(): profiles_list.append(profiles.text) if profiles_list: opt_value['profiles'] = profiles_list else: opt_value = opt.text elif section_name == 'syscheck' and opt_name == 'directories': opt_value = [] json_attribs = {} for a in opt.attrib: json_attribs[a] = opt.attrib[a] if opt.text: for path in opt.text.split(','): json_path = json_attribs.copy() json_path['path'] = path.strip() opt_value.append(json_path) elif section_name == 'syscheck' and opt_name in ('synchronization', 'whodata'): opt_value = {} for child in opt: child_section, child_config = _read_option(child.tag.lower(), child) opt_value[child_section] = child_config.split( ',') if child_config.find(',') > 0 else child_config elif (section_name == 'cluster' and opt_name == 'nodes') or \ (section_name == 'sca' and opt_name == 'policies'): opt_value = [child.text for child in opt] elif section_name == 'labels' and opt_name == 'label': opt_value = {'value': opt.text} for a in opt.attrib: opt_value[a] = opt.attrib[a] elif section_name == 'localfile' and opt_name == 'query': # Remove new lines, empty spaces and backslashes regex = rf'<{opt_name}>(.*)</{opt_name}>' opt_value = re.match( regex, re.sub( '(?:(\n) +)', '', tostring(opt, encoding='unicode').replace('\\<', '<').replace( '\\>', '>')).strip()).group(1) elif section_name == 'remote' and opt_name == 'protocol': opt_value = [elem.strip() for elem in opt.text.split(',')] else: if opt.attrib: opt_value = {} for a in opt.attrib: opt_value[a] = opt.attrib[a] if list(opt): for child in opt: child_section, child_config = _read_option( child.tag.lower(), child) opt_value[child_section] = child_config else: opt_value['item'] = opt.text else: opt_value = opt.text return opt_name, _replace_custom_values(opt_value)
def _prettify(elem): """Return a pretty-printed XML string for the Element.""" raw_string = tostring(elem, 'utf-8') reparsed = minidom.parseString(raw_string) return reparsed.toprettyxml(indent=" ")
def main(): # For py2exe builds freeze_support() # Handle SIGINT to terminate processes signal.signal(signal.SIGINT, sigint_handler) start_time = time() #--PLUGINS INITIALIZATION-- sslyze_plugins = PluginsFinder() available_plugins = sslyze_plugins.get_plugins() available_commands = sslyze_plugins.get_commands() # Create the command line parser and the list of available options sslyze_parser = CommandLineParser(available_plugins, __version__) online_servers_list = [] invalid_servers_list = [] # Parse the command line try: good_server_list, bad_server_list, args_command_list = sslyze_parser.parse_command_line() invalid_servers_list.extend(bad_server_list) except CommandLineParsingError as e: print e.get_error_msg() return should_print_text_results = not args_command_list.quiet and args_command_list.xml_file != '-' \ and args_command_list.json_file != '-' if should_print_text_results: print '\n\n\n' + _format_title('Available plugins') for plugin in available_plugins: print ' ' + plugin.__name__ print '\n\n' #--PROCESSES INITIALIZATION-- if args_command_list.https_tunnel: # Maximum one process to not kill the proxy plugins_process_pool = PluginsProcessPool(sslyze_plugins, args_command_list.nb_retries, args_command_list.timeout, max_processes_nb=1) else: plugins_process_pool = PluginsProcessPool(sslyze_plugins, args_command_list.nb_retries, args_command_list.timeout) #--TESTING SECTION-- # Figure out which hosts are up and fill the task queue with work to do if should_print_text_results: print _format_title('Checking host(s) availability') connectivity_tester = ServersConnectivityTester(good_server_list) connectivity_tester.start_connectivity_testing(network_timeout=args_command_list.timeout) SERVER_OK_FORMAT = u' {host}:{port:<25} => {ip_address} {client_auth_msg}' SERVER_INVALID_FORMAT = u' {server_string:<35} => WARNING: {error_msg}; discarding corresponding tasks.' # Store and print servers we were able to connect to for server_connectivity_info in connectivity_tester.get_reachable_servers(): online_servers_list.append(server_connectivity_info) if should_print_text_results: client_auth_msg = '' client_auth_requirement = server_connectivity_info.client_auth_requirement if client_auth_requirement == ClientAuthenticationServerConfigurationEnum.REQUIRED: client_auth_msg = ' WARNING: Server REQUIRED client authentication, specific plugins will fail.' elif client_auth_requirement == ClientAuthenticationServerConfigurationEnum.OPTIONAL: client_auth_msg = ' WARNING: Server requested optional client authentication' print SERVER_OK_FORMAT.format(host=server_connectivity_info.hostname, port=server_connectivity_info.port, ip_address=server_connectivity_info.ip_address, client_auth_msg=client_auth_msg) # Send tasks to worker processes for plugin_command in available_commands: if getattr(args_command_list, plugin_command): # Get this plugin's options if there's any plugin_options_dict = {} for option in available_commands[plugin_command].get_interface().get_options(): # Was this option set ? if getattr(args_command_list,option.dest): plugin_options_dict[option.dest] = getattr(args_command_list, option.dest) plugins_process_pool.queue_plugin_task(server_connectivity_info, plugin_command, plugin_options_dict) for tentative_server_info, exception in connectivity_tester.get_invalid_servers(): invalid_servers_list.append((tentative_server_info.server_string, exception)) # Print servers we were NOT able to connect to if should_print_text_results: for server_string, exception in invalid_servers_list: if isinstance(exception, ServerConnectivityError): print SERVER_INVALID_FORMAT.format(server_string=server_string, error_msg=exception.error_msg) else: # Unexpected bug in SSLyze raise exception print '\n\n' # Keep track of how many tasks have to be performed for each target task_num = 0 for command in available_commands: if getattr(args_command_list, command): task_num += 1 # --REPORTING SECTION-- # XML output xml_output_list = [] # Each host has a list of results result_dict = {} # We cannot use the server_info object directly as its address will change due to multiprocessing RESULT_KEY_FORMAT = u'{hostname}:{ip_address}:{port}'.format for server_info in online_servers_list: result_dict[RESULT_KEY_FORMAT(hostname=server_info.hostname, ip_address=server_info.ip_address, port=server_info.port)] = [] # Process the results as they come for plugin_result in plugins_process_pool.get_results(): server_info = plugin_result.server_info result_dict[RESULT_KEY_FORMAT(hostname=server_info.hostname, ip_address=server_info.ip_address, port=server_info.port)].append(plugin_result) result_list = result_dict[RESULT_KEY_FORMAT(hostname=server_info.hostname, ip_address=server_info.ip_address, port=server_info.port)] if len(result_list) == task_num: # Done with this server; print the results and update the xml doc if args_command_list.xml_file: xml_output_list.append(_format_xml_target_result(server_info, result_list)) if should_print_text_results: print _format_txt_target_result(server_info, result_list) # --TERMINATE-- exec_time = time()-start_time # Output JSON to a file if needed if args_command_list.json_file: json_output = {'total_scan_time': str(exec_time), 'network_timeout': str(args_command_list.timeout), 'network_max_retries': str(args_command_list.nb_retries), 'invalid_targets': [], 'accepted_targets': []} # Add the list of invalid targets for server_string, exception in invalid_servers_list: if isinstance(exception, ServerConnectivityError): json_output['invalid_targets'].append({server_string: exception.error_msg}) else: # Unexpected bug in SSLyze raise exception # Add the output of the plugins for each server for host_str, plugin_result_list in result_dict.iteritems(): server_info = plugin_result_list[0].server_info json_output['accepted_targets'].append(_format_json_result(server_info, plugin_result_list)) final_json_output = json.dumps(json_output, default=_object_to_json_dict, sort_keys=True, indent=4) if args_command_list.json_file == '-': # Print XML output to the console if needed print final_json_output else: # Otherwise save the XML output to the console with open(args_command_list.json_file, 'w') as json_file: json_file.write(final_json_output) # Output XML doc to a file if needed if args_command_list.xml_file: result_xml_attr = {'totalScanTime': str(exec_time), 'networkTimeout': str(args_command_list.timeout), 'networkMaxRetries': str(args_command_list.nb_retries)} result_xml = Element('results', attrib = result_xml_attr) # Sort results in alphabetical order to make the XML files (somewhat) diff-able xml_output_list.sort(key=lambda xml_elem: xml_elem.attrib['host']) for xml_element in xml_output_list: result_xml.append(xml_element) xml_final_doc = Element('document', title="SSLyze Scan Results", SSLyzeVersion=__version__, SSLyzeWeb=PROJECT_URL) # Add the list of invalid targets invalid_targets_xml = Element('invalidTargets') for server_string, exception in invalid_servers_list: if isinstance(exception, ServerConnectivityError): error_xml = Element('invalidTarget', error=exception.error_msg) error_xml.text = server_string invalid_targets_xml.append(error_xml) else: # Unexpected bug in SSLyze raise exception xml_final_doc.append(invalid_targets_xml) # Add the output of the plugins xml_final_doc.append(result_xml) # Remove characters that are illegal for XML # https://lsimons.wordpress.com/2011/03/17/stripping-illegal-characters-out-of-xml-in-python/ xml_final_string = tostring(xml_final_doc, encoding='UTF-8') illegal_xml_chars_RE = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]') xml_sanitized_final_string = illegal_xml_chars_RE.sub('', xml_final_string) # Hack: Prettify the XML file so it's (somewhat) diff-able xml_final_pretty = minidom.parseString(xml_sanitized_final_string).toprettyxml(indent=" ", encoding="utf-8" ) if args_command_list.xml_file == '-': # Print XML output to the console if needed print xml_final_pretty else: # Otherwise save the XML output to the console with open(args_command_list.xml_file, 'w') as xml_file: xml_file.write(xml_final_pretty) if should_print_text_results: print _format_title('Scan Completed in {0:.2f} s'.format(exec_time))
def tostring(self, *args, **kwargs): return tostring(self)
import xml.etree.ElementTree from xml.etree import ElementTree as et from xml.etree.ElementTree import fromstring, tostring path = '/home/pfb16181/NetBeansProjects/extractTitleAbstractPubMedData_generateWordDoc2vec/data/pubmed_filter_2017_2018_2019/' temp_list = [] for filename in os.listdir(path): print('parsing '+filename) temp_str = '' if not filename.endswith('.xml'): continue fullname = os.path.join(path, filename) et = xml.etree.ElementTree.parse(fullname) tree = et.parse(fullname) string_tree = tostring(tree) n_string_tree = str(string_tree, 'utf-8') root = xml.etree.ElementTree.fromstring(n_string_tree) try: temp_str += et.find('.//ArticleTitle').text temp = root.findall('.//AbstractText') for j in temp: temp_str += j.text except: print('error') pass text_file = open("/home/pfb16181/NetBeansProjects/extractTitleAbstractPubMedData_generateWordDoc2vec/parsed_data/parsed_pubmed_filter_2017_2018_2019/"+ "".join(filename.partition('.xml')[0].split())+".txt", "w") text_file.write(temp_str)
def __str__(self): return tostring(self.xml()).decode('utf-8') def xml(self): item = Element(u'item', self.unicode(self.attributes)) for attribute in (u'title', u'subtitle', u'icon'): value = getattr(self, attribute) if value is None: continue if len(value) == 2 and isinstance(value[1], dict): (value, attributes) = value else: attributes = {} SubElement(item, attribute, self.unicode(attributes)).text = self.unicode(value) return item
''' Turn a simple dict of key/value pairs into XML ''' elem = Element(tag) for key, val in d.items(): child = Element(key) child.text = str(val) elem.append(child) return elem s = { 'name': 'GOOG', 'shares': 100, 'price':490.1 } e = dict_to_xml('stock', s) e from xml.etree.ElementTree import tostring tostring(e) e.set('_id','1234') tostring(e) def dict_to_xml_str(tag, d): ''' Turn a simple dict of key/value pairs into XML ''' parts = ['<{}>'.format(tag)] for key, val in d.items(): parts.append('<{0}>{1}</{0}>'.format(key,val)) parts.append('</{}>'.format(tag)) return ''.join(parts) d = { 'name' : '<spam>' }
def populate05(inputFile): #cleaning XML file try: f = open(inputFile, 'r') except Exception as e: return #mdb.connect('localhost', 'username', 'password', 'database'); con = mdb.connect('localhost', 'root', 'password', 'Patents') with con: cur = con.cursor() cur.execute( "CREATE TABLE IF NOT EXISTS USPatents(patentNumber INT, abstract TEXT, inventor TEXT, \ currentUSClass TEXT, primaryExaminer TEXT, assistantExaminer TEXT, attorney TEXT, claims MEDIUMTEXT, description MEDIUMTEXT)" ) cur.execute( "CREATE TABLE IF NOT EXISTS InternationalClass(patentNumber VARCHAR(8), interClass TEXT)" ) cur.execute("CREATE TABLE IF NOT EXISTS \ FurtherUSClass(patentNumber INT, furtherUSClass TEXT)") cur.execute("CREATE TABLE IF NOT EXISTS \ FieldOfResearch(patentNumber INT, class TEXT)") cur.execute("CREATE TABLE IF NOT EXISTS \ PublicationReference(patentNumber INT, country TEXT, kind TEXT, name TEXT, date BIGINT)" ) cur.execute("CREATE TABLE IF NOT EXISTS \ ApplicationReference(patentNumber INT, docNumber TEXT, country TEXT, date BIGINT)" ) cur.execute("CREATE TABLE IF NOT EXISTS \ ReferPatcit(patentNumber INT, docNumber TEXT, country TEXT, kind TEXT, category TEXT, name TEXT, date BIGINT)" ) cur.execute("CREATE TABLE IF NOT EXISTS \ ReferNplcit(patentNumber INT, value TEXT, category TEXT)") xmld = '<TOP>\n' codecs.register_error('spacer', lambda ex: (u' ', ex.start + 1)) for _t in f: _t = _t.replace("&", "") _t = _t.decode('utf8', 'spacer') if _t.find('<?xml') == -1 and _t.find('<!DOCTYPE') == -1: xmld = xmld + _t try: if _t.find('</us-patent-grant>') > -1: #print _t xmld = xmld + '</TOP>' g = open("tmp", "w") g.write(xmld) g.close() xmld = '<TOP>\n' #start parsing doc = ElementTree() doc.parse("tmp") root = doc.getroot() #loop for main patent for n in root.iter("us-patent-grant"): #patent number patentNumber = 0 #appplication Number and publication Number for ar in n.iter("publication-reference"): dateFiled = None country = '' kind = '' name = '' for j in ar.iter("document-id"): for k in j.iter("doc-number"): if k.text.isdigit(): patentNumber = int(k.text) if not patentNumber: continue for k in j.iter("date"): if k.text.isdigit(): dateFiled = int(k.text) for k in j.iter("country"): country = k.text for k in j.iter("kind"): kind = k.text for k in j.iter("name"): name = k.text with con: cur = con.cursor() cur.execute( "INSERT INTO PublicationReference(patentNumber, country, kind, name, date) VALUES(%s, %s, %s, %s, %s)", (patentNumber, country, kind, name, dateFiled)) if not patentNumber: continue for ar in n.iter("application-reference"): docNumber = 0 dateFiled = None country = '' for j in ar.iter("document-id"): for k in j.iter("doc-number"): docNumber = k.text for k in j.iter("date"): if k.text.isdigit(): dateFiled = int(k.text) for k in j.iter("country"): country = k.text with con: cur = con.cursor() cur.execute( "INSERT INTO ApplicationReference(patentNumber, docNumber, country, date) VALUES(%s, %s, %s, %s)", (patentNumber, docNumber, country, dateFiled)) #inventor inventor = '' for ar in n.iter("parties"): for pr in ar.iter("applicants"): for j in pr.iter("applicant"): for k in j.iter("addressbook"): firstName = '' lastName = '' address = '' for ln in k.iter("last-name"): if ln.text: lastName = ln.text for ln in k.iter("first-name"): if ln.text: firstName = ln.text for add in k.iter("address"): for street in add.iter("street"): if street.text: address = street.text for city in add.iter("city"): if city.text: address += ' - ' + city.text for country in add.iter("country"): if country.text: address += ' - ' + country.text inventor += (firstName + ' ' + lastName + ' (' + address + ');') #international for ic in n.iter("classification-locarno"): for t in ic.iterfind("main-classification"): inter = t.text with con: cur = con.cursor() cur.execute( "INSERT INTO InternationalClass(patentNumber, interClass) VALUES(%s, %s)", (patentNumber, inter)) #field of research for rs in n.iter("field-of-search"): for k in rs.iter("classification-national"): fieldOfResearch = '' for j in k.iter("main-classification"): fieldOfResearch = j.text with con: cur = con.cursor() cur.execute( "INSERT INTO FieldOfResearch(patentNumber, class) VALUES(%s, %s)", (patentNumber, fieldOfResearch.strip())) #examiner primaryExaminer = '' assitantExaminer = '' for E in n.iter("examiners"): for pr in E.iter("primary-examiner"): firstName = '' lastName = '' for ln in pr.iter("last-name"): lastName = ln.text for ln in pr.iter("first-name"): firstName = ln.text if firstName == None: firstName = '' if lastName == None: lastName = '' primaryExaminer += lastName + ' ' + firstName + '; ' for pr in E.iter("assistant-examiner"): firstName = '' lastName = '' for ln in pr.iter("last-name"): lastName = ln.text for ln in pr.iter("first-name"): firstName = ln.text if firstName == None: firstName = '' if lastName == None: lastName = '' assitantExaminer += lastName + ' ' + firstName + '; ' #Attorney, Agent or Firm, attorney = '' p = n.find( "us-bibliographic-data-grant/parties/agents/agent/addressbook" ) if p: for ar in p.iter("orgname"): attorney += ar.text + ';' #Abstract text abstr = '' for abst in n.iter("abstract"): p = abst.find("p") _p = tostring(p) abstr = convertToHTMLView(_p.encode('UTF-8')) #US Class #furtherUSClass usClass = '' p = n.find( "us-bibliographic-data-grant/classification-national") for ma in p.iter("main-classification"): if ma.text: usClass = ma.text for fu in p.iter("further-classification"): if fu.text: furtherClass = fu.text.strip() with con: cur = con.cursor() cur.execute( "INSERT INTO FurtherUSClass(patentNumber, furtherUSClass) VALUES(%s, %s)", (patentNumber, furtherClass)) #patent references for _n in n.iter("references-cited"): for j in _n.iter("citation"): for tmp in j.iter("patcit"): country = '' kind = '' name = '' dateFiled = '' for k in tmp.iter("document-id"): _t = k.find("country") if _t.text == "US": country = "US" t = k.find("kind") if t: kind = t.text t = k.find("date") if t.text.isdigit(): dateFiled = int(t.text) t = k.find("name") if t: name = t.text number = k.find("doc-number") if number.text: num = number.text with con: cur = con.cursor() cur.execute( "INSERT INTO ReferPatcit(patentNumber, docNumber, country, kind, name, date) VALUES(%s, %s, %s, %s, %s, %s)", (patentNumber, num, country, kind, name, dateFiled)) value = '' for tmp in j.iter("nplcit"): cat = j.find("category") t = tmp.find("othercit") value = tostring(t) if value: with con: cur = con.cursor() cur.execute( "INSERT INTO ReferNplcit(patentNumber, value, category) VALUES(%s, %s, %s)", (patentNumber, convertToHTMLView( value.encode("UTF-8")), cat.text)) #claims claim = '' cl = n.find("claims") if cl: _cl = tostring(cl) claim = convertToHTMLView(_cl.encode('UTF-8')).strip() #descriptions description = '' des = n.find("description") if des: _des = tostring(des) description = convertToHTMLView( _des.encode('UTF-8')).strip() with con: cur = con.cursor() cur.execute( "Select * from USPatents where patentNumber = %s", (patentNumber)) if cur.rowcount == 0: cur.execute( "INSERT INTO USPatents(patentNumber, abstract, inventor, currentUSClass,\ primaryExaminer, assistantExaminer, attorney, claims, description) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)", (patentNumber, abstr, inventor.encode('UTF-8'), usClass.encode('UTF-8'), primaryExaminer.encode('UTF-8'), assitantExaminer.encode('UTF-8'), attorney.encode('UTF-8'), claim, description)) except Exception as e: print e
}).text = fileinfo['guid'] SubElement( item, 'pubDate').text = fileinfo['pubdate'].strftime("%a, %d %b %Y %T %z") SubElement(item, 'itunes:duration').text = time.strftime( "%H:%M:%S", time.gmtime(int(fileinfo['duration']))) chapters = SubElement(item, 'psc:chapters', attrib={ 'version': "1.2", 'xmlns:psc': 'http://podlove.org/simple-chapters' }) for chapter in sorted(fileinfo['chapters'], key=lambda x: x['start']): SubElement(chapters, 'psc:chapter', attrib={ 'start': time.strftime("%H:%M:%S", time.gmtime(chapter['start'])), 'title': chapter['title'] }) ### save feed file = open(path.join(DIR, 'podcast.xml'), "w") file.write(tostring(root, encoding='UTF-8', method='xml').decode()) # xml in single line #file.write(xml.dom.minidom.parseString(tostring(root, encoding='UTF-8', method='xml').decode()).toprettyxml()) # pretty-print, requires xml.dom.minidom
def animal(request,pk): t = Template(""" <head> <link rel="stylesheet" type="text/css" href="/static/style.css" /> </head> <div id= 'notepad'> <div id= 'content'> <div id= 'text'> <h3>Animal's common name:</h3> <p class="sansseriff"><div>{{ animal.common_names }}</div></p> <h3>Species ID:</h3> <p class="sansseriff"><div>{{ animal.species_ID }}</div></p> <h3>Animal's scientific name:</h3> <p class="sansseriff"><div>{{ animal.scientific_name }}</div></p> <h3>Animal's other names:</h3> {% if not animal.other_names %}-{% endif %} <p class="sansseriff"><div>{{ animal.other_names }}</div></p> <h3>Summary of the animal:</h3> {% if not animal.summary %}-{% endif %} <p class="sansseriff"><div>{{ animal.summary }}</div></p> <h3>Animal's GE score:</h3> <p class="sansseriff"><div>{{ animal.ge_score }}</div></p> <h3>Possibly exinct?:</h3> <p class="sansseriff"><div>{{ animal.extinct }}</div></p> <h3>Animal's origin country:</h3> {% if not animal.country_found_in %}-{% endif %} <p class="sansseriff"><div>{{ animal.country_found_in }}</div></p> </div><!--text--> <div id= 'funstuff'> <img src="{{image}}" /> {% if not animal.country_found_in %} {% else %} <iframe id= 'map' width="400" height="300" frameborder="0" scrolling="no" marginheight="0" marginwidth="0" src="http://maps.google.com/maps?f=q&source=s_q&hl=en&geocode=&q=uk&aq=&ie=UTF8&hq=&hnear={{location}}&t=m&z=5&output=embed"></iframe> {% endif %} <img src="/static/status.png" class= 'status'/> </div><!--endfun --> </div><!--content--> </div><!--notepad--> """) animal= Animal.objects.get(pk=pk) # this part is used to carry out a search of the animal thats being selected by the user using the search results from wikimedia url= urllib2.urlopen('https://commons.wikimedia.org/w/api.php?action=query&generator=search&gsrnamespace=6&gsrsearch=%22'+ urllib.quote_plus(animal.common_names ) +'%22&gsrlimit=1&prop=imageinfo&iiprop=url&format=xml' ) tree = ElementTree() tree.parse(url) print tostring(tree.getroot()) # this tells try: images= tree.find('query').find('pages').find('page').find('imageinfo').find('ii').attrib['url'] except AttributeError: images= "" # this is the variable thats used to store the country an animal is originally found in which is then used to find a location in which the animal was originally found in using an embedded google maps image location= urllib.quote_plus(animal.country_found_in ) c = Context({"animal": animal, "image": images, "location": location}) # this is used to display the image of the animal thats found onto the page print images return HttpResponse(t.render(c))
def getNfo(self, movie_info={}, data={}): # return imdb url only if self.conf('meta_url_only'): return 'http://www.imdb.com/title/%s/' % toUnicode( data['library']['identifier']) nfoxml = Element('movie') # Title try: el = SubElement(nfoxml, 'title') el.text = toUnicode(getTitle(data['library'])) except: pass # IMDB id try: el = SubElement(nfoxml, 'id') el.text = toUnicode(data['library']['identifier']) except: pass # Runtime try: runtime = SubElement(nfoxml, 'runtime') runtime.text = '%s min' % movie_info.get('runtime') except: pass # Other values types = [ 'year', 'mpaa', 'originaltitle:original_title', 'outline', 'plot', 'tagline', 'premiered:released' ] for type in types: if ':' in type: name, type = type.split(':') else: name = type try: if data['library'].get(type): el = SubElement(nfoxml, name) el.text = toUnicode(movie_info.get(type, '')) except: pass # Rating for rating_type in ['imdb', 'rotten', 'tmdb']: try: r, v = movie_info['rating'][rating_type] rating = SubElement(nfoxml, 'rating') rating.text = str(r) votes = SubElement(nfoxml, 'votes') votes.text = str(v) break except: log.debug('Failed adding rating info from %s: %s', (rating_type, traceback.format_exc())) # Genre for genre in movie_info.get('genres', []): genres = SubElement(nfoxml, 'genre') genres.text = toUnicode(genre) # Actors for actor in movie_info.get('actors', []): actors = SubElement(nfoxml, 'actor') name = SubElement(actors, 'name') name.text = toUnicode(actor) # Directors for director_name in movie_info.get('directors', []): director = SubElement(nfoxml, 'director') director.text = toUnicode(director_name) # Writers for writer in movie_info.get('writers', []): writers = SubElement(nfoxml, 'credits') writers.text = toUnicode(writer) # Clean up the xml and return it nfoxml = xml.dom.minidom.parseString(tostring(nfoxml)) xml_string = nfoxml.toprettyxml(indent=' ') text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+</', re.DOTALL) xml_string = text_re.sub('>\g<1></', xml_string) return xml_string.encode('utf-8')
}, '0137143419':{ 'title': 'Python Fundamentals', 'year': 2009, }, } books = Element('books') for isbn,info in BOOKS.iteritems(): book = SubElement(books, 'book') info.setdefault('authors', 'Wesley Chun') info.setdefault('edition', 1) for key, val in info.iteritems(): SubElement(book, key).text = ', '.join(str(val) .split(':')) xml = tostring(books) print '*** RAW XML ***' print xml print '\n*** PRETTY-PRINTED XML ***' dom = parseString(xml) print dom.toprettyxml(' ') print '*** FLAT STRUCTURE ***' for elmt in books.getiterator(): print elmt.tag, '-', elmt.text print '\n*** TITLES ONLY ***' for book in books.findall('.//title'): print book.text
"parameter", name="BrowserType1", value="firefox") node2 = SubElement(child, "classes") if test_labels[test] != []: node3 = SubElement(node2, 'class', name="msol." + test_labels[test_names[i]][0] + "." + class_names[i]) else: print( "Found a test case without label in JIRA. Please add class name as label." ) sys.exit(1) rough_string = tostring(root, 'utf-8') reparsed = minidom.parseString(rough_string) pretty_out = reparsed.toprettyxml(indent="\t") with open("first.xml", "w") as f1: f1.write(pretty_out) f1.close() with open("first.xml") as f2: with open("testng.xml", "w") as f3: for line in f2: if "classname" in line: output_line = line.replace("classname", "class-name") elif "threadcount" in line: output_line = line.replace("threadcount", "thread-count") else:
def parse(self, pdf): try: fp = open(pdf.path, 'rb') except IOError as e: logging.error("PDFMinerParser.parse unable to open PDF: %s" % e) return parser = PDFParser(fp) doc = PDFDocument(parser) if doc.found_eof and doc.eof_distance > 3: pdf.blob = parser.read_from_end(doc.eof_distance) visited = set() self.treebuild.start("pdf", {"path": pdf.path}) for xref in doc.xrefs: for objid in xref.get_objids(): if objid in visited: continue visited.add(objid) obj_attrs = {"id": str(objid), "type": "normal"} obj_data = '' obj_xml = self.treebuild.start("object", obj_attrs) try: self.dump(doc.getobj(objid)) except pdftypes.PDFObjectNotFound as e: obj_xml.set("type", "malformed") obj_data = parser.read_n_from(xref.get_pos(objid)[1], 4096) obj_data = obj_data.replace('<', '0x3C') except TypeError: obj_xml.set("type", "unknown") obj_data = parser.read_n_from(xref.get_pos(objid)[1], 512) except Exception as e: obj_xml.set("type", "exception") obj_data = parser.read_n_from(xref.get_pos(objid)[1], 512) self.add_xml_node("exception", {}, e.message) self.treebuild.data(obj_data) try: self.treebuild.end("object") except AssertionError as e: logging.error("Parse end object error: %s" % e) sys.stderr.write("%s\n" % tostring(obj_xml)) self.treebuild.start("trailer", {}) self.dump(xref.trailer) self.treebuild.end("trailer") self.treebuild.end("pdf") pdf.xml = self.treebuild.close() pdf.errors = doc.errors pdf.bytes_read = parser.BYTES pdf.parsed = True fp.close()
def search(self, params = {}, header = 0, count = False): if not isinstance(params, dict): raise biomart.BiomartException("'params' argument must be a dict") if self.verbose: print("[BiomartDataset:'%s'] Searching using following params:" % self.name) pprint.pprint(params) # read filters and attributes from params filters = params.get('filters', {}) attributes = params.get('attributes', []) # check filters for filter_name, filter_value in filters.items(): dataset_filter = self.filters.get(filter_name, None) if not dataset_filter: self.show_filters() raise biomart.BiomartException("The filter '%s' does not exist." % filter_name) if len(dataset_filter.accepted_values) > 0 and filter_value not in dataset_filter.accepted_values: error_msg = "The value '%s' for filter '%s' cannot be used." % (filter_value, filter_name) error_msg += " Use one of: [%s]" % ", ".join(map(str, dataset_filter.accepted_values)) raise biomart.BiomartException(error_msg) # check attributes unless we're only counting if not count: # no attributes given, use default attributes if not attributes: attributes = [a.name for a in self.attributes.values() if a.is_default] # if no default attributes have been defined, raise an exception if not attributes: raise biomart.BiomartException("at least one attribute is required, none given") for attribute_name in attributes: if attribute_name not in self.attributes.keys(): self.show_attributes() raise biomart.BiomartException("The attribute '%s' does not exist." % attribute_name) # selected attributes must belong to the same attribute page. if len(set([self.attributes[a].attribute_page for a in attributes])) > 1: self.show_attributes() raise biomart.BiomartException("You must use attributes that belong to the same attribute page.") # filters and attributes looks ok, start building the XML query root = Element('Query') root.attrib.update({ 'virtualSchemaName': 'default', # TODO: use database virtualSchemaName instead (if any error) 'formatter': 'TSV', 'header': str(header), 'uniqueRows': '1', 'datasetConfigVersion': '0.6', 'count': count is True and '1' or '' }) dataset = SubElement(root, "Dataset") dataset.attrib.update({ 'name': self.name, 'interface': self.interface }) # Add filters to the XML query for filter_name, filter_value in filters.items(): dataset_filter = self.filters[filter_name] filter_elem = SubElement(dataset, "Filter") filter_elem.set('name', filter_name) if 'boolean_list' == dataset_filter.filter_type: if filter_value is True or filter_value.lower() in ('included', 'only'): filter_elem.set('excluded', '0') elif filter_value is False or filter_value.lower() == 'excluded': filter_elem.set('excluded', '1') else: if isinstance(filter_value, list) or isinstance(filter_value, tuple): filter_value = ",".join(map(str, filter_value)) filter_elem.set('value', str(filter_value)) # Add attributes to the XML query, unless we're only counting if not count: for attribute_name in attributes: attribute_elem = SubElement(dataset, "Attribute") attribute_elem.set('name', str(attribute_name)) if self.verbose: print("[BiomartDataset] search query:\n%s" % tostring(root)) return self.server.get_request(query = tostring(root))