def authorize(self, account="domain", password=None):
if not password:
raise SessionException("A password is required for authorization")
cp = ClientPassword()
key = cp.genClientKey()
self.send(Session.__AUTH_REQ % self.__service)
response = self.__sendAndReceive(Session.__SRP_REQ % (key, account))
root = ElementTree.fromstring(response)
if root.tag != "SRPResponse":
self.close()
raise SessionException("Request for authorization was denied")
salt = root.get("Salt")
serverKey = root.get("ServerKey")
sessionKey = cp.computeSessionKey(account, password, salt, serverKey)
self._setCiphers(RC4Cipher(sessionKey), RC4Cipher(sessionKey))
verifyMessage = self.recv()
try:
root = ElementTree.fromstring(verifyMessage)
except:
self.close()
raise SessionException("Failed to establish session with password")
if root.tag != "PasswordVerify":
self.close()
raise SessionException("Unexpected verification response: " + root.tag)
self.send(verifyMessage)
def test_affiliate_with_supported_seats(self):
""" Verify that endpoint returns course runs for verified and professional seats only. """
response = self.client.get(self.affiliate_url)
self.assertEqual(response.status_code, 200)
root = ET.fromstring(response.content)
self.assertEqual(1, len(root.findall('product')))
self.assert_product_xml(
root.findall('product/[pid="{}-{}"]'.format(self.course_run.key, self.seat_verified.type))[0],
self.seat_verified
)
# Add professional seat.
seat_professional = SeatFactory(course_run=self.course_run, type=Seat.PROFESSIONAL)
response = self.client.get(self.affiliate_url)
root = ET.fromstring(response.content)
self.assertEqual(2, len(root.findall('product')))
self.assert_product_xml(
root.findall('product/[pid="{}-{}"]'.format(self.course_run.key, self.seat_verified.type))[0],
self.seat_verified
)
self.assert_product_xml(
root.findall('product/[pid="{}-{}"]'.format(self.course_run.key, seat_professional.type))[0],
seat_professional
)
def data_import(self, odm):
"""
Return result of request to import data to an instance.
Required params:
:param odm: odm xml data for import, must be utf-8 encoded and unescaped.
:type odm: str.
:returns: contents of the method response in an OrderedDict, or None.
"""
envelope_copy = et.fromstring(self.envelope)
body = [i for i in envelope_copy.iterfind(
'.//se:Body', {'se': self.ns_se})][0]
method_name = 'importRequest'
import_request = et.Element(et.QName(self.ns_data, method_name))
body.append(import_request)
odm_node = et.Element('odm')
odm_node.append(et.fromstring(odm))
import_request.append(odm_node)
envelope = et.tostring(envelope_copy)
response = self.request(
self.ocws_url, envelope, method_name, self.ns_data)
return response
def append_to(self, element):
branch_part = ""
if not self.is_on_master:
branch_part = ' branch="%s"' % self.__branch
material_name_part = ""
if self.__material_name is not None:
material_name_part = ' materialName="%s"' % self.__material_name
polling_part = ''
if not self.__polling:
polling_part = ' autoUpdate="false"'
destination_directory_part= ''
if self.__destination_directory:
destination_directory_part = ' dest="%s"' % self.__destination_directory
new_element = ET.fromstring(('<git url="%s"' % self.__url) + branch_part + material_name_part + polling_part + destination_directory_part + ' />')
if self.ignore_patterns:
filter_element = ET.fromstring("<filter/>")
new_element.append(filter_element)
sorted_ignore_patterns = list(self.ignore_patterns)
sorted_ignore_patterns.sort()
for ignore_pattern in sorted_ignore_patterns:
filter_element.append(ET.fromstring('<ignore pattern="%s"/>' % ignore_pattern))
element.append(new_element)
def set_owner(conn, uuid, owner):
"""Set ON_OWNER by uuid.
@param uuid: uuid of the VM
@param owner: string representing owner
"""
vmid = get_id_by_uuid(conn, uuid)
if not vmid:
return
vm = conn.lookupByID(vmid)
domain = ET.fromstring(vm.XMLDesc(0))
metadata = domain.find('./metadata') or ET.SubElement(domain, 'metadata')
owner_e = ET.SubElement(metadata, ET.QName('urn:opennode-tui', 'owner'))
owner_e.text = owner
## TODO: cleanup
open('/etc/libvirt/qemu/%s.xml' % (vm.name()), 'w').write(ET.tostring(domain))
data = open('/etc/libvirt/qemu/%s.xml' % (vm.name()), 'r').read()
domain_n = ET.fromstring(data)
owner_e = domain_n.find('./metadata/{urn:opennode-tui}owner')
assert owner_e is not None
assert owner_e.text == owner
return owner_e.text
def main():
count = 1
start_title = parse_args()
print "%i. %s" % (count, start_title)
count += 1
next_page = start_title
link_list = []
next_page = next_page.replace(" ", "+").encode('utf-8')
xml = urllib2.urlopen(
'https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=links&indexpageids=1&titles=' + next_page + '&plnamespace=&pllimit=5000')
root = ET.fromstring(xml.read())
for child in root.iter('pl'):
if ':' not in child.get('title'):
link_list.append(child.get('title'))
next_page = random.choice(link_list)
while next_page:
last_list = link_list
link_list = []
while not link_list:
next_page = random.choice(last_list)
print "%i. %s" % (count, next_page)
count += 1
next_page = next_page.replace(" ", "+").encode('utf-8')
xml = urllib2.urlopen(
'https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=links&indexpageids=1&titles=' + next_page + '&plnamespace=&pllimit=5000')
root = ET.fromstring(xml.read())
for child in root.iter('pl'):
if ':' not in child.get('title'):
link_list.append(child.get('title'))
def test_remove_doubletheme(self):
context = ET.fromstring("""
<contexte xmlns="http://olst.ling.umontreal.ca/dicoenviro/">
<contexte-texte>I compare apples and oranges</contexte-texte>
<participant type="Act" role="Agent">
<fonction-syntaxique nom="Subject">
<groupe-syntaxique nom="NP">I</groupe-syntaxique>
</fonction-syntaxique>
</participant>
<lexie-att>compare</lexie-att>
<participant type="Act" role="Theme">
<fonction-syntaxique nom="Object">
<groupe-syntaxique nom="NP">apples</groupe-syntaxique>
</fonction-syntaxique>
</participant>
and
<participant type="Act" role="Theme">
<fonction-syntaxique nom="Object">
<groupe-syntaxique nom="NP">oranges</groupe-syntaxique>
</fonction-syntaxique>
</participant>
</contexte>
""")
sentence_text, frame = xmlcontext_to_frame(
"http://olst.ling.umontreal.ca/dicoenviro/", 'compare', context)
vn_frame = ET.fromstring(
"""<SYNTAX><NP value="Agent" /><VERB /><NP value="Theme" /></SYNTAX>""")
self.assertEqual(syntax_to_str(frame), syntax_to_str(vn_frame))
def test_get_xml(self):
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject', value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element('MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
# test if xml from get_xml() is well formed
ET.fromstring(self.resModelProgram.metadata.get_xml())
xml_doc = self.resModelProgram.metadata.get_xml()
# check to see if the specific metadata are in the xml doc
self.assertTrue('5.1.011' in xml_doc)
self.assertTrue('Fortran' in xml_doc)
self.assertTrue('Windows' in xml_doc)
self.assertTrue(release_date in xml_doc)
self.assertTrue('http://www.hydroshare.org' in xml_doc)
self.assertTrue('http://www.github.com' in xml_doc)
self.assertTrue('releaseNote.pdf' in xml_doc)
self.assertTrue('manual.pdf' in xml_doc)
self.assertTrue('utilities.exe' in xml_doc)
self.assertTrue('sourceCode.zip' in xml_doc)
def _get_game_xml(self, steam_id, mode):
"""Get the XML list of games"""
fname = "games_{}.xml".format(steam_id)
if mode == "read":
log.info("Reading from {}".format(fname))
with open(fname, 'rb') as f:
content = f.read()
else:
url = self.userUrl.format(steam_id=steam_id)
log.debug("Requesting {}".format(url))
f = urlopen(url)
content = f.read()
if mode == "write":
log.info("Writing to {}".format(fname))
with open(fname, 'wb') as f:
f.write(content)
try:
root = ET.fromstring(content)
except UnicodeEncodeError:
root = ET.fromstring(content.encode('utf-8'))
return root
def do_manage_video_for_album(video_id, provider, context, id_filter, add):
client = provider.get_client(context)
items = []
root = ET.fromstring(client.get_albums(page=1))
do_xml_error(context, provider, root)
albums = root.find('albums')
if albums is not None:
for album in albums:
album_id = album.get('id')
if (add and album_id not in id_filter) or (not add and album_id in id_filter):
album_name = album.find('title').text
items.append((album_name, album_id))
pass
pass
pass
if not items:
if add:
context.get_ui().show_notification(context.localize(provider._local_map['vimeo.adding.no-album']), time_milliseconds=5000)
else:
context.get_ui().show_notification(context.localize(provider._local_map['vimeo.removing.no-album']), time_milliseconds=5000)
pass
return False
result = context.get_ui().on_select(context.localize(provider._local_map['vimeo.select']), items)
if result != -1:
root = ''
if add:
root = ET.fromstring(client.add_video_to_album(video_id, result))
else:
root = ET.fromstring(client.remove_video_from_album(video_id, result))
pass
return do_xml_error(context, provider, root)
return True
def _create_html_file(self, sourcefile, htmlfile, errors):
name = self.generator.get_name()
root = ElementTree.fromstring(CPPCHECK_HTML_FILE)
title = root.find("head/title")
title.text = "cppcheck - report - %s" % name
body = root.find("body")
for div in body.findall("div"):
if div.get("id") == "page":
page = div
break
for div in page.findall("div"):
if div.get("id") == "header":
h1 = div.find("h1")
h1.text = "cppcheck report - %s" % name
if div.get("id") == "content":
content = div
srcnode = self.generator.bld.root.find_node(sourcefile)
hl_lines = [e["line"] for e in errors if e.has_key("line")]
formatter = CppcheckHtmlFormatter(linenos=True, style="colorful", hl_lines=hl_lines, lineanchors="line")
formatter.errors = [e for e in errors if e.has_key("line")]
css_style_defs = formatter.get_style_defs(".highlight")
lexer = pygments.lexers.guess_lexer_for_filename(sourcefile, "")
s = pygments.highlight(srcnode.read(), lexer, formatter)
table = ElementTree.fromstring(s)
content.append(table)
s = ElementTree.tostring(root, method="html")
s = CCPCHECK_HTML_TYPE + s
node = self.generator.path.get_bld().find_or_declare(htmlfile)
node.write(s)
return css_style_defs
def render(self, context):
path = self.path.resolve(context)
if not re.search(r'\.svg$', path):
raise template.TemplateSyntaxError(
"SVG template nodes must be given a path to a SVG file to return.") # NOQA
fullpath = finders.find(path)
if fullpath is None:
raise template.TemplateSyntaxError(
"SVG file ({}) not found.".format(fullpath))
content = open(fullpath, 'r').read()
# Test if it is valid XML
ET.fromstring(content)
if self.kwargs:
kwargs = {x: y.resolve(context) for x, y in self.kwargs.items()}
content = re.sub(
r'(<svg[^>]*)', r'\1{}'.format(flatatt(kwargs)), content,
flags=re.MULTILINE)
if self.varname is None:
return content
context[self.varname] = mark_safe(content)
return ''
def walk(deps, p_node, depth, p_text):
global max_depth, w_freq
global w_connection_matrix
kids = deps.findall("./dep/governor[@idx=\""+str(p_node)+"\"]/..")
if depth > max_depth:
max_depth = depth
i = 1
for k in kids:
#ET.dump(k)
k_node = k.find("./dependent")
# ET.dump(k_node)
k_id = k_node.attrib["idx"]
# print int(p_node), int(k_id)
#w_connection_matrix[int(p_node)][int(k_id)] = 1
if k.attrib["type"] in ["ccomp", "root"]:
walk(deps, k_id, depth+1, k_node.text)
k.append(ET.fromstring("<depth>"+str(depth+1)+"</depth>"))
#elif k.attrib["type"].endswith("cl"):
# walk(deps, k_id, depth-1, k_node.text)
# k.append(ET.fromstring("<depth>"+str(depth-1)+"</depth>"))
else:
walk(deps, k_id, depth, k_node.text)
k.append(ET.fromstring("<depth>"+str(depth)+"</depth>"))
#k.append(ET.fromstring("<l_h_weight>"+str(gram_prob[p_text+" "+k.attrib["type"]])+"</l_h_weight>"))
k.append(ET.fromstring("<l_h_weight>"+cal_l_h_weight(p_text, k.attrib["type"])+"</l_h_weight>"))
k.append(ET.fromstring("<d_weight>"+str(d_weight(k_node.text))+"</d_weight>"))
i+=1
def next(self):
if self.doc_current >= self.docs_total:
if self.docs_fetched != self.docs_total:
raise ValueError(
"Number of documents fetched ({0}) doesn't match \
total reported by server ({1})"
.format(self.docs_fetched, self.docs_total))
else:
raise StopIteration
if hasattr(self, 'page_end') and self.page_current > self.page_end:
raise StopIteration
# for the given page of public photos results,
# for each <photo> tag, create an object with id, server & farm saved
# then get the info for the photo and add to object
# return the full list of objects to the harvest controller
tree = ET.fromstring(urllib.urlopen(self.url_current).read())
photo_list = tree.findall('.//photo')
objset = []
for photo in photo_list:
photo_obj = photo.attrib
url_photo_info = self.url_get_photo_info_template.format(
api_key=self.api_key, photo_id=photo_obj['id'])
ptree = ET.fromstring(urllib.urlopen(url_photo_info).read())
photo_info = ptree.find('.//photo')
photo_obj.update(photo_info.attrib)
photo_obj.update(
self.parse_tags_for_photo_info(photo_info.getchildren()))
self.docs_fetched += 1
objset.append(photo_obj)
self.page_current += 1
self.doc_current += len(objset)
return objset
def play(item):
logger.info("[mundonick.py] play video: " + item.url)
itemlist=[]
permalink = 'uri=mgid:uma:video:mundonick.com:' + item.url
data = scrapertools.cachePage(__urlconfig__ + permalink)
if (data == ''):
return itemlist
#logger.info(data)
import xml.etree.ElementTree as xmlet
configuration = xmlet.fromstring(data)
swfurl = configuration.find('.//player//URL').text
feedurl = configuration.find('.//player//feed').text
data = scrapertools.cachePage(feedurl)
#logger.info(data)
feed = xmlet.fromstring(data)
description = feed.find('.//item/description').text.encode("utf8","ignore").replace('<i>', '').replace('</i>', ' |').replace('<br/>', ' ').replace('LA', '');
#mediacontent = feed.find('{http://search.yahoo.com/mrss/}content').get('url')
patron = '<media:content type="text/xml" isDefault="true"\nurl="([^"]+)">'
matches = re.compile(patron,re.DOTALL).findall(data)
if DEBUG: scrapertools.printMatches(matches)
if matches: mediacontent = matches[0]
#data = scrapertools.cachePage(mediacontent)
#logger.info(data)
logger.info(description)
itemlist.append( Item(channel=__channel__, action="play", title=description, url=mediacontent, server="mundonick", thumbnail=item.thumbnail, folder=False) )
return itemlist
def decode(as_xml):
"""Decodes a document from an xml string.
:param as_xml: Document xml representation.
:type as_xml: unicode | str | ET.Element
:returns: A pyesdoc document instance.
:rtype: object
"""
# Convert to etree.
if isinstance(as_xml, unicode):
as_xml = ET.fromstring(convert.unicode_to_str(as_xml))
elif isinstance(as_xml, str):
as_xml = ET.fromstring(as_xml)
if not isinstance(as_xml, ET.Element):
raise TypeError("Cannot decode non xml documents")
# Get document type key.
doc_type_key = as_xml.find("meta/type").text
# Get document type.
doc_type = ontologies.get_type_from_key(doc_type_key)
if doc_type is None:
raise ValueError("meta.type key cannot be mapped to an ontology type.")
return _decode(as_xml, doc_type, False)
def _parse_log(self, log):
"""
Parse the "log" section produced by BoostTest.
This is always a XML file, and from this we produce most of the
failures possible when running BoostTest.
"""
# Fatal errors apparently generate invalid xml in the form:
# <FatalError>...</FatalError><TestLog>...</TestLog>
# so we have to manually split it into two xmls if that's the case.
parsed_elements = []
if log.startswith('<FatalError'):
fatal, log = log.split('</FatalError>')
fatal += '</FatalError>' # put it back, removed by split()
fatal_root = ElementTree.fromstring(fatal)
fatal_root.text = 'Fatal Error: %s' % fatal_root.text
parsed_elements.append(fatal_root)
log_root = ElementTree.fromstring(log)
parsed_elements.extend(log_root.findall('Exception'))
parsed_elements.extend(log_root.findall('Error'))
parsed_elements.extend(log_root.findall('FatalError'))
result = []
for elem in parsed_elements:
filename = elem.attrib['file']
linenum = int(elem.attrib['line'])
result.append(BoostTestFailure(filename, linenum, elem.text))
return result
def load_settings():
global settings_dic
defaults = {}
from xml.etree import ElementTree
settings=""
encontrado = False
#Lee el archivo XML (si existe)
if os.path.exists(configfilepath):
while len(settings)<> os.path.getsize(configfilepath) or len(settings)==0:
settings = open(configfilepath, 'rb').read()
root = ElementTree.fromstring(settings)
for target in root.findall("setting"):
settings_dic[target.get("id")] = target.get("value")
defaultsettings=""
while len(defaultsettings)<> os.path.getsize(menufilepath) or len(defaultsettings)==0:
defaultsettings = open(menufilepath, 'rb').read()
root = ElementTree.fromstring(defaultsettings)
for category in root.findall("category"):
for target in category.findall("setting"):
if target.get("id"):
defaults[target.get("id")] = target.get("default")
for key in defaults:
if not key in settings_dic:
settings_dic[key] = defaults[key]
set_settings(settings_dic)
def AddFile(self, ALMUrl, files, *args):
if self.__headers["Cookie"] is not None:
fileToSend = {}
for file in files:
fileToSend[file.filename] = file.body
locheaders = self.__headers.copy()
#locheaders[u'Content-Type'] = u'application/octet-stream'
r = requests.post(ALMUrl.__getattr__(*args),
headers=locheaders,
files=fileToSend,
auth=self.__user_pass)
if r.status_code == 201:
self.__Logging.info(u"[ALMSession] Create success", u"URL:", ALMUrl.__getattr__(*args))
return 0, r.headers._store['location'][1].split('/')[-1]
elif r.status_code == 500:
if isinstance(r.text, unicode):
exceptionxml = ET.fromstring(r.text.encode('utf8','ignore'))
else:
exceptionxml = ET.fromstring(r.text)
self.__Logging.error(u"[ALMSession] Create ALM function with errors", exceptionxml[0].text, exceptionxml[1].text, u"PATH:", ALMUrl.__getattr__(*args), u"DATA:", data, u"HEADERS:", self.__headers)
return int(r.status_code), ""
else:
self.__Logging.error(u"[ALMSession] Create ALM function with errors", r.status_code, r.reason, u"PATH:", ALMUrl.__getattr__(*args), u"HEADERS:", self.__headers)
return int(r.status_status), ""
else:
self.__Logging.error(u"[ALMSession] Create ALM function with errors", u"1", u"httplib2.Http not initialized")
return 1, ""
def test_define_and_retrieve(self):
conn = self.get_openAuth_curry_func()('qemu:///system')
self.assertEquals(conn.listDomainsID(), [])
conn.defineXML(get_vm_xml())
dom = conn.lookupByName('testname')
xml = dom.XMLDesc(0)
ElementTree.fromstring(xml)
def call(self):
"""
Makes a request to cghub server.
Returns generator that returns Result objects.
"""
self.patch_input_data()
query = self.build_query()
url = '%s%s' % (self.server_url, self.uri)
if query:
url = '%s?%s' % (url, query)
xml = self.get_source_file(url)
if self.format == self.FORMAT_JSON:
results = ijson.items(xml, 'result_set.results.item')
for item in results:
yield item
else:
# http://docs.python.org/dev/library/xml.dom.pulldom.html
doc = pulldom.parse(xml)
for event, node in doc:
if event == pulldom.START_ELEMENT:
if node.tagName == 'Result':
doc.expandNode(node)
# convert to python object
# http://docs.python.org/2/library/xml.etree.elementtree.html
result_xml = node.toxml(encoding='utf-8')
tree = ElementTree.fromstring(result_xml)
result = Result(tree)
yield self.patch_result(result, result_xml)
elif node.tagName == 'Hits':
doc.expandNode(node)
tree = ElementTree.fromstring(node.toxml())
self.hits = int(tree.text)
def as_html():
table = CountryTable(MEMORY_DATA)
root = ET.fromstring(table.as_html())
assert len(root.findall('.//thead/tr'))== 1
assert len(root.findall('.//thead/tr/th')) == 4
assert len(root.findall('.//tbody/tr')) == 4
assert len(root.findall('.//tbody/tr/td')) == 16
# no data with no empty_text
table = CountryTable([])
root = ET.fromstring(table.as_html())
assert 1 == len(root.findall('.//thead/tr'))
assert 4 == len(root.findall('.//thead/tr/th'))
assert 0 == len(root.findall('.//tbody/tr'))
# no data WITH empty_text
table = CountryTable([], empty_text='this table is empty')
root = ET.fromstring(table.as_html())
assert 1 == len(root.findall('.//thead/tr'))
assert 4 == len(root.findall('.//thead/tr/th'))
assert 1 == len(root.findall('.//tbody/tr'))
assert 1 == len(root.findall('.//tbody/tr/td'))
assert int(root.find('.//tbody/tr/td').attrib['colspan']) == len(root.findall('.//thead/tr/th'))
assert root.find('.//tbody/tr/td').text == 'this table is empty'
# with custom template
table = CountryTable([], template="django_tables2/table.html")
table.as_html()
def setup_github_authentication():
auth_strategy = ET.fromstring('''
<authorizationStrategy class="org.jenkinsci.plugins.GithubAuthorizationStrategy">
<rootACL>
<organizationNameList class="linked-list" />
<adminUserNameList class="linked-list">
<string>mfournier</string>
<string>octo</string>
</adminUserNameList>
<authenticatedUserReadPermission>true</authenticatedUserReadPermission>
<useRepositoryPermissions>false</useRepositoryPermissions>
<authenticatedUserCreateJobPermission>false</authenticatedUserCreateJobPermission>
<allowGithubWebHookPermission>true</allowGithubWebHookPermission>
<allowCcTrayPermission>true</allowCcTrayPermission>
<allowAnonymousReadPermission>true</allowAnonymousReadPermission>
</rootACL>
</authorizationStrategy>
''')
security_realm = ET.fromstring('''
<securityRealm class="org.jenkinsci.plugins.GithubSecurityRealm">
<githubWebUri>https://github.com</githubWebUri>
<githubApiUri>https://api.github.com</githubApiUri>
<clientID />
<clientSecret />
</securityRealm>
''')
root.find('useSecurity').text = 'true'
root.remove(root.find('authorizationStrategy'))
root.append(auth_strategy)
root.remove(root.find('securityRealm'))
root.append(security_realm)
root.find('securityRealm').find('clientID').text = vars['github_clientid']
root.find('securityRealm').find('clientSecret').text = vars['github_clientsecret']
def mine_disease_to_nct(ldisease, fout=None, ctmin=100):
url = 'http://clinicaltrials.gov/search?cond=%s&displayxml=true&count=%s'
log.info('found %d disease to process \n' % len(ldisease))
ldisease = sorted(map(lambda x: ' '.join(x.lower().split()), ldisease))
nct_disease = defaultdict(list)
c = 1
for d in sorted(ldisease):
log.info('processing: "%s"' % d)
d = d.replace(',', '')
fd = d.replace(' ', '+')
# number of trials
xmltree = xml_parser.fromstring(download_web_data(url % (fd, '0')))
nres = xmltree.get('count')
try:
if int(nres) < ctmin:
log.info(' --- found only %s trials - skipping \n' % nres)
continue
except Exception as e:
log.error(e)
continue
# list of trials
xmltree = xml_parser.fromstring(download_web_data(url % (fd, nres)))
lnct = xmltree.findall('clinical_study')
# nct = set()
for ct in lnct:
try:
cod = ct.find('nct_id')
# nct.add(cod.text)
nct_disease[cod.text].append(d)
except Exception as e:
log.error(e)
return nct_disease
def _process_case_block(domain, case_block, attachments, old_case_id):
def get_namespace(element):
m = re.match('\{.*\}', element.tag)
return m.group(0)[1:-1] if m else ''
def local_attachment(attachment, old_case_id, tag):
mime = attachment['server_mime']
size = attachment['attachment_size']
src = attachment['attachment_src']
cached_attachment = get_cached_case_attachment(domain, old_case_id, tag)
attachment_meta, attachment_stream = cached_attachment.get()
return UploadedFile(attachment_stream, src, size=size, content_type=mime)
# Remove namespace because it makes looking up tags a pain
root = ET.fromstring(case_block)
xmlns = get_namespace(root)
case_block = re.sub(' xmlns="[^"]+"', '', case_block, count=1)
root = ET.fromstring(case_block)
tag = "attachment"
xml_attachments = root.find(tag)
ret_attachments = {}
if xml_attachments:
for attach in xml_attachments:
attach.attrib['from'] = 'local'
attach.attrib['src'] = attachments[attach.tag]['attachment_src']
ret_attachments[attach.attrib['src']] = local_attachment(attachments[attach.tag], old_case_id, attach.tag)
# Add namespace back in without { } added by ET
root.attrib['xmlns'] = xmlns
return ET.tostring(root), ret_attachments
def listen(self):
while True:
data = self.recv_XML()
tag = data[1:data.find(" ")]
if tag == "u": #User joined
user_data = ET.fromstring(data).attrib
# Sometimes users re-login without properly logging out (e.g.
# when their rank is changed), so we don't want to have any
# duplicates.
self.users.remove(user_data["u"])
self.users.append(user_data)
print(self.users)
if tag == "l": #User left
user_id = ET.fromstring(data).attrib["u"]
self.users.remove(id)
print(self.users)
if tag == "m": #User spoke
content = ET.fromstring(data).attrib
user_id = content["u"]
user_message = content["t"]
if is_foul(user_message):
self.kick(user_id, "Offensive language")
elif user_message[0] == "!":
self.parse_command(user_message, user_id)
if tag == "logout":
return
def test_make_xml(self):
hostname, servicename, instruction = 'MixedCase', 'Mixed Case', '/cpu/count'
check = ncpacheck.NCPACheck(self.config, instruction, hostname, servicename)
xml = nrdp.make_xml(check).toxml()
tree = ET.fromstring(xml)
servicename = tree.findall('servicename')[0]
hostname = tree.findall('hostname')[0]
stdout = tree.findall('output')[0]
state = tree.findall('state')[0]
self.assertEquals(tree.tag, 'checkresult')
self.assertEquals(tree.attrib, {'type': 'service'})
self.assertEquals(servicename.text, 'Mixed Case')
self.assertEquals(hostname.text, 'MixedCase')
self.assertIsNotNone(stdout)
self.assertIsNotNone(state)
hostname, servicename, instruction = 'testing_host', '__HOST__', '/cpu/count'
check = ncpacheck.NCPACheck(self.config, instruction, hostname, servicename)
xml = nrdp.make_xml(check).toxml()
tree = ET.fromstring(xml)
hostname = tree.findall('hostname')[0]
stdout = tree.findall('output')[0]
state = tree.findall('state')[0]
self.assertEquals(tree.tag, 'checkresult')
self.assertEquals(tree.attrib, {'type': 'host'})
self.assertEquals(tree.findall('servicename'), [])
self.assertEquals(hostname.text, 'testing_host')
self.assertIsNotNone(stdout)
self.assertIsNotNone(state)
def _create_html_file(self, sourcefile, htmlfile, errors):
name = self.generator.get_name()
root = ElementTree.fromstring(CPPCHECK_HTML_FILE)
title = root.find('head/title')
title.text = 'cppcheck - report - %s' % name
body = root.find('body')
for div in body.findall('div'):
if div.get('id') == 'page':
page = div
break
for div in page.findall('div'):
if div.get('id') == 'header':
h1 = div.find('h1')
h1.text = 'cppcheck report - %s' % name
if div.get('id') == 'content':
content = div
srcnode = self.generator.bld.root.find_node(sourcefile)
hl_lines = [e['line'] for e in errors if e.has_key('line')]
formatter = CppcheckHtmlFormatter(linenos=True, style='colorful', hl_lines=hl_lines, lineanchors='line')
formatter.errors = [e for e in errors if e.has_key('line')]
css_style_defs = formatter.get_style_defs('.highlight')
lexer = pygments.lexers.guess_lexer_for_filename(sourcefile, "")
s = pygments.highlight(srcnode.read(), lexer, formatter)
table = ElementTree.fromstring(s)
content.append(table)
s = ElementTree.tostring(root, method='html')
s = CCPCHECK_HTML_TYPE + s
node = self.generator.path.get_bld().find_or_declare(htmlfile)
node.write(s)
return css_style_defs
def test_get_banks_details_not_found(self, mock_get):
with self.app as c:
with c.session_transaction() as session:
session["application_type"] = "cancel"
session["images"] = ["/document/1/image/1"]
session["original_image_data"] = ["/document/1/image/1"]
response = self.app.post("/get_details", data={"reg_no": "50001"})
html = response.data.decode("utf-8")
tree = ET.fromstring(html)
assert tree.find('.//*[@id="class_data"]/h4').text == "Retrieve original details"
assert tree.find('.//*[@id="class_data"]/p/strong').text == "Registration not found please re-enter"
# Check Rectify leg
with self.app as c:
with c.session_transaction() as session:
session["application_type"] = "rectify"
session["images"] = ["/document/1/image/1"]
session["original_image_data"] = ["/document/1/image/1"]
response = self.app.post("/get_details", data={"reg_no": "50001"})
html = response.data.decode("utf-8")
tree = ET.fromstring(html)
assert tree.find('.//*[@id="main"]/div[2]/div/h4').text == "Bankruptcy Rectification"
assert tree.find('.//*[@id="class_data"]/p/strong').text == "Registration not found please re-enter"
def get_job_status(self, http_object):
"""
performs the get of job status and verifies whether
the job request is completetd successfully
Args:
http_object : HTTP Request object of a job request
"""
root = etree.fromstring(http_object.response.content)
job_id = root.findall(".//{%s}JobID"%(self.ns))[0].text
http_job_object = HTTPClient.HTTPClient("uom", self.ip,
"jobs", self.content_type,
self.session )
job_status = " "
print("Processing...")
while job_status != "COMPLETED_OK" or job_status != "COMPLETED_WITH_WARNINGS":
http_job_object.HTTPGet(append=job_id)
if http_job_object.response_b:
root = etree.fromstring(http_job_object.response.content)
job_status = root.findall(".//{%s}Status"%(self.ns))[0].text
if job_status == "FAILED_BEFORE_COMPLETION" or job_status == "COMPLETED_WITH_ERROR":
try:
error = root.findall(".//{%s}Message"%(self.ns))[0].text
log.log_error(error)
except IndexError:
log.log_error("An error occured while performing job request")
break
print("Job Status :",job_status)
import requests
import xml.etree.ElementTree as ET
target_url = 'https://jlp.yahooapis.jp/MAService/V1/parse'
client_id = 'dj00aiZpPWdKaWZIMDFZMklzdCZzPWNvbnN1bWVyc2VjcmV0Jng9ZWU-'
sentence = 'すもももももももものうち'
data = {"appid": client_id, "results": "ma", "sentence": sentence}
response = requests.post(target_url, data=data)
root = ET.fromstring(response.text)
print(root)
def get_element_tree(self):
if self._et is None:
resp = requests.get(self.url)
self._et = ET.fromstring(resp.content)
return self._et
def searchXML(self, cityName,storeName="",roadAddress="",localAddress=""):
#self.listBox.delete(0,"end")
self.treeview.delete(*self.treeview.get_children())
cityCode = spam.queryCode(cityName.split('/')[1])
cnt=0
#encText = urllib.parse.quote("부천시")
storeName=urllib.parse.quote(storeName)
roadAddress=urllib.parse.quote(roadAddress)
localAddress=urllib.parse.quote(localAddress)
for i in range(1, 100): #원래 100에 1000개많큼 읽기
url = "https://openapi.gg.go.kr/RegionMnyFacltStus?KEY=2902618a276345c78da7557883182ca9&pIndex=" + str(
i) + "&pSize=1000&SIGUN_CD=" + str(cityCode)+"&CMPNM_NM="+storeName+"&REFINE_ROADNM_ADDR="+roadAddress+"&REFINE_LOTNO_ADDR="+localAddress
req = urllib.request.Request(url)
resp = urllib.request.urlopen(req)
strXml = resp.read().decode('utf-8')
if 'INFO-200' in strXml:
break
tree = ElementTree.fromstring(strXml)
items = tree.iter("row")
for j in items:
rStr = []
if j.find('SIGUN_NM').text: # 시이름 0
rStr.append(j.find('SIGUN_NM').text)
else:
rStr.append("정보 없음")
if j.find('CMPNM_NM').text: # 상점이름 1
rStr.append(j.find('CMPNM_NM').text)
else:
rStr.append("정보 없음")
if j.find('INDUTYPE_NM').text: # 업종이름 2
rStr.append(j.find('INDUTYPE_NM').text)
else:
rStr.append("정보 없음")
if j.find('REFINE_ROADNM_ADDR').text: # 도로명 주소 3
rStr.append(j.find('REFINE_ROADNM_ADDR').text)
else:
rStr.append("정보 없음")
if j.find('REFINE_LOTNO_ADDR').text: # 지번 주소 4
rStr.append(j.find('REFINE_LOTNO_ADDR').text)
else:
rStr.append("정보 없음")
if j.find('TELNO').text: # 전화 번호 5
rStr.append(j.find('TELNO').text)
else:
rStr.append("정보 없음")
if j.find('REFINE_ZIP_CD').text: # 우편 번호 6
rStr.append(j.find('REFINE_ZIP_CD').text)
else:
rStr.append("정보 없음")
if j.find('REFINE_WGS84_LAT').text: # 위도 7
rStr.append(j.find('REFINE_WGS84_LAT').text)
else:
rStr.append("정보 없음")
if j.find('REFINE_WGS84_LOGT').text: # 경도 8
rStr.append(j.find('REFINE_WGS84_LOGT').text)
else:
rStr.append("정보 없음")
temp = ""
tuple(rStr)
self.treeview.insert("",'end', text=str(cnt), values=rStr, iid=str(cnt) + "번")
cnt += 1
'''for k in range(len(rStr)):
def parse_xml(xml_string): return ElementTree.fromstring(xml_string)
action="store_true")
args=parser.parse_args()
################################################################################
name=args.author.replace('.','=')
names=name.split()
last=names[len(names)-1]
first='_'.join(names[:len(names)-1])
initial=last[0].lower()
publications=list(map(str.strip, args.publication.split(',')))
################################################################################
import urllib.request
url='http://dblp.uni-trier.de/pers/xx/{}/{}:{}.xml'.format(initial,last,first)
xmlstr=urllib.request.urlopen(url).read().decode('utf-8')
################################################################################
if args.debug:
print('AUTHOR: {}'.format(args.author))
print('SEARCHING FOR: {}'.format(args.publication))
print('URL: {}'.format(url))
################################################################################
import xml.etree.ElementTree as ET
root = ET.fromstring(xmlstr)
count = 0
types={'article':'journal',
'inproceedings':'booktitle'}
for t in types.keys():
for x in root.iter(t):
for publication in publications:
if publication in x.find(types[t]).text:
count+=1
print(count)
def parse_xml(config):
if os.path.exists(config):
return etree.parse(config)
else:
return etree.fromstring(config)
# go through revised files, all the *.original.txt ones.
revisions = collect_filenames(basedir)
for revision in revisions: # doc textarnir
with open(revision.original, encoding='utf-8') as f:
original_texts = f.read().split(';;')
with open(revision.output, encoding='utf-8') as f:
output_texts = f.read().split(';;')
if not len(original_texts) == len(original_texts):
print('mismatch between text versions')
raise Exception
texts = [Revision(*text) for text in zip(original_texts, output_texts)]
for idx, text in enumerate(texts):
outfile = '{}/{}-t{}.xml'.format(outputdir,
revision.original.split('.')[0], idx)
print(outfile)
teidoc = error_tei(*text)
try:
root = ET.fromstring(teidoc)
except:
print('Bad XML')
with open(outfile, 'w', encoding='utf-8') as f:
f.write(teidoc)
import urllib.request, urllib.parse, urllib.error
import xml.etree.ElementTree as ET
import ssl
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
#url = 'http://py4e-data.dr-chuck.net/comments_42.xml'
url = 'http://py4e-data.dr-chuck.net/comments_268799.xml'
uh = urllib.request.urlopen(url, context=ctx)
data = uh.read()
print('Retrieved', len(data), 'characters')
#print(data.decode())
tree = ET.fromstring(data)
results = tree.findall('comments/comment')
# we could replace these process by tree.findall(''.//count')
s = 0
for i in results:
num = i.find('count').text
num = float(num)
s += num
print(s)
if any(args.start_branch[0] in s for s in local_branches):
needs_abandon = True
if needs_abandon:
# Perform the abandon only if the branch already exists
if not args.quiet:
print('Abandoning branch: %s' % args.start_branch[0])
subprocess.check_output(['repo', 'abandon', args.start_branch[0]])
if not args.quiet:
print('')
# Get the master manifest from repo
# - convert project name and revision to a path
project_name_to_data = {}
manifest = subprocess.check_output(['repo', 'manifest'])
xml_root = ElementTree.fromstring(manifest)
projects = xml_root.findall('project')
remotes = xml_root.findall('remote')
default_revision = xml_root.findall('default')[0].get('revision')
#dump project data into the a list of dicts with the following data:
#{project: {path, revision}}
for project in projects:
name = project.get('name')
path = project.get('path')
if path is None:
path=name
revision = project.get('revision')
if revision is None:
for remote in remotes:
def main(sys_argv=None):
if sys_argv is None:
sys_argv = sys.argv[1:]
short_options = ""
long_options = ["piggyback", "servername=", "port=", "username="******"password="******"%s\n" % err)
return 1
opt_servername = None
opt_port = None
opt_username = None
opt_password = None
opt_piggyback_mode = False
for o, a in opts:
if o in ['--piggyback']:
opt_piggyback_mode = True
elif o in ['--servername']:
opt_servername = a
elif o in ['--port']:
opt_port = a
elif o in ['--username']:
opt_username = a
elif o in ['--password']:
opt_password = a
if not opt_servername or not opt_port:
usage()
return 1
url = "http://%s:%s/admin/xml/queues.jsp" % (opt_servername, opt_port)
auth = None
if opt_username:
auth = HTTPBasicAuth(opt_username, opt_password)
try:
response = requests.get(url, auth=auth)
if response.status_code == 401:
raise Exception("Unauthorized")
xml = response.text
data = ET.fromstring(xml)
except Exception as e:
sys.stderr.write("Unable to connect. Credentials might be incorrect: %s\n" % e)
return 1
attributes = ['size', 'consumerCount', 'enqueueCount', 'dequeueCount']
count = 0
output_lines = []
try:
if not opt_piggyback_mode:
output_lines.append("<<<mq_queues>>>")
for line in data:
count += 1
if opt_piggyback_mode:
output_lines.append("<<<<%s>>>>" % line.get('name'))
output_lines.append("<<<mq_queues>>>")
output_lines.append("[[%s]]" % line.get('name'))
stats = line.findall('stats')
values = ""
for job in attributes:
values += "%s " % stats[0].get(job)
output_lines.append(values)
if opt_piggyback_mode:
output_lines.append("<<<<>>>>")
output_lines.append("<<<local:sep(0)>>>")
output_lines.append("0 Active_MQ - Found %s Queues in total" % count)
except Exception as e: # Probably an IndexError
sys.stderr.write("Unable to process data. Returned data might be incorrect: %r" % e)
return 1
print("\n".join(output_lines))
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Download minecraft skins from planet minecraft."
)
parser.add_argument(
"sitemap", type=str, help="the planet minecraft sitemap url to use"
)
parser.add_argument("skins", type=int, help="the number of skins to download")
args = parser.parse_args()
# Get skin page urls
root = ET.fromstring(
requests.get(
args.sitemap,
headers={
"User-Agent": "Mozilla/5.0 (Windows NT x.y; rv:10.0) Gecko/20100101 Firefox/10.0"
},
).text
)
urls = [
el.find("{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text
for el in root.findall("{http://www.sitemaps.org/schemas/sitemap/0.9}url")
if el.find("{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text.startswith(
"https://www.planetminecraft.com/skin"
)
][: args.skins]
with Manager() as manager:
page_queue = manager.Queue()
for url in urls:
page_queue.put(url)
import requests
import urllib.request
import urllib.parse
import xml.etree.ElementTree as ET
import json
import sqlite3
url = "https://www.openstreetmap.org/api/0.6/changeset/71779142"
urlnumber = url.strip("https://www.openstreetmap.org/api/0.6/changeset/")
req = urllib.request.Request(url)
with urllib.request.urlopen(req) as response:
xml_string = response.read()
root = ET.fromstring(xml_string)
child = root[0]
osm = str(child.attrib)
osm2 = osm.split()
lat_p = osm2[17].strip("','")
lon_p = osm2[19].strip("','")
conn = sqlite3.connect("changesetgeo.db")
cur = conn.cursor()
item = (urlnumber, lat_p, lon_p)
cur.execute("INSERT INTO geolatlon VALUES (? , ? , ?)", item)
conn.commit
def download_refs(organism, ref_fpath):
ncbi_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
quast_fields = '&tool=quast&[email protected]'
organism = organism.replace('_', '+')
response = try_send_request(
ncbi_url +
'esearch.fcgi?db=assembly&term=%s+[Organism]&retmax=100' % organism +
quast_fields)
if not response:
return None
xml_tree = ET.fromstring(response)
if xml_tree.find('Count').text == '0': # Organism is not found
return None
ref_id_list = xml_tree.find('IdList').findall('Id')
best_ref_links = []
for id in ref_id_list:
databases = ['assembly_nuccore_refseq', 'assembly_nuccore_insdc']
for db in databases:
response = try_send_request(
ncbi_url +
'elink.fcgi?dbfrom=assembly&db=nuccore&id=%s&linkname="%s"' %
(id.text, db) + quast_fields)
if not response:
continue
xml_tree = ET.fromstring(response)
link_set = xml_tree.find('LinkSet')
if link_set is None:
continue
link_db = xml_tree.find('LinkSet').find('LinkSetDb')
if link_db is None:
continue
ref_links = link_db.findall('Link')
if best_ref_links and len(ref_links) > len(best_ref_links):
continue
best_ref_links = ref_links
if best_ref_links:
break
if best_ref_links and len(best_ref_links) < 3:
break
if not best_ref_links:
return None
if len(best_ref_links) > 500:
logger.info(
'%s has too fragmented reference genome! It will not be downloaded.'
% organism.replace('+', ' '))
return None
ref_ids = sorted(link.find('Id').text for link in best_ref_links)
is_first_piece = False
fasta_files = []
for ref_id in ref_ids:
fasta = try_send_request(
ncbi_url +
'efetch.fcgi?db=sequences&id=%s&rettype=fasta&retmode=text' %
ref_id)
if fasta and fasta[0] == '>':
fasta_files.append(fasta)
fasta_names = [f.split('|')[-1] for f in fasta_files]
with open(ref_fpath, "w") as fasta_file:
for name, fasta in sorted(zip(fasta_names, fasta_files),
key=natural_sort_key):
if not is_first_piece:
is_first_piece = True
else:
fasta = '\n' + fasta.rstrip()
fasta_file.write(fasta.rstrip())
if not os.path.isfile(ref_fpath):
return None
if not is_non_empty_file(ref_fpath):
os.remove(ref_fpath)
return None
return ref_fpath
def table_automaticDictionary(graph, dirName):
#====================================================
#
# BUILD HTML TABLE
#
#====================================================
dataIn = json.loads(graph)
## HEAD OF THE TABLE
# there is an x inside the script pointing to jquery because if it's empty it will
# automatically be written as a closed empty tag in the output and would not work
html = """<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta>
<title>Test collation with normalized tokens</title>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js">x</script>
<script type="text/javascript">
$(document).ready(function(){
$("tr[type='orig']").click(function(){
$(this).next().toggle();
});
});
</script>
<style>
tr[type="normAll"] {display: none;}
tr[type="norm"] {display: none;}
td, th {padding: 20px;border: 1px solid grey; width:100px}
table {border-collapse: collapse;}
.variant {background-color:yellow}
.invariant {background-color:lightgray}
.formal_variant {background-color:darkgray}
</style>
</head>"""
html += """<body>
<div style="font-size:1.1em; margin: 50px">
<p>This table has been automatically generated.
<br/>The code and the documentation are available <a href="https://github.com/elespdn/collation_spelling" target="_blank">here</a>.
<br/>In order to see the token used for the alignment, just click on a cell.
</p>
<p>Legend:
<ul>
<li>
<span class="variant">Substantive variant</span>
</li>
<li>
<span class="formal_variant">Formal variant</span>
</li>
<li>
<span class="invariant">Invariant</span>
</li>
</ul>
</p>
</div><table border="1"><thead><tr>"""
for x in dataIn['table']:
if x[0] is not None:
witName = x[0][0]['_sigil'] # define the witness name
html += "<th>"+witName+"</th>" # write the witness name in the head of the table
html += "</tr></thead><tbody>" ## close thead
for i in range(len(x)): # for 'i' in the length of the witness
istr = str(i) # from int to string, otherwise the following does not work
## CREATE ROW FOR ORIGINAL TOKEN (@type='orig') -
## EACH <td> HAS THE ORIGINAL TOKEN AS CONTENT AND THE NORMALIZED FORM IN @ana
html += "<tr type='orig' id='row"+istr+"_orig'>"
for x in dataIn['table']: # for each witness
element = x[i] # take the first (then at the next iteration take the second, etc.)
if element is not None:
for elementList in element:
origToken = elementList['t'].strip() # strip used for deleting whitespaces
normToken = elementList['n'].strip()
else:
origToken = ' - '
normToken = '_'
html += '<td ana="'+normToken+'">' + origToken+'</td>' # write the original token in a cell
## close tbody
html += '</tr>'
## CREATE ROW FOR NORMALIZED TOKEN - DISPLAY @type='norm'
html += "<tr type='norm' id='row"+istr+"_norm'>"
for x in dataIn['table']: # for each witness
element = x[i] # take the first (then at the next iteration take the second, etc.)
if element is not None:
for elementList in element:
normToken = elementList['n'].strip()
else:
normToken = '_'
html += "<td>"+normToken+"</td>" # write the original token in a cell
html += "</tr>"
## CLOSE BODY - END OF THE TABLE
html += "</tbody></table></body></html>"
#====================================================
#
# ANALYSE AND RENDER HTML TABLE
#
#====================================================
## One classes for each row should be added: variant, invariant, formal_variant
createdTable = html
root = ET.fromstring(createdTable)
for tr in root.iter('tr'): ## iterate over rows
if ((tr.get('type'))== "orig") and (tr is not None):
listTdText = [] ## open empty lists
listTdPos = []
listTdLemma = []
for td in tr.iter('td'): ## iterate over cells in a row
tdText = td.text
listTdText.append(tdText) ## and put their texts in a list (listTdText)
tdPos = td.get('ana').split('_')[0]
listTdPos.append(tdPos) ## and put their pos in a list (listTdPos)
tdLemma = td.get('ana').split('_')[1]
listTdLemma.append(tdLemma) ## and put their lemmas in a list (listTdLemma)
numberOfWits = len(listTdText)
## number of witnesses, we need it later
listTdLemmaAlternatives = []
## create list of all alternatives lemma (can be more than the number of the
## witnesses if there is ambiguity for some lemmas)
for item in listTdLemma:
for subitem in item.split('|'):
listTdLemmaAlternatives.append(subitem)
if listTdText.count(listTdText[0]) == len(listTdText): ## if the texts (original token) are equal
tr.set('class', 'invariant') ## invariant
else:
for i in range(len(listTdLemmaAlternatives)): # for each item in the list of all alternative lemmas
if (listTdLemmaAlternatives.count(listTdLemmaAlternatives[i]) == numberOfWits) and (listTdPos.count(listTdPos[0]) == len(listTdPos)):
## if pos are equal and
## if one lemma occurs a number of time equal to the number of the witnesses
tr.set('class', 'formal_variant')
break
else:
tr.set('class', 'variant')
tree = ET.tostring(root, encoding="unicode")
outFile = open('results/'+ dirName + '_table_automaticDictionary.html', 'w')
outFile.write(tree)
def collectdhcp():
#
myssl = ssl.create_default_context()
myssl.check_hostname = False
myssl.verify_mode = ssl.CERT_NONE
conn = pymysql.connect(host=host,
port=port,
user=user,
passwd=passwd,
db=db)
typeop = "op"
cmd = "<show><dhcp><server><lease><interface>%s</interface></lease></server></dhcp></show>" % (
interface)
cmd1 = "%s?key=%s&type=%s&cmd=%s" % (base, key, typeop, cmd)
# print (cmd1)
req = urllib.request.Request(cmd1, data=None)
# req.add_header( 'key', key )
# req.add_header( 'type', typeop )
# req.add_header( 'cmd', cmd )
resp_str = urllib.request.urlopen(req, context=myssl)
result = resp_str.read()
print(result)
tree = ET.fromstring(result)
for child in tree.iter('entry'):
ip = child.find('ip').text
mac = child.find('mac')
if mac is None:
mac = 'blank'
else:
mac = child.find('mac').text
hostname = child.find('hostname')
if hostname is None:
hostname = 'blank'
else:
hostname = child.find('hostname').text
name = child.get('name')
leasetime = child.find('leasetime')
if leasetime is None:
leasetime = 'Jan 1 00:00:01 1970'
leasetime = datetime.strptime(leasetime, '%b %d %H:%M:%S %Y')
else:
leasetime = child.find('leasetime').text
leaselen = len(leasetime)
leasetime = leasetime[:leaselen - 1]
leasetime = datetime.strptime(leasetime, '%a %b %d %H:%M:%S %Y')
# print(ip, mac, hostname, leasetime )
state = (
"insert into DHCP (IPaddr, MacAddr, Hostname, Leasetime , Source) values (INET_ATON('%s'),'%s','%s','%s' , 'FW' ) ON DUPLICATE KEY UPDATE IPaddr=INET_ATON('%s'), Hostname='%s' , Leasetime='%s' ;"
) % (ip, mac, hostname, leasetime, ip, hostname, leasetime)
cur = conn.cursor()
cur.execute(state)
cur.close()
# conn.commit()
state2 = (
"SELECT MacAddr FROM DHCP where `source`= 'FW' and Vendor is null;")
cur2 = conn.cursor()
cur2.execute(state2)
results2 = cur2.fetchall()
for row in results2:
mac = row[0]
# print (mac)
myssl = ssl.create_default_context()
myssl.check_hostname = False
myssl.verify_mode = ssl.CERT_NONE
url = "https://api.macvendors.com/%s" % (mac)
req = urllib.request.Request(url, data=None)
try:
resp_str = urllib.request.urlopen(req, context=myssl)
result3 = resp_str.read().decode('utf-8')
result3 = result3.replace('\uff0c', '')
print(mac, ' = ', result3)
except urllib.error.HTTPError as error:
print(error.code)
result3 = 'Unknown'
# print(mac ,' = ' , result3 )
cur3 = conn.cursor()
state3 = ("UPDATE DHCP set vendor = \"%s\" where MacAddr = \"%s\";"
) % (result3, mac)
cur3.execute(state3)
cur3.close()
# print(mac ,' + ' , result3 )
time.sleep(1)
# conn.commit()
cur2.close()
conn.commit()
conn.close()
def receipt(self,xml):
"""
Prints an xml based receipt definition
"""
def strclean(string):
if not string:
string = ''
string = string.strip()
string = re.sub('\s+',' ',string)
return string
def format_value(value, decimals=3, width=0, decimals_separator='.', thousands_separator=',', autoint=False, symbol='', position='after'):
decimals = max(0,int(decimals))
width = max(0,int(width))
value = float(value)
if autoint and math.floor(value) == value:
decimals = 0
if width == 0:
width = ''
if thousands_separator:
formatstr = "{:"+str(width)+",."+str(decimals)+"f}"
else:
formatstr = "{:"+str(width)+"."+str(decimals)+"f}"
ret = formatstr.format(value)
ret = ret.replace(',','COMMA')
ret = ret.replace('.','DOT')
ret = ret.replace('COMMA',thousands_separator)
ret = ret.replace('DOT',decimals_separator)
if symbol:
if position == 'after':
ret = ret + symbol
else:
ret = symbol + ret
return ret
def print_elem(stylestack, serializer, elem, indent=0):
elem_styles = {
'h1': {'bold': 'on', 'size':'double'},
'h2': {'size':'double'},
'h3': {'bold': 'on', 'size':'double-height'},
'h4': {'size': 'double-height'},
'h5': {'bold': 'on'},
'em': {'font': 'b'},
'b': {'bold': 'on'},
}
stylestack.push()
if elem.tag in elem_styles:
stylestack.set(elem_styles[elem.tag])
stylestack.set(elem.attrib)
if elem.tag in ('p','div','section','article','receipt','header','footer','li','h1','h2','h3','h4','h5'):
serializer.start_block(stylestack)
serializer.text(elem.text)
for child in elem:
print_elem(stylestack,serializer,child)
serializer.start_inline(stylestack)
serializer.text(child.tail)
serializer.end_entity()
serializer.end_entity()
elif elem.tag in ('span','em','b','left','right'):
serializer.start_inline(stylestack)
serializer.text(elem.text)
for child in elem:
print_elem(stylestack,serializer,child)
serializer.start_inline(stylestack)
serializer.text(child.tail)
serializer.end_entity()
serializer.end_entity()
elif elem.tag == 'value':
serializer.start_inline(stylestack)
serializer.pre(format_value(
elem.text,
decimals=stylestack.get('value-decimals'),
width=stylestack.get('value-width'),
decimals_separator=stylestack.get('value-decimals-separator'),
thousands_separator=stylestack.get('value-thousands-separator'),
autoint=(stylestack.get('value-autoint') == 'on'),
symbol=stylestack.get('value-symbol'),
position=stylestack.get('value-symbol-position')
))
serializer.end_entity()
elif elem.tag == 'line':
width = stylestack.get('width')
if stylestack.get('size') in ('double', 'double-width'):
width = width / 2
lineserializer = XmlLineSerializer(stylestack.get('indent')+indent,stylestack.get('tabwidth'),width,stylestack.get('line-ratio'))
serializer.start_block(stylestack)
for child in elem:
if child.tag == 'left':
print_elem(stylestack,lineserializer,child,indent=indent)
elif child.tag == 'right':
lineserializer.start_right()
print_elem(stylestack,lineserializer,child,indent=indent)
serializer.pre(lineserializer.get_line())
serializer.end_entity()
elif elem.tag == 'ul':
serializer.start_block(stylestack)
bullet = stylestack.get('bullet')
for child in elem:
if child.tag == 'li':
serializer.style(stylestack)
serializer.raw(' ' * indent * stylestack.get('tabwidth') + bullet)
print_elem(stylestack,serializer,child,indent=indent+1)
serializer.end_entity()
elif elem.tag == 'ol':
cwidth = len(str(len(elem))) + 2
i = 1
serializer.start_block(stylestack)
for child in elem:
if child.tag == 'li':
serializer.style(stylestack)
serializer.raw(' ' * indent * stylestack.get('tabwidth') + ' ' + (str(i)+')').ljust(cwidth))
i = i + 1
print_elem(stylestack,serializer,child,indent=indent+1)
serializer.end_entity()
elif elem.tag == 'pre':
serializer.start_block(stylestack)
serializer.pre(elem.text)
serializer.end_entity()
elif elem.tag == 'hr':
width = stylestack.get('width')
if stylestack.get('size') in ('double', 'double-width'):
width = width / 2
serializer.start_block(stylestack)
serializer.text('-'*width)
serializer.end_entity()
elif elem.tag == 'br':
serializer.linebreak()
elif elem.tag == 'img':
if 'src' in elem.attrib and 'data:' in elem.attrib['src']:
self.print_base64_image(elem.attrib['src'])
elif elem.tag == 'barcode' and 'encoding' in elem.attrib:
serializer.start_block(stylestack)
self.barcode(strclean(elem.text),elem.attrib['encoding'])
serializer.end_entity()
elif elem.tag == 'cut':
self.cut()
elif elem.tag == 'partialcut':
self.cut(mode='part')
elif elem.tag == 'cashdraw':
self.cashdraw(2)
self.cashdraw(5)
stylestack.pop()
try:
stylestack = StyleStack()
serializer = XmlSerializer(self)
root = ET.fromstring(xml.encode('utf-8'))
self._raw(stylestack.to_escpos())
print_elem(stylestack,serializer,root)
if 'open-cashdrawer' in root.attrib and root.attrib['open-cashdrawer'] == 'true':
self.cashdraw(2)
self.cashdraw(5)
if not 'cut' in root.attrib or root.attrib['cut'] == 'true' :
self.cut()
except Exception as e:
errmsg = str(e)+'\n'+'-'*48+'\n'+traceback.format_exc() + '-'*48+'\n'
self.text(errmsg)
self.cut()
raise e
def _get_nics(vm_object):
tree = ElementTree.fromstring(vm_object.XMLDesc(0))
return [_recurse(item) for item in tree.findall('devices/interface')]
# This allows us to connect over https to our firewall with it's self-signed cert.
ssl._create_default_https_context = ssl._create_unverified_context
# Store the location of the user XML file to paloURL
paloURL = "https://" + fwAddress + "/api/?key=" + fwAPIKey + "&type=op&cmd=<show><user><ip-user-mapping><all></all></ip-user-mapping></user></show>"
# Get the name we're looking for from user input.
userName = str.lower(
input("Input user\'s name(or leave blank for all users and hosts): "))
# Go and get the XML file.
paloData = str(urllib.request.urlopen(paloURL).read())[2:-1]
# This makes an Element Tree from our XML file.
paloTree = ET.fromstring(paloData)
# Take all the IP elements from the tree, then loop through them and append to our IP list.
ips = paloTree.findall('.//ip')
for each in ips:
ipList.append(each.text)
# Take all the user elements from the tree, then loop through them and append to our user list.
users = paloTree.findall('.//user')
for each in users:
userList.append(each.text)
# Loop through the IP list
for i in range(0, len(ipList)):
# Checks to see if the current entry on the user's list is the one we were supplied.
if userName in userList[i]:
def decryptBook(userkey, inpath, outpath):
if AES is None:
raise ADEPTError(u"PyCrypto or OpenSSL must be installed.")
rsa = RSA(userkey)
with closing(ZipFile(open(inpath, 'rb'))) as inf:
namelist = set(inf.namelist())
if 'META-INF/rights.xml' not in namelist or \
'META-INF/encryption.xml' not in namelist:
print u"{0:s} is DRM-free.".format(os.path.basename(inpath))
return 1
for name in META_NAMES:
namelist.remove(name)
try:
rights = etree.fromstring(inf.read('META-INF/rights.xml'))
adept = lambda tag: '{%s}%s' % (NSMAP['adept'], tag)
expr = './/%s' % (adept('encryptedKey'), )
bookkey = ''.join(rights.findtext(expr))
if len(bookkey) != 172:
print u"{0:s} is not a secure Adobe Adept ePub.".format(
os.path.basename(inpath))
return 1
bookkey = rsa.decrypt(bookkey.decode('base64'))
# Padded as per RSAES-PKCS1-v1_5
if bookkey[-17] != '\x00':
print u"Could not decrypt {0:s}. Wrong key".format(
os.path.basename(inpath))
return 2
encryption = inf.read('META-INF/encryption.xml')
decryptor = Decryptor(bookkey[-16:], encryption)
kwds = dict(compression=ZIP_DEFLATED, allowZip64=False)
with closing(ZipFile(open(outpath, 'wb'), 'w', **kwds)) as outf:
zi = ZipInfo('mimetype')
zi.compress_type = ZIP_STORED
try:
# if the mimetype is present, get its info, including time-stamp
oldzi = inf.getinfo('mimetype')
# copy across fields to be preserved
zi.date_time = oldzi.date_time
zi.comment = oldzi.comment
zi.extra = oldzi.extra
zi.internal_attr = oldzi.internal_attr
# external attributes are dependent on the create system, so copy both.
zi.external_attr = oldzi.external_attr
zi.create_system = oldzi.create_system
except:
pass
outf.writestr(zi, inf.read('mimetype'))
for path in namelist:
data = inf.read(path)
zi = ZipInfo(path)
zi.compress_type = ZIP_DEFLATED
try:
# get the file info, including time-stamp
oldzi = inf.getinfo(path)
# copy across useful fields
zi.date_time = oldzi.date_time
zi.comment = oldzi.comment
zi.extra = oldzi.extra
zi.internal_attr = oldzi.internal_attr
# external attributes are dependent on the create system, so copy both.
zi.external_attr = oldzi.external_attr
zi.create_system = oldzi.create_system
except:
pass
outf.writestr(zi, decryptor.decrypt(path, data))
except:
print u"Could not decrypt {0:s} because of an exception:\n{1:s}".format(
os.path.basename(inpath), traceback.format_exc())
return 2
return 0
def __init__(self, master, *args, **kwargs):
tk.Frame.__init__(self, master, *args, **kwargs)
table = tk.Frame(self)
table.pack(side="top", fill="both", expand=True, ipadx=2, ipady=2)
table.grid_rowconfigure(0, weight=1)
table.grid_columnconfigure(0, weight=1)
master.resizable(width=False, height=False)
element_tree = ET.ElementTree(ET.fromstring(requests.get("http://tcmb.gov.tr/kurlar/today.xml").text)).getroot()
self.widgets = {}
row = 1
tk.Label(table, text="Date: "+element_tree.get("Date")).grid(row=0, column=0, sticky="nsew", padx=5, pady=2)
tk.Label(table, text="Tarih: "+element_tree.get("Tarih")).grid(row=0, column=1, sticky="nsew", padx=5, pady=2)
tk.Label(table, text="Bulten No: "+element_tree.get("Bulten_No")).grid(row=0, column=2, sticky="nsew", padx=5, pady=2)
tk.Label(table, text="Currency Code", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=0, sticky="nsew")
tk.Label(table, text="Unit", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=1, sticky="nsew")
tk.Label(table, text="Isim", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=2, sticky="nsew")
tk.Label(table, text="Currency Name", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=3, sticky="nsew")
tk.Label(table, text="Forex Buying", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=4, sticky="nsew")
tk.Label(table, text="Forex Selling", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=5, sticky="nsew")
tk.Label(table, text="Banknote Buying", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=6, sticky="nsew")
tk.Label(table, text="Banknote Selling", relief=tk.RIDGE, font=("Helvetica", 10, "bold"), bg="orange").grid(row=1, column=7, sticky="nsew")
for currency in element_tree.findall('Currency'):
row += 1
currencycode = currency.get('CurrencyCode')
unit = currency.find('Unit').text
isim = currency.find('Isim').text
currencyname = currency.find('CurrencyName').text
forexbuying = currency.find('ForexBuying').text
if forexbuying is None: forexbuying = "-"
forexselling = currency.find('ForexSelling').text
if forexselling is None: forexselling = "-"
banknotebuying = currency.find('BanknoteBuying').text
if banknotebuying is None: banknotebuying = "-"
banknoteselling = currency.find('BanknoteSelling').text
if banknoteselling is None: banknoteselling = "-"
self.widgets[currencycode] = {
"currencycode": tk.Label(table, text=str(currencycode).strip(), relief=tk.RIDGE),
"unit": tk.Label(table, text=str(unit).strip(), relief=tk.RIDGE),
"isim": tk.Label(table, text=str(isim).strip(), relief=tk.RIDGE),
"currencyname": tk.Label(table, text=str(currencyname).strip(), relief=tk.RIDGE),
"forexbuying": tk.Label(table, text=str(forexbuying).strip(), relief=tk.RIDGE),
"forexselling": tk.Label(table, text=str(forexselling).strip(), relief=tk.RIDGE),
"banknotebuying": tk.Label(table, text=str(banknotebuying).strip(), relief=tk.RIDGE),
"banknoteselling": tk.Label(table, text=str(banknoteselling).strip(), relief=tk.RIDGE)
}
self.widgets[currencycode]["currencycode"].grid(row=row, column=0, sticky="nsew")
self.widgets[currencycode]["unit"].grid(row=row, column=1, sticky="nsew")
self.widgets[currencycode]["isim"].grid(row=row, column=2, sticky="nsew")
self.widgets[currencycode]["currencyname"].grid(row=row, column=3, sticky="nsew")
self.widgets[currencycode]["forexbuying"].grid(row=row, column=4, sticky="nsew")
self.widgets[currencycode]["forexselling"].grid(row=row, column=5, sticky="nsew")
self.widgets[currencycode]["banknotebuying"].grid(row=row, column=6, sticky="nsew")
self.widgets[currencycode]["banknoteselling"].grid(row=row, column=7, sticky="nsew")
url = 'http://ws.spotify.com/search/1/track?q=' + artist_name_esc
try:
self.lock.acquire()
logger.debug("Loading " + url)
result = urllib2.urlopen(url)
tracks_string = result.read()
except Exception, e:
logger.warning("Failed to load " + artist_name + ": " + str(e))
# TODO
#self.loading_failures.append(artist_name + ": " + str(e))
return []
finally:
self.lock.release()
tracks = etree.fromstring(tracks_string)
# F*****g namespaces!
tracks = filter(lambda c: c.tag == '{http://www.spotify.com/ns/music/1}track', tracks)
matching_tracks = []
for track in tracks:
track_artist_name = self.get_artist_name(track)
if track_artist_name != None and track_artist_name.lower() == artist_name.lower() and self.track_available_in_req_territories(track):
matching_tracks.append(track.attrib['href'])
if len(matching_tracks) >= self.MAX_TRACKS:
break
self.write_to_cache(artist_name, matching_tracks)
return matching_tracks
def update():
add_datasets = []
return_datasets = []
logger.debug( "Start updating of DB" )
for layer in wfs_layers:
if not layer in config_dataset_names:
continue
else:
logger.debug( "Found layer %s from config" % layer )
section = config_sections[config_dataset_names.index(layer)]
# check that tiff directory exists
tiff_dir = "%s/%s" % (tiff_dir_base,layer)
if not os.path.exists(tiff_dir):
logger.debug( "GeoTIFF directory does not exist. Create it." )
os.makedirs(tiff_dir)
# get WFS to get WMS urls
try:
wfs_url = wfs_layers[layer].replace("{key}",api_key)
response = urlopen( wfs_url )
logger.debug( "Data from url %s fetched" % wfs_url )
# ignore network related problems
except:
logger.error( "Network error occurred while fetching url %s" % wfs_url )
continue
logger.debug( "Parse WFS response for layer %s" % layer )
wfs_response = ElementTree.fromstring(response.read())
file_references = wfs_response.findall('.//{%s}fileReference' % gml_namespace)
for ref in file_references:
url = ref.text
query = urlparse(url).query
query = query.split("&")
for q in query:
if q[0:4].lower()=="time":
time_value = q.split("=")[-1]
elif q[0:3].lower()=="srs":
projdef = q.split("=")[-1]
elif q[0:4].lower()=="bbox":
bbox = q.split("=")[-1]
timestamp = datetime.strptime(time_value,\
"%Y-%m-%dT%H:%M:%SZ")
if ( timestamp<read_expiration_time(int( config.get(section,"cleanup_time") )) ):
# do not store old files
logger.debug ( "Skip expired dataset %s:%s" % (layer,str(timestamp)))
continue
# search if dataset already exists
radar_datasets = session.query(RadarDataset)\
.filter(RadarDataset.timestamp==timestamp)\
.filter(RadarDataset.name==layer)
# skip file fetching if it already exists
if radar_datasets.count()>0:
logger.debug ( "Dataset %s:%s already in database" % (layer,str(timestamp) ) )
continue
output_filename = tiff_dir + "/" + time_value.replace(":","")\
.replace("-","") + ".tif"
# save file to disk
try:
response = urlopen( url )
logger.debug( "Fetched data from url %s" % url )
# ignore network related problems
except:
logger.error( "Network error or invalid api-key occurred while fetching url %s" % url )
continue
f = open(output_filename,"wb")
f.write( response.read() )
f.close()
# import dataset to db
logger.info( "Add new dataset: %s:%s to DB." % (layer,str(timestamp)) )
add_datasets.append(
RadarDataset(
name = layer,
timestamp = timestamp,
geotiff_path = output_filename,
projdef = projdef,
bbox_original = bbox,
dataset_type = config.get(section,"dataset_type"),
unit = config.get(section,"unit"),
style = config.get(section,"style"),
title = config.get(section,"title")
)
)
return_datasets.append({"name": layer,
"timestamp": timestamp })
# finally add new datasets to DB
if (len(add_datasets)>0):
session.add_all(add_datasets)
session.commit()
logger.info ( "Added %i results." % len(add_datasets) )
logger.debug( "Updating of DB finished" )
session.close()
return return_datasets
def get_english_wikipedia_pages(num_chunks=1000000000, debug=False):
try:
ymd, jobs = get_most_recent_available_dump()
url = "https://dumps.wikimedia.org/enwiki/" + ymd + "/enwiki-" + ymd + "-pages-articles.xml.bz2"
print("url:", url)
decompressor = BZ2Decompressor()
print("decompressor:", decompressor)
req = urlopen(url)
print("req:", req)
start_page = b"<page>"
end_page = b"</page>"
text = b""
for n in range(num_chunks):
if debug: print("chunk")
chunk = req.read(16 * 1024)
if not chunk:
break
if debug: print("read")
text += decompressor.decompress(chunk)
if debug: print("text:", type(text))
#print("text:", text)
#with open("/tmp/output.txt", "a") as f:
#f.write(text.decode("utf-8"))
num_pages = text.count(start_page)
#print("num_pages:", num_pages)
for n in range(num_pages):
start_page_index = text.find(start_page)
if start_page_index != -1:
#print("start_page_index:", start_page_index)
#print("text:", text[:100])
# dump any text before start_page
text = text[start_page_index:]
#print("text:", text[:100])
end_page_index = text.find(end_page)
#print("end_page_index:", end_page_index)
if end_page_index != -1:
end_page_index += len(end_page)
page = text[:end_page_index]
text = text[end_page_index:]
#print("page:", page[:20], "...", page[-20:])
parsed = ET.fromstring(page)
yield parsed
except Exception as e:
print("[wake.get_english_wikipedia_pages] found exception:", e)
raise (e)
def parse_xml(data):
import xml.etree.ElementTree as ET
return ET.fromstring(data)
def parse(self, response):
base_url = get_base_url(response)
transport = paramiko.Transport((CLIENTS_SFTP_HOST, CLIENTS_SFTP_PORT))
password = "******"
username = "******"
transport.connect(username = username, password = password)
sftp = paramiko.SFTPClient.from_transport(transport)
files = sftp.listdir_attr()
last = get_last_file("CRC_PRICEFEED_Germany", files)
date_file = datetime.fromtimestamp(last.st_mtime)
hours_diff = (datetime.now() - date_file).total_seconds() / 3600
# Check file updates
if hours_diff >= 72:
self.errors.append('WARNING: No Update for 3 days')
zip_path = HERE+'/CRC_PRICEFEED_Germany.zip'
xml_path = HERE+'/CRC_PRICEFEED_Germany.xml'
sftp.get(last.filename, zip_path)
unzip(zip_path, xml_path)
xmlfeed_sku = ''
with open(xml_path) as f:
xmlfeed_sku = f.read()
sku_prices = {}
tree = et.fromstring(xmlfeed_sku)
for item in tree.find('priceList[@id="GermanyRP"]').find('prices').findall('price'):
sku = item.find('skuId').text
price = item.find('listPrice').text
sku_prices[sku] = price
last = get_last_file("PriceMonitorHandler", files)
zip_path = HERE+'/PriceMonitorHandler.zip'
xml_path = HERE+'/PriceMonitorHandler.xml'
sftp.get(last.filename, zip_path)
unzip(zip_path, xml_path)
xmlfeed_products = ''
with open(xml_path) as f:
xmlfeed_products = f.read()
sku_products = {}
tree = et.fromstring(xmlfeed_products)
for item in tree.find('skus').findall('sku'):
sku_products[item.find('skuID').text] = {'identifier':item.find('skuID').text,
'category': item.find('CategoryDescription').text,
'brand':item.find('BrandDescription').text,
'image_url': item.find('ImageURL').text,
'url': item.find('ProductURL').text,
'name': item.find('SkuDescription').text,
'sku': item.find('skuID').text,
'stock': item.find('SkuQuantity').text}
for sku, price in sku_prices.iteritems():
try:
product = sku_products[sku]
except KeyError:
log.msg('SKU not found:' + sku)
continue
product['price'] = price
product = Product(product)
loader = ProductLoader(response=response, item=product)
yield loader.load_item()
def from_pics(self, element=None, filename=None, pathlist=None):
"""The PICS information for the device can be either an Element Tree
element for a device from a document already being processed or the file
name of document in the file system. Populates the device based on the
elements within the device element.
Parameters:
element :
Element Tree device element.
filename :
File name of the PICS document.
pathlist :
Pathlist object containing alternate paths to the PICS document.
"""
global file_pathlist
pics_data = ''
try:
if element is None:
# try supplied path list
if pathlist is not None:
try:
pics_data = pathlist.read(filename)
except NameError:
pass
# try device file path list
if not pics_data and file_pathlist is not None:
try:
pics_data = file_pathlist.read(filename)
except NameError:
pass
# try local directory
if not pics_data:
f = open(filename, 'r')
pics_data = f.read()
f.close()
root = ET.fromstring(pics_data)
if root.tag != pics.PICS_ROOT:
raise SunSpecError("Unexpected root element: %s" % (root.tag))
d = root.find(pics.PICS_DEVICE)
if d is None:
raise SunSpecError("No '%s' elements found in '%s' element" % (pics.PICS_DEVICE, root.tag))
else:
d = element
if d.tag != pics.PICS_DEVICE:
raise SunSpecError("Unexpected device tag: '%s'" % (d.tag))
self.base_addr = d.attrib.get(pics.PICS_ATTR_BASE_ADDR, pics.PICS_BASE_ADDR_DEFAULT)
addr = self.base_addr + 2
for m in d.findall('*'):
if m is None:
raise SunSpecError("No '%s' elements found in '%s' element" % (pics.PICS_MODEL, d.tag))
if m.tag != pics.PICS_MODEL:
raise SunSpecError("Unexpected '%s' element in '%s' element" % (m.tag, d.tag))
model_id = m.attrib.get(pics.PICS_ATTR_ID)
if model_id is None:
raise SunSpecError('Module id error')
model_len = m.attrib.get(pics.PICS_ATTR_LEN)
if model_len is not None:
# raise SunSpecError('Module len error in model %d' % (model_id))
model_len = int(model_len)
# move address past model id and length (even though address is not real in the case)
model = Model(self, model_id, addr + 2, model_len)
try:
model.load()
except Exception as e:
model.load_error = str(e)
model.from_pics(m)
self.add_model(model)
addr += model.len + 2
except Exception as e:
raise SunSpecError('Error loading PICS: %s' % str(e))
#!/usr/bin/env python3
import gzip
import hashlib
import xml.etree.ElementTree as ET
import requests
base_url = 'https://repository.egi.eu/sw/production/cas/1/current/'
metapackage_name = 'ca-policy-lcg'
# Parse the repo manifest
response = requests.get(base_url+'repodata/repomd.xml')
assert response.ok
root = ET.fromstring(response.content)
for element in list(root):
if element.attrib.get('type') == 'primary':
location = element.find('./{http://linux.duke.edu/metadata/repo}location')
primary_url = base_url+location.attrib['href']
if element.attrib.get('type') == 'filelists':
location = element.find('./{http://linux.duke.edu/metadata/repo}location')
filelists_url = base_url+location.attrib['href']
assert filelists_url and primary_url
# Parse the RPM requirements
response = requests.get(primary_url)
assert response.ok
root = ET.fromstring(gzip.decompress(response.content))
dependency_map = {}
for package in root.findall('{http://linux.duke.edu/metadata/common}package'):
ticket = AuthClient.getst(tgt)
query = {'id': id, 'ticket': ticket, 'version': profile}
r = requests.get(uri, params=query)
print(r.url)
return r.text.encode('utf-8')
with open(inputfile, 'r') as f:
for line in f:
oid, name, memberOid, memberOidName = line.strip().split("|")
if len(memberOid) == 0:
codeSystem = getCodeSystem(oid)
else:
codeSystem = getCodeSystem(memberOid)
xml = ET.fromstring(codeSystem)
ns = {"ns0": "urn:ihe:iti:svs:2008"}
codeSystemInfo = {}
for concepts in xml.findall(".//ns0:ConceptList/ns0:Concept", ns):
cs = concepts.get('codeSystemName')
codeSystemVersion = concepts.get('codeSystemVersion')
codeSystemInfo.setdefault(cs, []).append(codeSystemVersion)
sys.stdout.write(oid + "|" + memberOid)
for k, v in codeSystemInfo.items():
sys.stdout.write(k + "|" + ','.join(set(v)))
#o.write(oid+"|"+name+"|"+memberOid+"|"+k+"|"+','.join(set(v)))
o.write(oid + "|" + name + "|" + memberOid + "|" + memberOidName +
"|" + k + "|" + ','.join(set(v)) + "\n")
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
artist_id INTEGER,
title TEXT UNIQUE
);
CREATE TABLE Track (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
title TEXT UNIQUE,
album_id INTEGER,
genre_id INTEGER,
len INTEGER, rating INTEGER, count INTEGER
);
''')
fd = open('library.xml')
lines = ET.fromstring(fd.read()).findall('dict/dict/dict')
fd.close()
for line in lines:
if (lookup(line, 'Track ID') is None): continue
name = lookup(line, 'Name')
artist = lookup(line, 'Artist')
genre = lookup(line, 'Genre')
album = lookup(line, 'Album')
count = lookup(line, 'Play Count')
rating = lookup(line, 'Rating')
length = lookup(line, 'Total Time')
#if name is None or artist is None or album is None or genre is None:
# continue
print(name, artist, genre, album, count, rating, length)
def _bazel_query_xml_tree(query):
"""Get xml output of bazel query invocation, parsed as XML tree"""
output = subprocess.check_output([
'tools/bazel', 'query', '--noimplicit_deps', '--output', 'xml', query
])
return ET.fromstring(output)
