def synchronize_archives(self, package):
log.info("Synhronizing %s package", package)
# Retrieve the simple page from the local directory.
tree = ElementTree.fromstring(
self.local_pypi_repo.load_simple_page(package))
# Parse each of the anchor tags, looking for a reference to
# `../../packages`. This is required because pep381 currently makes no
# assertion that the anchor tags have any specific attributes (i.e.,
# "download").
for anchor_tag in tree.iter(".//a"):
href = anchor_tag.get('href')
if not href.startswith("../../packages"):
continue
# If the local file already exists, skip it
local_package_path = self.local_pypi_repo.resource_path(href)
if os.path.exists(local_package_path):
log.debug("%s already synchronized.", local_package_path)
continue
# Create the directory structure as well (i.e.,
# /packages/source/2.6/w/whatever)
basedir = os.path.basename(local_package_path)
if not os.path.exists(basedir):
os.makedirs(basedir)
# Save the remote archive locally
remote_package = self.remote_pypi_repo.load_archive(href)
self.local_pypi_repo.write_archive(
local_package_path, remote_package)
log.info("Synchronized %s", local_package_path)
def xml2dict(xml_str):
"""xml to dict
@:param xml_str: string in XML format
@:return: Dictionary
"""
# return xmltodict.parse(xml_str)['xml']
root = ElementTree.fromstring(xml_str)
assert as_text(root.tag) == as_text('xml')
result = {}
for child in root:
tag = child.tag
text = child.text
result[tag] = text
return result
def readVersion( self, pkg_path ):
# unpack the package to get at the PackageInfo file
temp_path = tempfile.mkdtemp(prefix="autopkg", dir="/private/tmp")
self.env["destination_path"] = tempfile.mkdtemp(prefix="autopkg", dir="/private/tmp")
self.env["skip_payload"] = True
self.unpackFlatPkg()
# read the "version" attribute from the top level of the PackageInfo file
try:
with open(self.env['destination_path'] + "/PackageInfo", "r") as f:
filedata = f.read(self.env['file_content'])
except BaseException as e:
raise ProcessorError("Can't read file at %s: %s" % (
self.env['destination_path'] + "/PackageInfo",
e))
xmldata = ElementTree.fromstring( filedata )
def gen_mac_address(self, offset):
mac_addresses = []
# If domains exist, get the highest MAC address and use it as a new base MAC
if len(self.domain_list) != 0:
for domain in self.domain_list:
# Convert domain descriptions to XML
xml_tree = ElementTree.fromstring(domain.XMLDesc(0))
domain_mac = xml_tree.find('devices/interface/mac').attrib
domain_mac = int(domain_mac['address'].replace(':', ''), 16)
mac_addresses.append(domain_mac)
# Format number as MAC address
new_base_mac = max(mac_addresses)
new_mac = new_base_mac + offset
new_mac = ':'.join(
format(s, '02x') for s in bytes.fromhex(new_mac))
return new_mac
# If no domains exist, start numbering from 0
else:
base_mac = '00000000'
new_mac = "{:012X}".format(int(base_mac, 16) + offset)
return new_mac
def message(self):
file = open(self.filename)
mydata = file.read()
file.close()
if ('blankname' in mydata):
rep = re.compile('blankname')
mydata = rep.sub(self.content, mydata)
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
self.reply = '请输入您的手机号：'
elif ('blanktel' in mydata):
rep = re.compile('blanktel')
mydata = rep.sub(self.content, mydata)
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
self.reply = '请输入您想到的任何句子：'
elif ('blankwx' in mydata):
rep = re.compile('blankwx')
mydata = rep.sub(self.wx, mydata)
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
self.reply = '请输入您想设置的密码：'
elif ('blankpass' in mydata):
rep = re.compile('blankpass')
word = 'select id,unitname from `system-group` where 1'
xmb = self.idc.searchWord(word, 0)
mydata = rep.sub(self.content, mydata)
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
self.reply = '请回复项目部名前的数字选择项目部：\n'
for i in xmb:
for j in i:
if (isinstance(j, str)):
self.reply = self.reply + j
else:
self.reply = self.reply + str(j) + '、'
self.reply = self.reply + '\n'
elif ('blankkey' in mydata):
rep = re.compile('blankkey')
mydata = rep.sub(self.content, mydata)
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
myxml = ElementTree.fromstring(mydata)
date = str(time.strftime('%Y-%m-%d', time.localtime(time.time())))
name = myxml.find('Name').text.strip(' ')
tel = myxml.find('Tel').text.strip(' ')
wx = myxml.find('Wx').text.strip(' ')
sha256 = hashlib.sha256()
mypass = myxml.find('Pass').text.strip(' ')
sha256.update(mypass.encode('utf-8'))
mypass = sha256.hexdigest()
keypass = myxml.find('Key').text.strip(' ')
word = 'SELECT id FROM `members-infor` WHERE tel="' + tel + '" and mininame="' + name + '"'
data = self.idc.searchWord(word, 0)
if (data == None):
if (name != '' and mypass != '' and tel != ''):
word = "insert into `members-infor` (mininame,tel,pass,userlimit,date,userlevel) VALUES ('" + name + "','" + tel + "','" + mypass + "','" + keypass + "','" + date + "','0')"
self.idc.insertWord(word, 0)
word = 'select id from `members-login` where loginuser="******"'
tmpdata = self.idc.searchWord(word, 0)
word = 'select id from `members-login` where loginuser="******"'
tmpteldata = self.idc.searchWord(word, 0)
word = 'select id from `members-infor` where tel="' + tel + '"'
iddata = self.idc.searchWord(word, 0)
if (tmpdata == None and len(iddata) == 1):
word = 'INSERT INTO `members-login`(`id`,`loginuser`) VALUES ("' + str(
iddata[0][0]) + '","' + wx + '")'
self.idc.insertWord(word, 0)
if (tmpteldata == None and len(iddata) == 1):
word = 'INSERT INTO `members-login`(`id`,`loginuser`) VALUES ("' + str(
iddata[0][0]) + '","' + tel + '")'
self.idc.insertWord(word, 0)
self.reply = '完成注册！！'
os.remove(self.filename)
else:
self.reply = '您输入的信息有不合规的，请重新注册！！'
os.remove(self.filename)
else:
self.reply = '您已经注册过了！！'
if (os.path.exists(self.filename)):
os.remove(self.filename)
else:
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
myxml = ElementTree.fromstring(mydata)
date = str(time.strftime('%Y-%m-%d', time.localtime(time.time())))
name = myxml.find('Name').text.strip(' ')
tel = myxml.find('Tel').text.strip(' ')
wx = myxml.find('Wx').text.strip(' ')
sha256 = hashlib.sha256()
mypass = myxml.find('Pass').text.strip(' ')
sha256.update(mypass.encode('utf-8'))
mypass = sha256.hexdigest()
keypass = myxml.find('Key').text.strip(' ')
word = 'SELECT id FROM `members-infor` WHERE tel="' + tel + '" and mininame="' + name + '"'
data = self.idc.searchWord(word, 0)
if (data == None):
if (name != '' and mypass != '' and tel != ''):
word = "insert into `members-infor` (mininame,tel,pass,userlimit,date,userlevel) VALUES ('" + name + "','" + tel + "','" + mypass + "','" + keypass + "','" + date + "','0')"
self.idc.insertWord(word, 0)
word = 'select id from `members-login` where loginuser="******"'
tmpdata = self.idc.searchWord(word, 0)
word = 'select id from `members-login` where loginuser="******"'
tmpteldata = self.idc.searchWord(word, 0)
word = 'select id from `members-infor` where tel="' + tel + '"'
iddata = self.idc.searchWord(word, 0)
if (tmpdata == None and len(iddata) == 1):
word = 'INSERT INTO `members-login`(`id`,`loginuser`) VALUES ("' + str(
iddata[0][0]) + '","' + wx + '")'
self.idc.insertWord(word, 0)
if (tmpteldata == None and len(iddata) == 1):
word = 'INSERT INTO `members-login`(`id`,`loginuser`) VALUES ("' + str(
iddata[0][0]) + '","' + tel + '")'
self.idc.insertWord(word, 0)
self.reply = '完成注册！！'
os.remove(self.filename)
else:
self.reply = '您输入的信息有不合规的，请重新注册！！'
os.remove(self.filename)
else:
self.reply = '您已经注册过了！！'
if (os.path.exists(self.filename)):
os.remove(self.filename)
self.text.append(1)
self.text.append(self.reply)
if len(self.text) > 1:
if self.text[0] == 1:
CreateTime = int(time.time())
out = self.reply_text(self.wx, self.touser, CreateTime,
self.text[1])
return out
def download_NCBI_records(id_df,
NCBI_records_dirpath,
tax_dict,
NCBI_API_key,
overwrite_fasta=[],
pid_outpath=''):
"""Downloads NCBI protein records for each NCBI Gene ID listed in ags_mapped_id_df.
:param: id_df: DataFrame object with required columns 'Gene Symbol' and 'AGS Gene ID.' Gene symbol
entries are used to name fasta files downloaded; AGS Gene IDs are queried using Entrez elink
to 1) match Gene ID to all corresponding Protein IDs and 2) download those Protein IDs into one fasta
file per gene symbol, saved into NCBI_records_dirpath
:param NCBI_records_dirpath: directory path where NCBI input data will be downloaded
:param gene_field_name: column name in ags_mapped_id_df where ortholog IDs are stored
:param protein_field_name: column name in ags_mapped_id_df where linked Protein ID(s) will be stored as comma separated list
:param NCBI_API_key: Must be valid API key or empty string
:return: modified ags_mapped_id_df containing a populated protein_field_name column with linked Protein IDs. Will download
corresponding Protein Sequences to files named by gene symbol into directory specified by NCBI_records_dirpath
"""
import subprocess
import requests
import xml.etree.ElementTree as ET
ENTREZ_BASE_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
api_key_ext = ""
if NCBI_API_key:
#Default value will be empty string if NCBIAPIKey field is empty or absent in config file, defaults
#to making API requests without API key
api_key_ext = "&api_key={0}".format(NCBI_API_key)
gene_field_name, protein_field_name = tax_dict['gid_column'], tax_dict[
'pid_column']
ags_mapped_df = id_df.loc[~id_df[gene_field_name].isnull(), :]
# Convert Gene ID to list of Protein IDs corresponding to transcript variant sequences
for idx, row in ags_mapped_df.iterrows():
symbol = row["gene_symbol"]
fasta_fpath = "{0}/{1}.fasta".format(NCBI_records_dirpath, symbol)
if not os.path.exists(fasta_fpath) or symbol in overwrite_fasta:
AGS_gid = row[gene_field_name]
# print("==={0}===".format(symbol))
# print("AGS Gene ID: {0}".format(AGS_gid))
elink_req = "elink.fcgi?dbfrom=gene&db=protein&id={0}{1}".format(
AGS_gid, api_key_ext)
gp_elink_url = ENTREZ_BASE_URL + elink_req
elink_response = requests.get(gp_elink_url)
xml_data = elink_response.content
root = ET.fromstring(xml_data)
# Check XML formatting of elink pages - update xpath accordingly if functionality breaks
# Pulls Record IDs for Protein specifically; use gene_protein_refseq for Protein RefSeqs
protein_IDs = [
link.text for link in root.findall(
".//LinkSetDb[LinkName='gene_protein']/Link/Id")
]
id_str = ','.join(protein_IDs)
id_df.loc[idx, protein_field_name] = id_str
efetch_req = "efetch.fcgi?db=protein&id={0}&rettype=fasta&retmode=text{1}".format(
id_str, api_key_ext)
efetch_url = ENTREZ_BASE_URL + efetch_req
subprocess.run(args=['wget', efetch_url, '-O', fasta_fpath])
if pid_outpath:
id_df.to_csv(pid_outpath, sep='\t')
return id_df
def __init__(self, xmp):
self._root = ElementTree.fromstring(xmp)
def message(self):
file = open(self.filename)
mydata = file.read()
file.close()
if ('blankyear' in mydata):
rep = re.compile('blankyear')
mydata = rep.sub(self.content, mydata)
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
self.reply = '请输入查询月份：（例如：04）'
elif ('blankmonth' in mydata):
rep = re.compile('blankmonth')
mydata = rep.sub(self.content, mydata)
fp = open(self.filename, 'w')
fp.write(mydata)
fp.close()
myxml = ElementTree.fromstring(mydata)
name = myxml.find('Name').text.strip(' ')
tel = myxml.find('Tel').text.strip(' ')
userlimit = myxml.find('Userlimit').text.strip(' ')
year = myxml.find('Year').text.strip(' ')
month = myxml.find('Month').text.strip(' ')
if (len(month) == 1): month = str(0) + month
word = 'select wages from `members-wages` where name="' + name + '" and year="' + year + '" and month="' + month + '" and userlimit="' + userlimit + '"'
#print(word)
wagesdata = self.idc.searchWord(word, 1)
if (not isinstance(wagesdata, list)):
os.remove(self.filename)
else:
exceptkeys = ['出勤天数', '双休', '法定假', '未出勤天数', '']
exceptvalues = ['0.00', '']
self.reply = self.reply + '尊敬的' + name + '您好！\n'
self.reply = self.reply + '您' + year + '年' + month + '月的工资明细如下\n'
if (len(wagesdata) != 0):
for i in wagesdata:
wagesjson = eval(i[0])
for j in wagesjson:
#print(j+'-----'+wagesjson[j])
if ((wagesjson[j] != '')
and (wagesjson[j] not in exceptvalues)
and (j not in exceptkeys)):
self.reply = self.reply + j + '： ' + wagesjson[
j] + '\n'
#print(self.reply)
os.remove(self.filename)
else:
self.reply = '查无信息！！！'
os.remove(self.filename)
else:
if (os.path.exists(self.filename)):
os.remove(self.filename)
#print(self.reply)
self.text.append(1)
self.text.append(self.reply)
if len(self.text) > 1:
if self.text[0] == 1:
CreateTime = int(time.time())
out = self.reply_text(self.wx, self.touser, CreateTime,
self.text[1])
return out
def get_content_ascii_format(self): #pylint: disable=too-many-statements
"""
from the content, write the old format .ion file. Necessary since siesta only reads
ion info in this format.
"""
from xml.etree import ElementTree
root = ElementTree.fromstring(self.get_content())
#preliminary check on lj_projs, necessary due to a problem in siesta.
#See "Add lj_projs and j support to ion xml files" commit to siesta in GitLab
found_lj_proj = root.find("lj_projs")
if found_lj_proj is not None:
have_lj_proj = "T" in found_lj_proj.text or "t" in found_lj_proj.text
else:
ln_list = []
for proj in root.find("kbs"):
ln_list.append((int(proj.attrib["l"]), int(proj.attrib["n"])))
if len(ln_list) == len(set(ln_list)):
have_lj_proj = False
else:
have_lj_proj = True
#start of the file content construction
string = ""
#Construct the preamble (basis spec and pseudo header)
preamble_el = root.find("preamble")
string = string + "<" + preamble_el.tag + ">" + preamble_el.text
string = string + xml_element_to_string(preamble_el[0]) #basis
string = string + xml_element_to_string(preamble_el[1]) #pseudo_header
string = string + "</" + preamble_el.tag + ">\n"
string = string + root.find("symbol").text + "\n"
string = string + root.find("label").text + "\n"
string = string + root.find("z").text + "\n"
string = string + root.find("valence").text + "\n"
string = string + root.find("mass").text + "\n"
string = string + root.find("self_energy").text + "\n"
string = string + root.find("lmax_basis").text + root.find(
"norbs_nl").text + "\n"
if have_lj_proj:
string = string + root.find("lmax_projs").text + root.find(
"nprojs_nl").text + "T\n"
else:
string = string + root.find("lmax_projs").text + root.find(
"nprojs_nl").text + "#\n"
#The Paos
string = string + "# PAOs:__________________________\n"
for orbital in root.find("paos"):
string = string + orbital.attrib["l"] + orbital.attrib[
"n"] + orbital.attrib["z"] + orbital.attrib[
"ispol"] + orbital.attrib["population"] + "\n"
radfunc = orbital.find("radfunc")
string = string + radfunc.find("npts").text + radfunc.find(
"delta").text + radfunc.find("cutoff").text
string = string + radfunc.find("data").text
#The KBs. Note that (in case of have_lj_proj) the j value is not read from the .ion.xml but calculated
#on site. This is because the j value for each projector was added only in recent version of siesta.
#The implementation assumes that j=l-1/2 is always the first listed, j=l+1/2 the second! Hope it is true!!
string = string + "# KBs:__________________________\n"
collect_ln = []
for projector in root.find("kbs"):
l_val = int(projector.attrib["l"])
n_val = int(projector.attrib["n"])
if have_lj_proj:
if (l_val, n_val) in collect_ln:
j_val = " " + str(l_val + 0.5) + " "
else:
j_val = " " + str(
abs(l_val - 0.5)) + " " #abs for the l=0 case
string = string + " " + str(l_val) + j_val + str(
n_val) + projector.attrib["ref_energy"] + "\n"
else:
string = string + " " + str(l_val) + " " + str(
n_val) + projector.attrib["ref_energy"] + "\n"
collect_ln.append((l_val, n_val))
radfunc = projector.find("radfunc")
string = string + radfunc.find("npts").text + radfunc.find(
"delta").text + radfunc.find("cutoff").text
string = string + radfunc.find("data").text
#Other quantities
string = string + "# Vna:__________________________\n"
radfunc = root.find("vna").find("radfunc")
string = string + radfunc.find("npts").text + radfunc.find(
"delta").text + radfunc.find("cutoff").text
string = string + radfunc.find("data").text
string = string + "# Chlocal:__________________________\n"
radfunc = root.find("chlocal").find("radfunc")
string = string + radfunc.find("npts").text + radfunc.find(
"delta").text + radfunc.find("cutoff").text
string = string + radfunc.find("data").text
core_info = radfunc = root.find("core")
if core_info is not None:
string = string + "# Core:__________________________\n"
radfunc = root.find("core").find("radfunc")
string = string + radfunc.find("npts").text + radfunc.find(
"delta").text + radfunc.find("cutoff").text
string = string + radfunc.find("data").text
return string
