def delete_question(question_id=None):
try:
user_id = data_manager.get_userid_by_username(session['username'])
except KeyError:
return redirect('/')
if user_id == data_manager.get_foreign_key_by_id(
data_manager.question_db, 'users_id', question_id)[0]['users_id']:
data_manager.delete_line_by_foreign_id(data_manager.comment_db,
'question_id', question_id)
filename = 'static/image_for_question' + str(question_id) + '.png'
if util.check_file(filename):
util.delete_file(filename)
answer_ids_to_delete = data_manager.get_ids_by_foreign_id(
data_manager.answer_db, 'question_id', question_id)
for answer_id in answer_ids_to_delete:
data_manager.delete_line_by_foreign_id(data_manager.comment_db,
'answer_id',
answer_id['id'])
filename = 'static/image_for_answer' + str(
answer_id['id']) + '.png'
if util.check_file(filename):
util.delete_file(filename)
data_manager.delete_answer_by_question_id(question_id)
data_manager.delete_question(question_id)
else:
flash('Invalid user')
return redirect('/list')
def map_around_compound(mapfile, coord, compid):
'''cut the ASU map to the residue level.
compid: model_compound_chainID_resnumber_alt_insertion.
mapfile: the CCP4 map in ASU.
coord: the coordinate file for the map (in cif/pdb)
'''
if (not util.check_file(100, mapfile) or not util.check_file(100, coord)):
err = 'Error: Either mapfile or coordinate file does not exist.'
config.ERRLOG.append(err)
print(err)
return
xyzlim, xyzcomp = find_xyzlim_compound(compid, coord)
if (len(xyzlim.strip()) < 2):
err = 'Error: compound boundary in fraction not found. check with compound ID'
config.ERRLOG.append(err)
print(err)
return
# below is cut map and get jmol
t = compid.split('_')
comp = '_'.join([t[0], t[1], t[2]])
mapout = comp + '_cut.map'
maphtml = comp + '.html'
mapscr = cut_map_bylimit(xyzlim)
util.delete_file(mapout)
arg = mapfile + ' ' + ' ' + mapout
command = "chmod +x %s ; ./%s " % (mapscr, mapscr) + arg
os.system(command)
min, max, mean, sigma = map_info(mapfile)
min1, max1, mean1, sigma1 = map_info(mapout)
cont = {
'0.5': 0.5,
'0.7': 0.7,
'1.0': 1.0,
'1.5': 1.5,
'2.0': 2.0
} #contour 4 map in asu.
cont1 = cont #contour 4 sub map.
scale = 1.0
if (float(sigma1) > 0):
scale = float(sigma) / float(sigma1)
for z in list(cont.keys()):
cont1[z] = cont[z] * scale
maphtml = get_html4jmol(comp, xyzcomp, mapout, cont1)
return maphtml, mapout
def get_dcc(dccfile):
'''put dccfile as list of list
'''
dcc = []
if (util.check_file(100, dccfile) == 0): return dcc
flist = open(dccfile, 'r').readlines()
items, values = cif.cifparse(flist, '_pdbx_rscc_mapman.')
nseq = cif.parse_values(items, values, "_pdbx_rscc_mapman.auth_seq_id")
chid = cif.parse_values(items, values, "_pdbx_rscc_mapman.auth_asym_id")
comp = cif.parse_values(items, values, "_pdbx_rscc_mapman.auth_comp_id")
alt = cif.parse_values(items, values, "_pdbx_rscc_mapman.label_alt_id")
ins = cif.parse_values(items, values, "_pdbx_rscc_mapman.label_ins_code")
cc = cif.parse_values(items, values, "_pdbx_rscc_mapman.correlation")
rsr = cif.parse_values(items, values, "_pdbx_rscc_mapman.real_space_R")
zrsr = cif.parse_values(items, values,
"_pdbx_rscc_mapman.real_space_Zscore")
biso = cif.parse_values(items, values, "_pdbx_rscc_mapman.Biso_mean")
occ = cif.parse_values(items, values, "_pdbx_rscc_mapman.occupancy_mean")
#model=cif.parse_values(items,values,"_pdbx_rscc_mapman.model_id");
pdbid = cif.parse_values(items, values, "_pdbx_rscc_mapman.pdb_id")
if not items: return dcc
for i in range(len(chid)):
a = [
nseq[i], chid[i], comp[i], alt[i], cc[i], rsr[i], biso[i], occ[i],
pdbid[i]
]
dcc.append(a)
return dcc
def map_info(mapfile):
'''get the min, max, mean, sigma from the map
'''
min, max, mean, sigma = '-1', '-1', '-1', '-1'
log = mapfile + '_header'
scr = mapfile + '.sh'
arg = 'mapdump mapin %s <<eof >%s \neof\n' % (mapfile, log)
fw = open(scr, 'w')
fw.write(arg)
fw.close()
os.system('chmod +x %s ; ./%s' % (scr, scr))
if not util.check_file(10, log): return min, max, mean, sigma
fp = open(log, 'r')
for x in fp:
if 'Minimum density ...' in x:
min = x.rstrip().split(' ')[-1]
elif 'Maximum density ...' in x:
max = x.rstrip().split(' ')[-1]
elif ' Mean density ..' in x:
mean = x.rstrip().split(' ')[-1]
elif 'deviation from mean density ..' in x:
sigma = x.rstrip().split(' ')[-1]
util.delete_file(scr, log)
return min, max, mean, sigma
def downloader(file_url):
""" TODO """
file_url = str(file_url)
dir_name = urllib.parse.unquote(file_url.split('/')[5])
zip_name = '.'.join([dir_name, 'zip'])
# tar_name = '.'.join([dir_name, 'tar', 'gz'])
try:
if not os.path.exists(dir_name):
os.mkdir(dir_name)
file_num = extract_file_number(get_file_name(file_url))
for i in range(int(file_num), -1, -1):
web_url = file_url[0:(file_url.rindex('-') + 1)] + str(i) + 'a.jpg'
if not check_file(web_url):
web_url = file_url[0:(file_url.rindex('-') +
1)] + str(i) + 'v.jpg'
file_object = requests.get(web_url, allow_redirects=True)
download_location = os.path.sep.join(
[os.getcwd(), dir_name,
get_file_name(web_url)])
with open(download_location, 'wb') as image:
image.write(file_object.content)
file_paths = get_all_file_paths(dir_name)
result = upload_files(file_paths, dir_name)
return jsonify(message=result)
except Exception:
pass
def get_user_admins(self):
if not os.path.exists(self.user_admin_file):
util.check_file(self.user_admin_file)
return []
with open(self.user_admin_file, 'r') as file:
if os.stat(self.user_admin_file).st_size == 0:
return []
user_admin_list = []
for d in json.load(file):
try:
user_admin_list.append(
user_admin.UserAdmin(d[self.json_user_id], d[self.json_server_id]))
except KeyError:
print("user_id or server_id doesnt exist: {}".format(d))
return user_admin_list
def get_channel_birthdays(self):
if not os.path.exists(self.channel_birthday_file):
util.check_file(self.channel_birthday_file)
return []
with open(self.channel_birthday_file, 'r') as file:
if os.stat(self.channel_birthday_file).st_size == 0:
return []
channel_birthday_list = []
for d in json.load(file):
try:
channel_birthday_list.append(
channel_birthday.ChannelBirthday(
d[self.json_channel_id], d[self.json_server_id]))
except KeyError:
print("channel_id or server_id doesnt exist: {}".format(d))
return channel_birthday_list
def read(self, raw, offset): self.data = dict() #int16_t graphic_id; #int8_t damage_percent; #int8_t unknown; #int8_t unknown; damage_graphic_struct = Struct(endianness + "h 3b") pc = damage_graphic_struct.unpack_from(raw, offset) offset_info(offset, pc, "damage graphic", damage_graphic_struct, mode=1) offset += damage_graphic_struct.size self.data["graphic_id"] = pc[0] check_file(pc[0]) self.data["damage_percent"] = pc[1] #self.data[""] = pc[2] #self.data[""] = pc[3] return offset
def pointless_log(logfile, dic):
'''parse stat from the log of pointless
'''
if util.check_file(100, logfile)==0 : return
for x in open(logfile, 'r').readlines():
if ('<L^2>:' in x[:11] or '< L^2 >' in x[:11]) and 'untwinned' in x:
dic['L2_pt']=x.split(':')[1].lstrip().split()[0]
elif 'Best Solution: ' in x[:17] and 'space group' in x:
dic['spg_pt']=x.split('group')[1].strip()
def get_message_birthdays(self):
if not os.path.exists(self.message_birthday_file):
util.check_file(self.message_birthday_file)
return []
with open(self.message_birthday_file, 'r') as file:
if os.stat(self.message_birthday_file).st_size == 0:
return []
message_birthday_list = []
for d in json.load(file):
try:
message_birthday_list.append(
message_birthday.MessageBirthday(
d[self.json_server_id],
d[self.json_birthday_message]))
except KeyError:
print("server_id or birthday_message doesnt exist: {}".
format(d))
return message_birthday_list
def sfcheck_cif(inpfile, dd, detail):
if not util.check_file(100,inpfile): return
fr=open(inpfile, "r")
for ln in fr:
if "_pdbx_map_overall.RSR" in ln: #sfcheck,
dd['realr']=util.get_value_after_id(ln,"RSR")
elif "_pdbx_map_overall.RSCC" in ln:
dd['dcorr']=util.get_value_after_id(ln,"RSCC")
fr.close()
def init():
# 加载配置文件
with open("config.yml", encoding="utf-8") as f:
config = yaml.safe_load(f)
# 创建日志文件
check_file(config["logger"])
# 创建缓存文件
check_file(config["ip_buffer"])
# logging 基本设置
logging.basicConfig(
filename=config['logger'],
level=logging.DEBUG,
format="%(asctime)s %(filename)s : %(levelname)s %(message)s",
datefmt="%Y-%m-%d %A %H:%M:%S")
# 同时为 console 创建 logging
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s %(filename)s : %(levelname)s %(message)s')
console.setFormatter(formatter)
logging.getLogger().addHandler(console)
return config
def sfcheck_log(inpfile, dd, detail):
'''extract calculated data from file and put them into dd
'''
if not util.check_file(100,inpfile): return
dd['prog']='SFCHECK'
if detail=='NOTLS':
dd['detail']='without TLS correction'
else:
dd['detail']='with TLS correction'
fr=open(inpfile, "r")
for ln in fr:
if "Resolution /input data/ :" in ln: #sfcheck
tmp=util.get_value_after_id(ln, ':')
dd['resl'],dd['resh'] = [x.strip() for x in tmp.split('-')]
elif 'Completeness :' in ln:
tmp = util.get_value_after_id(ln, ':')
dd['comp']= tmp.split()[0]
elif 'R-factor :' in ln:
dd['rall']= util.get_value_after_id(ln, ':')
dd['rfact']=dd['rall']
elif 'Correlation factor :' in ln:
dd['fcc']= util.get_value_after_id(ln, ':')
#elif 'N_refls :' in ln:
#dd['nref']= util.get_value_after_id(ln, ':')
elif 'Number of reflections :' in ln and 'in_file ' in ln :
dd['nref']= util.get_value_after_id(ln, ':').split('(')[0]
elif 'Nfree,Nrest :' in ln:
dd['nfree']= util.get_value_after_id(ln, ':').split()[0]
elif 'Rfree,Rrest :' in ln:
tmp=util.get_value_after_id(ln, ':')
dd['rfree'],t = [x.strip() for x in tmp.split()]
if (util.is_number(t)): dd['rfact'] = t
elif 'Boverall /estimated by Wilson plot/ : ' in ln:
dd['bwilson']= util.get_value_after_id(ln, ':')
elif 'PDB_code :' in ln:
dd['pdbid']= util.get_value_after_id(ln, ':')
fr.close()
def sf_convertor(pdbfile, sffile, type1, add):
'''convert to any type of formats,
type1: the output format; add: extra options
'''
print ('\nConverting SF file to %s format ... ' %type1)
out=sffile + '.' + type1
if not util.check_file(500,pdbfile):
arg="sf_convert -o %s -sf %s -out %s %s >/dev/null " %(type1, sffile, out, add)
else:
arg="sf_convert -o %s -sf %s -pdb %s -out %s %s >/dev/null " %(type1,sffile, pdbfile, out,add)
os.system(arg)
return out
def val_from_dcc(outf):
''' The format of outf can not be changed !!
index (0,1,2,3,4,5,6,7...) corresponds to
(resh,resl,Rwork,Rfree,Comp,FCC, Real_R, Dcc)
res_rep: the reported; res_not: no TLS; res_tls: with TLS
'''
if not util.check_file(500, outf): return '', ''
flist = open(outf, "r").readlines()
items, values = cif.cifparse(flist, '_pdbx_density.')
res = cif.parse_values(items, values, '_pdbx_density.ls_d_res_high')
rw = cif.parse_values(items, values, '_pdbx_density.R_value_R_work')
rf = cif.parse_values(items, values, '_pdbx_density.R_value_R_free')
biso = cif.parse_values(items, values, '_pdbx_density.Biso_mean')
bwil = cif.parse_values(items, values, '_pdbx_density.B_wilson')
l2 = cif.parse_values(items, values,
'_pdbx_density.Padilla-Yeates_L2_mean')
z = cif.parse_values(items, values, '_pdbx_density.Z_score_L_test')
fom = cif.parse_values(items, values, '_pdbx_density.fom')
items, values = cif.cifparse(flist, '_pdbx_density_corr.')
prog = cif.parse_values(items, values, '_pdbx_density_corr.program')
resh = cif.parse_values(items, values, '_pdbx_density_corr.ls_d_res_high')
rwork = cif.parse_values(items, values,
'_pdbx_density_corr.ls_R_factor_R_work')
rfree = cif.parse_values(items, values,
'_pdbx_density_corr.ls_R_factor_R_free')
fcc = cif.parse_values(items, values,
'_pdbx_density_corr.correlation_coeff_Fo_to_Fc')
rsr = cif.parse_values(items, values, '_pdbx_density_corr.real_space_R')
dcc = cif.parse_values(items, values, '_pdbx_density_corr.correlation')
detail = cif.parse_values(items, values, '_pdbx_density_corr.details')
nr, nb = 0, 0
for i, x in enumerate(detail):
if 'Best' in x:
nb = i
break
rep = '%8s %4s %6s %6s %6s ' % (prog[nr], resh[nr], rwork[nr],
rfree[nr], biso[0])
val = '%8s %4s %6s %6s %6s %6s %6s %6s %6s %6s %6s ' % (
prog[nb], resh[nb], rwork[nb], rfree[nb], bwil[0], l2[0], z[0], fom[0],
fcc[nb], rsr[nb], dcc[nb])
return rep, val
def sf_symmetry(sffile, pdbfile):
''' get the best space group by pointless.
'''
sfmtz = prog.sf_convertor(pdbfile, sffile, 'mtz', '')
if not util.check_file(500, sfmtz):
print("Error: MTZ file not generated, check symmetry/cell in sf file.")
sys.exit()
print('Getting the best space group by pointless...')
out = prog.run_ccp4(pdbfile, sfmtz, 'pointless')
print('For details, please see the output file =%s' % out)
util.delete_file(sfmtz, 'sf_format_guess.text', 'sf_information.txt')
def log_from_shelx(shelx_log, dic):
'''parse stat from the log of shelx
'''
if util.check_file(100, shelx_log)==0 :
util.perror('Error: SHELX failed!')
return
dic['prog']='SHELX'
fp = open(shelx_log, 'r')
for x in fp:
if ' R1 = ' in x and 'unique reflections' in x:
t=x.split('=',1)[1].split()
dic['rfact'], dic['nref'] = t[0], t[2]
fp.close()
def __init__(self, path="./config.ini"):
"""
检查配置文件是否存在，不存在则直接return
检查输出文件夹是否存在，不存在则创建
config.ini必须要有的字段
- dump_dir
- base_url
"""
print u'\n in Spider __init__'
if not util.check_file(path):
print '\n\t no file ' + path
return
self.config = util.init(path)
if self.config.has_key('ua'):
self.config['ua'] = util.init(self.config['ua'])
self.config['requests_dir'] = self.config['dump_dir'] + '/requests'
print self.config
util.check_path(self.config['dump_dir'])
def __init__(self, path="./config.ini"):
"""
检查配置文件是否存在，不存在则直接return
检查输出文件夹是否存在，不存在则创建
config.ini必须要有的字段
- dump_dir
- base_url
"""
print u'\n in Spider __init__'
if not util.check_file(path):
print '\n\t no file ' + path
return
self.config = util.init(path)
if self.config.has_key('ua'):
self.config['ua'] = util.init(self.config['ua'])
self.config['requests_dir'] = self.config['dump_dir'] + '/requests'
print self.config
util.check_path(self.config['dump_dir'])
def get_atominfo(file):
'''Load atom information (pdb or cif)
'''
atominfo = {
'chain': [],
'nres': [],
'comp': [],
'atom': [],
'symbol': [],
'alt': [],
'ins': [],
'x': [],
'y': [],
'z': [],
'xf': [],
'yf': [],
'zf': [],
'occ': [],
'mtr': [],
'symop': [],
'spg': '',
'cell': [],
'mtrix': [],
'atom_o': [],
'group': [],
'natm': [],
'natm_c': [],
'segid': []
}
#print 'Loading atom information.'
if not util.check_file(100, file):
print('Error: file (%s) not exist' % file)
return atominfo
if util.is_cif(file):
dic = data_from_cif(file, atominfo)
else:
dic = data_from_pdb(file, atominfo)
# print 'spg=', dic['spg'], dic['cell']
atominfo.update(dic)
return atominfo
def contact(file):
'''calculate close contact with grid method check the NCS matrix if exist.
'''
if not util.check_file(100, file):
print('Error: file (%s) does not exist' % file)
return
atominfo = get_atominfo(file)
# expand_sym_cello(atominfo) #testing
print 'All close contacts of atoms in the file =%s' % file
netdis = check_dis(atominfo, 0)
if atominfo['mtrix']: #do extra for atoms by mtrix
print '\nClose contacts of atoms generated by matrix in file=%s' % file
atominfo_mtr = atom_by_mtrix(atominfo, atominfo['mtrix'])
netdis = check_dis(atominfo_mtr, 0)
def parse_edstat(out):
'''parse the output file generated by EDSTAT. use 1st line items as key.
'''
edst={}
if util.check_file(100, out)==0: return edst
flist=open(out, 'r').readlines()
first=[]
for x in flist:
if 'RT ' in x and 'RN ' in x and 'CI ' in x :
first=x.split()
for y in first : edst[y]=[]
else:
t=x.split()
for i, y in enumerate(t): edst[first[i]].append(y)
return edst
def values_from_dcc(outf):
''' The format of outf can not be changed !!
index (0,1,2,3,4,5,6,7...) corresponds to
(resh,resl,Rwork,Rfree,Comp,FCC, Real_R, Dcc)
res_rep: the reported; res_not: no TLS; res_tls: with TLS
'''
bstatus, prog, prog_rep, value = '?', 'REFMAC', '?', '?'
res_rep = [
-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0
]
res_not = [
-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0
]
res_tls = [
-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0
]
if not util.check_file(500, outf):
return value, res_rep, res_not, res_tls, bstatus, prog_rep
fr = open(outf, "r")
for line in fr:
if "'PDB reported'" in line:
get_value(res_rep, line)
prog_rep = line.split()[1].strip()
elif prog in line and 'Best ' in line:
get_value(res_not, line)
elif '_pdbx_density.iso_B_value_type' in line:
bstatus = 'P'
if ' FULL' in line: bstatus = 'T'
fr.close()
value="%4.2f %5.3f %5.3f %5.3f %5.3f %5.3f %5.3f " \
%(res_rep[0], res_rep[2], res_rep[3], res_rep[5],
res_not[2], res_not[3], res_not[5])
return value, res_rep, res_not, res_tls, bstatus, prog_rep
def delete_answer(answer_id=None):
try:
user_id = data_manager.get_userid_by_username(session['username'])
except KeyError:
return redirect('/')
question_id = data_manager.get_foreign_key_by_id(
data_manager.answer_db, 'question_id', answer_id)[0]['question_id']
if user_id == data_manager.get_foreign_key_by_id(data_manager.answer_db,
'users_id',
answer_id)[0]['users_id']:
data_manager.delete_line_by_foreign_id(data_manager.comment_db,
'answer_id', answer_id)
question_id = data_manager.delete_answer_by_id(
answer_id)['question_id']
filename = 'static/image_for_answer' + str(answer_id) + '.png'
if util.check_file(filename):
util.delete_file(filename)
else:
flash('Invalid user')
return redirect('/question/{}'.format(question_id))
def plot_items(plot_data, id, title, xlabel, ylabel, plot):
'''
'''
if id == 'twin':
if len(title) == 0:
title = 'The Padilla-Yeates cumulative probability (acentric data)'
if len(xlabel) == 0: xlabel = '|L|'
if len(ylabel) == 0: ylabel = 'N(|L|)'
if len(plot) == 0:
plot = ''' using 1:2 t "Observed data", '' u 1:3 t "Untwinned by theory" , '' u 1:4 t "Twinned by theory" '''
elif id == 'I/sigI':
title = 'The change of <I/sigI> with resolution (1/resol^2)'
xlabel, ylabel = '1/resol^2', '<I/sigI>'
plot = ' using 1:2 t "<I/sigI> vs resolution" '
# print(plot_data)
prog.gnu_plot1(plot_data, xlabel, ylabel, title, plot)
if not util.check_file(500,
plot_data + '.png'): #png not generate, try ps file
print('Trying to plot in postscript')
prog.gnu_plotps(plot_data, xlabel, ylabel, title, plot)
def get_rsr_database(res, rfact, file):
'''parse the mean and dev of the RsR for the res range for each
residue.
'''
if not util.check_file(30, file):
print('Warning: no RsR data base is found. ')
flist=open(file, 'r').readlines()
items,values = cif.cifparse(flist, '_rsr_shell.') # a loop
resname=cif.parse_values(items,values,"_rsr_shell.residue");
resh=cif.parse_values(items,values,"_rsr_shell.d_res_high");
resl=cif.parse_values(items,values,"_rsr_shell.d_res_low");
rfh=cif.parse_values(items,values,"_rsr_shell.rfact_high");
rfl=cif.parse_values(items,values,"_rsr_shell.rfact_low");
sst=cif.parse_values(items,values,"_rsr_shell.secondary_structure");
mean_all=cif.parse_values(items,values,"_rsr_shell.mean_all");
dev_all= cif.parse_values(items,values,"_rsr_shell.deviation_all");
num_all=cif.parse_values(items,values,"_rsr_shell.number_all");
mean_fil=cif.parse_values(items,values,"_rsr_shell.mean_filter");
dev_fil=cif.parse_values(items,values,"_rsr_shell.deviation_filter");
num_fil=cif.parse_values(items,values,"_rsr_shell.number_filter");
resn={}
for i,x in enumerate (resname) :
# if float(resh[i]) <=res <= float(resl[i]):
id='%s_%s' %(resname[i], sst[i])
# if float(resh[i]) <= res <= float(resl[i]) and float(rfl[i]) <= rfact <= float(rfh[i]):
if float(resh[i]) <= res <= float(resl[i]) :
t= [float(mean_all[i]),float(dev_all[i]),int(num_all[i]),
float(mean_fil[i]),float(dev_fil[i]), int(num_fil[i])]
if id not in resn : resn[ id ] = t
return resn
def test_trailing_backslash(filename, string, expected):
warnings = util.check_file(m.TrailingBackslash, filename, string)
assert warnings == expected
def filename(self, filename):
check_file(filename)
self._filename = filename
def test_hash_number_of_fields(filename, string, expected):
warnings = util.check_file(m.HashNumberOfFields, filename, string)
assert warnings == expected
def test_hash_type(filename, string, expected):
warnings = util.check_file(m.HashType, filename, string)
assert warnings == expected
def read(self, raw, offset):
self.data = dict()
#unit type==:
#object = 10
#flag = 20
#dead_fish = 30
#bird = 40
#projectile = 60
#creatable = 70
#building = 80
#tree = 90
#int8_t type;
#uint16_t name_length;
#int16_t id0;
#uint16_t language_dll_name;
#uint16_t language_dll_creation;
#int16_t unit_class;
#int16_t graphic_standing0;
#int16_t graphic_standing1;
#int16_t graphic_dying0;
#int16_t graphic_dying1;
#int8_t death_mode;
#int16_t hit_points;
#float line_of_sight;
#int8_t garnison_capacity;
#float radius_size0;
#float radius_size1;
#float hp_bar_height0;
#int16_t sound_train0;
#int16_t sound_train1;
#int16_t dead_unit_id;
#int8_t placement_mode;
#int8_t air_mode;
#int16_t icon_id;
#int8_t hidden_in_editor;
#int16_t unknown;
#int16_t enabled;
#int16_t placement_by_pass_terrain0;
#int16_t placement_by_pass_terrain1;
#int16_t placement_terrain0;
#int16_t placement_terrain1;
#float editor_radius0;
#float editor_radius1;
#int8_t building_mode;
#int8_t visible_in_fog;
#int16_t terrain_restriction;
#int8_t fly_mode;
#int16_t ressource_capacity;
#float ressource_decay;
#int8_t blast_type;
#int8_t unknown;
#int8_t interaction_mode;
#int8_t minimap_mode;
#int16_t command_attribute;
#int16_t unknown;
#int16_t unknown;
#uint16_t language_dll_help;
#int16_t[4] hot_keys;
#int8_t unknown;
#int8_t unknown;
#bool unselectable;
#int8_t unknown;
#int8_t unknown;
#int8_t unknown;
#int8_t selection_mask;
#int8_t selection_shape_type;
#int8_t selection_shape;
#int8_t attribute;
#int8_t civilisation;
#int16_t unknown;
#int8_t selection_effect;
#uint8_t editor_selection_color;
#float selection_radius0;
#float selection_radius1;
#float hp_bar_height1;
unit0_struct = Struct(endianness + "b H h 2H 5h b h f b 3f 3h 2b h b 6h 2f 2b h b h f 4b 3h H 4h 2b ? 8b h b B 3f")
pc = unit0_struct.unpack_from(raw, offset)
offset_info(offset, pc, "unit header0", unit0_struct)
offset += unit0_struct.size
self.data["type"] = pc[0]
self.data["name_length"] = pc[1]
if self.data["name_length"] > 100:
raise Exception("name is longer than 100, very unlikely: %d" % self.data["name_length"])
self.data["id0"] = pc[2]
self.data["language_dll_name"] = pc[3]
self.data["language_dll_creation"] = pc[4]
self.data["unit_class"] = pc[5]
self.data["graphic_standing0"] = pc[6]
check_file(pc[6])
self.data["graphic_standing1"] = pc[7]
check_file(pc[7])
self.data["graphic_dying0"] = pc[8]
self.data["graphic_dying1"] = pc[9]
self.data["death_mode"] = pc[10]
self.data["hit_points"] = pc[11]
self.data["line_of_sight"] = pc[12]
self.data["garnison_capacity"] = pc[13]
self.data["radius_size0"] = pc[14]
self.data["radius_size1"] = pc[15]
self.data["hp_bar_height0"] = pc[16]
self.data["sound_train0"] = pc[17]
check_file(pc[17])
self.data["sound_train1"] = pc[18]
check_file(pc[18])
self.data["dead_unit_id"] = pc[19]
self.data["placement_mode"] = pc[20]
self.data["air_mode"] = pc[21]
self.data["icon_id"] = pc[22]
self.data["hidden_in_editor"] = pc[23]
#self.data[""] = pc[24]
self.data["enabled"] = pc[25]
self.data["placement_by_pass_terrain0"] = pc[26]
self.data["placement_by_pass_terrain1"] = pc[27]
self.data["placement_terrain0"] = pc[28]
self.data["placement_terrain1"] = pc[29]
self.data["editor_radius0"] = pc[30]
self.data["editor_radius1"] = pc[31]
self.data["building_mode"] = pc[32]
self.data["visible_in_fog"] = pc[33]
self.data["terrain_restriction"] = pc[34]
self.data["fly_mode"] = pc[35]
self.data["ressource_capacity"] = pc[36]
self.data["ressource_decay"] = pc[37]
self.data["blast_type"] = pc[38]
#self.data[""] = pc[39]
self.data["interaction_mode"] = pc[40]
self.data["minimap_mode"] = pc[41]
self.data["command_attribute"] = pc[42]
#self.data[""] = pc[43]
#self.data[""] = pc[44]
self.data["language_dll_help"] = pc[45]
self.data["hot_keys"] = pc[46:(46+4)]
#self.data[""] = pc[50]
#self.data[""] = pc[51]
self.data["unselectable"] = pc[52]
#self.data[""] = pc[53]
#self.data[""] = pc[54]
#self.data[""] = pc[55]
self.data["selection_mask"] = pc[56]
self.data["selection_shape_type"] = pc[57]
self.data["selection_shape"] = pc[58]
self.data["attribute"] = pc[59]
self.data["civilisation"] = pc[60]
#self.data[""] = pc[61]
self.data["selection_effect"] = pc[61]
self.data["editor_selection_color"] = pc[62]
self.data["selection_radius0"] = pc[63]
self.data["selection_radius1"] = pc[64]
self.data["hp_bar_height1"] = pc[65]
self.data["ressource_storage"] = list()
for i in range(3):
t = RessourceStorage()
offset = t.read(raw, offset)
self.data["ressource_storage"] += [t.data]
#int8_t damage_graphic_count
unit1_struct = Struct(endianness + "b")
pc = unit1_struct.unpack_from(raw, offset)
offset += unit1_struct.size
self.data["damage_graphic_count"], = pc
self.data["damage_graphic"] = list()
for i in range(self.data["damage_graphic_count"]):
t = DamageGraphic()
offset = t.read(raw, offset)
self.data["damage_graphic"] += [t.data]
#int16_t selection_sound;
#int16_t dying_sound;
#int16_t attack_sound;
#char name[name_length];
#int16_t id1;
#int16_t id2;
unit2_struct = Struct(endianness + "3h %ds 2h" % self.data["name_length"])
pc = unit2_struct.unpack_from(raw, offset)
offset_info(offset, pc, "unit header 2", unit2_struct)
offset += unit2_struct.size
self.data["sound_selection"] = pc[0]
check_file(pc[0])
self.data["sound_dying"] = pc[1]
check_file(pc[1])
self.data["sound_attack"] = pc[2]
check_file(pc[2])
self.data["name"] = zstr(pc[3])
self.data["id1"] = pc[4]
self.data["id2"] = pc[5]
dbg("unit type = %d\nname = %s" % (self.data["type"], self.data["name"]), 2)
#every thing else only exists for non-trees (!= 90)
#and things with id >= 20
if self.data["type"] != 90 and self.data["type"] >= 20:
#float speed;
tmp_struct = Struct(endianness + "f")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=20 SPEED", tmp_struct)
offset += tmp_struct.size
self.data["speed"], = pc
if math.isnan(self.data["speed"]):
raise Exception("Speed is NaN!")
dbg("speed:" + str(pc), 2)
if self.data["type"] >= 30:
#int16_t walking_graphics0;
#int16_t walking_graphics1;
#float rotation_speed;
#int8_t unknown;
#int16_t tracking_unit;
#bool tracking_unit_used;
#float tracking_unit_density;
#float unknown;
#int8_t unknown[17];
tmp_struct = Struct(endianness + "2h f b h ? 2f 17b")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=30", tmp_struct)
offset += tmp_struct.size
self.data["walking_graphics0"] = pc[0]
check_file(pc[0])
self.data["walking_graphics1"] = pc[1]
check_file(pc[1])
self.data["rotation_speed"] = pc[2]
#self.data[""] = pc[2]
self.data["tracking_unit"] = pc[3]
self.data["tracking_unit_used"] = pc[4]
self.data["tracking_unit_density"] = pc[5]
#self.data[""] = pc[6]
#self.data[""] = pc[7:(7+17)]
if self.data["type"] >= 40:
#int16_t sheep_conversion;
#float search_radius;
#float work_rate;
#int16_t drop_site0;
#int16_t drop_site1;
#int8_t villager_mode;
#int16_t move_sound;
#int16_t stop_sound;
#int8_t animal_mode;
tmp_struct = Struct(endianness + "h 2f 2h b 2h b")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=40", tmp_struct)
offset += tmp_struct.size
self.data["sheep_conversion"] = pc[0]
self.data["search_radius"] = pc[1]
self.data["work_rate"] = pc[2]
self.data["drop_site0"] = pc[3]
self.data["drop_site1"] = pc[4]
self.data["villager_mode"] = pc[5]
self.data["move_sound"] = pc[6]
check_file(pc[6])
self.data["stop_sound"] = pc[7]
check_file(pc[7])
self.data["animal_mode"] = pc[8]
if self.data["type"] >= 60:
#int8_t unknown;
#int8_t unknown;
tmp_struct = Struct(endianness + "2b")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=60 header0", tmp_struct)
offset += tmp_struct.size
#self.data[""] = pc[0]
#self.data[""] = pc[1]
#uint16_t attack_count;
tmp_struct = Struct(endianness + "H")
pc = tmp_struct.unpack_from(raw, offset)
offset += tmp_struct.size
self.data["attack_count"] = pc[0]
self.data["attack"] = list()
for i in range(self.data["attack_count"]):
t = HitType()
offset_tmp = t.read(raw, offset)
self.data["attack"] += [t.data]
offset = offset_tmp
#uint16_t armor_count;
tmp_struct = Struct(endianness + "H")
pc = tmp_struct.unpack_from(raw, offset)
offset += tmp_struct.size
self.data["armor_count"] = pc[0]
self.data["armor"] = list()
for i in range(self.data["armor_count"]):
t = HitType()
offset_tmp = t.read(raw, offset)
self.data["armor"] += [t.data]
offset = offset_tmp
#int16_t unknown;
#float max_range;
#float blast_radius;
#float reload_time0;
#int16_t projectile_unit_id;
#int16_t accuracy_percent;
#int8_t tower_mode;
#int16_t delay;
#float graphics_displacement[3];
#int8_t unknown;
#float min_range;
#float garnison_recovery_rate;
#int16_t attack_graphic;
#int16_t melee_armor_displayed;
#int16_t attack_displayed;
#int16_t range_displayed;
#float reload_time1;
tmp_struct = Struct(endianness + "h 3f 2h b h 3f b 2f 4h xx f")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=60 data", tmp_struct)
offset += tmp_struct.size - 2
#self.data[""] = pc[0]
self.data["max_range"] = pc[1]
self.data["blast_radius"] = pc[2]
self.data["reload_time0"] = pc[3]
self.data["projectile_unit_id"] = pc[4]
self.data["accuracy_percent"] = pc[5]
self.data["tower_mode"] = pc[6]
self.data["delay"] = pc[7]
self.data["graphics_displacement"] = pc[8:(8+3)]
#self.data[""] = pc[11]
self.data["min_range"] = pc[12]
self.data["garnison_recovery_rate"] = pc[13]
self.data["attack_graphic"] = pc[14]
self.data["melee_armor_displayed"] = pc[15]
self.data["attack_displayed"] = pc[16]
self.data["range_displayed"] = pc[17]
self.data["reload_time1"] = pc[18]
if self.data["type"] == 60:
#int8_t stretch_mode;
#int8_t compensation_mode;
#int8_t drop_animation_mode;
#int8_t penetration_mode;
#int8_t unknown;
#float projectile_arc;
tmp_struct = Struct(endianness + "5b x x f")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, "==60", tmp_struct)
offset += tmp_struct.size
self.data["stretch_mode"] = pc[0]
self.data["compensation_mode"] = pc[1]
self.data["drop_animation_mode"] = pc[2]
self.data["penetration_mode"] = pc[3]
#self.data[""] = pc[4]
self.data["projectile_arc"] = pc[5]
if self.data["type"] >= 70:
self.data["ressource_cost"] = list()
for i in range(3):
t = RessourceCost()
offset = t.read(raw, offset)
self.data["ressource_cost"] += [t.data]
#int16_t train_time;
#int16_t train_location_id;
#int8_t button_id;
#int32_t unknown;
#int32_t unknown;
#int8_t missile_graphic_delay;
#int8_t hero_mode;
#int16_t garnison_graphic0;
#int16_t garnison_graphic1;
#float attack_missile_duplication0;
#int8_t attack_missile_duplication1;
#float attack_missile_duplication_spawning[3];
#int32_t attack_missile_duplication_unit;
#int32_t attack_missile_duplication_graphic;
#int8_t unknown;
#int16_t pierce_armor_displayed;
tmp_struct = Struct(endianness + "2h b 2i 2b 2h xx f b 3f 2i b h")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=70", tmp_struct)
offset += tmp_struct.size
self.data["train_time"] = pc[0]
self.data["train_location_id"] = pc[1]
self.data["button_id"] = pc[2]
#self.data[""] = pc[3]
#self.data[""] = pc[4]
self.data["missile_graphic_delay"] = pc[5]
self.data["hero_mode"] = pc[6]
self.data["garnison_graphic0"] = pc[7]
self.data["garnison_graphic1"] = pc[8]
self.data["attack_missile_duplication0"] = pc[9]
self.data["attack_missile_duplication1"] = pc[10]
self.data["attack_missile_duplication_spawning"] = pc[11:(11+3)]
self.data["attack_missile_duplication_unit"] = pc[14]
self.data["attack_missile_duplication_graphic"] = pc[15]
#self.data[""] = pc[16]
self.data["pierce_armor_displayed"] = pc[17]
#raise Exception()
if self.data["type"] >= 80:
#int16_t construction_graphic_id;
#int16_t snow_graphic_id;
#int16_t adjacent_mode;
#int8_t unknown;
#int8_t unknown;
#int16_t stack_unit_id;
#int16_t terrain_id;
#int16_t unknown;
#int16_t research_id;
#int8_t unknown;
tmp_struct = Struct(endianness + "3h 2b 4h b")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=80 header 0", tmp_struct)
offset += tmp_struct.size
self.data["construction_graphic_id"] = pc[0]
check_file(pc[0])
self.data["snow_graphic_id"] = pc[1]
check_file(pc[1])
self.data["adjacent_mode"] = pc[2]
#self.data[""] = pc[3]
#self.data[""] = pc[4]
self.data["stack_unit_id"] = pc[5]
self.data["terrain_id"] = pc[6]
#self.data[""] = pc[7]
#self.data["research_id"] = pc[8]
#self.data[""] = pc[9]
self.data["building_annex"] = list()
for i in range(4):
t = BuildingAnnex()
offset_tmp = t.read(raw, offset)
offset = offset_tmp
self.data["building_annex"] += [t.data]
#int16_t head_unit;
#int16_t transform_unit;
#int16_t unknown;
#int16_t construction_sound_id;
#int8_t garnison_type;
#float garnison_heal_rate;
#int32_t unknown;
#int16_t unknown;
#int8_t unknown[6];
tmp_struct = Struct(endianness + "4h b f i h 6b")
pc = tmp_struct.unpack_from(raw, offset)
offset_info(offset, pc, ">=80 header 1", tmp_struct)
offset += tmp_struct.size
self.data["head_unit"] = pc[0]
self.data["transform_unit"] = pc[1]
#self.data[""] = pc[2]
self.data["construction_sound_id"] = pc[3]
check_file(pc[3])
self.data["garnison_type"] = pc[4]
self.data["garnison_heal_rate"] = pc[5]
#self.data[""] = pc[6]
#self.data[""] = pc[7]
#self.data[""] = pc[8:(8+6)]
return offset
def test_overridden_variable(filename, string, expected):
warnings = util.check_file(m.OverriddenVariable, filename, string)
assert warnings == expected
def test_indent(filename, string, expected):
warnings = util.check_file(m.Indent, filename, string)
assert warnings == expected
def test_typo_in_package_variable(filename, string, expected):
warnings = util.check_file(m.TypoInPackageVariable, filename, string)
assert warnings == expected
'subject','language',iso,''])
else:
record[element] = newline.sub(r'\\n',record[element])
if '\n' in record[element]:
print record[element]
sys.exit(1)
print>>outstream, '\t'.join([archive_id,item_id,elt_id,oai_id,\
element,'NULL','NULL',record[element]])
if __name__=="__main__":
if len(sys.argv)!=4:
print "Usage: python create_test_set.py olac_records test_set gold_standard"
sys.exit(1)
records, test_set, gold_std = sys.argv[1:]
test_set = check_file(test_set,'w',utf=True)
gold_std = check_file(gold_std,'w',utf=True)
reader = TabDBCorpusReader('.', '.*db\.tab')
olac_records = reader.records(records)
iso639_records = []
for record in olac_records:
if 'iso639' in record:
isos = record['iso639'].split()
if isos and reduce(iand, [len(i)==3 and i.islower() for i in isos]): # if it is a list of three letter codes
iso639_records.append(record)
print_records(iso639_records,test_set)
for record in iso639_records:
try:
def __init__(self, **kwargs):
self.db = 'templog.db' if 'db' not in kwargs else kwargs['db']
dbstat = check_file(db)
os.environ['PIMMS_DB'] = db
if dbstat == 2: self.createdb(self.db)
