def copy_config():
config = utils.get_json_file_data('config.json')
planConfigRootPath = utils.format_path(
os.path.join(os.getcwd(), config['planConfigDir']))
serverSrcPath = utils.format_path(
os.path.join(planConfigRootPath, config['serverOnlyDirName']))
commonSrcPath = utils.format_path(
os.path.join(planConfigRootPath, config['commonDirName']))
activitiesSrcPath = utils.format_path(
os.path.join(planConfigRootPath, config['activitiesDirName']))
dstPath = utils.format_path(
os.path.join(os.getcwd(), config['serverConfigDir']))
filterList = []
for filterFile in config['serverNotCopy']:
filterPath = utils.format_path(
os.path.join(planConfigRootPath, filterFile))
filterList.append(filterPath)
utils.logger_info("更新配置......")
os.system("svn update " + (planConfigRootPath).encode('gbk'))
utils.logger_info("配置更新完成，准备拷贝！")
utils.logger("配置源目录：")
utils.logger(serverSrcPath.encode('utf-8'))
utils.logger(commonSrcPath.encode('utf-8'))
utils.logger(activitiesSrcPath.encode('utf-8'))
utils.logger("拷贝到：" + dstPath.encode('utf-8'))
utils.cover_copy_files(commonSrcPath, dstPath, filterList)
utils.cover_copy_files(serverSrcPath, dstPath, filterList)
utils.cover_copy_files(activitiesSrcPath, dstPath, filterList)
utils.logger_info('配置拷贝结束！')
async def create_disk_worker(project, rg_name, uri, disk_name, location, f):
con = create_db_con()
client_id = Project.objects(name=project)[0]['client_id']
secret = Project.objects(name=project)[0]['secret']
tenant_id = Project.objects(name=project)[0]['tenant_id']
subscription_id = Project.objects(name=project)[0]['subscription_id']
creds = ServicePrincipalCredentials(client_id=client_id,
secret=secret,
tenant=tenant_id)
compute_client = ComputeManagementClient(creds, subscription_id)
con.close()
async_creation = compute_client.images.create_or_update(
rg_name, disk_name, {
'location': location,
'storage_profile': {
'os_disk': {
'os_type': 'Linux',
'os_state': "Generalized",
'blob_uri': uri,
'caching': "ReadWrite",
'storage_account_type': 'StandardSSD_LRS'
}
},
'hyper_vgeneration': 'V1'
})
image_resource = async_creation.result()
try:
BluePrint.objects(project=project,
host=disk_name).update(image_id=disk_name,
status='40')
except Exception as e:
print("disk creation updation failed: " + repr(e))
logger("Disk creation updation failed: " + repr(e), "warning")
finally:
con.close()
def parse_mail(mailStr):
original = email.message_from_string(mailStr)
org_subj = original['Subject']
try:
subj = decode_header(original['Subject'])[0][0]
win_subj = subj.decode('utf-8')
fin_subj = win_subj.encode("ascii", "ignore")
logger(['subject', org_subj, subj, fin_subj])
except:
fin_subj = org_subj
parsed_mail_from = parse_mail_from(original['From'])
parsed_mail_to = parse_mail_from(original['To'])
parsed_mail = {
"from": parsed_mail_from,
"sub": fin_subj,
"to": parsed_mail_to,
"date": adaptTime(original['Date']),
"phoneNumber": '+919722761117'
}
for part in original.walk():
if part.get_content_type() == "text/plain":
parsed_mail["body"] = part.get_payload(decode=True)
else:
continue
logger(
['parsed mail time is', original['Date'],
adaptTime(original['Date'])])
return parsed_mail
def create_vnet(rg_name, vnet_name, cidr, location, project):
print("Provisioning a vnet...some operations might take a minute or two.")
con = create_db_con()
client_id = Project.objects(name=project)[0]['client_id']
secret = Project.objects(name=project)[0]['secret']
tenant_id = Project.objects(name=project)[0]['tenant_id']
subscription_id = Project.objects(name=project)[0]['subscription_id']
creds = ServicePrincipalCredentials(client_id=client_id,
secret=secret,
tenant=tenant_id)
network_client = NetworkManagementClient(creds, subscription_id)
poller = network_client.virtual_networks.create_or_update(
rg_name, vnet_name, {
"location": location,
"address_space": {
"address_prefixes": [cidr]
}
})
vnet_result = poller.result()
print(
"Provisioned virtual network {vnet_result.name} with address prefixes {vnet_result.address_space.address_prefixes}"
)
try:
BluePrint.objects(network=cidr).update(vpc_id=vnet_result.name,
status='43')
except Exception as e:
print("Vnet creation failed to save: " + repr(e))
logger("Vnet creation failed to save: " + repr(e), "warning")
return False
finally:
con.close()
return True
def handle_file(args, name, now):
meta = {}
count = 0
col_size = len(args.columnName.split(','))
try:
rf = open(args.sourceFile, 'r', encoding='utf-8')
except:
utils.edit_list(now, 'err_generate\n')
logger().error(['err_data_source', args.sourceFile])
return {}
with open(name, 'a', encoding='utf-8') as wf:
for line in rf:
count = count + 1
rows = line.strip().split(args.separator)
if len(rows) != col_size:
utils.edit_list(now, 'err_generate\n')
logger().error(['err_source_info', now+'/'+str(count)])
return {}
wf.write('|||'.join(('%s' % i).replace('|||', '---') for i in rows).replace('\n', '') + '\n')
with open(name, 'r', encoding='utf-8') as f:
content = f.read()
meta.__setitem__("size", count)
meta.__setitem__("md5", utils.get_md5(content))
meta.__setitem__("colName", args.columnName)
return meta
def parse_mail_from(fromStr):
logger(fromStr)
reg_grp = re.search("<(.+?)>", fromStr)
mail_from = fromStr
if reg_grp is not None:
mail_from = reg_grp.group(1)
return mail_from
def run(args):
# 准备阶段
now = str(int(round(time.time() * 1000)))
args.tagName = utils.decodeURL(args.tagName)
utils.write_to_list(now + '|' + args.settingsActive.replace('|', '-') + '|' + args.tagName.replace('|', '-') + '|' + 'generating' + '\n')
path = args.path + "/" + now
utils.mkdir(path)
name = args.tagOwner + "." + now
real_path = path + "/" + name + ".data"
# 生成数据包文件
meta = source_filter()[args.sourceType](args, real_path, now)
# 生成meta文件
if meta != {}:
real_path = path + "/" + name + ".meta"
meta.__setitem__("timestamp", now)
# meta.__setitem__("dataName", real_path)
meta.__setitem__("tagName", args.tagName)
meta_json = utils.to_json(meta)
generate_meta(real_path, utils.encode(meta_json))
utils.edit_list(now, 'generated' + '\n')
logger().info(['generated', now])
else:
logger().error(['err_generate', now])
def get_vm_types(project):
client = ''
location = ''
machine_types = []
try:
con = create_db_con()
subscription_id = Project.objects(name=project)[0]['subscription_id']
client_id = Project.objects(name=project)[0]['client_id']
tenant_id = Project.objects(name=project)[0]['tenant_id']
secret_id = Project.objects(name=project)[0]['secret']
location = Project.objects(name=project)[0]['location']
creds = ServicePrincipalCredentials(client_id=client_id,
secret=secret_id,
tenant=tenant_id)
client = ComputeManagementClient(creds, subscription_id)
machine_types = list_available_vm_sizes(client,
region=location,
minimum_cores=1,
minimum_memory_MB=768)
flag = True
except Exception as e:
print(repr(e))
logger("Fetching vm details failed: " + repr(e), "warning")
flag = False
con.close()
return machine_types, flag
async def download_worker(osdisk_raw, project, host):
con = create_db_con()
account_name = Storage.objects(project=project)[0]['storage']
container_name = Storage.objects(project=project)[0]['container']
access_key = Storage.objects(project=project)[0]['access_key']
sas_token = sas.generate_sas_token(account_name, access_key)
pipe_result = ''
file_size = '0'
try:
cur_path = os.getcwd()
path = cur_path + "/osdisks/" + osdisk_raw
if not os.path.exists(path):
os.popen(
'echo "download started"> ./logs/ansible/migration_log.txt')
url = "https://" + account_name + ".blob.core.windows.net/" + container_name + "/" + osdisk_raw + "?" + sas_token
command1 = "azcopy copy --recursive '" + url + "' '" + path + "'"
os.popen('echo ' + command1 +
'>> ./logs/ansible/migration_log.txt')
process1 = await asyncio.create_subprocess_shell(command1,
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT)
await process1.wait()
BluePrint.objects(project=project, host=host).update(status='32')
except Exception as e:
print(repr(e))
logger(str(e), "warning")
finally:
con.close()
def E(level=1):
if level == 0:
from common import level1 as P
P = partial(P, FOnly=True) # high order function, here we only test LEVEL-1 F CNN
elif level == 1:
from level import level1 as P
elif level == 2:
from level import level2 as P
else:
from level import level3 as P
data = getDataFromTxt(TXT)
error = np.zeros((len(data), 5))
for i in range(len(data)):
imgPath, bbox, landmarkGt = data[i]
img = cv2.imread(imgPath, cv2.CV_LOAD_IMAGE_GRAYSCALE)
assert(img is not None)
logger("process %s" % imgPath)
landmarkP = P(img, bbox)
plot_point[i] = landmarkP
# real landmark
landmarkP = bbox.reprojectLandmark(landmarkP)
landmarkGt = bbox.reprojectLandmark(landmarkGt)
error[i] = evaluateError(landmarkGt, landmarkP, bbox)
return error
def fixfilebase(self, f):
assert f.startswith(self.expectedpath), [f, self.expectedpath]
f = self.newpath + f[len(self.expectedpath):]
schemafile = f.split(':')[-1].split('#')[0]
if not cmn.fexists(schemafile):
logger('#err ...schema file {0} not found\n'.format(schemafile))
return f
def begin_backend_session(self, sysid, password):
self.logging.info(('start login', sysid, password))
logger().info(('start login', sysid, password))
user = self.application.dbutil.isloginsuccess(sysid,
password) # mysql数据库
if not user:
print("login failed")
return False
self.logging.info(('login checked', sysid, password))
# 添加用户登录记录
name = "登陆用户：" + sysid if type(
sysid) != bytes else "登陆用户：" + sysid.decode()
ip_info = self.request.remote_ip
logtime = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(time.time()))
sql = "insert into tb_login_record values(null, '%s', '%s', '%s')" % (
name, ip_info, logtime)
self.application.dbutil.execute(sql)
# 查找该用户的所有权限
# permission = self.application.dbutil.getFindPermission(sysid)
# arr = []
# for p in permission:
# arr.append(p["title"])
self.session["data"] = user
self.session["sysid"] = sysid
# self.session['permission'] = arr
self.session.save()
return True
def create_subnet(rg_name, vnet_name, subnet_name, cidr, project):
print(
"Provisioning a subnet...some operations might take a minute or two.")
con = create_db_con()
client_id = Project.objects(name=project)[0]['client_id']
secret = Project.objects(name=project)[0]['secret']
tenant_id = Project.objects(name=project)[0]['tenant_id']
subscription_id = Project.objects(name=project)[0]['subscription_id']
creds = ServicePrincipalCredentials(client_id=client_id,
secret=secret,
tenant=tenant_id)
network_client = NetworkManagementClient(creds, subscription_id)
con.close()
poller = network_client.subnets.create_or_update(rg_name, vnet_name,
subnet_name,
{"address_prefix": cidr})
subnet_result = poller.result()
print(
"Provisioned virtual subnet {subnet_result.name} with address prefix {subnet_result.address_prefix}"
)
try:
con = create_db_con()
print(subnet_result.id)
BluePrint.objects(subnet=cidr).update(subnet_id=str(subnet_result.id),
status='60')
except Exception as e:
print("Subnet creation failed to save: " + repr(e))
logger("Subnet creation failed to save: " + repr(e), "warning")
return False
finally:
con.close()
return True
def handle_db(args, name, now):
try:
conn = utils.get_mysql_conn(args.host, args.dbUser, args.dbPassword, args.database)
cursor = conn.cursor()
cursor.execute(args.sql)
except:
utils.edit_list(now, 'err_generate\n')
logger().error(['err_data_source', now+'/'+args.host])
return {}
meta = {}
count = 0
rows = cursor.fetchmany(int(args.fetchSize))
with open(name, 'a', encoding='utf-8') as f:
while rows:
for row in rows:
v = '|||'.join(('%s' % i).replace('|||', '---') for i in list(row))
f.write(v.replace('\n', '') + '\n')
count = count + 1
if count % int(args.fetchSize) == 0:
rows = cursor.fetchmany(int(args.fetchSize))
else:
rows = False
with open(name, 'r', encoding='utf-8') as f:
content = f.read()
index = cursor.description
cols = []
for i in range(len(index)):
cols.append(index[i][0])
meta.__setitem__("size", count)
meta.__setitem__("md5", utils.get_md5(content))
meta.__setitem__("colName", ','.join(cols))
return meta
def runVideo(cap):
Frame.frame_width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
Frame.frame_height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
first_go = True
frame = None
current_frame = None
previous_frame = None
while (True):
if first_go != True:
previous_frame = Frame(current_frame.getFrame())
ret, frame = cap.read()
if ret == False:
break
else:
current_frame = Frame(frame)
if first_go != True:
difference_frame = current_frame.getDifferenceFrame(previous_frame)
thresholded_frame = difference_frame.getBinary(threshold=100)
dilated_frame = thresholded_frame.getDilated(iterations=10)
valid_contours = dilated_frame.findObjects(
minContourZone=settings.MIN_COUNTOUR_ZONE)
Tracker.registerNewObjects(valid_contours, current_frame)
# set which frame to display for user
ready_frame = current_frame
ready_frame.addBoundingBoxesFromContours(Tracker)
st.write("test")
ready_frame.putText("Threads: " + str(threading.active_count()),
(7, 20))
ready_frame.putText(
"Object Bufor size: " + str(len(Tracker.objectsToClassify)),
(7, 50))
ready_frame.putText("FPS: " + FPS.tick(), (7, 80))
ready_frame.putText(
"Cars passed: " + str(len(Tracker.lostObjects)), (7, 110))
ready_frame.putText("Test var: " + str(12), (7, 140))
ready_frame.show()
else:
first_go = False
current_frame.show()
if cv2.waitKey(1) & 0xFF == ord('q'):
logger('cars found: ' + str(Tracker.lostObjects), settings.LOG)
break
cap.release()
cv2.destroyAllWindows()
stopThreads()
async def conversion_worker(osdisk_raw, project, host):
con = create_db_con()
account_name = Storage.objects(project=project)[0]['storage']
container_name = Storage.objects(project=project)[0]['container']
access_key = Storage.objects(project=project)[0]['access_key']
sas_token = sas.generate_sas_token(account_name, access_key)
pipe_result = ''
try:
osdisk_vhd = osdisk_raw.replace(".raw", ".vhd")
cur_path = os.getcwd()
path = cur_path + "/osdisks/" + osdisk_raw
vhd_path = cur_path + "/osdisks/" + osdisk_vhd
print("Start converting")
print(path)
os.popen('echo "start converting">> ./logs/ansible/migration_log.txt')
command2 = "qemu-img convert -f raw -o subformat=fixed -O vpc " + path + " " + vhd_path
process2 = await asyncio.create_subprocess_shell(command2,
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT)
await process2.wait()
BluePrint.objects(project=project, host=host).update(status='34')
os.popen(
'echo "Conversion completed" >> ./logs/ansible/migration_log.txt')
except Exception as e:
print(str(e))
logger(str(e), "warning")
file_size = '0'
finally:
con.close()
def init_messages_dict(self):
messages = {}
count = 0
for language_dir in [
d for d in listdir(self.translations_dir)
if path.isdir(path.join(cwd[0], self.translations_dir, d))
]:
temp_messages = {}
for fileName in listdir('{}/{}'.format(self.translations_dir,
language_dir)):
if self.translations_extension in fileName:
try:
filePath = '{}/{}/{}'.format(self.translations_dir,
language_dir, fileName)
with open(filePath, 'r',
encoding='utf-8') as translation_file:
temp_messages[fileName.replace(
self.translations_extension,
'')] = load(translation_file)
count += 1
except Exception as err:
logger('文件[{}] 添加失败 ({})'.format(filePath, err))
messages[language_dir] = temp_messages
self.messages.update(messages)
logger('支持语言数({})，共添加翻译文件共({})个..完毕!'.format(len(messages), count))
def main(args):
if not args.has('-config'):
args.add_key('-config', "./config.json")
logger(str([args.keys, args.args()]) + '\n')
if not args.has('-out'):
logger('#__noOutFileGiven___\n')
return
if not args.has('-dbg') and (cmn.fexists(args['-out']) and not args.has('-overwrite-outfile')):
logger('#__outfile:{0} exists\n'.format(args['-out']))
return
#try:
if True:
if args.has('-extract'):
import sraparse
return sraparse.SRAParseObjSet.extract_attributes_to_json(args.args())
elif args.has("-test-sample"):
testargs = ["./examples/samples.xml", "-config:{0}".format(args['-config']), "-out:./examples/samples.versioned.xml"]
validate_sample.main(Config(testargs))
elif args.has("-sample"):
validate_sample.main(args)
elif args.has("-experiment"):
validate_experiment.main(args)
else:
raise NotImplementedError("#__unknownArguments__")
else:
#except Exception as err:
logger('#__unexpected__\n')
logger(str(err.message) + '\n')
def from_csv(self, path, doc_id):
"""
read CSV and created inverted index for 'Snippet' column
"""
try:
csv = read_csv(path)
except Exception as e:
logger(str(e) + str(path))
return
# for i_row in tqdm(range(len(csv['Snippet'])),
# bar_format='{l_bar}{bar:50}{r_bar}{bar:-50b}',
# desc=("parsing csv file : " + path.split("/")[-1])):
for i_row in range(len(csv['Snippet'])):
tokens = self._tokenizer.tokenize(csv['Snippet'][i_row])
if REMOVE_STOP_WORDS:
tokens = [i for i in tokens if i not in self._stop_words]
doc = self._n_docs + i_row
self._post2doc_mapper[doc] = [doc_id, i_row]
self._tmp_doc[doc] = []
for token in tokens:
if token in self._vocab.keys():
self._vocab[token] += 1
else:
self._vocab[token] = 1
term = self._stemmer.stem(self._lemmatizer.lemmatize(token))
self._tmp_doc[doc].append(term)
self._trie.add_string(term, doc)
self._n_docs += len(csv['Snippet'])
def xoauth_authenticate(emailId, access_token):
logger([access_token, emailId])
def _auth(*args, **kwargs):
return 'user=%s\1auth=Bearer %s\1\1' % (emailId, access_token)
return 'XOAUTH2', _auth
def start(self):
# self.start.__doc__ = self._thread.start.__doc__
if not self.started:
self.started = True
self.return_value = self.target(*self.args, **self.kwargs)
logger('fake_thread_started', self.target.__name__)
else:
raise RuntimeError()
def obj_id(self, e):
try:
idblock = e.get('@idblock', dict())
tags = [idblock[k] for k in ['alias', 'refname', 'accession'] if k in idblock]
return 'unknown' if not tags else self.sanitizer.filter_alphan('.'.join(tags), '.-_')
except Exception as e:
logger('#__couldNotExactId__:{0}\n'.format(e ))
return 'unknown'
def get_user_list():
users = requests.get('http://localhost:8082/users').json()
verified_users = []
for user in users:
if user['verified'] is True:
verified_users.append(user)
logger(['verified users', verified_users])
return verified_users
def get_words(terminals, landmarks, rel=None):
words = []
probs = []
entropy = []
for n,lmk in zip(terminals, landmarks):
# if we could not get an expansion for the LHS, we just pass down the unexpanded nonterminal symbol
# it gets the probability of 1 and entropy of 0
if n in NONTERMINALS:
words.append(n)
probs.append(1.0)
entropy.append(0.0)
continue
lmk_class = (lmk.object_class if lmk else None)
lmk_color = (lmk.color if lmk else None)
rel_class = rel_type(rel)
dist_class = (rel.measurement.best_distance_class if hasattr(rel, 'measurement') else None)
deg_class = (rel.measurement.best_degree_class if hasattr(rel, 'measurement') else None)
cp_db = CWord.get_word_counts(pos=n,
lmk_class=lmk_class,
lmk_ori_rels=get_lmk_ori_rels_str(lmk),
lmk_color=lmk_color,
rel=rel_class,
rel_dist_class=dist_class,
rel_deg_class=deg_class)
if cp_db.count() <= 0:
logger( 'Could not expand %s (lmk_class: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (n, lmk_class, lmk_color, rel_class, dist_class, deg_class) )
terminals.append( n )
continue
logger( 'Expanded %s (lmk_class: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (n, lmk_class, lmk_color, rel_class, dist_class, deg_class) )
ckeys, ccounts = zip(*[(cword.word,cword.count) for cword in cp_db.all()])
ccounter = {}
for cword in cp_db.all():
if cword.word in ccounter: ccounter[cword.word] += cword.count
else: ccounter[cword.word] = cword.count
ckeys, ccounts = zip(*ccounter.items())
# print 'ckeys', ckeys
# print 'ccounts', ccounts
ccounts = np.array(ccounts, dtype=float)
ccounts /= ccounts.sum()
w, w_prob, w_entropy = categorical_sample(ckeys, ccounts)
words.append(w)
probs.append(w_prob)
entropy.append(w_entropy)
p, H = np.prod(probs), np.sum(entropy)
# print 'expanding %s to %s (p: %f, H: %f)' % (terminals, words, p, H)
return words, p, H
def begin_backend_session1(self, sysid, password):
self.logging.info(('start login', sysid, password))
logger().info(('start login', sysid, password))
if not self.application.backend_auth.login(sysid,
password): # mongodb数据库
print("login failed")
return False
self.logging.info(('login checked', sysid, password))
user = self.db.tb_system_user.find_one({'userid': sysid}, {
'passwd': 0,
'_id': 0
})
if not user:
print("no user exists!", sysid)
return False
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
last = self.db.tb_login_record.find_one({}, {
"id": 1,
"_id": 0
},
sort=[("id",
pymongo.DESCENDING)])
id = int(last.get("id", 0)) + 1 if last else 1
ip_info = self.request.remote_ip
self.db.tb_login_record.insert({
"userid": sysid,
"ip": ip_info,
"atime": now,
"id": id
})
# user["db"] = self.application.settings["database"]
# user["system"] = self.application.settings["system"]
logininfos = user.get('login', [])
logininfos.append({"ip": ip_info, "time": now})
self.db.tb_system_user.update(
{'userid': sysid},
{'$set': {
'status': 'online',
"login": logininfos[-10:]
}})
user['status'] = "online"
user['login'] = logininfos[-10:]
print("==========================", logininfos, logininfos[-10:])
print(user['login'])
# 查找该用户的所有权限
# permission = self.application.dbutil.getFindPermission(sysid)
# arr = []
# for p in permission:
# arr.append(p["title"])
self.session["data"] = user
self.session["sysid"] = sysid
# self.session['permission'] = arr
self.session.save()
return True
def get_sentence_meaning_likelihood(sentence, lmk, rel):
modparse = get_modparse(sentence)
t = ParentedTree.parse(modparse)
print '\n%s\n' % t.pprint()
probs, entropies, lrpc, tps = get_tree_probs(t, lmk, rel)
if np.prod(probs) == 0.0:
logger('ERROR: Probability product is 0 for sentence: %s, lmk: %s, rel: %s, probs: %s' % (sentence, lmk, rel, str(probs)))
return np.prod(probs), sum(entropies), lrpc, tps
def validate_semantics(self, attrs):
attributes = attrs['attributes']
if 'donor_age_unit' in attributes and attributes['donor_age_unit'] == 'year' and isinstance(attributes['donor_age'], int):
age = int(attributes['donor_age'])
if age > 90:
logger('#__error: Donors over 90 years of age should be entered as "90+"\n')
return False
return True
def __init__(self, sra, validators):
super(SampleValidator, self).__init__(validators)
self.normalize = lambda t: t.lower().replace(' ', '_')
self.sra = sra
self.xmljson = self.sra.obj_xmljson()
for (xml, attrs) in self.xmljson:
logger('\n#__normalizingTags:{0}\n'.format(attrs['title']))
attrs['attributes'] = self.normalize_tags(attrs['attributes'])
logger("\n\n")
def run_once(self, make_thread=True, last_update_id=None, update_timeout=30):
""" Check the the messages for commands and make a Thread or FakeThread with the command depending on make_thread.
Args:
make_thread:
True: the function returns a list with threads. Which didn't start yet.
False: the function returns a list with FakeThreads. Which did'nt start yet.
last_update_id:
the offset arg from getUpdates and is kept up to date within this function
update_timeout:
timeout for updates. can be None for no timeout.
Returns:
A tuple of two elements. The first element is a list with Threads or FakeThreads which didn't start yet.
The second element is the updated las_update_id
"""
if make_thread:
ch_Thread = threading.Thread
else:
ch_Thread = FakeThread
bot_name = self.bot.username
threads = {}
self._getupdates_can_write.append(True)
get_updates_index = len(self._getupdates_can_write) - 1
get_updates_thread = threading.Thread(target=self.get_updates,
kwargs={'index': get_updates_index,
'offset': last_update_id})
get_updates_thread.start()
get_updates_thread.join(timeout=update_timeout)
if get_updates_thread.isAlive():
logger('ERROR getupdates timed out, using empty list')
self._getupdates_can_write[get_updates_index] = False
self._last_updates = []
updates = self._last_updates
for update in updates:
last_update_id = update.update_id + 1
message = update.message
if len(message.text) == 0:
message.text = ' '
if message.text[0] == '/':
command, username = message.text.split(' ')[0], bot_name
if '@' in command:
command, username = command.split('@')
if username == bot_name:
command_func = self._get_command_func(command)
if command_func is not None:
self.bot.sendChatAction(chat_id=update.message.chat.id, action=telegram.ChatAction.TYPING)
if self.isValidCommand is None or self.isValidCommand(update):
t = ch_Thread(target=command_func, args=(update,))
threads[(message.text, update.message.chat.id)] = t
else:
t = ch_Thread(target=self._command_not_valid, args=(update,))
threads[(message.text + ' unauthorized', update.message.chat.id)] = t
else:
t = ch_Thread(target=self._command_not_found, args=(update,))
threads[(message.text + ' not found', update.message.chat.id)] = t
return threads, last_update_id
def send_complete(self, server, recipient, error_text='', status=-1, error=None):
if status == 1:
self.send_success += 1
logger('邮件发送成功 (server [{}] to {}) 成功({}) [{}]'.format(server.name, recipient, self.send_success, self.translator.language))
return self.translator.tran('mail.complete.success')
else:
self.send_failed += 1
logger('邮件发送失败 (server [{}] to {}) [{}] ({}) 失败({}) [{}]'.format(server.name, recipient, error_text, str(error), self.send_failed, self.translator.language))
return self.translator.tran('mail.complete.failed') + error_text
def metric_compute(val_labels, predict_labels):
logger("the result of prediction of validation is as follow:")
occlu_ratio = occlu_ratio_compute(val_labels)
accuracy = accuracy_compute(val_labels, predict_labels)
occlu_recall = recall_compute(val_labels, predict_labels, mode="occlu")
clear_recall = recall_compute(val_labels, predict_labels, mode="clear")
print("occlu_ratio is {}".format(occlu_ratio))
print("accuracy is {}".format(accuracy))
print("occlu_recall is {}".format(occlu_recall))
print("clear_recall is {}".format(clear_recall))
def extract_additional_experiment_attributes(self, obj, hashed):
strategy = hashed.get("library_strategy", "" ).strip()
if not strategy:
strategy = self.extract_optional(obj, ".//SEQUENCING_LIBRARY_STRATEGY")
if not strategy or len(strategy) > 1:
logger("#warn__: cannot parse 'library_strategy' or 'library_sequencing_strategy'.. {0}\n ".format(str(strategy)))
else:
logger("#warn__: updated 'library_strategy' with 'library_sequencing_strategy'.. {0}\n ".format(str(strategy[0].text)))
hashed["library_strategy"] = strategy[0].text.strip()
return hashed
def from_sra_main_to_attributes(self, hashed):
if 'library_strategy' in hashed:
if 'LIBRARY_STRATEGY' in hashed['attributes'] or 'library_strategy' in hashed['attributes']:
lib_strat_attr = 'LIBRARY_STRATEGY' if 'LIBRARY_STRATEGY' in hashed['attributes'] else 'library_strategy'
hashed['attributes']['LIBRARY_STRATEGY_IHEC'] = hashed['attributes'][lib_strat_attr]
old_lib_start = hashed['attributes'].pop(lib_strat_attr)
logger("#warn:__library_strategy__ defined in both SRA block and as IHEC attribute:{0}, value pushed into 'LIBRARY_STRATEGY_IHEC'\n".format(old_lib_start))
hashed['attributes']['LIBRARY_STRATEGY'] = [hashed['library_strategy']]
#hashed['attributes']['@idblock'] = hashed['@idblock']
return hashed
def __init__(self, name, account, password, host, port, sender_name='osu!Kafuu', client_type='default', reply_address=None):
self.name = name
self.account = account
self.password = password
self.host = host
self.port = port
self.sender_name = sender_name
self.client_type = client_type
self.reply_address = reply_address or account
logger('增加邮件服务器：{} <{}>'.format(name, account))
def init_templates(self):
templates = {}
count = 0
for fileName in listdir(self.template_dir):
if self.template_extension in fileName:
with open('{}/{}'.format(self.template_dir, fileName), 'r', encoding='utf-8') as template_file:
templates[fileName.replace(self.template_extension, '')] = Template(template_file.read())
count += 1
self.templates = templates
logger('初始化邮件模板({})..成功!'.format(count))
def adaptTime(date):
logger(['date to adapt', date])
pattern = '%a, %d %b %Y %H:%M:%S'
try:
return int(time.mktime(time.strptime(split_date_str(date),
pattern))) * 1000
except:
pattern = '%d %b %Y %H:%M:%S'
return int(time.mktime(time.strptime(split_date_str(date),
pattern))) * 1000
def send_mail(self, recipient, title, content, sender_name='', server_name='', template_data={}, lang='cn', tried_servers=[]):
send_status = -1
tried_servers_temp = tried_servers if len(tried_servers) > 0 else []
self.translator.language = lang
if len(tried_servers) == len(self.mail_servers):
logger('已达到最大重试次数..放弃此次发信任务！')
return send_status, self.translator.tran('mail.errors.service_unavailable')
if server_name:
server = [s for s in self.mail_servers if s.name == server_name][0]
else:
if len(tried_servers) > 0:
logger('发信失败，更换服务器重试... [当前重试次数({})]'.format(len(tried_servers_temp)))
server = [s for s in self.mail_servers if s.name not in tried_servers][0]
else: server = self.mail_servers[0]
client = smtplib.SMTP_SSL(host=server.host) if server.port == 465 or server.client_type == 'ssl' else smtplib.SMTP(host=server.host)
temp = self.templates.get(content)
if temp != None:
content = self.template_render(temp, template_data)
msg = MIMEText(content, 'html', 'utf-8')
msg['Subject'] = Header(title, 'utf-8')
msg['From'] = '{} <{}>'.format(sender_name or server.sender_name, server.account)
msg['To'] = '{} <{}>'.format('osu!Kafuu User', recipient)
msg['Reply-to'] = server.reply_address
try:
client.connect(server.host, server.port)
client.login(server.account, server.password)
client.sendmail(server.account, [recipient], msg.as_string())
client.quit()
send_status = 1
re_msg = self.send_complete(server, recipient, status=1)
except smtplib.SMTPConnectError as err:
re_msg = self.send_complete(server, recipient, self.send_error(0), error=err)
except smtplib.SMTPAuthenticationError as err:
re_msg = self.send_complete(server, recipient, self.send_error(1), error=err)
except smtplib.SMTPSenderRefused as err:
re_msg = self.send_complete(server, recipient, self.send_error(2), error=err)
except smtplib.SMTPRecipientsRefused as err:
re_msg = self.send_complete(server, recipient, self.send_error(3), error=err)
except smtplib.SMTPDataError as err:
re_msg = self.send_complete(server, recipient, self.send_error(4), error=err)
except smtplib.SMTPException as err:
re_msg = self.send_complete(server, recipient, self.send_error(5, reason=err), error=err)
except Exception as err:
re_msg = self.send_complete(server, recipient, self.send_error(5, reason=err), error=err)
if send_status == -1:
tried_servers_temp.append(server.name)
send_status, re_msg = self.send_mail(recipient, title, content, sender_name, server_name, template_data, lang, tried_servers_temp)
return send_status, re_msg
def get_koinex_stats(self) -> None:
logger("################ Koinex Statastics #################")
stats_dict = self.koinex_client.get_market_stats()
for coin in stats_dict:
max_value = stats_dict.get(coin).get('max_24hrs')
min_value = stats_dict.get(coin).get('min_24hrs')
avg_value = (float(min_value) + float(max_value)) / 2
print("{coin_name}:".format(coin_name=coin) +
" 24-Highest: {highest},".format(highest=max_value) +
" 24-Lowest: {lowest},".format(lowest=min_value) +
"Average price: {average}".format(average=avg_value))
def _generate_help_list(self):
logger('methods', [attr[0] for attr in getmembers(self, predicate=ismethod)])
command_functions = [attr[1] for attr in getmembers(self, predicate=ismethod) if attr[0][:8] == 'command_' and
attr[0] not in self.skip_in_help]
help_message = ''
for command_function in command_functions:
if command_function.__doc__ is not None:
help_message += ' /' + command_function.__name__[8:] + ' - ' + command_function.__doc__ + '\n'
else:
help_message += ' /' + command_function.__name__[8:] + ' - ' + '\n'
return help_message
def get_updates(self, *args, index, offset, **kwargs):
try:
temp = self.bot.getUpdates(*args, offset=offset, **kwargs)
except Exception as e:
temp = []
logger('because an error occoured updates will be empty id:', index, type(e), e.args, e)
if self._getupdates_can_write[index]:
self._last_updates = temp
else:
logger('error get_updates done. but not able to send output.', index)
return temp
def create_vm_worker(rg_name, vm_name, location, username, password, vm_type,
nic_id, subscription_id, image_name, project):
con = create_db_con()
client_id = Project.objects(name=project)[0]['client_id']
secret = Project.objects(name=project)[0]['secret']
tenant_id = Project.objects(name=project)[0]['tenant_id']
subscription_id = Project.objects(name=project)[0]['subscription_id']
creds = ServicePrincipalCredentials(client_id=client_id,
secret=secret,
tenant=tenant_id)
compute_client = ComputeManagementClient(creds, subscription_id)
con.close()
print(
"Provisioning virtual machine {vm_name}; this operation might take a few minutes."
)
print(nic_id)
poller = compute_client.virtual_machines.create_or_update(
rg_name, vm_name, {
"location": location,
"storage_profile": {
"image_reference": {
'id':
'/subscriptions/' + subscription_id + '/resourceGroups/' +
rg_name + '/providers/Microsoft.Compute/images/' +
image_name
}
},
"hardware_profile": {
"vm_size": vm_type
},
"os_profile": {
"computer_name": vm_name,
"admin_username": username,
"admin_password": password
},
"network_profile": {
"network_interfaces": [{
"id": nic_id,
}]
}
})
vm_result = poller.result()
print("Provisioned virtual machine")
try:
con = create_db_con()
BluePrint.objects(project=project,
image_id=image_name).update(vm_id=vm_result.name,
status='100')
except Exception as e:
print("VM creation updation failed: " + repr(e))
logger("VM creation updation failed: " + repr(e), "warning")
finally:
con.close()
def probs_metric(inverse=False):
rand_p = Vec2(random()*table.width+table.min_point.x, random()*table.height+table.min_point.y)
try:
bestmeaning, bestsentence = generate_sentence(rand_p, False, scene, speaker, usebest=True, golden=inverse, printing=printing)
sampled_landmark, sampled_relation = bestmeaning.args[0], bestmeaning.args[3]
golden_posteriors = get_all_sentence_posteriors(bestsentence, meanings, golden=(not inverse), printing=printing)
# lmk_prior = speaker.get_landmark_probability(sampled_landmark, landmarks, PointRepresentation(rand_p))[0]
all_lmk_probs = speaker.all_landmark_probs(landmarks, Landmark(None, PointRepresentation(rand_p), None))
all_lmk_probs = dict(zip(landmarks, all_lmk_probs))
lmk_prior = all_lmk_probs[sampled_landmark]
head_on = speaker.get_head_on_viewpoint(sampled_landmark)
rel_prior = speaker.get_probabilities_points( np.array([rand_p]), sampled_relation, head_on, sampled_landmark)
lmk_post = golden_posteriors[sampled_landmark]
rel_post = golden_posteriors[sampled_relation]
ps = np.array([golden_posteriors[lmk]*golden_posteriors[rel] for lmk, rel in meanings])
rank = None
for i,p in enumerate(ps):
lmk,rel = meanings[i]
# logger( '%f, %s' % (p, m2s(lmk,rel)))
head_on = speaker.get_head_on_viewpoint(lmk)
# ps[i] *= speaker.get_landmark_probability(lmk, landmarks, PointRepresentation(rand_p))[0]
ps[i] *= all_lmk_probs[lmk]
ps[i] *= speaker.get_probabilities_points( np.array([rand_p]), rel, head_on, lmk)
if lmk == sampled_landmark and rel == sampled_relation:
idx = i
ps += epsilon
ps = ps/ps.sum()
prob = ps[idx]
rank = sorted(ps, reverse=True).index(prob)
entropy = entropy_of_probs(ps)
except (ParseError,RuntimeError) as e:
logger( e )
lmk_prior = 0
rel_prior = 0
lmk_post = 0
rel_post = 0
prob = 0
rank = len(meanings)-1
entropy = 0
distances = [[None]]
head_on = speaker.get_head_on_viewpoint(sampled_landmark)
all_descs = speaker.get_all_meaning_descriptions(trajector, scene, sampled_landmark, sampled_relation, head_on, 1)
distances = []
for desc in all_descs:
distances.append([edit_distance( bestsentence, desc ), desc])
distances.sort()
return lmk_prior,rel_prior,lmk_post,rel_post,\
prob,entropy,rank,distances[0][0],type(sampled_relation)
def parse_mail_list(fetched_mails):
cnt = 0 # hack in iterating, array type [mail, flags....]
mails = []
for mail in fetched_mails:
# toPrint = if mail[0]: mail[0]
# only store 0,2,4..
if cnt % 2 != 1 and (cnt is not len(fetched_mails) - 1 and re.search(
'\\Seen', fetched_mails[cnt + 1]) is not None):
logger(['mail to parse', mail[0]])
mails.append(parse_mail(mail[1]))
cnt += 1
return mails
def _load_checkpoint_from_file(config):
abs_path_ckpt = os.path.abspath(config.checkpoint_file)
# Return checkpoint if valid
if os.path.isfile(abs_path_ckpt):
try:
checkpoint = torch.load(abs_path_ckpt)
return checkpoint
except Exception as e:
logger(f"Failed with exception {e}.")
else:
raise RuntimeError("Please specify a PyTorch checkpoint file.")
def accept_correction( meaning, correction, update_func='geometric', update_scale=10 ):
(lmk, lmk_prob, lmk_ent,
rel, rel_prob, rel_ent,
rel_exp_chain, rele_prob_chain, rele_ent_chain, rel_terminals, rel_landmarks,
lmk_exp_chain, lmke_prob_chain, lmke_ent_chain, lmk_terminals, lmk_landmarks,
rel_words, relw_prob, relw_ent,
lmk_words, lmkw_prob, lmkw_ent) = meaning.args
old_meaning_prob, old_meaning_entropy, lrpc, tps = get_sentence_meaning_likelihood( correction, lmk, rel )
update = update_funcs[update_func](lmk_prob * rel_prob, old_meaning_prob, lmk_ent + rel_ent, old_meaning_entropy) * update_scale
logger('Update functions is %s and update value is: %f' % (update_func, update))
# print 'lmk_prob, lmk_ent, rel_prob, rel_ent, old_meaning_prob, old_meaning_entropy, update', lmk_prob, lmk_ent, rel_prob, rel_ent, old_meaning_prob, old_meaning_entropy, update
# print lmk.object_class, type(rel)
dec_update = -update
for lhs,rhs,parent,_ in rel_exp_chain:
# print 'Decrementing production - lhs: %s, rhs: %s, parent: %s' % (lhs,rhs,parent)
update_expansion_counts( dec_update, lhs, rhs, parent, rel=rel )
for lhs,rhs,parent,lmk in lmk_exp_chain:
# print 'Decrementing production - lhs: %s, rhs: %s, parent: %s' % (lhs,rhs,parent)
update_expansion_counts( dec_update, lhs, rhs, parent, lmk_class=(lmk.object_class if lmk else None),
lmk_ori_rels=get_lmk_ori_rels_str(lmk),
lmk_color=(lmk.color if lmk else None) )
for term,word in zip(rel_terminals,rel_words):
# print 'Decrementing word - pos: %s, word: %s, rel: %s' % (term, word, rel)
update_word_counts( dec_update, term, word, rel=rel )
for term,word,lmk in zip(lmk_terminals,lmk_words,lmk_landmarks):
# print 'Decrementing word - pos: %s, word: %s, lmk_class: %s' % (term, word, lmk.object_class)
update_word_counts( dec_update, term, word, lmk_class=lmk.object_class,
lmk_ori_rels=get_lmk_ori_rels_str(lmk),
lmk_color=(lmk.color if lmk else None) )
# reward new words with old meaning
for lhs,rhs,parent,lmk,rel in lrpc:
# print 'Incrementing production - lhs: %s, rhs: %s, parent: %s' % (lhs,rhs,parent)
update_expansion_counts( update, lhs, rhs, parent, rel=rel,
lmk_class=(lmk.object_class if lmk else None),
lmk_ori_rels=get_lmk_ori_rels_str(lmk),
lmk_color=(lmk.color if lmk else None) )
for lhs,rhs,lmk,rel in tps:
# print 'Incrementing word - pos: %s, word: %s, lmk_class: %s' % (lhs, rhs, (lmk.object_class if lmk else None) )
update_word_counts( update, lhs, rhs, lmk_class=(lmk.object_class if lmk else None),
rel=rel,
lmk_ori_rels=get_lmk_ori_rels_str(lmk),
lmk_color=(lmk.color if lmk else None) )
def make_diffstring(content_ab, separator):
raw_text_input_a, raw_text_input_b = content_ab
text_input_a = raw_text_input_a.split(separator)
text_input_b = raw_text_input_b.split(separator)
# http://docs.python.org/library/difflib.html
diff_object = difflib.HtmlDiff(wrapcolumn=87)
diff_string = diff_object.make_table( text_input_a, text_input_b)
if not type(diff_string) == unicode:
logger('make_table failed')
return
return ''.join(diff_string)
def E():
data = getDataFromTxt(TXT)
error = np.zeros((len(data), 3))
for i in range(len(data)):
imgPath, bbox, landmarkGt = data[i]
landmarkGt = landmarkGt[:3, :]
img = cv2.imread(imgPath, cv2.CV_LOAD_IMAGE_GRAYSCALE)
assert(img is not None)
logger("process %s" % imgPath)
landmarkP = EN(img, bbox)
# real landmark
landmarkP = bbox.reprojectLandmark(landmarkP)
landmarkGt = bbox.reprojectLandmark(landmarkGt)
error[i] = evaluateError(landmarkGt, landmarkP, bbox)
return error
def heatmaps_for_sentences(sentences, all_meanings, loi_infos, xs, ys, scene, speaker, step=0.02):
printing=False
x = np.array( [list(xs-step*0.5)]*len(ys) )
y = np.array( [list(ys-step*0.5)]*len(xs) ).T
scene_bb = scene.get_bounding_box()
scene_bb = scene_bb.inflate( Vec2(scene_bb.width*0.5,scene_bb.height*0.5) )
combined_heatmaps = []
for obj_lmk, ms, heatmapss in loi_infos:
combined_heatmap = None
for sentence in sentences:
posteriors = None
while not posteriors:
try:
posteriors = get_all_sentence_posteriors(sentence, all_meanings, printing=printing)
except ParseError as pe:
raise pe
except Exception as e:
print e
sleeptime = random()*0.5
logger('Sleeping for %f and retrying "%s"' % (sleeptime,sentence))
time.sleep(sleeptime)
continue
big_heatmap1 = None
for m,(h1,h2) in zip(ms, heatmapss):
lmk,rel = m
p = posteriors[rel]*posteriors[lmk]
if big_heatmap1 is None:
big_heatmap1 = p*h1
else:
big_heatmap1 += p*h1
if combined_heatmap is None:
combined_heatmap = big_heatmap1
else:
combined_heatmap *= big_heatmap1
combined_heatmaps.append(combined_heatmap)
return combined_heatmaps
def main(args):
print args['-config']
outfile = args['-out']
config = json2.loadf(args['-config'])
xml_validator = XMLValidator(config["sra"]["sample"])
ihec_validators = cmn.safedict([(schema["version"] , JsonSchema(schema["schema"], args)) for schema in config["ihec"]["sample"]])
objtype = 'SAMPLE'
objset = 'SAMPLE_SET'
validated = list()
xmllist = args.args()
nObjs = 0
for e in xmllist:
sra = SRAParseObjSet.from_file(e)
nObjs += sra.nOffspring()
assert sra.xml.getroot().tag == objset, ['__Expected:' + objset]
assert sra.is_valid__xml(xml_validator) or args.has('-not-sra-xml-but-try')
v = SampleValidator(sra, ihec_validators)
validated.extend(v.is_valid_ihec())
versioned_xml = ['<{0}>'.format(objset) ]
for e in validated:
(version, xml) = e
sra_versioned = SRAParseObj(xml)
sra_versioned.add_attribute("VALIDATED_AGAINST_METADATA_SPEC", "{0}/{1}".format(version, objtype))
versioned_xml.append(sra_versioned.tostring())
versioned_xml.append('</{0}>'.format(objset))
validated_xml_file = cmn.writel(outfile, versioned_xml)
print 'written:' + validated_xml_file
print 'validated:', len(validated)
print 'failed:', nObjs - len(validated)
if validated:
validated_xml_set = SRAParseObjSet.from_file(validated_xml_file)
assert validated_xml_set.is_valid__xml(xml_validator) or args.has("-skip-updated-xml-validation")
logger('ok\n')
else:
logger('..no valid objects found\n')
def probs_metric():
meaning, sentence = generate_sentence(rand_p, consistent, scene, speaker, usebest=True, printing=printing)
sampled_landmark,sampled_relation = meaning.args[0],meaning.args[3]
print meaning.args[0],meaning.args[3], len(sentence)
if sentence == "":
prob = 0
entropy = 0
else:
logger( 'Generated sentence: %s' % sentence)
try:
golden_posteriors = get_all_sentence_posteriors(sentence, meanings, golden=True, printing=printing)
epsilon = 1e-15
ps = np.array([golden_posteriors[lmk]*golden_posteriors[rel] for lmk, rel in meanings])
temp = None
for i,p in enumerate(ps):
lmk,rel = meanings[i]
# logger( '%f, %s' % (p, m2s(lmk,rel)))
head_on = speaker.get_head_on_viewpoint(lmk)
ps[i] *= speaker.get_landmark_probability(lmk, landmarks, PointRepresentation(rand_p))[0]
ps[i] *= speaker.get_probabilities_points( np.array([rand_p]), rel, head_on, lmk)
if lmk == meaning.args[0] and rel == meaning.args[3]:
temp = i
ps += epsilon
ps = ps/ps.sum()
prob = ps[temp]
rank = sorted(ps, reverse=True).index(prob)
entropy = entropy_of_probs(ps)
except ParseError as e:
logger( e )
prob = 0
rank = len(meanings)-1
entropy = 0
head_on = speaker.get_head_on_viewpoint(sampled_landmark)
all_descs = speaker.get_all_meaning_descriptions(trajector, scene, sampled_landmark, sampled_relation, head_on, 1)
distances = []
for desc in all_descs:
distances.append([edit_distance( sentence, desc ), desc])
distances.sort()
return prob,entropy,rank,distances[0][0]
def ready(self):
print '############initialization###############'
log=logger()
#m=monitor(log=log,name='test1',interval=10,mailInterval=10,smsInterval=30,smsBegin=1,smsEnd=1,describe=1,contacts=1,retryTimes=3,url='http://www.baidu.com',timeout=1000,retryInterval=5,status=True,statusCode=200)
#m.start()
from models import items
result=items.objects.all()
for i in result:
m=monitor(logger=log,id=i.id,name=i.name,interval=i.interval,mailInterval=i.mailInterval,smsInterval=i.smsInterval,smsBegin=i.smsBegin,smsEnd=i.smsEnd,retryTimes=i.retryTimes,describe=i.describe,url=i.url,timeout=i.timeout,retryInterval=i.retryInterval,contacts=i.contacts,status=i.status,statusCode=i.statusCode)
m.setName(i.name)
m.start()
def choosing_object_metric():
trajector = choice(loi)
sentence, sampled_relation, sampled_landmark = speaker.describe(trajector, scene, max_level=1)
lmk_probs = []
try:
combined_heatmaps = heatmaps_for_sentence(sentence, all_meanings, loi_infos, xs, ys, scene, speaker, step=step)
for combined_heatmap,obj_lmk in zip(combined_heatmaps, loi):
ps = [p for (x,y),p in zip(list(product(xs,ys)),combined_heatmap) if obj_lmk.representation.contains_point( Vec2(x,y) )]
# print ps, xs.shape, ys.shape, combined_heatmap.shape
lmk_probs.append( (sum(ps)/len(ps), obj_lmk) )
lmk_probs = sorted(lmk_probs, reverse=True)
top_p, top_lmk = lmk_probs[0]
lprobs, lmkss = zip(*lmk_probs)
logger( sorted(zip(np.array(lprobs)/sum(lprobs), [(l.name, l.color, l.object_class) for l in lmkss]), reverse=True) )
logger( 'I bet %f you are talking about a %s %s %s' % (top_p/sum(lprobs), top_lmk.name, top_lmk.color, top_lmk.object_class) )
# objects.append(top_lmk)
except Exception as e:
logger( 'Unable to get object from sentence. %s' % e, 'fail' )
print traceback.format_exc()
exit()
return loi.index(trajector), [ (lprob, loi.index(lmk)) for lprob,lmk in lmk_probs ]
def generate(ftxt, mode, argument=False):
"""
Generate Training Data for LEVEL-2
mode = train or test
"""
data = getDataFromTxt(ftxt)
trainData = defaultdict(lambda: dict(patches=[], landmarks=[]))
for (imgPath, bbox, landmarkGt) in data:
img = cv2.imread(imgPath, cv2.CV_LOAD_IMAGE_GRAYSCALE)
assert(img is not None)
logger("process %s" % imgPath)
landmarkPs = randomShiftWithArgument(landmarkGt, 0.05)
if not argument:
landmarkPs = [landmarkPs[0]]
for landmarkP in landmarkPs:
for idx, name, padding in types:
patch, patch_bbox = getPatch(img, bbox, landmarkP[idx], padding)
patch = cv2.resize(patch, (15, 15))
patch = patch.reshape((1, 15, 15))
trainData[name]['patches'].append(patch)
_ = patch_bbox.project(bbox.reproject(landmarkGt[idx]))
trainData[name]['landmarks'].append(_)
for idx, name, padding in types:
logger('writing training data of %s'%name)
patches = np.asarray(trainData[name]['patches'])
landmarks = np.asarray(trainData[name]['landmarks'])
patches = processImage(patches)
shuffle_in_unison_scary(patches, landmarks)
with h5py.File('train/2_%s/%s.h5'%(name, mode), 'w') as h5:
h5['data'] = patches.astype(np.float32)
h5['landmark'] = landmarks.astype(np.float32)
with open('train/2_%s/%s.txt'%(name, mode), 'w') as fd:
fd.write('train/2_%s/%s.h5'%(name, mode))
def scan_once(self, last_update_id, update_timeout):
bot_name = self.bot.username
self._getupdates_can_write.append(True)
get_updates_index = len(self._getupdates_can_write) - 1
get_updates_thread = threading.Thread(target=self.get_updates,
kwargs={'index': get_updates_index,
'offset': last_update_id})
get_updates_thread.start()
get_updates_thread.join(timeout=update_timeout)
if get_updates_thread.isAlive():
logger('ERROR getupdates timed out, using empty list')
self._getupdates_can_write[get_updates_index] = False
self._last_updates = []
updates = self._last_updates
for update in updates:
last_update_id = update.update_id + 1
message = update.message
if message.text[0] == '/':
command, username = message.text.split(' ')[0], bot_name
if '@' in command:
command, username = command.split('@')
if username == bot_name:
events['on_command'].trigger(update.message)
def get_raw_url_from_gist_id(gist_id, gist_name_propper=None):
gist_id = str(gist_id)
url = 'https://api.github.com/gists/' + gist_id
found_json = urllib.urlopen(url).read()
wfile = json.JSONDecoder()
wjson = wfile.decode(found_json)
files_flag = 'files'
file_names = wjson[files_flag].keys()
logger(file_names)
logger(gist_name_propper)
if not gist_name_propper:
file_name = file_names[0]
else:
# this is a little crude.
if gist_name_propper.startswith('file_'):
gist_name_propper = gist_name_propper[5:]
file_name = gist_name_propper
return wjson[files_flag][file_name]['raw_url']
def validate(self, jsonObj, details):
try:
jsonschema.validate(jsonObj, self.schema, format_checker=jsonschema.FormatChecker())
return True
except jsonschema.ValidationError as err:
if self.verbose:
self.errs.append(err)
logfile = self.errlog(len(self.errs), self.obj_id(details))
with open(logfile, "w") as errs:
context_size = len(err.context)
if context_size > 0:
errs.write('Multiple sub-schemas can apply. This is what prevents successful validation in each:\n')
prev_schema = -1
for suberror in sorted(err.context, key=lambda e: e.schema_path):
schema_index = suberror.schema_path[0]
if prev_schema < schema_index:
errs.write('Schema %d:\n' % (schema_index + 1))
prev_schema = schema_index
errs.write(' %s\n' % (suberror.message))
else:
errs.write(err.message)
logger('#__validationFailuresFound: see {0}\n'.format(logfile))
return False
def generate_sentence(loc, consistent, scene=None, speaker=None, usebest=False, golden=False, meaning=None, printing=True):
utils.scene = utils.ModelScene(scene, speaker)
if meaning is None:
(lmk, lmk_prob, lmk_ent), (rel, rel_prob, rel_ent) = get_meaning(loc=loc, usebest=usebest)
else:
lmk, rel = meaning
lmk_prob = lmk_ent = rel_prob = rel_ent = None
meaning1 = m2s(lmk, rel)
logger( meaning1 )
while True:
rel_exp_chain, rele_prob_chain, rele_ent_chain, rel_terminals, rel_landmarks = get_expansion('RELATION', rel=rel, usebest=usebest, golden=golden, printing=printing)
lmk_exp_chain, lmke_prob_chain, lmke_ent_chain, lmk_terminals, lmk_landmarks = get_expansion('LANDMARK-PHRASE', lmk=lmk, usebest=usebest, golden=golden, printing=printing)
rel_words, relw_prob, relw_ent, rel_a = get_words(rel_terminals, landmarks=rel_landmarks, rel=rel, usebest=usebest, golden=golden, printing=printing)
lmk_words, lmkw_prob, lmkw_ent, lmk_a = get_words(lmk_terminals, landmarks=lmk_landmarks, prevword=(rel_words[-1] if rel_words else None), usebest=usebest, golden=golden, printing=printing)
sentence = ' '.join(rel_words + lmk_words)
if printing: logger( 'rel_exp_chain: %s' % rel_exp_chain )
if printing: logger( 'lmk_exp_chain: %s' % lmk_exp_chain )
meaning = Meaning((lmk, lmk_prob, lmk_ent,
rel, rel_prob, rel_ent,
rel_exp_chain, rele_prob_chain, rele_ent_chain, rel_terminals, rel_landmarks,
lmk_exp_chain, lmke_prob_chain, lmke_ent_chain, lmk_terminals, lmk_landmarks,
rel_words, relw_prob, relw_ent,
lmk_words, lmkw_prob, lmkw_ent))
meaning.rel_a = rel_a
meaning.lmk_a = lmk_a
if consistent:
# get the most likely meaning for the generated sentence
try:
posteriors = get_sentence_posteriors(sentence, iterations=10, extra_meaning=(lmk,rel))
except:
print 'try again ...'
continue
meaning2 = max(posteriors, key=itemgetter(1))[0]
# is the original meaning the best one?
if meaning1 != meaning2:
print
print 'sentence:', sentence
print 'original:', meaning1
print 'interpreted:', meaning2
print 'try again ...'
print
continue
for m,p in sorted(posteriors, key=itemgetter(1)):
print m, p
return meaning, sentence
def is_valid_ihec(self):
validated = list()
for (xml, attrs) in self.xmljson:
(version, title_sanitized) = self.latest_valid_spec(attrs)
semantics_ok = self.validate_semantics(attrs)
if version and semantics_ok:
validated.append((version, xml))
logger("# is valid ihec spec:{0} version:{1} [{2}]\n".format('True', version, title_sanitized))
else:
logger("# is valid ihec spec:{0} version:{1} [{2}]\n".format('False', '__invalid__', title_sanitized))
if version and not semantics_ok:
logger("# found a valid spec version but failed semantic validation:{2}\n".format('', '', title_sanitized))
return validated
def autocorrect(scene, speaker, num_iterations=1, scale=1000, num_processors=7, num_samples=10,
best_samples=False, consistent=False, initial_training=False, cheating=False,
explicit_pointing=False, ambiguous_pointing=False, multiply=False,
golden_metric=True, mass_metric=True, student_metric=True, choosing_metric=True):
plt.ion()
assert initial_training + cheating + explicit_pointing + ambiguous_pointing == 1, \
'Must choose Initial Training, Cheating, Explicit or Ambiguous'
printing=False
scene_bb = scene.get_bounding_box()
scene_bb = scene_bb.inflate( Vec2(scene_bb.width*0.5,scene_bb.height*0.5) )
table = scene.landmarks['table'].representation.get_geometry()
step = 0.04
if choosing_metric:
loi = [lmk for lmk in scene.landmarks.values() if lmk.name != 'table']
all_heatmaps_tupless, xs, ys = speaker.generate_all_heatmaps(scene, step=step, loi=loi)
loi_infos = []
all_meanings = set()
for obj_lmk,all_heatmaps_tuples in zip(loi, all_heatmaps_tupless):
lmks, rels, heatmapss = zip(*all_heatmaps_tuples)
meanings = zip(lmks,rels)
# print meanings
all_meanings.update(meanings)
loi_infos.append( (obj_lmk, meanings, heatmapss) )
all_heatmaps_tupless, xs, ys = speaker.generate_all_heatmaps(scene, step=step)
all_heatmaps_tuples = all_heatmaps_tupless[0]
x = np.array( [list(xs-step*0.5)]*len(ys) )
y = np.array( [list(ys-step*0.5)]*len(xs) ).T
# all_heatmaps_tuples = []
# for lmk, d in all_heatmaps_dict.items():
# for rel, heatmaps in d.items():
# all_heatmaps_tuples.append( (lmk,rel,heatmaps) )
# all_heatmaps_tuples = all_heatmaps_tuples[:100]
lmks, rels, heatmapss = zip(*all_heatmaps_tuples)
graphmax1 = graphmax2 = 0
meanings = zip(lmks,rels)
landmarks = list(set(lmks))
relations = list(set(rels))
demo_sentences = ['near to the left edge of the table',
'somewhat near to the right edge of the table',
'on the table',
'on the middle of the table',
'at the lower left corner of the table',
'far from the purple prism']
epsilon = 0.0001
def heatmaps_for_sentence(sentence, all_meanings, loi_infos, xs, ys, scene, speaker, step=0.02):
printing=False
scene_bb = scene.get_bounding_box()
scene_bb = scene_bb.inflate( Vec2(scene_bb.width*0.5,scene_bb.height*0.5) )
x = np.array( [list(xs-step*0.5)]*len(ys) )
y = np.array( [list(ys-step*0.5)]*len(xs) ).T
posteriors = get_all_sentence_posteriors(sentence, all_meanings, printing=printing)
combined_heatmaps = []
for obj_lmk, meanings, heatmapss in loi_infos:
big_heatmap1 = None
for m,(h1,h2) in zip(meanings, heatmapss):
lmk,rel = m
p = posteriors[rel]*posteriors[lmk]
if big_heatmap1 is None:
big_heatmap1 = p*h1
else:
big_heatmap1 += p*h1
combined_heatmaps.append(big_heatmap1)
return combined_heatmaps
def loop(num_iterations):
min_dists = []
lmk_priors = []
rel_priors = []
lmk_posts = []
rel_posts = []
golden_log_probs = []
golden_entropies = []
golden_ranks = []
rel_types = []
total_mass = []
student_probs = []
student_entropies = []
student_ranks = []
student_rel_types = []
object_answers = []
object_distributions = []
epsilon = 1e-15
for iteration in range(num_iterations):
logger(('Iteration %d' % iteration),'okblue')
rand_p = Vec2(random()*table.width+table.min_point.x, random()*table.height+table.min_point.y)
trajector = Landmark( 'point', PointRepresentation(rand_p), None, Landmark.POINT )
if initial_training:
sentence, sampled_relation, sampled_landmark = speaker.describe(trajector, scene, False, 1)
if num_samples:
for i in range(num_samples):
landmark, relation, _ = speaker.sample_meaning(trajector, scene, 1)
train((landmark,relation), sentence, update=1, printing=printing)
else:
for (landmark,relation),prob in speaker.all_meaning_probs( trajector, scene, 1 ):
train((landmark,relation), sentence, update=prob, printing=printing)
else:
meaning, sentence = generate_sentence(rand_p, consistent, scene, speaker, usebest=True, printing=printing)
logger( 'Generated sentence: %s' % sentence)
if cheating:
landmark, relation = meaning.args[0],meaning.args[3]
else:
if explicit_pointing:
landmark = meaning.args[0]
if ambiguous_pointing:
pointing_point = landmark.representation.middle + Vec2(random()*0.1-0.05,random()*0.1-0.05)
#_, bestsentence = generate_sentence(rand_p, consistent, scene, speaker, usebest=True, printing=printing)
try:
golden_posteriors = get_all_sentence_posteriors(sentence, meanings, golden=True, printing=printing)
except ParseError as e:
logger( e )
prob = 0
rank = len(meanings)-1
entropy = 0
ed = len(sentence)
golden_log_probs.append( prob )
golden_entropies.append( entropy )
golden_ranks.append( rank )
min_dists.append( ed )
continue
epsilon = 1e-15
ps = [[golden_posteriors[lmk]*golden_posteriors[rel],(lmk,rel)] for lmk, rel in meanings if ((not explicit_pointing) or lmk == landmark)]
if not explicit_pointing:
all_lmk_probs = speaker.all_landmark_probs(landmarks, Landmark(None, PointRepresentation(rand_p), None))
all_lmk_probs = dict(zip(landmarks, all_lmk_probs))
if ambiguous_pointing:
all_lmk_pointing_probs = speaker.all_landmark_probs(landmarks, Landmark(None, PointRepresentation(pointing_point), None))
all_lmk_pointing_probs = dict(zip(landmarks, all_lmk_pointing_probs))
temp = None
for i,(p,(lmk,rel)) in enumerate(ps):
# lmk,rel = meanings[i]
# logger( '%f, %s' % (p, m2s(lmk,rel)))
head_on = speaker.get_head_on_viewpoint(lmk)
if not explicit_pointing:
# ps[i][0] *= speaker.get_landmark_probability(lmk, landmarks, PointRepresentation(rand_p))[0]
ps[i][0] *= all_lmk_probs[lmk]
if ambiguous_pointing:
# ps[i][0] *= speaker.get_landmark_probability(lmk, landmarks, PointRepresentation(pointing_point))[0]
ps[i][0] *= all_lmk_pointing_probs[lmk]
ps[i][0] *= speaker.get_probabilities_points( np.array([rand_p]), rel, head_on, lmk)[0]
if lmk == meaning.args[0] and rel == meaning.args[3]:
temp = i
ps,_meanings = zip(*ps)
print ps
ps = np.array(ps)
ps += epsilon
ps = ps/ps.sum()
temp = ps[temp]
ps = sorted(zip(ps,_meanings),reverse=True)
logger( 'Attempted to say: %s' % m2s(meaning.args[0],meaning.args[3]) )
logger( 'Interpreted as: %s' % m2s(ps[0][1][0],ps[0][1][1]) )
logger( 'Attempted: %s vs Interpreted: %s' % (str(temp), str(ps[0][0])))
# logger( 'Golden entropy: %f, Max entropy %f' % (golden_entropy, max_entropy))
landmark, relation = ps[0][1]
head_on = speaker.get_head_on_viewpoint(landmark)
all_descs = speaker.get_all_meaning_descriptions(trajector, scene, landmark, relation, head_on, 1)
distances = []
for desc in all_descs:
distances.append([edit_distance( sentence, desc ), desc])
distances.sort()
print distances
correction = distances[0][1]
if correction == sentence:
correction = None
logger( 'No correction!!!!!!!!!!!!!!!!!!', 'okgreen' )
accept_correction( meaning, correction, update_scale=scale, eval_lmk=(not explicit_pointing), multiply=multiply, printing=printing )
def probs_metric(inverse=False):
bestmeaning, bestsentence = generate_sentence(rand_p, consistent, scene, speaker, usebest=True, golden=inverse, printing=printing)
sampled_landmark, sampled_relation = bestmeaning.args[0], bestmeaning.args[3]
try:
golden_posteriors = get_all_sentence_posteriors(bestsentence, meanings, golden=(not inverse), printing=printing)
# lmk_prior = speaker.get_landmark_probability(sampled_landmark, landmarks, PointRepresentation(rand_p))[0]
all_lmk_probs = speaker.all_landmark_probs(landmarks, Landmark(None, PointRepresentation(rand_p), None))
all_lmk_probs = dict(zip(landmarks, all_lmk_probs))
lmk_prior = all_lmk_probs[sampled_landmark]
head_on = speaker.get_head_on_viewpoint(sampled_landmark)
rel_prior = speaker.get_probabilities_points( np.array([rand_p]), sampled_relation, head_on, sampled_landmark)
lmk_post = golden_posteriors[sampled_landmark]
rel_post = golden_posteriors[sampled_relation]
ps = np.array([golden_posteriors[lmk]*golden_posteriors[rel] for lmk, rel in meanings])
rank = None
for i,p in enumerate(ps):
lmk,rel = meanings[i]
# logger( '%f, %s' % (p, m2s(lmk,rel)))
head_on = speaker.get_head_on_viewpoint(lmk)
# ps[i] *= speaker.get_landmark_probability(lmk, landmarks, PointRepresentation(rand_p))[0]
ps[i] *= all_lmk_probs[lmk]
ps[i] *= speaker.get_probabilities_points( np.array([rand_p]), rel, head_on, lmk)
if lmk == sampled_landmark and rel == sampled_relation:
idx = i
ps += epsilon
ps = ps/ps.sum()
prob = ps[idx]
rank = sorted(ps, reverse=True).index(prob)
entropy = entropy_of_probs(ps)
except ParseError as e:
logger( e )
lmk_prior = 0
rel_prior = 0
lmk_post = 0
rel_post = 0
prob = 0
rank = len(meanings)-1
entropy = 0
distances = [[None]]
head_on = speaker.get_head_on_viewpoint(sampled_landmark)
all_descs = speaker.get_all_meaning_descriptions(trajector, scene, sampled_landmark, sampled_relation, head_on, 1)
distances = []
for desc in all_descs:
distances.append([edit_distance( bestsentence, desc ), desc])
distances.sort()
return lmk_prior,rel_prior,lmk_post,rel_post,\
prob,entropy,rank,distances[0][0],type(sampled_relation)
def db_mass():
total = CProduction.get_production_sum(None)
total += CWord.get_word_sum(None)
return total
def choosing_object_metric():
trajector = choice(loi)
sentence, sampled_relation, sampled_landmark = speaker.describe(trajector, scene, max_level=1)
lmk_probs = []
try:
combined_heatmaps = heatmaps_for_sentence(sentence, all_meanings, loi_infos, xs, ys, scene, speaker, step=step)
for combined_heatmap,obj_lmk in zip(combined_heatmaps, loi):
ps = [p for (x,y),p in zip(list(product(xs,ys)),combined_heatmap) if obj_lmk.representation.contains_point( Vec2(x,y) )]
# print ps, xs.shape, ys.shape, combined_heatmap.shape
lmk_probs.append( (sum(ps)/len(ps), obj_lmk) )
lmk_probs = sorted(lmk_probs, reverse=True)
top_p, top_lmk = lmk_probs[0]
lprobs, lmkss = zip(*lmk_probs)
logger( sorted(zip(np.array(lprobs)/sum(lprobs), [(l.name, l.color, l.object_class) for l in lmkss]), reverse=True) )
logger( 'I bet %f you are talking about a %s %s %s' % (top_p/sum(lprobs), top_lmk.name, top_lmk.color, top_lmk.object_class) )
# objects.append(top_lmk)
except Exception as e:
logger( 'Unable to get object from sentence. %s' % e, 'fail' )
print traceback.format_exc()
exit()
return loi.index(trajector), [ (lprob, loi.index(lmk)) for lprob,lmk in lmk_probs ]
if golden_metric:
lmk_prior,rel_prior,lmk_post,rel_post,prob,entropy,rank,ed,rel_type = probs_metric()
else:
lmk_prior,rel_prior,lmk_post,rel_post,prob,entropy,rank,ed,rel_type = \
None, None, None, None, None, None, None, None, None
lmk_priors.append( lmk_prior )
rel_priors.append( rel_prior )
lmk_posts.append( lmk_post )
rel_posts.append( rel_post )
golden_log_probs.append( prob )
golden_entropies.append( entropy )
golden_ranks.append( rank )
min_dists.append( ed )
rel_types.append( rel_type )
if mass_metric:
total_mass.append( db_mass() )
else:
total_mass.append( None )
if student_metric:
_,_,_,_,student_prob,student_entropy,student_rank,_,student_rel_type = probs_metric(inverse=True)
else:
_,_,_,_,student_prob,student_entropy,student_rank,_,student_rel_type = \
None, None, None, None, None, None, None, None, None
student_probs.append( student_prob )
student_entropies.append( student_entropy )
student_ranks.append( student_rank )
student_rel_types.append( student_rel_type )
if choosing_metric:
answer, distribution = choosing_object_metric()
else:
answer, distribution = None, None
object_answers.append( answer )
object_distributions.append( distribution )
return zip(lmk_priors, rel_priors, lmk_posts, rel_posts,
golden_log_probs, golden_entropies, golden_ranks,
min_dists, rel_types, total_mass, student_probs,
student_entropies, student_ranks, student_rel_types,
object_answers, object_distributions)
filename = ''
if initial_training: filename += 'initial_training'
if cheating: filename+= 'cheating'
if explicit_pointing: filename+='explicit_pointing'
if ambiguous_pointing: filename+='ambiguous_pointing'
if multiply: filename+='_multiply'
filename += ('_p%i_n%i_u%i.shelf' % (num_processors,num_iterations,scale))
import shelve
f = shelve.open(filename)
f['lmk_priors'] = []
f['rel_priors'] = []
f['lmk_posts'] = []
f['rel_posts'] = []
f['golden_log_probs'] = []
f['golden_entropies'] = []
f['golden_ranks'] = []
f['min_dists'] = []
f['rel_types'] = []
f['total_mass'] = []
f['student_probs'] = []
f['student_entropies'] = []
f['student_ranks'] = []
f['student_rel_types'] = []
f['object_answers'] = []
f['object_distributions'] = []
f['initial_training'] = initial_training
f['cheating'] = cheating
f['explicit_pointing'] = explicit_pointing
f['ambiguous_pointing'] = ambiguous_pointing
f.close()
chunk_size = 10
num_each = int(num_iterations/num_processors)
n = int(num_each / chunk_size)
extra = num_each % chunk_size
logger( "num_each: %i, chunk_size: %i, n: %i, extra: %i" % (num_each, chunk_size, n, extra) )
for i in range(n):
lists = parmap(loop,[chunk_size]*num_processors)
# lists = map(loop,[chunk_size]*num_processors)
result = []
for i in range(chunk_size):
for j in range(num_processors):
result.append( lists[j][i] )
lmk_priors, rel_priors, lmk_posts, rel_posts, \
golden_log_probs, golden_entropies, golden_ranks, \
min_dists, rel_types, total_mass, student_probs, student_entropies, \
student_ranks, student_rel_types, object_answers, object_distributions = zip(*result)
f = shelve.open(filename)
f['lmk_priors'] += lmk_priors
f['rel_priors'] += rel_priors
f['lmk_posts'] += lmk_posts
f['rel_posts'] += rel_posts
f['golden_log_probs'] += golden_log_probs
f['golden_entropies'] += golden_entropies
f['golden_ranks'] += golden_ranks
f['min_dists'] += min_dists
f['rel_types'] += rel_types
f['total_mass'] += total_mass
f['student_probs'] += student_probs
f['student_entropies'] += student_entropies
f['student_ranks'] += student_ranks
f['student_rel_types'] += student_rel_types
f['object_answers'] += object_answers
f['object_distributions'] += object_distributions
f.close()
if extra:
lists = parmap(loop,[extra]*num_processors)
# lists = map(loop,[extra]*num_processors)
result = []
for i in range(extra):
for j in range(num_processors):
result.append( lists[j][i] )
lmk_priors, rel_priors, lmk_posts, rel_posts, \
golden_log_probs, golden_entropies, golden_ranks, \
min_dists, rel_types, total_mass, student_probs, student_entropies, \
student_ranks, student_rel_types, object_answers, object_distributions = zip(*result)
f = shelve.open(filename)
f['lmk_priors'] += lmk_priors
f['rel_priors'] += rel_priors
f['lmk_posts'] += lmk_posts
f['rel_posts'] += rel_posts
f['golden_log_probs'] += golden_log_probs
f['golden_entropies'] += golden_entropies
f['golden_ranks'] += golden_ranks
f['min_dists'] += min_dists
f['rel_types'] += rel_types
f['total_mass'] += total_mass
f['student_probs'] += student_probs
f['student_entropies'] += student_entropies
f['student_ranks'] += student_ranks
f['student_rel_types'] += student_rel_types
f['object_answers'] += object_answers
f['object_distributions'] += object_distributions
f.close()
exit()
