def get_pdos(out_file='dos.msg'):
"""Calculate and save the projected DOS. Requires a previously relaxed
calculation.
Parameters
----------
out_file : str
Name of the output file to save the results to.
"""
atoms = get_relaxed_calculation()
calc = atoms.get_calculator()
# Calculate the pdos and write to disk
dos = calc.calc_pdos(nscf=True,
kpts=atoms.info['kpts'] * [2, 2, 1],
DeltaE=0.01,
slab=True,
Emin=-40,
Emax=40,
tetrahedra=False,
sigma=0.2)
dos = list(dos)
array_to_list(dos)
# If outfile, write a MessagePack encoded version to disk
if out_file:
with open(out_file, 'w') as f:
msgpack.dump(dos, f)
# Return a BSON friendly version
return json.dumps(dos, encoding='utf-8')
def get_or_build(path, build_fn, *args, **kwargs):
"""
Load from serialized form or build an object, saving the built
object.
Remaining arguments are provided to `build_fn`.
"""
save = False
obj = None
if path is not None and os.path.isfile(path):
with open(path, 'rb') as obj_f:
obj = msgpack.load(obj_f, use_list=False, encoding='utf-8')
else:
save = True
if obj is None:
obj = build_fn(*args, **kwargs)
if save and path is not None:
with open(path, 'wb') as obj_f:
msgpack.dump(obj, obj_f)
return obj
def wedge(cluster_set, report_cluster_status=False, force_wedge_thrsh=False):
# The lower bound of the edges being processed by the wedge algorithm.
global edge_cut_prob
global wedge_thrsh
if not force_wedge_thrsh:
edge_cut_prob = bconfig.WEDGE_THRESHOLD / 4.
wedge_thrsh = bconfig.WEDGE_THRESHOLD
else:
edge_cut_prob = force_wedge_thrsh / 4.
wedge_thrsh = force_wedge_thrsh
matr = ProbabilityMatrix(cluster_set.last_name)
matr.load()
global h5file
h5filepath = bconfig.TORTOISE_FILES_PATH + 'wedge_cache_' + str(PID())
h5file = h5py.File(h5filepath)
convert_cluster_set(cluster_set, matr)
del matr # be sure that this is the last reference!
do_wedge(cluster_set)
report = []
if report_cluster_status:
msg = []
for cl1 in cluster_set.clusters:
for cl2 in cluster_set.clusters:
if cl2 > cl1:
id1 = cluster_set.clusters.index(cl1)
id2 = cluster_set.clusters.index(cl2)
c12 = _compare_to(cl1, cl2)
c21 = _compare_to(cl2, cl1)
report.append((id1, id2, c12 + c21))
msg.append(' %s vs %s : %s + %s = %s -- %s' %
(id1, id2, c12, c21, c12 + c21, cl1.hates(cl2)))
msg = 'Wedge final clusters for %s: \n' % str(wedge_thrsh) + '\n'.join(
msg)
logger.log(msg)
restore_cluster_set(cluster_set)
if bconfig.DEBUG_CHECKS:
assert cluster_set._debug_test_hate_relation()
assert cluster_set._debug_duplicated_recs()
if report_cluster_status:
destfile = '/tmp/baistats/cluster_status_report_pid_%s_lastname_%s_thrsh_%s' % (
str(PID()), str(cluster_set.last_name), str(wedge_thrsh))
f = filehandler.open(destfile, 'w')
SER.dump([
wedge_thrsh, cluster_set.last_name, report,
cluster_set.num_all_bibs
], f)
f.close()
gc.collect()
h5file.close()
os.remove(h5filepath)
def main(args):
logging.basicConfig(level=logging.INFO)
model = json.load(args.ifile)
fn = fn_from_args(args)
if fn:
weight_layers = [layer for layer in model
if layer['layerName'] in weight_first_list]
if fn.needs_two_step:
for weights in [layer['parameters'][0] for layer in weight_layers]:
fn.consume(weights)
fn.done()
for i, layer in enumerate(weight_layers):
layer['parameters'][0] = transform(layer['parameters'][0], fn)
if fn.needs_two_step:
model = {
'codebook' : fn.serialize_codebook(),
'model' : model
}
if args.ubjson_format:
args.ofile.write(simpleubjson.encode(model))
elif args.json:
args.ofile.write(json.dumps(model).encode('utf8'))
else:
msgpack.dump(model, args.ofile, use_bin_type=True,
use_single_float=args.single_precision_float)
args.ofile.close()
args.ifile.close()
def to_local_file(obj, filename, to_save_path=None):
"""
Save an object as file.
Params
------
obj : object
Serialized object or any object which be built-in Python types.
root_dir : str
Root directory where the object will be saved.
filename : str
Object filename.
"""
if to_save_path is None:
_init_data_dir()
to_save_path = DEFAULT_DATA_DIR_NAME
fpath = path.join(to_save_path, filename)
try:
with open(fpath, mode='wb') as file:
mpk.dump(obj, file)
except TypeError:
raise Exception('`obj` could not be saved because it ' +
'is not serialized.')
def load_embeddings(self):
"""generate embeddings suited for the current vocab or load previously cached ones."""
embedding_file = os.path.join(self.args.output_dir,
'embedding.msgpack')
if not os.path.exists(embedding_file):
if self.args.language == "chinese":
embeddings = load_embeddings_Chinese(
self.args.pretrained_embeddings,
self.vocab,
self.args.embedding_dim,
mode=self.args.embedding_mode,
lower=self.args.lower_case)
else:
embeddings = load_embeddings_English(
self.args.pretrained_embeddings,
self.vocab,
self.args.embedding_dim,
mode=self.args.embedding_mode,
lower=self.args.lower_case)
with open(embedding_file, 'wb') as f:
msgpack.dump(embeddings, f)
else:
with open(embedding_file, 'rb') as f:
embeddings = msgpack.load(f)
return embeddings
def setUp(self):
self.dir = tempfile.mkdtemp()
self.runner = click.testing.CliRunner()
if self.keep_tree:
print 'KmlTrackTest is running in %s' % self.dir
self.infile_msgpack = os.path.join(self.dir, 'in.msgpack')
with open(self.infile_msgpack, 'w') as f:
for row in self.test_track:
msgpack.dump(row, f)
self.infile_json = os.path.join(self.dir, 'in.json')
with open(self.infile_json, 'w') as f:
for row in self.test_track:
json.dump(row, f)
f.write('\n')
self.infile_csv = os.path.join(self.dir, 'in.csv')
with open(self.infile_csv, 'w') as f:
f = csv.DictWriter(
f, fieldnames=['lat', 'lon', 'timestamp', 'course', 'color'])
f.writeheader()
for row in self.test_track:
f.writerow(row)
self.outfile = os.path.join(self.dir, 'out.kml')
def dump(data, filepath):
'''
Write data as as type self.ext to filepath. json or msgpack
'''
if ' ' in filepath:
raise raeting.KeepError("Invalid filepath '{0}' "
"contains space".format(filepath))
root, ext = os.path.splitext(filepath)
if ext == '.json':
with aiding.ocfn(filepath, "w+") as f:
json.dump(data, f, indent=2, encoding='utf-8')
f.flush()
os.fsync(f.fileno())
elif ext == '.msgpack':
if not msgpack:
raise raeting.KeepError("Invalid filepath ext '{0}' "
"needs msgpack installed".format(filepath))
with aiding.ocfn(filepath, "w+b", binary=True) as f:
msgpack.dump(data, f, encoding='utf-8')
f.flush()
os.fsync(f.fileno())
else:
raise raeting.KeepError("Invalid filepath ext '{0}' "
"not '.json' or '.msgpack'".format(filepath))
def put_password(policy_pubkey,
username,
password,
save_as_file: bool = False):
data_source = Enrico(policy_encrypting_key=policy_pubkey)
data_source_public_key = bytes(data_source.stamp)
kits = list()
actual_data = {
'username': username,
'password': password,
}
plaintext = msgpack.dumps(actual_data, use_bin_type=True)
message_kit, _signature = data_source.encrypt_message(plaintext)
kit_bytes = message_kit.to_bytes()
kits.append(kit_bytes)
data = {
'data_source': data_source_public_key,
'kits': kits,
}
if save_as_file:
with open(HEART_DATA_FILENAME, "wb") as file:
msgpack.dump(data, file, use_bin_type=True)
return data
def crawler_fans():
all_person_set = set()
f_fans = open('fans.txt', 'w')
for page_index in range(1, 345, 1):
for class_index in [1, 2, 4, 5, 6]:
start = time()
try:
url = 'https://123fans.cn/results.php?qi=%d&c=%d'\
% (page_index, class_index)
req = urllib2.Request(url, None)
response = urllib2.urlopen(req)
html_doc = response.read()
soup = BeautifulSoup(html_doc)
children = [k for k in soup.children][1]
lis = children.select('.odd')
for k in lis:
try:
name = k.select('.name')[0].string
all_person_set.add(name)
f_fans.write(name + '\n')
except:
traceback.print_exc()
continue
print page_index, class_index, \
len(all_person_set), time()-start
except:
print 'error', page_index, class_index
sleep(5)
all_person_set = list(all_person_set)
msgpack.dump(all_person_set, open('fans.p', 'w'))
def msg_pack(file_name, contents):
try:
with open(path + file_name, 'wb') as f:
msgpack.dump(contents, f)
except Exception, e:
logging.info(e)
return False
def main():
dialect_name, working_dir = sys.argv[1:]
executor = sqlexecutor.executor(dialect_name, working_dir)
print "Ready"
sys.stdout.flush()
try:
for message in msgpack.Unpacker(sys.stdin, read_size=1):
command = message[0]
args = message[1:]
if command == "execute":
(creation_sql, query, ) = args
result = executor.execute(creation_sql, query)
if result.table is None:
column_names = None
rows = None
else:
column_names = result.table.column_names
rows = result.table.rows
msgpack.dump((result.error, column_names, rows), sys.stdout)
sys.stdout.flush()
else:
return
finally:
executor.close()
def prepare():
args, _ = setup()
train_data = load_data("train.txt")
dev_data = load_data("test.txt")
word_dict = build_dict(train_data[0] + dev_data[0])
x, y, e1, e2, dist1, dist2, e1_pos, e2_pos = vectorize(
train_data, word_dict)
train = list(zip(x, e1, e2, dist1, dist2, e1_pos, e2_pos, y))
e_x, e_y, e_e1, e_e2, e_dist1, e_dist2, e_e1_pos, e_e2_pos = vectorize(
dev_data, word_dict)
valid = list(
zip(e_x, e_e1, e_e2, e_dist1, e_dist2, e_e1_pos, e_e2_pos, e_y))
embed_file = 'embeddings.txt'
vac_file = 'words.lst'
embedding = load_embedding(embed_file, vac_file, word_dict)
# save
meta = {"embeddings": embedding.tolist()}
result = {"train": train, "valid": valid}
with open("meta.msgpack", "wb") as f:
msgpack.dump(meta, f)
with open("data.msgpack", "wb") as f:
msgpack.dump(result, f)
def dump(self, directory):
"""
Utility function to dump/save model
:param directory: directory to output to
:return:
"""
data = {"identity_to_values_small": self.identity_to_values_small,
"n_users ": self.n_users,
"n_identity_sent_values ": self.n_identity_sent_values,
"nu": self.nu,
"kappa": self.kappa,
"beta": self.beta,
"index_to_ids": self.index_to_ids,
"ids_to_index": self.ids_to_index,
"iteration": self.iteration,
"train_perplexity": self.train_perplexity,
"test_perplexity": self.test_perplexity
}
iter_str = str(self.iteration)
msgpack.dump(data, open(os.path.join(directory, iter_str + "_sent_basic.mpack"), "wb"))
np.save(os.path.join(directory, "sent_mu_0"), self.mu_0)
np.save(os.path.join(directory, "sent_sigma_0"), self.sigma_0)
np.save(os.path.join(directory, iter_str + "_sent_sigma"), self.sigma)
np.save(os.path.join(directory, iter_str + "_sent_mu"), self.mu)
np.save(os.path.join(directory, iter_str + "_sent_precision_matrix"), self.precision_matrix)
np.save(os.path.join(directory, iter_str + "_sent_phi"), self.phi)
def create_bond_dict(components_pdbx_file_path, msgpack_file_path):
pdbx_file = pdbx.PDBxFile()
pdbx_file.read(components_pdbx_file_path)
components = pdbx_file.get_block_names()
bond_dict = {}
for component in components:
print(component)
cif_bonds = pdbx_file.get_category("chem_comp_bond", block=component)
if cif_bonds is None:
# No bond info for this compound
continue
if isinstance(cif_bonds["comp_id"], str):
# Single string -> single bond
group_bonds = {
(cif_bonds["atom_id_1"], cif_bonds["atom_id_2"]):
BOND_ORDERS[cif_bonds["value_order"]]
}
else:
# Looped values -> multiple bonds
group_bonds = {(atom1, atom2): BOND_ORDERS[order]
for atom1, atom2, order in zip(
cif_bonds["atom_id_1"], cif_bonds["atom_id_2"],
cif_bonds["value_order"])}
bond_dict[component] = group_bonds
with open(msgpack_file_path, "wb") as msgpack_file:
msgpack.dump(bond_dict, msgpack_file)
def data_to_token_ids(source_data, id_data_path, vocab_to_id, cache=True):
"""Tokenize data file and turn into token-ids using given vocabulary file.
This function loads data line-by-line from data_path, calls the above
sentence_to_token_ids, and saves the result to target_path. See comment
for sentence_to_token_ids on the details of token-ids format.
Args:
source_data:
id_data_path:
vocab_to_id:
cache: Boolean;
"""
if not gfile.Exists(id_data_path) or not cache:
print("Creating id tokenized data %s" % id_data_path)
id_data = []
for source in source_data:
id_source = [[START_LINE_ID]]
for line in source:
id_line = [vocab_to_id.get(word[1], UNK_ID) for word in line]
id_source.append(id_line)
id_source.append([END_LINE_ID])
id_data.append(id_source)
with gfile.GFile(id_data_path, mode="w") as id_data_file:
pickle.dump(id_data, id_data_file)
else:
with gfile.GFile(id_data_path, mode="r") as token_file:
id_data = pickle.load(token_file)
return id_data
def save_to_file(self, filename):
"""Save only the bare minimum needed to reconstruct this CoverageDB.
This serializes the data to a single file and cab reduce the disk footprint of
block coverage significantly (depending on overlap and number of files)."""
if file_backing_disabled:
raise Exception(
"[!] Can't save/load coverage db files without msgpack. Try `pip install msgpack`"
)
save_dict = dict()
save_dict["version"] = 1 # serialized covdb version
save_dict["module_name"] = self.module_name
save_dict["module_base"] = self.module_base
save_dict["coverage_files"] = self.coverage_files
# save tighter version of block dict {int: int} vice {int: str}
block_dict_to_save = {}
file_index_map = {
filepath: self.coverage_files.index(filepath)
for filepath in self.coverage_files
}
for block, trace_list in self.block_dict.items():
trace_id_list = [file_index_map[name] for name in trace_list]
block_dict_to_save[block] = trace_id_list
save_dict["block_dict"] = block_dict_to_save
# write packed version to file
with open(filename, "wb") as f:
msgpack.dump(save_dict, f)
self.filename = filename
def main():
lfw_folder = '/data/liubo/face/lfw_face'
pair_file = '/data/liubo/face/lfw_pair.txt'
same_dist_list = []
no_same_dist_list = []
for line in open(pair_file):
tmp = line.rstrip().split()
if len(tmp) == 3:
person = tmp[0]
person_path = os.path.join(lfw_folder, person)
pic_list = os.listdir(person_path)
if len(pic_list) == 1:
print 'error person :', person
continue
else:
np.random.shuffle(pic_list)
pic_path1 = os.path.join(person_path, pic_list[0])
pic_path2 = os.path.join(person_path, pic_list[1])
dist = cal_two_pic_distance(pic_path1, pic_path2)
same_dist_list.append(dist)
elif len(tmp) == 4:
person1 = tmp[0]
person1_path = os.path.join(lfw_folder, person1)
pic1_list = os.listdir(person1_path)
person2 = tmp[2]
person2_path = os.path.join(lfw_folder, person2)
pic2_list = os.listdir(person2_path)
if len(pic1_list) > 0 and len(pic2_list) > 0:
np.random.shuffle(pic1_list)
np.random.shuffle(pic2_list)
pic_path1 = os.path.join(person1_path, pic1_list[0])
pic_path2 = os.path.join(person2_path, pic2_list[0])
dist = cal_two_pic_distance(pic_path1, pic_path2)
no_same_dist_list.append(dist)
msgpack.dump((same_dist_list, no_same_dist_list), open('dist.p', 'wb'))
def select_and_dump_wizard_findings(wizard_findings, target_file):
cfg = config
if len(wizard_findings) == 0:
print_l("[!] No wizard findings!")
sys.exit()
index = cfg["function_number"]
if index in range(0, len(wizard_findings)):
pass
else:
print_l("Functions found:")
for i, finding in enumerate(wizard_findings):
if "source" in finding:
print_l("{}) {func_name} from {source}:{start}-{end}".format(
i, **finding))
else:
print_l("{}) {func_name}".format(i, **finding))
buffer = bytearray(finding["buffer"])
hexdump(buffer)
# Let the user select a finding, add it to the config
index = select_from_range(len(wizard_findings),
"Choose a function to fuzz> ")
wizard_findings[index]["selected"] = True
with open(target_file.replace("targets.msg", "all_targets.msg"),
"wb") as msg_file:
msgpack.dump(wizard_findings, msg_file)
with open(target_file, "wb") as msg_file:
msgpack.dump(list(filter(lambda k: k["selected"], wizard_findings)),
msg_file)
return wizard_findings
def create_dict(components_pdbx_file_path, msgpack_file_path,
subcategory, expected_type):
pdbx_file = pdbx.PDBxFile()
pdbx_file.read(components_pdbx_file_path)
components = pdbx_file.get_block_names()
data_dict = {}
for i, component in enumerate(components):
print(f"{((i+1) / len(components) * 100):4.1f} %", end="\r")
try:
cif_dict = pdbx_file.get_category("chem_comp", block=component)
except ValueError:
# The 'chem_comp' category may contain unparsable names
# with wrong quote escaping
# In this case the PDBx file parser raises an Exception
cif_dict = None
if cif_dict is None:
# No or erroneous info for this compound
data_dict[component] = None
else:
try:
data = expected_type(cif_dict[subcategory])
except ValueError:
# Unparsable data, e.g. '?' as float
data = None
data_dict[component] = data
print()
with open(msgpack_file_path, "wb") as msgpack_file:
msgpack.dump(data_dict, msgpack_file)
def save_msgpack(cls, filename, data):
"""Save data into MSGPACK file
Parameters
----------
filename : str
Filename path
data :
Data to be stored
Returns
-------
None
"""
try:
import msgpack
except ImportError:
message = '{name}: Unable to import msgpack module. You can install it with `pip install msgpack-python`.'.format(
name=cls.__class__.__name__)
cls.logger().exception(message)
raise ImportError(message)
msgpack.dump(data, open(filename, 'wb'), use_bin_type=True)
def main(args):
logging.basicConfig(level=logging.INFO)
model = json.load(args.ifile)
fn = fn_from_args(args)
if fn:
weight_layers = [
layer for layer in model if layer['layerName'] in weight_first_list
]
if fn.needs_two_step:
for weights in [layer['parameters'][0] for layer in weight_layers]:
fn.consume(weights)
fn.done()
for i, layer in enumerate(weight_layers):
layer['parameters'][0] = transform(layer['parameters'][0], fn)
if fn.needs_two_step:
model = {'codebook': fn.serialize_codebook(), 'model': model}
if args.ubjson_format:
args.ofile.write(simpleubjson.encode(model))
elif args.json:
args.ofile.write(json.dumps(model).encode('utf8'))
else:
msgpack.dump(model,
args.ofile,
use_bin_type=True,
use_single_float=args.single_precision_float)
args.ofile.close()
args.ifile.close()
def encrypt_patient_data(policy_pubkey, data_fields,
label: bytes = DEFAULT_LABEL,
save_as_file: bool = False):
data_source = DataSource(policy_pubkey_enc=policy_pubkey,
label=label)
data_source_public_key = bytes(data_source.stamp)
ipfs_api = ipfsapi.connect()
kits = list()
with open("Merkle_json.json", "r") as read_file:
data = json.load(read_file)
share=data_fields
share_data = {}
for i in share:
share_data[i] = {'Value' : data[0][i]['Value'], 'Hash' : data[0][i]['Hash']}
plaintext = msgpack.dumps(share_data, use_bin_type=True)
message_kit, _signature = data_source.encrypt_message(plaintext)
kit_bytes = message_kit.to_bytes()
kits.append(kit_bytes)
data = {
'data_source': data_source_public_key,
'kits': kits,
}
if save_as_file:
with open(PATIENT_DETAIL, "wb") as file:
msgpack.dump(data, file, use_bin_type=True)
res = ipfs_api.add(PATIENT_DETAIL)
return res
def dump(data, filepath):
'''
Write data as as type self.ext to filepath. json or msgpack
'''
if ' ' in filepath:
raise raeting.KeepError("Invalid filepath '{0}' "
"contains space".format(filepath))
if hasattr(data, 'get'):
for key, val in data.items(): # P3 json.dump no encoding parameter
if isinstance(val, (bytes, bytearray)):
data[key] = val.decode('utf-8')
root, ext = os.path.splitext(filepath)
if ext == '.json':
with ocfn(filepath, "w+") as f:
json.dump(data, f, indent=2)
f.flush()
os.fsync(f.fileno())
elif ext == '.msgpack':
if not msgpack:
raise raeting.KeepError("Invalid filepath ext '{0}' "
"needs msgpack installed".format(filepath))
with ocfn(filepath, "w+b", binary=True) as f:
msgpack.dump(data, f, encoding='utf-8')
f.flush()
os.fsync(f.fileno())
else:
raise raeting.KeepError("Invalid filepath ext '{0}' "
"not '.json' or '.msgpack'".format(filepath))
def dump(data, filepath):
'''
Write data as as type self.ext to filepath. json or msgpack
'''
if ' ' in filepath:
raise raeting.KeepError("Invalid filepath '{0}' "
"contains space".format(filepath))
with aiding.ocfn(filepath, "w+") as f:
root, ext = os.path.splitext(filepath)
if ext == '.json':
json.dump(data, f, indent=2)
elif ext == '.msgpack':
if not msgpack:
raise raeting.KeepError(
"Invalid filepath ext '{0}' "
"needs msgpack installed".format(filepath))
msgpack.dump(data, f)
else:
raise raeting.KeepError(
"Invalid filepath ext '{0}' "
"not '.json' or '.msgpack'".format(filepath))
f.flush()
os.fsync(f.fileno())
def dump(data: object, path: str):
if type(path) != str:
msgpack.dump(data, path)
return
with open(path, "wb") as f:
kwargs['ensure_ascii'] = False
msgpack.dump(data, f)
def sort_out(filename,
from_year,
heading_size_for_statistics,
top_k,
min_n_events=2000,
sheet_name="Sheet1"):
# why is xlrd being deprecated in favor of openpyxl when the latter cannot even open certain .xlsx files??
df = pd.read_excel(f"input/{filename}",
sheet_name=sheet_name,
engine="xlrd")
for country, country_df in tqdm(df.groupby("COUNTRY")):
current_entries = country_df["YEAR"] >= int(from_year)
reference_date = dt.datetime(year=int(from_year), month=1, day=1)
event_types, event_times, labels = parse_events(
country_df[current_entries], int(heading_size_for_statistics),
int(top_k), reference_date)
prefix = filename.split('.')[0]
n_types = len(labels)
n_events = len(event_types)
if n_events < min_n_events:
continue
tqdm.write(' '.join([
colored(country, "red"),
'; '.join(', '.join(label) for label in labels)
]))
name = f"{prefix}-{country}-since{from_year}-top{top_k}_{n_types}-{n_events//1000}k-events"
with open(f"output/{name}.msgpack", "wb") as f:
msgpack.dump([event_types, event_times], f)
with open(f"output/{name}-labels.json", "w") as f:
json.dump(labels, f)
def load_check_result_url(dic_file, check_url_file):
person_result_dic = {} # {person:([](right_set),[](wrong_set))} # 肯定正确和肯定错的的图片
right_url_count = wrong_url_count = error_format_count = no_baike_count = no_meaning_count = 0
if os.path.exists(dic_file):
person_result_dic = msgpack.load(open(dic_file, 'rb'))
for line in open(check_url_file):
tmp = line.rstrip().split('\t')
# [person_name, pic_index, pic_url, baike_name, baike_sim, newbaike_sim, guess_info]
person_name = tmp[0]
right_list, wrong_list = person_result_dic.get(person_name, ([], []))
if len(tmp) == 7:
if tmp[3] not in no_meaning_list:
if tmp[3] == no_find_baike:
no_baike_count += 1
continue
else:
if get_newbaike_sim(tmp[4]) > sim_threshold:
if tmp[0] == tmp[3]:
right_list.append(tmp[1])
right_url_count += 1
else:
wrong_url_count += 1
wrong_list.append(tmp[1])
else: # 小于某概率时结果不可信,需要标注
no_baike_count += 1
continue
else:
no_meaning_count += 1
continue
else:
error_format_count += 1
continue
person_result_dic[person_name] = (right_list, wrong_list)
print right_url_count, wrong_url_count, no_baike_count, no_meaning_count, error_format_count
msgpack.dump(person_result_dic, open('person_result_dic.p', 'w'))
def crawler_baidu_person_list():
# 从百度人气榜上获取人名
all_person_list = set()
for index in range(50):
url = 'http://baike.baidu.com/operation/api/starflowerstarlist?' \
'rankType=thisWeek&pg=%d' % index
req = urllib2.Request(url, None)
response = urllib2.urlopen(req)
html_doc = response.read()
content = json.loads(html_doc)
this_page_list = content.get('data').get('thisWeek')
for person_content in this_page_list:
all_person_list.add(person_content.get('name'))
print len(all_person_list)
for index in range(50):
url = 'http://baike.baidu.com/operation/api/starflowerstarlist?' \
'rankType=lastWeek&pg=%d' % index
req = urllib2.Request(url, None)
response = urllib2.urlopen(req)
html_doc = response.read()
content = json.loads(html_doc)
this_page_list = content.get('data').get('lastWeek')
for person_content in this_page_list:
all_person_list.add(person_content.get('name'))
print len(all_person_list)
all_person_set = list(all_person_list)
msgpack.dump(all_person_set, open('baidu_fans.p', 'w'))
def get_total_potential(out_file='potential.msg'):
"""Calculate and save the total potential. Requires a previously relaxed
calculation.
Parameters
----------
out_file : str
Name of the output file to save the results to.
"""
atoms = get_relaxed_calculation()
calc = atoms.get_calculator()
# Collect the total potential and write to disk
potential = calc.extract_total_potential()
potential = list(potential)
array_to_list(potential)
# If outfile, write a MessagePack encoded version to disk
if out_file:
with open(out_file, 'w') as f:
msgpack.dump(potential, f)
# Return a BSON friendly version
return json.dumps(potential, encoding='utf-8')
def freqs_to_cBpack(input_file, output_file, cutoff=600):
"""
Convert a frequency list into the idiosyncratic 'cBpack' format that
will be loaded by wordfreq: a list in msgpack format of frequency
tiers, each tier being one centibel (a factor of 10^(1/100))
less frequent than the previous tier.
"""
cBpack = []
for line in input_file:
word, strfreq = line.rstrip().split('\t', 1)
if word == '__total__':
raise ValueError("This is a count file, not a frequency file")
freq = float(strfreq)
neg_cB = -(round(math.log10(freq) * 100))
if neg_cB >= cutoff:
break
while neg_cB >= len(cBpack):
cBpack.append([])
cBpack[neg_cB].append(word)
for sublist in cBpack:
sublist.sort()
cBpack_data = [{'format': 'cB', 'version': 1}] + cBpack
msgpack.dump(cBpack_data, output_file)
def crawler_fans():
all_person_set = set()
f_fans = open('fans.txt', 'w')
for page_index in range(1, 345, 1):
for class_index in [1, 2, 4, 5, 6]:
start = time()
try:
url = 'https://123fans.cn/results.php?qi=%d&c=%d'\
% (page_index, class_index)
req = urllib2.Request(url, None)
response = urllib2.urlopen(req)
html_doc = response.read()
soup = BeautifulSoup(html_doc)
children = [k for k in soup.children][1]
lis = children.select('.odd')
for k in lis:
try:
name = k.select('.name')[0].string
all_person_set.add(name)
f_fans.write(name+'\n')
except:
traceback.print_exc()
continue
print page_index, class_index, \
len(all_person_set), time()-start
except:
print 'error', page_index, class_index
sleep(5)
all_person_set = list(all_person_set)
msgpack.dump(all_person_set, open('fans.p', 'w'))
def freqs_to_cBpack(in_filename, out_filename, cutoff=-600):
"""
Convert a csv file of words and their frequencies to a file in the
idiosyncratic 'cBpack' format.
Only words with a frequency greater than `cutoff` centibels will be
written to the new file.
This cutoff should not be stacked with a cutoff in `read_freqs`; doing
so would skew the resulting frequencies.
"""
freqs = read_freqs(in_filename, cutoff=0, lang=None)
cBpack = []
for token, freq in freqs.items():
cB = round(math.log10(freq) * 100)
if cB <= cutoff:
continue
neg_cB = -cB
while neg_cB >= len(cBpack):
cBpack.append([])
cBpack[neg_cB].append(token)
for sublist in cBpack:
sublist.sort()
# Write a "header" consisting of a dictionary at the start of the file
cBpack_data = [{'format': 'cB', 'version': 1}] + cBpack
with gzip.open(out_filename, 'wb') as outfile:
msgpack.dump(cBpack_data, outfile)
def crawler_baidu_person_list():
# 从百度人气榜上获取人名
all_person_list = set()
for index in range(50):
url = 'http://baike.baidu.com/operation/api/starflowerstarlist?' \
'rankType=thisWeek&pg=%d' % index
req = urllib2.Request(url, None)
response = urllib2.urlopen(req)
html_doc = response.read()
content = json.loads(html_doc)
this_page_list = content.get('data').get('thisWeek')
for person_content in this_page_list:
all_person_list.add(person_content.get('name'))
print len(all_person_list)
for index in range(50):
url = 'http://baike.baidu.com/operation/api/starflowerstarlist?' \
'rankType=lastWeek&pg=%d' % index
req = urllib2.Request(url, None)
response = urllib2.urlopen(req)
html_doc = response.read()
content = json.loads(html_doc)
this_page_list = content.get('data').get('lastWeek')
for person_content in this_page_list:
all_person_list.add(person_content.get('name'))
print len(all_person_list)
all_person_set = list(all_person_list)
msgpack.dump(all_person_set, open('baidu_fans.p', 'w'))
def get_or_build(path, build_fn, *args, **kwargs):
"""
Load from serialized form or build an object, saving the built
object.
Remaining arguments are provided to `build_fn`.
"""
save = False
obj = None
if path is not None and os.path.isfile(path):
with open(path, 'rb') as obj_f:
obj = msgpack.load(obj_f, use_list=False, encoding='utf-8')
else:
save = True
if obj is None:
obj = build_fn(*args, **kwargs)
if save and path is not None:
with open(path, 'wb') as obj_f:
msgpack.dump(obj, obj_f)
return obj
def freqs_to_cBpack(in_filename, out_filename, cutoff=-600):
"""
Convert a csv file of words and their frequencies to a file in the
idiosyncratic 'cBpack' format.
Only words with a frequency greater than `cutoff` centibels will be
written to the new file.
This cutoff should not be stacked with a cutoff in `read_freqs`; doing
so would skew the resulting frequencies.
"""
freqs = read_freqs(in_filename, cutoff=0, lang=None)
cBpack = []
for token, freq in freqs.items():
cB = round(math.log10(freq) * 100)
if cB <= cutoff:
continue
neg_cB = -cB
while neg_cB >= len(cBpack):
cBpack.append([])
cBpack[neg_cB].append(token)
for sublist in cBpack:
sublist.sort()
# Write a "header" consisting of a dictionary at the start of the file
cBpack_data = [{'format': 'cB', 'version': 1}] + cBpack
with gzip.open(out_filename, 'wb') as outfile:
msgpack.dump(cBpack_data, outfile)
def dump(data, path):
"""
Serialize data dict and write to file given by path where serialization is
given by path's extension of either JSON, MsgPack, or CBOR for extension
.json, .mgpk, or .cbor respectively
"""
if ' ' in path:
raise IOError(f"Invalid file path '{path}' contains space.")
root, ext = os.path.splitext(path)
if ext == '.json':
with ocfn(path, "w+b") as f:
json.dump(data, f, indent=2)
f.flush()
os.fsync(f.fileno())
elif ext == '.mgpk':
with ocfn(path, "w+b") as f:
msgpack.dump(data, f)
f.flush()
os.fsync(f.fileno())
elif ext == '.cbor':
with ocfn(path, "w+b") as f:
cbor.dump(data, f)
f.flush()
os.fsync(f.fileno())
else:
raise IOError(f"Invalid file path ext '{path}' "
f"not '.json', '.mgpk', or 'cbor'.")
def encrypt_track_segments(policy_pubkey, dir_path):
data_source = Enrico(policy_encrypting_key=policy_pubkey)
data_source_public_key = bytes(data_source.stamp)
print(dir_path)
target_path = "/".join(dir_path.split('/')[:-1])
target_path = os.path.join(target_path, 'segments_encrypted')
if not os.path.exists(target_path):
try:
os.makedirs(target_path)
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
track_files = os.scandir(dir_path)
for track_segment in track_files:
with open(track_segment, "rb") as f:
plaintext = f.read()
ciphertext, signature = data_source.encrypt_message(plaintext)
print("Signature", signature)
data = {
'track_segment_data': ciphertext.to_bytes(),
'data_source': data_source_public_key
}
with open(os.path.join(target_path, track_segment.name), "wb") as f:
msgpack.dump(data, f, use_bin_type=True)
return True
def wedge(cluster_set, report_cluster_status=False, force_wedge_thrsh=False):
# The lower bound of the edges being processed by the wedge algorithm.
global edge_cut_prob
global wedge_thrsh
if not force_wedge_thrsh:
edge_cut_prob = bconfig.WEDGE_THRESHOLD / 4.
wedge_thrsh = bconfig.WEDGE_THRESHOLD
else:
edge_cut_prob = force_wedge_thrsh / 4.
wedge_thrsh = force_wedge_thrsh
matr = ProbabilityMatrix(cluster_set.last_name)
matr.load()
global h5file
h5filepath = bconfig.TORTOISE_FILES_PATH+'wedge_cache_'+str(PID())
h5file = h5py.File(h5filepath)
convert_cluster_set(cluster_set, matr)
del matr # be sure that this is the last reference!
do_wedge(cluster_set)
report = []
if bconfig.DEBUG_WEDGE_PRINT_FINAL_CLUSTER_COMPATIBILITIES or report_cluster_status:
msg = []
for cl1 in cluster_set.clusters:
for cl2 in cluster_set.clusters:
if cl2 > cl1:
id1 = cluster_set.clusters.index(cl1)
id2 = cluster_set.clusters.index(cl2)
c12 = _compare_to(cl1,cl2)
c21 = _compare_to(cl2,cl1)
report.append((id1,id2,c12+c21))
msg.append( ' %s vs %s : %s + %s = %s -- %s' % (id1, id2, c12, c21, c12+c21, cl1.hates(cl2)))
msg = 'Wedge final clusters for %s: \n' % str(wedge_thrsh) + '\n'.join(msg)
if not bconfig.DEBUG_WEDGE_OUTPUT and bconfig.DEBUG_WEDGE_PRINT_FINAL_CLUSTER_COMPATIBILITIES:
print
print msg
print
wedge_print(msg)
restore_cluster_set(cluster_set)
if bconfig.DEBUG_CHECKS:
assert cluster_set._debug_test_hate_relation()
assert cluster_set._debug_duplicated_recs()
if report_cluster_status:
destfile = '/tmp/baistats/cluster_status_report_pid_%s_lastname_%s_thrsh_%s' % (str(PID()),str(cluster_set.last_name),str(wedge_thrsh))
f = filehandler.open(destfile, 'w')
SER.dump([wedge_thrsh,cluster_set.last_name,report,cluster_set.num_all_bibs],f)
f.close()
gc.collect()
h5file.close()
os.remove(h5filepath)
def dump(cls, records, filepath=None):
"""Dump in the same fashion as load."""
if filepath is None:
filepath = os.path.join(config['current_snapshot'], 'repos.msgpack')
with utils.FaultTolerantFile(filepath) as f:
msgpack.dump(records, f, default=cls._dumper)
def __call__(self):
if self.filename == None:
self.filename = msgpackmemoized_basedir + '/' + self.f.__name__ + '.msgpack'
if path.exists(self.filename):
return msgpack.load(open(self.filename))
result = self.f()
msgpack.dump(result, open(self.filename, 'w'))
return result
def serializeToFile(self, fname, annotations):
"""
Overwritten to write Msgpack files.
"""
# TODO make all image filenames relative to the label file
import msgpack
f = open(fname, "w")
msgpack.dump(annotations, f)
def dump(self, outfile):
"""Write a serialized version of the database to filehandle."""
db_dict = {
'meta_prints': self.meta_prints,
'content_prints': self.content_prints,
'series_id': self.series_id,
}
msgpack.dump(db_dict, outfile)
def dump_file(self, obj, fp):
try:
msgpack.dump(obj, fp)
return True
except Exception as ex:
log.warn('Unable to dump object to file: %s', ex, exc_info=True)
return False
def match_one_file(midi_filename, embed_fn, hash_fn, msd_embeddings,
msd_sequences, msd_feature_paths, msd_ids, output_filename):
"""
Match one MIDI file to the million song dataset by computing its CQT,
pruning by matching its embedding, re-pruning by matching its downsampled
hash sequence, and finally doing DTW on CQTs on the remaining entries.
Parameters
----------
midi_filename : str
Path to a MIDI file to match to the MSD
embed_fn : function
Function which takes in a CQT and produces a fixed-length embedding
hash_fn : function
Function which takes in a CQT and produces a sequence of binary vectors
msd_embeddings : np.ndarray
(# MSD entries x embedding dimension) matrix of all embeddings for all
entries from the MSD
msd_sequences : list of np.ndarray
List of binary vector sequences (represented as ints) for all MSD
entries
msd_feature_paths : list of str
Path to feature files (containing CQT) for each MSD entry
msd_ids : list of str
MSD ID of each corresponding entry in the above lists
output_filename : str
Where to write the results file, which includes the DTW scores for all
of the non-pruned MSD entries
"""
# Try to compute a CQT for the MIDI file
try:
m = pretty_midi.PrettyMIDI(midi_filename)
except Exception as e:
print 'Could not parse {}: {}'.format(
os.path.split(midi_filename)[1], traceback.format_exc(e))
return
try:
midi_gram = feature_extraction.midi_cqt(m)
except Exception as e:
print "Error creating CQT for {}: {}".format(
os.path.split(midi_filename)[1], traceback.format_exc(e))
return
# Skip this file if the MIDI gram is very long, to avoid memory issues
if midi_gram.shape[0] > MAX_FRAMES:
return
# Compute the embedding of the CQT
midi_embedding = embed_fn(midi_gram.reshape(1, 1, *midi_gram.shape))
# Compute the hash sequence
midi_hash_sequence = hash_fn(midi_gram.reshape(1, 1, *midi_gram.shape))
# Convert to sequence of integers
midi_hash_sequence = dhs.vectors_to_ints(midi_hash_sequence > 0)
midi_hash_sequence = midi_hash_sequence.astype(np.uint32)
matches = match_one_midi(
midi_gram, midi_embedding, midi_hash_sequence, msd_embeddings,
msd_sequences, msd_feature_paths, msd_ids)
# Write out the result
with open(output_filename, 'wb') as f:
msgpack.dump(matches, f)
def SavePermResults(path,name,method,*args):
data = []
for thing in enumerate(args):
data.append(thing)
if method=='msgpack':
with open(os.path.join(path,name+'.mpac'),'wb') as f:
msgpack.dump(data,f)
else:
with open(os.path.join(path,name+'.pickle'),'wb') as f:
pickle.dump(data,f)
def make_hanzi_converter(table_in, msgpack_out):
table = {}
with open(table_in, encoding='utf-8') as infile:
for line in infile:
hexcode, char = line.rstrip('\n').split('\t')
codept = int(hexcode, 16)
assert len(char) == 1
if chr(codept) != char:
table[codept] = char
with gzip.open(msgpack_out, 'wb') as outfile:
msgpack.dump(table, outfile, raw=False)
def make_hanzi_converter(table_in, msgpack_out):
table = {}
with open(table_in, encoding="utf-8") as infile:
for line in infile:
hexcode, char = line.rstrip("\n").split("\t")
codept = int(hexcode, 16)
assert len(char) == 1
if chr(codept) != char:
table[codept] = char
with gzip.open(msgpack_out, "wb") as outfile:
msgpack.dump(table, outfile, encoding="utf-8")
def write_http_response_to_temp_file(http_response):
"""
Write an HTTPResponse instance to a temp file using msgpack
:param http_response: The HTTP response
:return: The name of the file
"""
temp = get_temp_file('http')
data = http_response.to_dict()
msgpack.dump(data, temp, use_bin_type=True)
temp.close()
return temp.name
def write_tags_to_temp_file(tag_list):
"""
Write an Tag list to a temp file using msgpack
:param tag_list: The Tag list
:return: The name of the file
"""
temp = get_temp_file('tags')
data = [t.to_dict() for t in tag_list]
msgpack.dump(data, temp, use_bin_type=True)
temp.close()
return temp.name
def pickle_object(path, targetobject, json_pickle=False):
"""
@type path: str or unicode
@type targetobject: object
@type json_pickle: bool
"""
msgpack.dump(targetobject, open(path, "wb"), msgpack.HIGHEST_PROTOCOL)
if json_pickle:
if isinstance(targetobject, dict):
json_object(path, targetobject)
else:
json_object(path, targetobject)
def save(self, filename):
tfn = '%s.inprog-%d' % (filename, random.randint(1, 10000000))
fh = open(tfn, 'w')
try:
msgpack.dump(self.todict(), fh)
finally:
fh.close()
if os.path.exists(filename):
os.rename(filename, '%s.bak' % filename)
os.rename(tfn, filename)
def create_train_valid_data(folder='/data/liubo/face/research_feature_self'):
# 根据已经存在的数据训练人脸验证模型
person_list = os.listdir(folder)
path_feature_dic = {} #
for person in person_list:
person_path = os.path.join(folder, person)
pic_feature_list = os.listdir(person_path)
for pic_feature_path in pic_feature_list:
pic_feature_path = os.path.join(person_path, pic_feature_path)
pic_feature = msgpack_numpy.load(open(pic_feature_path, 'rb'))
path_feature_dic[pic_feature_path] = pic_feature
msgpack.dump(path_feature_dic, open('research_feature.p', 'wb'))
def save(self):
"""Save object into DB."""
resp = self.response
values = []
values.append(resp.get_id())
values.append(self.request.get_uri().url_string)
values.append(resp.get_code())
values.append(self.tag)
values.append(int(self.mark))
values.append(str(resp.info()))
values.append(resp.get_wait_time())
values.append(resp.get_msg())
values.append(resp.content_type)
ch = resp.charset
values.append(ch)
values.append(self.request.get_method())
values.append(len(resp.body))
code = int(resp.get_code()) / 100
values.append(code)
values.append(resp.get_alias())
values.append(int(self.request.get_uri().has_query_string()))
if not self.id:
sql = ('INSERT INTO %s '
'(id, url, code, tag, mark, info, time, msg, content_type, '
'charset, method, response_size, codef, alias, has_qs) '
'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)' % self._DATA_TABLE)
self._db.execute(sql, values)
self.id = self.response.get_id()
else:
values.append(self.id)
sql = ('UPDATE %s'
' SET id = ?, url = ?, code = ?, tag = ?, mark = ?, info = ?, '
'time = ?, msg = ?, content_type = ?, charset = ?, '
'method = ?, response_size = ?, codef = ?, alias = ?, has_qs = ? '
' WHERE id = ?' % self._DATA_TABLE)
self._db.execute(sql, values)
#
# Save raw data to file
#
fname = self._get_fname_for_id(self.id)
req_res = open(fname, 'wb')
data = (self.request.to_dict(),
self.response.to_dict(),
self._MSGPACK_CANARY)
msgpack.dump(data, req_res)
req_res.close()
return True
def _write(self):
'''
Write out to disk
'''
if not HAS_MSGPACK:
return
# TODO Add check into preflight to ensure dir exists
# TODO Dir hashing?
with salt.utils.fopen(self._path, 'w+') as fp_:
cache = {
"CacheDisk_data": self._dict,
"CacheDisk_cachetime": self._key_cache_time
}
msgpack.dump(cache, fp_)
def find_url_list():
pic_face_index_dic = msgpack.load(open('pic_face_index_dic.p', 'rb'))
url_folder = '/data/url'
for person in pic_face_index_dic:
print person
url_list = open(os.path.join(url_folder, person+'.txt'),
'r').read().split('\n')
need_check_url_index_list = pic_face_index_dic.get(person)
for index in range(len(need_check_url_index_list)):
tmp = url_list[int(need_check_url_index_list[index])].split('\t')
need_check_url_index_list[index] = \
(need_check_url_index_list[index], tmp[-1])
pic_face_index_dic[person] = need_check_url_index_list
msgpack.dump(pic_face_index_dic, open('pic_face_index_url_dic.p', 'wb'))
def convert(infile, outfile):
if not outfile:
ext = infile.split('.')[-1]
outfile = '%s%s' % (infile[:-len(ext)-1], EXT)
print('%s > %s' % (infile, outfile))
print('reading in JSON')
with open(infile) as op:
data = json.load(op)
print('writing to msgpack')
with open(outfile, 'wb') as op:
msgpack.dump(data, op)
def start(interval=3600, expire=604800):
ck = salt.utils.minions.CkMinions(__opts__)
presence_file = "{0}/minions/presence.p".format(__opts__["cachedir"])
wheel = salt.wheel.WheelClient(__opts__)
while True:
log.debug("Checking for present minions")
minions = {}
if os.path.exists(presence_file):
try:
with salt.utils.fopen(presence_file, "r") as f:
minions = msgpack.load(f)
except IOError as e:
log.error("Could not open presence file {0}: {1}".format(presence_file, e))
time.sleep(interval)
continue
minion_keys = _get_keys()
now = time.time()
present = ck.connected_ids()
# For our existing keys, check which are present
for m in minion_keys:
# If we have a key that's not in the presence file, it may be a new minion
# It could also mean this is the first time this engine is running and no
# presence file was found
if m not in minions:
minions[m] = now
elif m in present:
minions[m] = now
log.debug("Finished checking for present minions")
# Delete old keys
stale_keys = []
for m, seen in minions.iteritems():
if now - expire > seen:
stale_keys.append(m)
if len(stale_keys):
for k in stale_keys:
log.info("Removing stale key for {0}".format(k))
wheel.cmd("key.delete", stale_keys)
del minions[k]
try:
with salt.utils.fopen(presence_file, "w") as f:
msgpack.dump(minions, f)
except IOError as e:
log.error("Could not write to presence file {0}: {1}".format(presence_file, e))
time.sleep(interval)
def get_textunits(filename):
out_fn = os.path.join(OUTPUT_DIRECTORY,os.path.basename(filename))
print out_fn+".mpack"
if os.path.exists(out_fn+".mpack"):
return 'done'
##### GET THE DATA
dep_parse = read_grouped_by_newline_file(filename)
print filename
to_write = []
for i,x in enumerate(dep_parse):
if i % 5000 == 0:
print i
spl = x[0].split("\t")
uid = spl[11]
tweet_id = spl[10]
date = x[0].split("\t")[-2]
try:
s = TextUnit(uid+ "\t" + tweet_id, date,
sent_to_id,identity_to_id,gram_list,
emoji_info=[emoji_data,emoji_regex],
emoticon_to_eval_dim=False,
dependency_parsed_conll=x,
sent_values=sent_values,
hashtag_epa_data=False,
vader_dict=False,
do_negation_on_full_sentence=True,
use_events=True,
use_behaviors=True,
use_isa=False,
use_clause_level= True,
use_parent_child=False,
use_own_full_sentence=False)
if len(s.identities):
to_write.append(s)
except:
print 'failed', i, filename
#pickle.dump(to_write, open(out_fn,"wb"),-1)
dat = [ [x.unit_id,x.date,x.identities,x.raw_text,
x.identities_to_constraint_string_map,
x.constraint_string_list,
x.full_deflection_string]
for x in to_write]
msgpack.dump(dat, open(out_fn+".mpack","wb"))
return 'done'
