def main(argv):
# set up logging
util.create_dir("logs")
util.configure_logging("gzip", "logs/gzip.log")
# check args
if len(argv) != 4:
print "Usage: {} <c/d> <source> <target>".format(argv[0])
return -1
# check if first argument is valid
if argv[1] != "c" and argv[1] != "d":
logging.error("First argument %s should be 'c' or 'd'", repr(argv[1]))
return -1
# check if files do (not) exist
source = argv[2]
target = argv[3]
if not os.path.isfile(source):
logging.error("Source %s does not exist", repr(source))
return -1
if os.path.isfile(target) and os.path.getsize(target) > 0:
logging.error("Target %s already exists and is non-empty",
repr(target))
return -1
# compress/decompress
if argv[1] == "c":
compress(source, target)
else:
decompress(source, target)
return 0
def init_album(self):
#album json
js = self.handler.read_link(url_album % self.album_id).json()['album']
#name
self.album_name = util.decode_html(js['name'])
#album logo
self.logo = js['picUrl']
# artist_name
self.artist_name = js['artists'][0]['name']
#handle songs
for jsong in js['songs']:
song = NeteaseSong(self.handler, song_json=jsong)
song.group_dir = self.artist_name + u'_' + self.album_name
song.group_dir = song.group_dir.replace('/', '_')
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
#creating the dir
LOG.debug(msg.head_163 + msg.fmt_create_album_dir % d)
util.create_dir(d)
#download album logo images
LOG.debug(msg.head_163 + msg.fmt_dl_album_cover % self.album_name)
downloader.download_url(
self.logo, path.join(d, 'cover.' + self.logo.split('.')[-1]))
def convert_to_lucene(question_answer_content, doc_type_verbose, source_path):
"""
:param question_answer_document content:
:return: yield matchzoo data
At initial version, we are just focusing on the context and question, nothing more,
therefore we are ignoring the answer part as of now
"""
word_counter, char_counter = Counter(), Counter()
examples, eval, questions, paragraphs, q_to_ps = process_squad_file(question_answer_content, word_counter, char_counter)
tokenized_paragraphs = tokenize_contexts(paragraphs, -1)
tokenized_questions = tokenize_contexts(questions, -1)
tokenized_questions, tokenized_paragraphs = fixing_the_token_problem(tokenized_questions, tokenized_paragraphs)
paragraphs_nontokenized = [" ".join(context) for context in tokenized_paragraphs]
questions_nontokenized = [" ".join(context) for context in tokenized_questions]
if doc_type_verbose == 1 or doc_type_verbose == 3:
# questions
print('Questions are getting dumped.')
dst_dir = UTIL.create_dir(os.path.join(source_path, 'lucene_questions'))
for indx, doc in tqdm(enumerate(questions_nontokenized)):
as_json = dict()
as_json['content'] = doc
#as_json['doc_id'] = indx
UTIL.dump_json_file(os.path.join(dst_dir, '{}.json'.format(indx)), as_json, None)
elif doc_type_verbose == 2 or doc_type_verbose == 3:
print('Paragraphs are getting dumped.')
dst_dir = UTIL.create_dir(os.path.join(source_path, 'lucene_paragraphs'))
for indx, doc in tqdm(enumerate(paragraphs_nontokenized)):
as_json = dict()
as_json['content'] = doc
#as_json['doc_id'] = indx
UTIL.dump_json_file(os.path.join(dst_dir, '{}.json'.format(indx)), as_json, None)
print('Completed.')
def generate_pmbuild_config(config, profile):
print(
"--------------------------------------------------------------------------------"
)
print(
"pmbuild live reload config -----------------------------------------------------"
)
print(
"--------------------------------------------------------------------------------"
)
if "data_dir" not in config:
print(
"[error]: did not generate pmbuild_config.json for live reloading")
return
print("writing " + config["data_dir"] + "/" + "pmbuild_config.json")
wd = os.getcwd()
pmd = util.sanitize_file_path(config["env"]["pmtech_dir"])
md = {
"profile": profile,
"pmtech_dir": pmd,
"pmbuild": "cd " + wd + " && " + pmd + "pmbuild " + profile + " "
}
util.create_dir(config["data_dir"])
f = open(os.path.join(config["data_dir"], "pmbuild_config.json"), "w+")
f.write(json.dumps(md, indent=4))
def gen_dataset(N, data_path, isMC=True):
data_path = os.path.abspath(data_path)
ut.create_dir(data_path)
if (isMC):
movie_dict = ut.unpickle(
os.path.join("data", "movies_single_index.pkl"))
else:
movie_dict = ut.unpickle(os.path.join(
"data",
"movies_multi_index.pkl")) #Need to make this new index thing
full_ids = list(movie_dict.keys())
movie_ids = random.sample(full_ids, N)
ids_l = len(movie_ids)
shuffle(movie_ids)
train_ids = movie_ids[:int(ids_l * SPLIT[0])]
val_ids = movie_ids[int(ids_l * SPLIT[0]):int(ids_l *
(SPLIT[0] + SPLIT[1]))]
test_ids = movie_ids[int(ids_l * (SPLIT[0] + SPLIT[1])):-1]
print("Generating Training Set...")
generate_set(train_ids, movie_dict, data_path, "train", isMC)
print("Generating Validation Set...")
generate_set(val_ids, movie_dict, data_path, "val", isMC)
print("Generating Test Set...")
generate_set(test_ids, movie_dict, data_path, "test", isMC)
def write_prediction(
self,
valid_df,
batch_size,
):
"""
Run prediction using given model on a list of images,
and write the result to a csv file.
:param batch_size: number of inputs in each batch.
:param valid_df: validation dataset table
:return:
path to result result csv
"""
predictions = self.model.predict_generator(
self.img_generator(valid_df, batch_size),
steps=math.ceil(valid_df.shape[0] / batch_size))
util.create_dir(self.result_path)
for i in range(len(predictions)):
valid_df.at[i, "prediction"] = predictions[i]
result_path = os.path.join(
self.result_path,
"vgg_{:%Y-%m-%d-%H%M}.csv".format(datetime.datetime.now()))
valid_df.to_csv(result_path)
return result_path
def init_album(self):
#album json
js = self.handler.read_link(url_album % self.album_id).json()['album']
#name
self.album_name = util.decode_html(js['name'])
#album logo
self.logo = js['picUrl']
# artist_name
self.artist_name = js['artists'][0]['name']
#handle songs
for jsong in js['songs']:
song = NeteaseSong(self.handler, song_json=jsong)
song.group_dir = self.artist_name + u'_' + self.album_name
song.group_dir = song.group_dir.replace('/', '_')
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
#creating the dir
LOG.debug(msg.head_163 + msg.fmt_create_album_dir % d)
util.create_dir(d)
#download album logo images
LOG.debug(msg.head_163 + msg.fmt_dl_album_cover % self.album_name)
downloader.download_url(self.logo, path.join(d,'cover.' +self.logo.split('.')[-1]))
def init_album(self):
#album json
js = self.m163.read_link(url_album % self.album_id).json()['album']
#name
self.album_name = util.decode_html(js['name'])
#album logo
self.logo = js['picUrl']
# artist_name
self.artist_name = js['artists'][0]['name']
#handle songs
for jsong in js['songs']:
song = NeteaseSong(self.m163, song_json=jsong)
song.group_dir = self.artist_name + u'_' + self.album_name
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
#creating the dir
LOG.debug(u'创建专辑目录[%s]' % d)
util.create_dir(d)
#download album logo images
LOG.debug(u'下载专辑[%s]封面' % self.album_name)
downloader.download_by_url(
self.logo, path.join(d, 'cover.' + self.logo.split('.')[-1]))
def write_prediction(self, valid_df, batch_size):
"""
Run prediction using given model on a list of images,
and write the result to a csv file.
:param batch_size: number of inputs in each batch.
:param valid_df: validation dataset table
:return:
path to result result csv
"""
predictions = self.model.predict_generator(
self.img_generator(valid_df, batch_size, False),
steps=self.calc_steps(valid_df, batch_size))
df = valid_df.groupby("study").agg({
"path": lambda x: x.tolist(),
"label": np.prod
}).reset_index()
util.create_dir(self.result_path)
for i in range(len(df)):
valid_df.loc[valid_df["study"] == df.iloc[i]["study"],
"prediction"] = predictions[i]
result_path = os.path.join(
self.result_path,
"{}_{:%Y-%m-%d-%H%M}.csv".format(self.__class__.__name__,
datetime.datetime.now()))
valid_df.to_csv(result_path)
return valid_df
def test_run_all():
'''Generate population, check it's the right size.'''
nloci = 400
create_dir('population')
os.chdir('population')
p = Population(TRUE, 'point', nloci)
p.gen()
assert len(run_all()) == p.size
def test_create_dir():
td = 'whatever'
create_dir(td)
assert os.path.isdir(td)
# second call should also succeed, even though td exists
create_dir(td)
assert os.path.isdir(td)
os.rmdir(td)
def init_album(self):
resp_json = self.handler.read_link(url_album % self.album_id).json()
j = resp_json['data']['trackList']
if not j:
LOG.error(resp_json['message'])
return
#description
html = self.handler.read_link(self.url).text
soup = BeautifulSoup(html, 'html.parser')
if soup.find('meta', property="og:title"):
self.album_desc = soup.find('span', property="v:summary").text
# name
self.album_name = soup.find('meta', property="og:title")['content']
# album logo
self.logo = soup.find('meta', property="og:image")['content']
# artist_name
self.artist_name = soup.find('meta',
property="og:music:artist")['content']
else:
aSong = j[0]
self.album_name = aSong['album_name']
self.logo = aSong['album_pic']
self.artist_name = aSong['artistVOs'][0]['artistName']
self.album_desc = None
#handle songs
for jsong in j:
song = XiamiSong(self.handler, song_json=jsong)
song.song_name = jsong['name'] # name or songName
song.group_dir = self.artist_name + u'_' + self.album_name
song.group_dir = song.group_dir.replace('/', '_')
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
#creating the dir
LOG.debug(msg.head_xm + msg.fmt_create_album_dir % d)
util.create_dir(d)
#download album logo images
LOG.debug(msg.head_xm + msg.fmt_dl_album_cover % self.album_name)
if self.logo:
self.logo = self.handler.add_http_prefix(self.logo)
downloader.download_url(
self.logo, path.join(d, 'cover.' + self.logo.split('.')[-1]))
LOG.debug(msg.head_xm + msg.fmt_save_album_desc % self.album_name)
if self.album_desc:
self.album_desc = re.sub(r'<\s*[bB][rR]\s*/>', '\n',
self.album_desc)
self.album_desc = re.sub(r'<.*?>', '', self.album_desc)
self.album_desc = util.decode_html(self.album_desc)
import codecs
with codecs.open(path.join(d, 'album_description.txt'), 'w',
'utf-8') as f:
f.write(self.album_desc)
def default_logger_creator(config):
"""Creates a Unified logger with a default logdir prefix
containing the agent name and the env id
"""
if config['multiagent']['policies_to_train'][0] == '0':
agent_path = os.path.join(EXPERIMENT_PATH, "player0")
elif config['multiagent']['policies_to_train'][0] == '1':
agent_path = os.path.join(EXPERIMENT_PATH, "player1")
create_dir(agent_path, clean=True)
return UnifiedLogger(config, agent_path, loggers=None)
def _save_image(step, img, experiment_path):
img_path = os.path.join(experiment_path, "render")
create_dir(img_path, clean=False)
if step == 0:
files = glob.glob(f'{img_path}/*')
for f in files:
os.remove(f)
img_path = os.path.join(img_path, f"{str(step).zfill(5)}.png")
img = img.resize((1000, 1000))
img.save(img_path)
def test_generate_population():
'''Generate population, check it's the right size.'''
nloci = 400
create_dir('population')
os.chdir('population')
p = Population(WILD_TYPE, 'point', nloci)
p.gen()
files = [file for file in os.walk('.') if file[2]]
assert len(files) == nloci
assert len(p.enum()) == nloci
assert p.size == nloci
def init_album(self):
resp_json = self.handler.read_link(url_album % self.album_id).json()
j = resp_json['data']['trackList']
if not j :
LOG.error(resp_json['message'])
return
#description
html = self.handler.read_link(self.url).text
soup = BeautifulSoup(html,'html.parser')
if soup.find('meta', property="og:title"):
self.album_desc = soup.find('span', property="v:summary").text
# name
self.album_name = soup.find('meta', property="og:title")['content']
# album logo
self.logo = soup.find('meta', property="og:image")['content']
# artist_name
self.artist_name = soup.find('meta', property="og:music:artist")['content']
else:
aSong = j[0]
self.album_name = aSong['album_name']
self.logo = aSong['album_pic']
self.artist_name = aSong['artistVOs'][0]['artistName']
self.album_desc = None
#handle songs
for jsong in j:
song = XiamiSong(self.handler, song_json=jsong)
song.song_name = jsong['name'] # name or songName
song.group_dir = self.artist_name + u'_' + self.album_name
song.group_dir = song.group_dir.replace('/','_')
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
#creating the dir
LOG.debug(msg.head_xm + msg.fmt_create_album_dir % d)
util.create_dir(d)
#download album logo images
LOG.debug(msg.head_xm + msg.fmt_dl_album_cover % self.album_name)
if self.logo:
self.logo = self.handler.add_http_prefix(self.logo)
downloader.download_url(self.logo, path.join(d,'cover.' +self.logo.split('.')[-1]))
LOG.debug(msg.head_xm + msg.fmt_save_album_desc % self.album_name)
if self.album_desc:
self.album_desc = re.sub(r'<\s*[bB][rR]\s*/>','\n',self.album_desc)
self.album_desc = re.sub(r'<.*?>','',self.album_desc)
self.album_desc = util.decode_html(self.album_desc)
import codecs
with codecs.open(path.join(d,'album_description.txt'), 'w', 'utf-8') as f:
f.write(self.album_desc)
def execute_test_sample(self, context, s, difficulty_test_base_path,
in_mem, out_mem, regs, timesteps):
if context.print_memories or context.print_circuits is not 0:
sample_difficulty_base_path = "%s/%s" % (difficulty_test_base_path,
s)
create_dir(sample_difficulty_base_path, True)
if context.info_is_active:
print(
"\nSample[%d], Initial memory: %s, Desired memory: %s, Initial registers: %s"
%
(s, in_mem[:].argmax(axis=1), out_mem, regs[:].argmax(axis=1)))
# Iterate for every timestep
debug = list() # Init debug dictionary for the sample
for t in range(timesteps):
dt = DebugTimestep(context, t, s)
coeffs, complete = self.__run_network(regs, dt)
regs, in_mem = self.__run_circuit(regs, in_mem, context.gates,
coeffs, dt)
debug.append(dt)
if self.context.stop_at_the_will and complete.sum() >= 1.0:
break
# Debug for the sample
if context.info_is_active:
for dt in debug:
print(dt)
print("\t• Expected mem => %s" % out_mem)
if context.print_memories:
with open("%s/memories.txt" % sample_difficulty_base_path,
"a+") as f:
f.write(
"\\textbf{Step} & %s & %s & Read & Write \\\\ \hline \n" %
(" & ".join(["%s" % r for r in range(out_mem.shape[0])
]), " & ".join([
"\\textit{r}%d" % r
for r in range(context.num_regs)
])))
for dt in debug:
dt.print_memory_to_file(sample_difficulty_base_path, timesteps)
if context.print_circuits is not 0:
# Create dir for the single example of a difficulty
for dt in debug:
if context.print_circuits is 1:
dt.print_circuit(sample_difficulty_base_path)
else:
dt.print_pruned_circuit(sample_difficulty_base_path)
return s, in_mem
def init_collection(self):
j = self.xm.read_link(url_collection % (self.collection_id) ).json()['collect']
self.collection_name = j['name']
for jsong in j['songs']:
song = Song(self.xm, song_json=jsong)
#rewrite filename, make it different
song.group_dir = self.collection_name
song.abs_path = path.join(config.DOWNLOAD_DIR, song.group_dir, song.filename)
self.songs.append(song)
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
def init_playlist(self):
j = self.handler.read_link(url_playlist % (self.playlist_id) ).json()['result']
self.playlist_name = j['name']
for jsong in j['tracks']:
song = NeteaseSong(self.handler, song_json=jsong)
#rewrite filename, make it different
song.group_dir = self.playlist_name
song.post_set()
self.songs.append(song)
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
def get_embedding_set(df, embedding_type, output_name, shaping, selection_tag="", overwrite=False):
logging.info("Creating %s embedding %s...", embedding_type, output_name)
path = "../data/cache/" + selection_tag + output_name + "_" + embedding_type
path_and_name = path + "/" + shaping + ".pkl.pkl"
if not util.check_output_necessary(path_and_name, overwrite):
# df = pickle.load(path_and_name)
with open(path_and_name, 'rb') as infile:
df = pickle.load(infile)
return df
util.create_dir(path)
embedding_df = None
logging.debug("Preparing to run vectorization")
if embedding_type == "w2v":
embedding_df = word2vec_creator.run_w2v(df, path_and_name, shaping=shaping, word_limit=-1, sentics=False)
elif embedding_type == "w2v_sentic":
embedding_df = word2vec_creator.run_w2v(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True)
elif embedding_type == "w2v_sentic_full":
embedding_df = word2vec_creator.run_w2v(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True, zero_pad=True)
elif embedding_type == "w2v_limit":
# NOTE: yes, it looks like sentics should be false, but this is how we limit w2v embeddings to just the patterns the sentics would have been calculated on
embedding_df = word2vec_creator.run_w2v(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True, model_only=True)
elif embedding_type == "glove":
embedding_df = word2vec_creator.run_glove(df, path_and_name, shaping=shaping, word_limit=-1, sentics=False)
elif embedding_type == "glove_sentic":
embedding_df = word2vec_creator.run_glove(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True)
elif embedding_type == "glove_sentic_full":
embedding_df = word2vec_creator.run_glove(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True, zero_pad=True)
elif embedding_type == "glove_limit":
# NOTE: yes, it looks like sentics should be false, but this is how we limit glove embeddings to just the patterns the sentics would have been calculated on
embedding_df = word2vec_creator.run_glove(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True, model_only=True)
elif embedding_type == "ft":
embedding_df = word2vec_creator.run_fasttext(df, path_and_name, shaping=shaping, word_limit=-1, sentics=False)
elif embedding_type == "ft_sentic":
embedding_df = word2vec_creator.run_fasttext(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True)
elif embedding_type == "ft_sentic_full":
embedding_df = word2vec_creator.run_fasttext(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True, zero_pad=True)
elif embedding_type == "fasttext_limit":
# NOTE: yes, it looks like sentics should be false, but this is how we limit fasttext embeddings to just the patterns the sentics would have been calculated on
embedding_df = word2vec_creator.run_fasttext(df, path_and_name, shaping=shaping, word_limit=-1, sentics=True, model_only=True)
elif embedding_type == "sentic":
embedding_df = word2vec_creator.vectorize_collection(df, path_and_name, model=None, shaping=shaping, word_limit=-1, sentics=True)
elif embedding_type == "tfidf":
# TODO: add ability to add sentics here too?
embedding_df = create_tfidf(df, path, overwrite=True)
# NOTE: would have to save it (and check for preexisting) separately as well then
return embedding_df
def init_collection(self):
j = self.xm.read_link(url_collection % (self.collection_id) ).json()['data']['trackList']
j_first_song = j[0]
#read collection name
self.collection_name = self.get_collection_name()
for jsong in j:
song = XiamiSong(self.xm, song_json=jsong)
#rewrite filename, make it different
song.group_dir = self.collection_name
song.post_set()
self.songs.append(song)
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
def init_fav(self):
""" parse html and load json and init Song object
for each found song url"""
page = 1
user = ''
total = 0
cur = 1 #current processing link
LOG.debug(msg.head_xm + msg.fmt_init_fav % self.uid)
while True:
html = self.handler.read_link(url_fav % (self.uid, page)).text
soup = BeautifulSoup(html, 'html.parser')
if not user:
user = soup.title.string
if not total:
total = soup.find('span', class_='counts').string
links = [
link.get('href') for link in soup.find_all(
href=re.compile(r'xiami.com/song/[^?]+')) if link
]
if links:
for link in links:
LOG.debug(msg.head_xm + msg.fmt_parse_song_url % link)
if self.verbose:
sys.stdout.write(
log.hl(
'[%d/%s] parsing song ........ ' %
(cur, total), 'green'))
sys.stdout.flush()
try:
cur += 1
song = XiamiSong(self.handler, url=link)
#time.sleep(2)
if self.verbose:
sys.stdout.write(log.hl('DONE\n', 'green'))
except:
sys.stdout.write(log.hl('FAILED\n', 'error'))
continue
#rewrite filename, make it different
song.group_dir = user
song.post_set()
self.songs.append(song)
page += 1
else:
break
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
LOG.debug(msg.head_xm + msg.fmt_init_fav_ok % self.uid)
def gist(input_file, output_folder):
img_dict, img_dict_path = load_images_from_file(
input_file) # take all images category by category
gist = gist_img(img_dict)
name = gist[0]
desc_list = gist[1] # Takes the descriptor list which is unordered one
cat_dict = gist[
2] # Takes the sift features that is seperated class by class for train data
create_dir(output_folder)
create_dir(f"{output_folder}/{name}/")
save(cat_dict, img_dict_path, f"{output_folder}/{name}/")
print(f"\nThe execution of gist is done!\n")
def init_topsong(self):
j = self.handler.read_link(url_artist_top_song % (self.artist_id)).json()
self.artist_name = j['artist']['name']
for jsong in j['hotSongs']:
song = NeteaseSong(self.handler, song_json=jsong)
song.group_dir = self.artist_name + '_TopSongs'
song.post_set()
self.songs.append(song)
#check config for top X
if len(self.songs) >= config.DOWNLOAD_TOP_SONG:
break
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
def init_topsong(self):
j = self.xm.read_link(url_artist_top_song % (self.artist_id)).json()
for jsong in j['songs']:
song = XiamiSong(self.xm, song_json=jsong)
song.group_dir = self.artist_name + '_TopSongs'
song.post_set()
self.songs.append(song)
#check config for top X
if len(self.songs) >= config.DOWNLOAD_TOP_SONG:
break
if len(self.songs):
#set the artist name
self.artist_name = self.songs[-1].artist_name
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
def save_img(img, filename):
"""
Utility method that convert a ndarray into image and save to a image file.
Args:
img: image in ndarray
filename: target filename including path
Returns:
"""
img = keras.preprocessing.image.array_to_img(img)
try:
img.save(filename)
except FileNotFoundError:
util.create_dir(os.path.dirname(filename))
img.save(filename)
def init_topsong(self):
j = self.xm.read_link(url_artist_top_song % (self.artist_id)).json()
for jsong in j['songs']:
song = XiamiSong(self.xm, song_json=jsong)
song.group_dir = self.artist_name + '_TopSongs'
song.post_set()
self.songs.append(song)
#check config for top X
if len(self.songs) >= config.DOWNLOAD_TOP_SONG:
break
if len(self.songs):
#set the artist name
self.artist_name = self.songs[-1].artist_name
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
def init_collection(self):
LOG.debug(msg.head_xm + msg.fmt_init_collect % self.collection_id)
j = self.handler.read_link(url_collection % (self.collection_id) ).json()['data']['trackList']
j_first_song = j[0]
#read collection name
self.collection_name = self.get_collection_name()
for jsong in j:
song = XiamiSong(self.handler, song_json=jsong)
#rewrite filename, make it different
song.group_dir = self.collection_name
song.post_set()
self.songs.append(song)
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
LOG.debug(msg.head_xm + msg.fmt_init_collect_ok % self.collection_id)
def run_textures(config):
print(
"--------------------------------------------------------------------------------"
)
print(
"textures -----------------------------------------------------------------------"
)
print(
"--------------------------------------------------------------------------------"
)
tool_cmd = tool_to_platform(config["tools"]["texturec"])
for task in config["textures"]:
files = get_task_files_containers(task)
for f in files:
copy_fmt = [".dds", ".pmv"]
conv_fmt = [".png", ".jpg", ".tga", ".bmp", ".txt"]
cont_fmt = [".txt"]
fext = os.path.splitext(f[0])[1]
if fext in copy_fmt:
util.copy_file_create_dir_if_newer(f[0], f[1])
if fext in conv_fmt:
export = export_config_for_file(f[0])
dep_inputs = [f[0]]
if fext in cont_fmt:
export = export_config_for_directory(f[0], "osx")
dep_inputs = get_container_dep_inputs(f[0], dep_inputs)
dst = util.change_ext(f[1], ".dds")
if not dependencies.check_up_to_date_single(dst):
if "format" not in export.keys():
export["format"] = "RGBA8"
dep_outputs = [dst]
dep_info = dependencies.create_dependency_info(
dep_inputs, dep_outputs)
dep = dict()
dep["files"] = list()
dep["files"].append(dep_info)
util.create_dir(dst)
cmd = tool_cmd + " "
cmd += "-f " + f[0] + " "
cmd += "-t " + export["format"] + " "
if "cubemap" in export.keys() and export["cubemap"]:
cmd += " --cubearray "
cmd += "-o " + dst
print("texturec " + f[0])
subprocess.call(cmd, shell=True)
dependencies.write_to_file_single(
dep, util.change_ext(dst, ".json"))
def main():
if len(sys.argv) < 7:
print(
'Usage: python3 sample.py [MODEL_PATH] [DATA_PATH] [SPLIT_INDEX] [OUT_PATH] [RECIPE_ID] [NUM_SAMPLES]'
)
exit()
model_path = os.path.abspath(sys.argv[1])
data_path = os.path.abspath(sys.argv[2])
split_index = int(sys.argv[3])
out_path = os.path.abspath(sys.argv[4])
recipe_id = sys.argv[5]
num_samples = int(sys.argv[6])
util.create_dir(out_path)
saved_model = torch.load(model_path)
data = GANstronomyDataset(data_path, split=opts.TVT_SPLIT)
data.set_split_index(split_index)
data_loader = DataLoader(data,
batch_size=1,
shuffle=False,
sampler=SequentialSampler(data))
num_classes = data.num_classes()
G = Generator(opts.LATENT_SIZE, opts.EMBED_SIZE).to(opts.DEVICE)
G.load_state_dict(saved_model['G_state_dict'])
G.eval()
embs = None
for ibatch, data_batch in enumerate(data_loader):
with torch.no_grad():
recipe_ids, recipe_embs, img_ids, imgs, classes, _, _ = data_batch
batch_size, recipe_embs, imgs = util.get_variables3(
recipe_ids, recipe_embs, img_ids, imgs)
if recipe_ids[0] == recipe_id:
embs = recipe_embs
break
assert embs is not None
z = torch.randn(num_samples, opts.LATENT_SIZE).to(opts.DEVICE)
imgs_gen = G(z, embs.expand(num_samples, opts.EMBED_SIZE)).detach()
for i in range(num_samples):
img_gen = imgs_gen[i]
save_results(out_path, recipe_id, img_gen, i)
def test():
cd_to_pigz()
util.create_dir("pigz_test_tmp")
shutil.copyfile("pigz.c", os.path.join("pigz_test_tmp", "pigz.c"))
os.chdir("pigz_test_tmp")
sha1_orig = util.file_sha1("pigz.c")
test_one_with_flag(sha1_orig, ".gz", "-f")
test_one_with_flag2(sha1_orig, ".gz", "-fb", "32")
test_one_with_flag2(sha1_orig, ".gz", "-fp", "1")
test_one_with_flag(sha1_orig, ".zz", "-fz")
test_one_with_flag(sha1_orig, ".zip", "-fK")
test_one_with_flag2(sha1_orig, ".gz", "-11", "-f")
# TODO: more tests for stdin/stdout
# TODO: compare with gzip/compress (if available in path)
cd_to_pigz()
rm_dirs(["pigz_test_tmp"])
def test():
cd_to_pigz()
util.create_dir("pigz_test_tmp")
shutil.copyfile("pigz.c", os.path.join("pigz_test_tmp", "pigz.c"))
os.chdir("pigz_test_tmp")
sha1_orig = util.file_sha1("pigz.c")
test_one_with_flag(sha1_orig, ".gz", "-f")
test_one_with_flag2(sha1_orig, ".gz", "-fb", "32")
test_one_with_flag2(sha1_orig, ".gz", "-fp", "1")
test_one_with_flag(sha1_orig, ".zz", "-fz")
test_one_with_flag(sha1_orig, ".zip", "-fK")
test_one_with_flag2(sha1_orig, ".gz", "-11", "-f")
# TODO: more tests for stdin/stdout
# TODO: compare with gzip/compress (if available in path)
cd_to_pigz()
rm_dirs(["pigz_test_tmp"])
def load_single_config(conf_parser, conf_key):
config_warn_msg = "Cannot load config [%s], use default value: %s"
try:
v = conf_parser.get("settings", conf_key)
if not v:
raise Exception("ConfigError", "invalid")
gkey = var_dict[conf_key][0]
ty = var_dict[conf_key][1]
if ty == "n":
globals()[gkey] = int(v)
else:
if ty == "p":
util.create_dir(v)
globals()[gkey] = v
except:
log.warn(config_warn_msg % (conf_key, str(globals()[var_dict[conf_key][0]])))
def init_fav(self):
page = 1
while True:
j = self.xm.read_link(url_fav % (self.uid, str(page)) ).json()
if j['songs'] :
for jsong in j['songs']:
song = Song(self.xm, song_json=jsong)
#rewrite filename, make it different
song.group_dir = 'favorite_%s' % self.uid
song.abs_path = path.join(config.DOWNLOAD_DIR, song.group_dir, song.filename)
self.songs.append(song)
page += 1
else:
break
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
def main():
# cli args checking
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="action")
parser_c = subparsers.add_parser("c")
parser_c.add_argument("source", type=str)
parser_c.add_argument("target", type=str)
parser_c.add_argument("reference", type=str)
parser_c.add_argument("--inner", type=str)
parser_c.add_argument("--delta", type=str)
parser_c.add_argument("--nointra", action="store_true")
parser_d = subparsers.add_parser("d")
parser_d.add_argument("source", type=str)
parser_d.add_argument("target", type=str)
args = parser.parse_args()
# create method name
if args.action == "c":
method = create_method_name(args.nointra, args.delta, args.inner)
elif args.action == "d":
with open(args.source, "rb") as f:
method = util.parse_header(f)[1]
# set up logging
util.create_dir("logs")
util.configure_logging(method, "logs/{}.log".format(method))
# check if files do (not) exist
if not os.path.isfile(args.source):
logging.error("Source %s does not exist", repr(args.source))
return -1
if os.path.isfile(args.target) and os.path.getsize(args.target) > 0:
logging.error("Target %s already exists and is non-empty",
repr(args.target))
return -1
if args.action == "c" and not os.path.isfile(args.reference):
logging.error("Reference %s does not exist", repr(args.reference))
return -1
# compress/decompress
if args.action == "c":
return compress(args.source, args.target, args.reference, args.nointra,
args.delta, args.inner)
elif args.action == "d":
return decompress(args.source, args.target)
def load_single_config(conf_parser, conf_key):
config_warn_msg = "Cannot load config [%s], use default value: %s"
try:
v = conf_parser.get('settings', conf_key)
if not v:
raise Exception('ConfigError', 'invalid')
gkey = var_dict[conf_key][0]
ty = var_dict[conf_key][1]
if ty == 'n':
globals()[gkey] = int(v)
else:
if ty == 'p':
util.create_dir(v)
globals()[gkey] = v
except:
log.warn(config_warn_msg %
(conf_key, str(globals()[var_dict[conf_key][0]])))
def draw_date_stats(date_stats):
langs_info = read_auxiliary_file('../aux_files/langs_all_info.csv')
code_to_lang = get_code_lang_mapping(langs_info)
code_to_lang['overall'] = 'Общее распределение по датам'
graphs_dir = create_dir('date_graphs')
# draw_each_lang_separately(date_stats, code_to_lang, graphs_dir)
# draw_all_langs_same_plot(date_stats, code_to_lang, graphs_dir)
draw_overall_dates_with_registrars(date_stats, graphs_dir)
def load_single_config(conf_parser, conf_key):
config_warn_msg = "Cannot load config [%s], use default value: %s"
try:
v = conf_parser.get('settings', conf_key)
if not v:
raise Exception('ConfigError','invalid')
gkey = var_dict[conf_key][0]
ty = var_dict[conf_key][1]
if ty == 'n':
globals()[gkey] = int(v)
else:
if ty =='p':
util.create_dir(v)
globals()[gkey] = v
except:
if not conf_key.find('proxy.http'):
log.warn(config_warn_msg % (conf_key, str(globals()[var_dict[conf_key][0]])))
def init_fav(self):
""" parse html and load json and init Song object
for each found song url"""
page = 1
user = ''
total = 0
cur = 1 #current processing link
LOG.debug(msg.head_xm + msg.fmt_init_fav % self.uid)
while True:
html = self.handler.read_link(url_fav%(self.uid,page)).text
soup = BeautifulSoup(html,'html.parser')
if not user:
user = soup.title.string
if not total:
total = soup.find('span', class_='counts').string
links = [link.get('href') for link in soup.find_all(href=re.compile(r'xiami.com/song/[^?]+')) if link]
if links:
for link in links:
LOG.debug(msg.head_xm + msg.fmt_parse_song_url % link)
if self.verbose:
sys.stdout.write(log.hl('[%d/%s] parsing song ........ '%(cur, total), 'green'))
sys.stdout.flush()
try:
cur += 1
song = XiamiSong(self.handler, url=link)
#time.sleep(2)
if self.verbose:
sys.stdout.write(log.hl('DONE\n', 'green'))
except:
sys.stdout.write(log.hl('FAILED\n', 'error'))
continue
#rewrite filename, make it different
song.group_dir = user
song.post_set()
self.songs.append(song)
page += 1
else:
break
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
LOG.debug(msg.head_xm + msg.fmt_init_fav_ok % self.uid)
def init_collection(self):
LOG.debug(msg.head_xm + msg.fmt_init_collect % self.collection_id)
j = self.handler.read_link(url_collection % (self.collection_id) ).json()['data']['trackList']
if not j:
LOG.warning(msg.head_xm + " cannot load data for url: %s "% self.url)
return
j_first_song = j[0]
#read collection name
self.collection_name = self.get_collection_name()
for jsong in j:
song = XiamiSong(self.handler, song_json=jsong)
#rewrite filename, make it different
song.group_dir = self.collection_name
song.post_set()
self.songs.append(song)
if len(self.songs):
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
LOG.debug(msg.head_xm + msg.fmt_init_collect_ok % self.collection_id)
def load_single_config(conf_parser, conf_key):
config_warn_msg = "Cannot load config [%s], use default value: %s"
try:
v = conf_parser.get('settings', conf_key)
if not v:
raise Exception('ConfigError','invalid')
gkey = var_dict[conf_key][0]
ty = var_dict[conf_key][1]
if ty == 'n':
#here check topx should less than 30
if conf_key == 'download.artist.topsong' and int(v) >30:
return
globals()[gkey] = int(v)
else:
if ty =='p':
util.create_dir(v)
globals()[gkey] = v
except:
log.warn(config_warn_msg % (conf_key, str(globals()[var_dict[conf_key][0]])))
def init_topsong(self):
LOG.debug(msg.head_xm + msg.fmt_init_artist% self.artist_id)
j = self.handler.read_link(url_artist_top_song % (self.artist_id)).json()['data']['trackList']
for jsong in j:
song = XiamiSong(self.handler, song_json=jsong)
if not self.artist_name:
self.artist_name = song.artist_name
song.group_dir = self.artist_name + '_TopSongs'
song.post_set()
self.songs.append(song)
#check config for top X
if config.DOWNLOAD_TOP_SONG>0 and len(self.songs) >= config.DOWNLOAD_TOP_SONG:
break
if len(self.songs):
#set the artist name
self.artist_name = self.songs[-1].artist_name
#creating the dir
util.create_dir(path.dirname(self.songs[-1].abs_path))
LOG.debug(msg.head_xm + msg.fmt_init_artist_ok % self.artist_id)
def init_album(self):
j = self.handler.read_link(url_album % self.album_id).json()['data']['trackList']
j_first_song = j[0]
#name
self.album_name = util.decode_html(j_first_song['album_name'])
#album logo
self.logo = j_first_song['album_pic']
# artist_name
self.artist_name = j_first_song['artist']
#description
html = self.handler.read_link(self.url).text
soup = BeautifulSoup(html,'html.parser')
self.album_desc = soup.find('span', property="v:summary").text
#handle songs
for jsong in j:
song = XiamiSong(self.handler, song_json=jsong)
song.group_dir = self.artist_name + u'_' + self.album_name
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
#creating the dir
LOG.debug(msg.head_xm + msg.fmt_create_album_dir % d)
util.create_dir(d)
#download album logo images
LOG.debug(msg.head_xm + msg.fmt_dl_album_cover % self.album_name)
downloader.download_url(self.logo, path.join(d,'cover.' +self.logo.split('.')[-1]))
LOG.debug(msg.head_xm + msg.fmt_save_album_desc % self.album_name)
if self.album_desc:
self.album_desc = re.sub(r'<\s*[bB][rR]\s*/>','\n',self.album_desc)
self.album_desc = re.sub(r'<.*?>','',self.album_desc)
self.album_desc = util.decode_html(self.album_desc)
import codecs
with codecs.open(path.join(d,'album_description.txt'), 'w', 'utf-8') as f:
f.write(self.album_desc)
def init_album(self):
j = self.xm.read_link(url_album % self.album_id).json()['album']
# name
self.album_name = util.decode_html(j['title'])
# album logo
self.logo = j['album_logo']
# artist_name
self.artist_name = j['artist_name']
# description
self.album_desc = j['description']
# handle songs
for jsong in j['songs']:
song = XiamiSong(self.xm, song_json=jsong)
song.group_dir = self.artist_name + u'_' + self.album_name
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
# creating the dir
LOG.debug(u'创建专辑目录[%s]' % d)
util.create_dir(d)
# download album logo images
LOG.debug(u'下载专辑[%s]封面' % self.album_name)
downloader.download_by_url(
self.logo, path.join(d, 'cover.' + self.logo.split('.')[-1]))
LOG.debug(u'保存专辑[%s]介绍' % self.album_name)
if self.album_desc:
self.album_desc = re.sub(
r'<\s*[bB][rR]\s*/>', '\n', self.album_desc)
self.album_desc = re.sub(r'<.*?>', '', self.album_desc)
self.album_desc = util.decode_html(self.album_desc)
import codecs
with codecs.open(path.join(d, 'album_description.txt'), 'w', 'utf-8') as f:
f.write(self.album_desc)
def init_album(self):
#album json
js = self.m163.read_link(url_album % self.album_id).json()['album']
#name
self.album_name = util.decode_html(js['name'])
#album logo
self.logo = js['picUrl']
# artist_name
self.artist_name = js['artists'][0]['name']
#handle songs
for jsong in js['songs']:
song = NeteaseSong(self.m163, song_json=jsong)
song.group_dir = self.artist_name + u'_' + self.album_name
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
#creating the dir
LOG.debug(u'创建专辑目录[%s]' % d)
util.create_dir(d)
#download album logo images
LOG.debug(u'下载专辑[%s]封面'% self.album_name)
downloader.download_by_url(self.logo, path.join(d,'cover.' +self.logo.split('.')[-1]))
def init_album(self):
# album json
js = self.handler.read_link(url_album % self.album_id).json()["album"]
# name
self.album_name = util.decode_html(js["name"])
# album logo
self.logo = js["picUrl"]
# artist_name
self.artist_name = js["artists"][0]["name"]
# handle songs
for jsong in js["songs"]:
song = NeteaseSong(self.handler, song_json=jsong)
song.group_dir = self.artist_name + u"_" + self.album_name
song.post_set()
self.songs.append(song)
d = path.dirname(self.songs[-1].abs_path)
# creating the dir
LOG.debug(msg.head_163 + msg.fmt_create_album_dir % d)
util.create_dir(d)
# download album logo images
LOG.debug(msg.head_163 + msg.fmt_dl_album_cover % self.album_name)
downloader.download_by_url(self.logo, path.join(d, "cover." + self.logo.split(".")[-1]))
def get_stats_cache_dir(): return create_dir(os.path.join(get_cache_dir(), "stats")) def get_logs_cache_dir(): return create_dir(os.path.join(get_cache_dir(), "logs"))
"analyze_out" : "",
}
fields_no_serialize = ["rel_build_log", "analyze_out"]
def __init__(self, read_from_file=None):
Serializable.__init__(self, Stats.fields, Stats.fields_no_serialize, read_from_file)
def file_size(p):
return os.path.getsize(p)
def str2bool(s):
if s.lower() in ("true", "1"): return True
if s.lower() in ("false", "0"): return False
assert(False)
g_cache_dir = create_dir(os.path.realpath(os.path.join("..", "sumatrapdfcache", "buildbot")))
g_stats_cache_dir = create_dir(os.path.join(g_cache_dir, "stats"))
g_logs_cache_dir = create_dir(os.path.join(g_cache_dir, "logs"))
def get_cache_dir(): return g_cache_dir
def get_stats_cache_dir(): return g_stats_cache_dir
def get_logs_cache_dir(): return g_logs_cache_dir
def logs_efi_out_path(ver):
return os.path.join(get_logs_cache_dir(), str(ver) + "_efi.txt.bz2")
# logs are only kept for potential troubleshooting and they're quite big,
# so we delete old files (we keep logs for the last $to_keep revisions)
def delete_old_logs(to_keep=10):
files = os.listdir(get_logs_cache_dir())
versions = []
for f in files:
def get_logs_cache_dir(): return create_dir(os.path.join(get_cache_dir(), "logs")) # logs are only kept for potential troubleshooting and they're quite big, # so we delete old files (we keep logs for the last $to_keep revisions) def delete_old_logs(to_keep=10):
def get_cache_dir(): return create_dir(os.path.join("..", "sumatrapdfcache", "buildbot"))
def get_stats_cache_dir(): return create_dir(os.path.join(get_cache_dir(), "stats"))
if m2: lossRate = float(m2.group(1)) lossRate /= 100. return throughput, lossRate if __name__ == '__main__': if len(sys.argv) != 1 and len(sys.argv) != 4 and len(sys.argv) != 5: print "Usage: %s [mode algorithm drive [iperf_script]]" % sys.argv[0] sys.exit(0) python_script = "../iperf/server_tcp.py" device = "" outputFile = None dirname = "../../data/drive_2_28_2016" create_dir(dirname) if len(sys.argv) >= 4: mode = sys.argv[1] alg = sys.argv[2] drive = sys.argv[3] outputFile = "%s/gps_throughput_%s_%s_%s.dat" % (dirname, mode, alg, drive) if len(sys.argv) == 5: python_script = sys.argv[4] gps_info = gps_parser.GPSInfo() gps_parse = gps_parser.GPSParser() thread = threading.Thread(target=gps_parser.GPSParser.read_gps, args=(gps_parse, gps_info, device,)) thread.start() time.sleep(2) outf = open(outputFile, 'w') if outputFile else sys.stdout
def sum_efi_cache_dir(ver):
# make it outside of sumatrapdf_efi directory?
d = os.path.join(sum_efi_dir(), "efi_cache", str(ver))
return util.create_dir(d)
def calculate_capacity(data_rates, arr):
throughput_arr = arr[:,3:].copy()
for no, rate in enumerate(data_rates):
throughput_arr[:, no] = (1 - throughput_arr[:, no]) * rate
capacity_arr = numpy.max(throughput_arr, axis=1)
return capacity_arr
if __name__ == '__main__':
data_rates = numpy.array([1, 2, 5.5, 6, 9, 11, 12, 18, 24, 36, 48, 54])
INTERVAL = 50
max_num_bs = 20
max_num_clients = 20
dirname_w = "trace"
filename_r = "packet_loss.dat"
create_dir(dirname_w)
arr = numpy.loadtxt(filename_r, dtype='f8')
for client_id in range(1, max_num_clients + 1):
capacity_arr_total = None
filename_w = "%s/client_%d.dat" % (dirname_w, client_id)
for bs_id in range(1, max_num_bs + 1):
#filename_loss = "%s/packet_loss_%d_%d.dat" % (dirname_w, bs_id, client_id)
filename_loss = None
synthesized_arr = shuffle_packet_loss(INTERVAL, arr, filename_loss)
capacity_arr = calculate_capacity(data_rates, synthesized_arr)
if capacity_arr_total is None:
capacity_arr_total = capacity_arr
else:
capacity_arr_total = numpy.vstack((capacity_arr_total, capacity_arr))
numpy.savetxt(filename_w, numpy.atleast_2d(capacity_arr_total).T, fmt="%g")
