def main(data_dir, config_dir, output_dir):
try:
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
summary = {}
for model in sorted(config['models']):
summary[model] = []
# the script will not be run if there is an error
cmd_id = 0
for _, _, exp_config in parse_commands(model, config):
for combo in unfold_settings(exp_config):
stats, msg = parse_data_file(exp_config['type'], model,
config, combo, data_dir,
cmd_id)
if stats is None:
write_status(output_dir, False, msg)
return 1
stats['command_id'] = cmd_id
summary[model].append(stats)
cmd_id += 1
write_json(output_dir, 'data.json', summary)
write_status(output_dir, True, 'success')
except Exception as e:
write_status(output_dir, False, render_exception(e))
def report_results(model_name, i, config, specific_params, num_retries,
out_file, use_dtr, trial_run, trial_run_outfile,
results_queue):
"""
Given a queue of results, do all the necessary reporting
"""
measurements = []
while not results_queue.empty():
measurements.append(results_queue.get())
# all we care about for a trial run is max memory usage
if trial_run:
write_json(
os.getcwd(), trial_run_outfile,
{'mem': max(map(lambda data: data['total_mem'], measurements))})
return
memory_budget = specific_params.get('memory_budget', -1)
dry_run = config['dry_run']
save_log = use_dtr and specific_params.get(
'save_logs', config['save_logs']) and i == config['n_inputs'] - 1
if save_log:
save_trial_log(config['log_dest'],
config.get('simrd_config', None),
model_name,
specific_params,
is_baseline=(memory_budget == -1))
# clean up after ourselves
delete_logs()
with open(out_file, 'a', newline='') as csvfile:
from pt_trial_util import create_csv_writer
writer = create_csv_writer(csvfile, specific_params)
for j in range(len(measurements)):
data = measurements[j]
# do unit conversions now: times in ms,
# memory in MB
entry = {
'num_retries': num_retries,
'time': data['time'] * 1e3,
'sync_time': data['sync_time'] * 1e3,
# pytorch's cuda elapsed time is already in ms
'gpu_time': float(data['gpu_time']),
# 'cuda_time' : float(data['cuda_time']) * 1e-6,
'input_mem': data['input_mem'] * 1e-6,
'model_mem': data['model_mem'] * 1e-6,
'total_mem': data['total_mem'] * 1e-6,
'memory_budget': memory_budget,
# profiling (reported in nanoseconds)
'base_compute_time': data['base_compute_time'] * 1e-6,
'remat_compute_time': data['remat_compute_time'] * 1e-6,
'search_time': data['search_time'] * 1e-6,
'cost_time': data['cost_time'] * 1e-6,
'rep': j - dry_run,
'input': i,
**specific_params
}
writer.writerow(entry)
def main(config_dir, output_dir):
"""Run the experiment."""
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
try:
# the experiment involves RPC calls that could potentially hang so we have a timeout on our end too
killswitch = Timer(config.get('timeout', 300),
lambda: timeout_failure(output_dir))
killswitch.start()
result = {}
config_iter = itertools.product(config['models'], config['targets'],
config['devices'])
for (model, target, device) in config_iter:
# TODO(weberlo): There has to be some idiom to get rid of this boilerplate.
if model not in result:
result[model] = {}
if target not in result[model]:
result[model][target] = {}
if device not in result[model][target]:
result[model][target][device] = {}
result[model][target][device] = run_single(model, target, device,
config)
killswitch.cancel()
except Exception as e:
write_status(output_dir, False,
'Exception encountered:\n' + render_exception(e))
return 1
write_json(output_dir, 'data.json', result)
write_status(output_dir, True, 'success')
def main(argv):
args = parse_arguments(argv)
if args.save_as is None:
out_dir = os.path.join(args.out_dir, base64_encode(args.keyword))
else:
out_dir = os.path.join(args.out_dir, base64_encode(args.save_as))
downloader = NiconicoDownloader(out_dir)
auth = read_json(args.auth_json)
auth = dict() if auth is None else auth
while True:
if 'niconico' not in auth:
auth['niconico'] = {
'username': input('Username >> '),
'password': getpass('Password >> '),
}
try:
downloader.authenticate(**auth['niconico'])
write_json(args.auth_json, auth)
break
except NoSuchElementException:
logger.error('Failed to login.')
del auth['niconico']
downloader(args.keyword)
def main(config_dir, output_dir):
"""Run the experiment."""
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
try:
result = {}
config_iter = itertools.product(
config['models'],
config['targets'],
config['devices'])
for (model, target, device) in config_iter:
# TODO(weberlo): There has to be some idiom to get rid of this boilerplate.
if model not in result:
result[model] = {}
if target not in result[model]:
result[model][target] = {}
if device not in result[model][target]:
result[model][target][device] = {}
result[model][target][device] = run_single(model, target, device, config)
except Exception as e:
write_status(output_dir, False, 'Exception encountered:\n' + render_exception(e))
return 1
write_json(output_dir, 'data.json', result)
write_status(output_dir, True, 'success')
def analyze_experiment(info, experiments_dir, tmp_data_dir,
date_str, exp_name):
exp_dir = os.path.join(experiments_dir, exp_name)
exp_data_dir = os.path.join(tmp_data_dir, exp_name)
tmp_analysis_dir = os.path.join(exp_data_dir, 'analysis')
idemp_mkdir(tmp_analysis_dir)
analyzed_data_dir = info.exp_data_dir(exp_name)
if not os.path.exists(analyzed_data_dir):
idemp_mkdir(analyzed_data_dir)
subprocess.call([os.path.join(exp_dir, 'analyze.sh'),
info.exp_config_dir(exp_name), exp_data_dir, tmp_analysis_dir],
cwd=exp_dir)
status = validate_status(tmp_analysis_dir)
# read the analyzed data, append a timestamp field, and copy over to the permanent data dir
if status['success']:
data_exists = check_file_exists(tmp_analysis_dir, 'data.json')
if not data_exists:
status = {'success': False, 'message': 'No data.json file produced by {}'.format(exp_name)}
else:
# collect data to dump to data_*.json
dump_data = {
'timestamp' : date_str,
}
dump_data.update(read_json(tmp_analysis_dir, 'data.json'))
# fetch time spent on the experiment
dump_data.update(get_timing_info(info, exp_name))
write_json(analyzed_data_dir, 'data_{}.json'.format(date_str), dump_data)
info.report_exp_status(exp_name, 'analysis', status)
return status['success']
def add_value_to_node():
nodes = common.read_json(OUTPUT_DIR, NODE_FILE_NAME)
edges = common.read_json(OUTPUT_DIR, EDGE_FILE_NAME)
for node in nodes:
count = len(list(filter(lambda x: x['from'] == node['id'], edges)))
count += len(list(filter(lambda x: x['to'] == node['id'], edges)))
node['value'] = count
common.write_json(nodes, OUTPUT_DIR, NODE_FILE_NAME)
def add_position_to_node():
nodes = common.read_json(OUTPUT_DIR, NODE_FILE_NAME)
positions = common.read_json(OUTPUT_DIR, POSITION_FILE_NAME)
for node in nodes:
position = positions[str(node['id'])]
node['x'] = position['x']
node['y'] = position['y']
common.write_json(nodes, OUTPUT_DIR, NODE_FILE_NAME)
def munge_queries(set_id):
print "Munging set_id:", set_id
common.use_set(set_id)
model = munger.munge_model(common.read_most_recent('model'))
state = munger.munge_state(common.read_most_recent('state'))
results = {"state":state, "model":model}
common.write_json(results, "results.json")
def main(data_dir, config_dir, output_dir):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
# No further analysis is required beyond the raw stats reported by the VTA
# simulator, so we just propagate the data to the next stage of the
# pipeline.
data = read_json(data_dir, 'data.json')
write_json(output_dir, 'data.json', data)
write_status(output_dir, True, 'success')
def make_topology():
entries = common.read_json(OUTPUT_DIR, ENTRIES_FILE_NAME)
nodes = []
edges = []
for entry in entries:
nodes.append(make_node(entry))
edges += make_edges(entry, entries)
common.write_json(nodes, OUTPUT_DIR, NODE_FILE_NAME)
common.write_json(edges, OUTPUT_DIR, EDGE_FILE_NAME)
def main(data_dir, config_dir, output_dir):
try:
config, msg = validate_trials_config(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
summary = {}
baseline_dict = {}
for model in sorted(config['models']):
summary[model] = []
baseline_dict[model] = {}
# the script will not be run if there is an error
cmd_id = 0
for _, _, exp_config in parse_commands(model, config):
baseline_params = None
for specific_params in unfold_settings(exp_config):
batch_size = specific_params['batch_size']
if specific_params['type'] == 'baseline':
baseline_dict[model][batch_size] = {
'type': 'baseline',
'specific_params': specific_params,
'cmd_id': cmd_id
}
# if there is a corresponding baseline,
# let's match using the dict
baseline_params = None
if (batch_size in baseline_dict[model]
and specific_params['type'] != 'baseline'):
baseline_params = baseline_dict[model][batch_size]
stats, msg = parse_data_file(
exp_config['type'],
model,
config,
specific_params,
data_dir,
cmd_id,
baseline_params=baseline_params)
if stats is None:
write_status(output_dir, False, msg)
return 1
stats['command_id'] = cmd_id
summary[model].append(stats)
cmd_id += 1
write_json(output_dir, 'data.json', summary)
write_status(output_dir, True, 'success')
except Exception as e:
write_status(output_dir, False, render_exception(e))
def main(argv):
args = parse_arguments(argv)
dirname = os.path.basename(args.input_dir)
valid_dir = os.path.join(args.output_dir, dirname, 'valid')
invalid_dir = os.path.join(args.output_dir, dirname, 'invalid')
os.makedirs(valid_dir, exist_ok=True)
os.makedirs(invalid_dir, exist_ok=True)
removed_json = os.path.join(args.output_dir, dirname, '.cache.json')
names = get_filenames(args.input_dir)
valid_names = get_filenames(valid_dir)
invalid_names = get_filenames(invalid_dir)
removed_names = read_json(removed_json)
removed_names = [] if removed_names is None else removed_names
names = sorted(
set(names) - set(valid_names) - set(invalid_names) -
set(removed_names))
# Instruction
sys.stdout.write('Key input instructions:\n'
'j: Accept current image\n'
'k: Reject current image\n'
'u: Undo recent validation\n'
'd: Exclude image \n'
'q: Quit validation\n')
i = 0
while i < len(names):
path = os.path.join(args.input_dir, names[i])
key = show_image(path, args.size)
if key == KeyStatus.UNDO and i > 1:
i -= 1
if os.path.exists(os.path.join(valid_dir, names[i])):
os.remove(os.path.join(valid_dir, names[i]))
elif os.path.exists(os.path.join(invalid_dir, names[i])):
os.remove(os.path.join(invalid_dir, names[i]))
else:
removed_names.pop()
elif key == KeyStatus.OK:
shutil.copyfile(path, os.path.join(valid_dir, names[i]))
i += 1
elif key == KeyStatus.FAIL:
shutil.copyfile(path, os.path.join(invalid_dir, names[i]))
i += 1
elif key == KeyStatus.REMOVE:
removed_names.append(names[i])
write_json(removed_json, removed_names)
i += 1
else:
exit()
def process_score(info, score_metric, data_dir, graph_dir, timestamp):
data = score_metric.compute_score(info)
data['timestamp'] = timestamp
write_json(data_dir, 'data_{}.json'.format(timestamp), data)
# graphs failing is not a fatal error, just an inconvenience
try:
score_metric.score_graph(data, graph_dir)
all_data = sort_data(data_dir)
score_metric.longitudinal_graphs(all_data, graph_dir)
except Exception as e:
print(render_exception(e))
finally:
return score_metric.score_text(data)
def main():
print "Processing"
movies = common.read_json("tunefind.json")
pool = Pool(5)
results = [pool.apply_async(process_movie, [m]) for m in movies]
updated_movies = []
for w in results:
w.wait()
updated_movies.append(w.get())
common.write_json("musicbrainz.json", updated_movies)
def main(argv):
args = parse_arguments(argv)
config = load_config(args.config)
ctime = time.strftime('%y%m%d_%H%M')
dst_dir = os.path.join(args.output_dir, '%s_%s2%s' % (ctime, *args.labels))
os.makedirs(dst_dir, exist_ok=True)
write_json(os.path.join(dst_dir, 'config.json'), config)
datasets = setup_dataset(config, args.labels[0], args.labels[1])
models = setup_model(config)
trainer = setup_trainer(config, dst_dir, args.gpu,
datasets, models, args.labels)
trainer.run(1000, args.max_epoch)
def delete_duplication():
entries = common.read_json(OUTPUT_DIR, ENTRIES_FILE_NAME)
fixed_entries = []
titles = []
index = 1
for entry in entries:
title = entry['title']
if title in titles: continue
fixed = entry
fixed['id'] = index
fixed_entries.append(fixed)
titles.append(title)
index += 1
common.write_json(fixed_entries, OUTPUT_DIR, 'entries_fixed.json')
def log_error(experiment_name, model_name, specific_params, inp, err_msg,
path_prefix):
err_info = {'input': inp, 'msg': err_msg}
logged_errors = {}
if check_file_exists(path_prefix, 'errors.json'):
logged_errors = read_json(path_prefix, 'errors.json')
if experiment_name not in logged_errors:
logged_errors[experiment_name] = {}
if model_name not in logged_errors[experiment_name]:
logged_errors[experiment_name][model_name] = []
logged_errors[experiment_name][model_name].append({
'err_info': err_info,
**specific_params
})
write_json(path_prefix, 'errors.json', logged_errors)
def load_from_web():
print "Loading from Web"
movies = common.read_json(JSON_IN_FILE)
pool = Pool(5)
worker = [pool.apply_async(process_movie, [m]) for m in movies]
imdb_movies = []
for w in worker:
w.wait()
result = w.get()
if result is not None:
imdb_movies.append(w.get())
common.write_json(JSON_OUT_FILE, imdb_movies)
def load_from_web():
print "Loading from Web"
movies = common.read_json(JSON_IN_FILE)
pool = Pool(5)
worker = [pool.apply_async(process_movie, [m]) for m in movies]
imdb_movies = []
for w in worker:
w.wait()
result = w.get()
if result is not None:
imdb_movies.append(w.get())
common.write_json(JSON_OUT_FILE, imdb_movies)
def get_video_info(video_id):
video_info_list = []
items = youtube.videos().list(part="snippet, contentDetails",
id=video_id).execute()["items"][0]
resp_save_dest = "data/resp/yt/v/"
timestamp = common.now_iso(1) # UTC+0
common.make_dir(resp_save_dest)
common.write_json(resp_save_dest + timestamp + " " + video_id + ".json",
items)
video_info_list.append(items["snippet"]["channelTitle"]) # 0
video_info_list.append(items["snippet"]["channelId"]) # 1
video_info_list.append(items["snippet"]["publishedAt"]) # 2
video_info_list.append(items["snippet"]["title"]) # 3
video_info_list.append(items["snippet"]["description"]) # 4
video_info_list.append(items["contentDetails"]["duration"]) # 5
return video_info_list # Throw video_info_list to fileproc.
def main(config_dir, home_dir, output_dir):
info = DashboardInfo(home_dir)
conf = read_config(config_dir)
data_dir = os.path.join(output_dir, 'data')
graph_dir = os.path.join(output_dir, 'graphs')
idemp_mkdir(data_dir)
idemp_mkdir(graph_dir)
timestamp = get_timestamp()
score_confs = conf['score_confs']
metrics = set(score_confs.keys())
metrics = metrics.intersection(set(SCORE_METRICS.keys()))
if not metrics:
write_status(output_dir, True, 'No scores to report')
return 0
score_data = {}
score_reports = {}
for metric in metrics:
score_metric = SCORE_METRICS[metric](score_confs[metric])
valid, msg = check_prerequisites(info, score_metric.prereq())
if not valid:
write_status(output_dir, False, msg)
return 1
score_data_dir = os.path.join(data_dir, metric)
score_graph_dir = os.path.join(graph_dir, metric)
idemp_mkdir(score_data_dir)
idemp_mkdir(score_graph_dir)
try:
report = process_score(info, score_metric, score_data_dir,
score_graph_dir, timestamp)
score_reports[metric] = report
except Exception as e:
write_status(
output_dir, False,
'Encountered exception while scoring {}:\n{}'.format(
metric, render_exception(e)))
return 1
report = {'title': 'Metric Scores', 'value': format_scores(score_reports)}
write_json(output_dir, 'report.json', report)
write_status(output_dir, True, 'success')
def crawl():
segs = get_entry_urls()
logger.write_log("start crawling")
try:
common.create_dir(OUTPUT_DIR)
entries = []
for entry in crawl_entries(segs):
if (not entry):
logger.write_log("error occurred")
return
entries.append(entry)
common.write_json(entries, OUTPUT_DIR, ENTRIES_FILE_NAME)
except Exception:
logger.write_log("error occurred")
logger.write_error(traceback.format_exc())
finally:
logger.write_log("end crawling")
def load_from_web():
print ""
print "Loading from Web..."
request = urllib2.Request(EP_TUNEFIND_MOVIES)
add_header(request)
moviedata = parse_response(urllib2.urlopen(request))
count = 0
movies = []
for m in moviedata["movies"]:
count += 1
print "#%02d: %s" % (count, m["name"])
movie = {"title": m["name"], "soundtrack": get_songs(m['id'])}
movies.append(movie)
# One request per second
sleep(1.1)
common.write_json(JSON_OUT_FILE, movies)
def load_from_web():
print "Loading from Web..."
network = pylast.LastFMNetwork(api_key=API_KEY, api_secret=API_SECRET)
movies = common.read_json(JSON_IN_FILE)
song_chunks = []
for m in movies:
if len(m["soundtrack"]) > 0:
song_chunks.append(m["soundtrack"])
pool = Pool(5)
worker = [pool.apply_async(process_songs, [chunk, network]) for chunk in song_chunks]
lastfm_songs = []
for w in worker:
w.wait()
for s in w.get():
lastfm_songs.append(s)
common.write_json(JSON_OUT_FILE, lastfm_songs)
def load_from_web():
print ""
print "Loading from Web..."
request = urllib2.Request(EP_TUNEFIND_MOVIES)
add_header(request)
moviedata = parse_response(urllib2.urlopen(request))
count = 0
movies = []
for m in moviedata["movies"]:
count += 1
print "#%02d: %s" % (count, m["name"])
movie = {"title": m["name"], "soundtrack": get_songs(m['id'])}
movies.append(movie)
# One request per second
sleep(1.1)
common.write_json(JSON_OUT_FILE, movies)
def match():
import deepmatching_wrapper as dm
import cv2
candidate_matching_database = common.load_pickle(Path("temp/candidate_matching_database.pickle"))
common.prepare_clean_dir(Path("output/"))
common.prepare_clean_dir(Path("output/images/"))
output = {}
for query_file, candidates in candidate_matching_database.items():
query_name = Path(query_file).stem
matching_result = []
for target_class_name, target_images in candidates.items():
for i, (target_path, similarity) in enumerate(target_images):
print("Matching", query_file, "with target image", target_path)
matches, name1, name2, qw, qh, tw, th, img1, img2 = dm.match(query_file, target_path)
src_pts = np.float32([[m[0], m[1]] for m in matches])
dst_pts = np.float32([[m[2], m[3]] for m in matches])
i = 0
inlier = []
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, setting.RANSAC_THRESHOLD)
for index, m in enumerate(mask):
if np.isclose(m, 1):
i += 1
inlier.append(matches[index])
output_name = "%s_%s_%02d.jpg" % (query_name, target_class_name, i)
dm.draw(img1, img2, inlier, Path("output/images/") / output_name)
matching_result.append({
"class_name": target_class_name,
"inlier": len(inlier)
})
output[query_file.name] = sorted(matching_result, key=lambda x: x["inlier"], reverse=True)
common.write_json(Path("output/result.json"), output)
def extract_face(self, img, basename):
filename = '%s_%s.jpg' % (basename, '%02d')
rects = self._detector(img)
self._rects[basename] = list()
if len(rects) > 0:
logger.info('Extract face from "%s"', basename)
src_h, src_w = img.shape[:2]
for i, (x, y, w, h) in enumerate(rects):
cnt_x, cnt_y = x + w // 2, y + h // 2
cnt_d = int(max(w, h) * (1.0 + self._margin) * 0.5)
x0, y0 = max(cnt_x - cnt_d, 0), max(cnt_y - cnt_d, 0)
x1, y1 = min(cnt_x + cnt_d, src_w), min(cnt_y + cnt_d, src_h)
if x1 - x0 < cnt_d * 2:
if x0 > 0:
x0 = max(x1 - cnt_d * 2, 0)
elif x1 < src_w:
x1 = min(x0 + cnt_d * 2, src_w)
if y1 - y0 < cnt_d * 2:
if y0 > 0:
y0 = max(y1 - cnt_d * 2, 0)
elif y1 < src_h:
y1 = min(y0 + cnt_d * 2, src_h)
if x1 - x0 < cnt_d * 2 and x1 - x0 < y1 - y0:
new_d = x1 - x0
y0 = cnt_y - new_d // 2
y1 = y0 + new_d
elif y1 - y0 < cnt_d * 2 and x1 - x0 > y1 - y0:
new_d = y1 - y0
x0 = cnt_x - new_d // 2
x1 = x0 + new_d
self._save_extract_face(img[y0:y1, x0:x1, :], filename % i)
self._rects[basename].append([int(x0), int(x1), int(y0), int(y1)])
write_json(os.path.join(self.dst_dir, self._CACHE_FILENAME),
self._rects)
def main(data_dir, config_dir, output_dir):
config, msg = validate_config(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
devs = config['devices']
listing_settings = generate_listing_settings(config)
ret = {}
for dev in devs:
ret[dev] = {}
for listing, settings in listing_settings.items():
if not use_networks:
query = generate_data_query(config, dev, settings)
summary, success, msg = trials_stat_summary(
data_dir, *query)
if not success:
write_status(output_dir, False, msg)
return 1
ret[dev][listing] = summary['mean']
add_detailed_summary(ret, summary, dev, listing)
continue
ret[dev][listing] = {}
for network in config['networks']:
query = generate_data_query(config, dev, network, settings)
summary, success, msg = trials_stat_summary(
data_dir, *query)
if not success:
write_status(output_dir, False, msg)
return 1
ret[dev][listing][network] = summary['mean']
add_detailed_summary(ret, summary, dev, listing, network)
write_json(output_dir, 'data.json', ret)
write_status(output_dir, True, 'success')
def process_telemetry_statistics(info,
exp_name,
output_dir,
time_str,
cpu_stat_parser=parse_cpu_stat,
gpu_stat_parser=parse_gpu_stat):
'''
Collect data of telemetry statistics and write to results directory
Note: The "parsing" logic procedure written in this file is specialized to deal with
telemetry collected at pipsqueak. They are not guaranteed to work on other platforms.
'''
telemetry_output_dir = info.subsys_telemetry_dir(exp_name)
if not os.path.exists(telemetry_output_dir):
idemp_mkdir(telemetry_output_dir)
data_dir = os.path.join(output_dir, f'telemetry/{exp_name}')
cpu_telemetry_dir = os.path.join(data_dir, 'cpu')
gpu_telemetry_dir = os.path.join(data_dir, 'gpu')
write_json(os.path.join(telemetry_output_dir, 'gpu'),
f'gpu-{time_str}.json',
gpu_stat_parser(gpu_telemetry_dir, time_str))
write_json(os.path.join(telemetry_output_dir, 'cpu'),
f'cpu-{time_str}.json',
cpu_stat_parser(cpu_telemetry_dir, time_str))
def load_from_web():
print "Loading from Web..."
start_date = datetime.strptime(STARTDATE, "%d.%m.%Y")
end_date = datetime.strptime(ENDDATE, "%d.%m.%Y")
date_delta = timedelta(days=7)
current_date = start_date
days = []
while current_date < end_date:
days.append(current_date)
current_date = current_date + date_delta
pool = Pool(5)
results = [pool.apply_async(get_charts, [d]) for d in days]
charts = []
for w in results:
w.wait()
charts.append(w.get())
common.write_json(JSON_OUT_FILE, charts)
def extend_simrd_config(dest_dir, sim_conf_filename, model_name,
specific_params, log_name):
if not check_file_exists(dest_dir, sim_conf_filename):
prepare_out_file(dest_dir, sim_conf_filename)
write_json(dest_dir, sim_conf_filename, dict())
conf = read_json(dest_dir, sim_conf_filename)
if model_name not in conf:
conf[model_name] = []
conf[model_name].append({
'name':
model_util.get_model_family(model_name),
'batch_size':
str(specific_params['batch_size']),
'layers':
specific_params.get('layers', model_util.get_model_layers(model_name)),
'type':
model_util.get_model_type(model_name),
'log':
log_name,
'has_start':
True
})
write_json(dest_dir, sim_conf_filename, conf)
def extend_simrd_config(dest_dir, sim_conf_filename, model_name,
specific_params, log_name):
import model_util
if not check_file_exists(dest_dir, sim_conf_filename):
prepare_out_file(dest_dir, sim_conf_filename)
write_json(dest_dir, sim_conf_filename, dict())
conf = read_json(dest_dir, sim_conf_filename)
if model_name not in conf:
conf[model_name] = []
name = model_util.format_model_name(model_name, specific_params)
conf[model_name].append({
'name':
name,
'title':
name,
'desc':
model_util.format_input_description(model_name, specific_params),
'log':
log_name,
'has_start':
True
})
write_json(dest_dir, sim_conf_filename, conf)
def load_from_web():
print "Loading from Web..."
network = pylast.LastFMNetwork(api_key=API_KEY, api_secret=API_SECRET)
movies = common.read_json(JSON_IN_FILE)
song_chunks = []
for m in movies:
if len(m["soundtrack"]) > 0:
song_chunks.append(m["soundtrack"])
pool = Pool(5)
worker = [
pool.apply_async(process_songs, [chunk, network])
for chunk in song_chunks
]
lastfm_songs = []
for w in worker:
w.wait()
for s in w.get():
lastfm_songs.append(s)
common.write_json(JSON_OUT_FILE, lastfm_songs)
def write_tasklist(path, taglist, tasklist):
out = []
for x in tasklist:
x['tags'] = list(x['tags'])
out.append(x)
write_json(path, {'tags': list(taglist), 'tasks': out})
def save_reply(prefix, parameters):
job_id = parameters['ids']['jobId']
project_id = parameters['ids']['projectId']
filename = "%s-%s.json" % (project_id, job_id)
common.write_json(parameters, "%s/%s" % (prefix, filename))
