def install_Windows(self):
# ctags
ctags_zip = os.path.join(DOWNLOADS, CTAGS_VER + '.zip')
if not isfile(ctags_zip):
wget('http://sourceforge.net/projects/ctags/files/ctags/5.8/{}.zip'.
format(CTAGS_VER), ctags_zip)
unzip(ctags_zip, CTAGS_VER + '/ctags.exe')
def update(self):
# Step 1: Get newest nightly folder
response = urllib2.urlopen(self.nightly_dir+"/?C=N;O=D")
html = response.read()
folder_id = re.findall("[0-9]{5,}", html)[0]
# Step 2: Find the correct file
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id)
html = response.read()
exec_file = re.findall("jsshell-win32.zip", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.json", html)[0]
# Step 3: Get build information
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id+"/"+json_file)
html = response.read()
info = json.loads(html)
# Step 4: Fetch archive
print "Retrieving", self.nightly_dir+"/"+folder_id+"/"+exec_file
urllib.urlretrieve(self.nightly_dir+"/"+folder_id+"/"+exec_file, self.tmp_dir + "shell.zip")
# Step 5: Unzip
utils.unzip(self.tmp_dir,"shell.zip")
# Step 6: Save info
self.updated = True
self.cset = info["moz_source_stamp"]
def install_ipfs(quiet=False):
'''
Installs the ipfs software for windows and linux systems.
'''
if ipfs_installed():
return True
os_name = os.name # either of ('posix', 'nt', 'java')
n_bits = 32 << bool(sys.maxsize >> 32)
ipfs_url = IPFS_LINKS.get(n_bits, dict()).get(os_name, None)
file_extension = ('zip', 'tar.gz')[os.name == 'posix']
output_file_name = 'ipfs.' + file_extension
ipfs_dir_name = 'go-ipfs' # dir after unzipping.
if ipfs_url is None:
raise NotImplementedError('Program doesn\'t support your OS yet.')
downloaded = utils.wget(ipfs_url,
output_document=output_file_name,
tries=math.inf,
timeout=100,
quiet=quiet)
if downloaded:
utils.unzip(output_file_name)
os.chdir(ipfs_dir_name)
if os_name == 'posix':
os.system('sudo ./install.sh')
else:
os.system('ipfs')
os.chdir('..')
if not quiet:
print('IPFS installed successfully.')
print('Removing intermediate files...')
os.remove(output_file_name)
return True
return False
def get_sport(sport):
global root
utils.logmessage("Getting: " + sport, "getdata", 1)
if sport == "baseball":
url = "http://seanlahman.com/files/database/lahman-csv_2014-02-14.zip"
path = root + "rawdata/lahman/"
saveas = "lahman.zip"
if sport == "basketball":
url = "http://www.databasebasketball.com/databasebasketball_2009_v1.zip"
path = root + "rawdata/basketball/"
saveas = "basketball.zip"
# download file
utils.logmessage("Downloading zip file", "getdata", 1)
utils.download_file(url, path+saveas)
# unzip file
utils.unzip(path+saveas, path)
utils.logmessage("Unzipping file", "getdata", 1)
pass
def deploy(machine=None):
print green("当前机器:" + env.host)
#如果指定机器,那就只上着一个
if machine and not machine == env.host:
return
#关闭slb
#print env.host
for slb in env.slbList:
utils.setSlb(env.slbServer, slb, 0)
utils.runCmd("sleep 5")
#备份
utils.zip(env.online)
utils.download(env.online, env.localPath)
#上传
utils.upload(env.source, env.target)
#杀掉进程
utils.stopProcess(env.aport)
utils.runCmd("sleep 10")
#解压缩
utils.unzip(env.online)
#开始进程
utils.runCmd(env.start_process)
#sleep 多少秒
utils.runCmd("sleep " + env.sleep_time)
#监控进程
run(env.monitor_url)
#上线slb
for slb in env.slbList:
utils.setSlb(env.slbServer, slb, 100)
def unzip(online=None, machine=None):
if not machine or not online:
abort("机器、online路径不能为空")
print green("当前机器:" + env.host)
if machine and not machine == env.host:
return
utils.unzip(online)
def download(target="all", verbose=False, **kwargs):
if target == "all":
download(target="code", verbose=verbose)
download(target="data", verbose=verbose)
elif target == "code":
for algorithm in GIT_REPOS.keys():
download(target=algorithm, verbose=verbose)
elif target in GIT_REPOS.keys():
download_path = os.path.join(SRC_DIR, target)
print(download_path)
if os.path.exists(download_path):
print("Download path {path} exists. Skipping downloading {algo}".format(path=download_path, algo=target))
return
if verbose:
print("Downloading {algo} code to {path}.".format(algo=target, path=download_path))
Repo.clone_from(GIT_REPOS[target], download_path)
elif target == "data":
for graph in GRAPHS:
download(graph, verbose=True)
elif target in GRAPHS.keys():
for url in GRAPHS[target].values():
if not url.startswith("http"):
continue
download_path = os.path.join(GRAPH_DIR, target, os.path.basename(url))
unzip_path = os.path.join(GRAPH_DIR, target, os.path.splitext(os.path.basename(url))[0])
utils.download(url, download_path)
utils.unzip(download_path, unzip_path)
else:
raise ValueError("Unknown target.")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
if self.platform != "win" and self.platform != "macos":
raise ValueError("Only Windows and Mac platforms are currently supported")
if dest is None:
dest = os.pwd
if channel is None:
version_url = "https://msedgedriver.azureedge.net/LATEST_DEV"
else:
version_url = "https://msedgedriver.azureedge.net/LATEST_%s" % channel.upper()
version = get(version_url).text.strip()
if self.platform == "macos":
bits = "mac64"
edgedriver_path = os.path.join(dest, self.edgedriver_name)
else:
bits = "win64" if uname[4] == "x86_64" else "win32"
edgedriver_path = os.path.join(dest, "%s.exe" % self.edgedriver_name)
url = "https://msedgedriver.azureedge.net/%s/edgedriver_%s.zip" % (version, bits)
# cleanup existing Edge driver files to avoid access_denied errors when unzipping
if os.path.isfile(edgedriver_path):
# remove read-only attribute
os.chmod(edgedriver_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
os.remove(edgedriver_path)
driver_notes_path = os.path.join(dest, "Driver_notes")
if os.path.isdir(driver_notes_path):
shutil.rmtree(driver_notes_path, ignore_errors=False, onerror=handle_remove_readonly)
self.logger.info("Downloading MSEdgeDriver from %s" % url)
unzip(get(url).raw, dest)
if os.path.isfile(edgedriver_path):
self.logger.info("Successfully downloaded MSEdgeDriver to %s" % edgedriver_path)
return find_executable(self.edgedriver_name, dest)
def download(_):
conf.log = setup_log(logging.DEBUG) if conf.args['debug'] else setup_log()
tempdir = tempfile.mkdtemp()
cdn_url = conf.checkpoints_cdn.format(conf.checkpoints_version)
temp_zip = os.path.join(tempdir, "{}.zip".format(conf.checkpoints_version))
try:
conf.log.info("Downloading {}".format(cdn_url))
dl_file(conf.checkpoints_cdn.format(conf.checkpoints_version),
temp_zip)
conf.log.info("Extracting {}".format(temp_zip))
unzip(temp_zip, conf.args['checkpoints'])
conf.log.info("Moving Checkpoints To Final Location")
for c in ("cm.lib", "mm.lib", "mn.lib"):
if os.path.isfile(os.path.join(conf.args['checkpoints'], c)):
os.remove(os.path.join(conf.args['checkpoints'], c))
shutil.move(
os.path.join(conf.args['checkpoints'], 'checkpoints', c),
conf.args['checkpoints'])
shutil.rmtree(os.path.join(conf.args['checkpoints'], 'checkpoints'))
except Exception as e:
conf.log.error(e)
conf.log.error(
"Something Gone Bad Download Downloading The Checkpoints")
shutil.rmtree(tempdir)
sys.exit(1)
shutil.rmtree(tempdir)
conf.log.info("Checkpoints Downloaded Successfully")
def mfold_result(task_id,
zipname="now.zip",
path="./results/mfold/",
verbose=True):
"""Gets mfold result via task_id
Args:
task_id: Id of task which was given by RESTful API
zipname: Name of zip file in which client will save results.
After save this file is removed
path: Path where results should be stored
verbose: Bool which tells if function should print what she actualy does
Returns:
None
"""
req = get_request("mfold_result", task_id)
with open(zipname, "wb") as f:
for chunk in req.iter_content():
f.write(chunk)
unzip(zipname, path)
os.remove(zipname)
if verbose:
print("Result under: {}/\n".format(path + task_id))
def update(self):
# Step 1: Get newest nightly folder
response = urllib2.urlopen(self.nightly_dir+"/?C=N;O=D")
html = response.read()
folder_id = re.findall("[0-9]{5,}", html)[0]
# Step 2: Find the correct file
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id)
html = response.read()
exec_file = re.findall("jsshell-win32.zip", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.json", html)[0]
# Step 3: Get build information
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id+"/"+json_file)
html = response.read()
info = json.loads(html)
# Step 4: Fetch archive
print "Retrieving", self.nightly_dir+"/"+folder_id+"/"+exec_file
urllib.urlretrieve(self.nightly_dir+"/"+folder_id+"/"+exec_file, self.tmp_dir + "shell.zip")
# Step 5: Unzip
utils.unzip(self.tmp_dir,"shell.zip")
# Step 6: Save info
self.updated = True
self.cset = info["moz_source_stamp"]
def parse_source(source, idx, header):
"""
Import data from a single source based on the data type.
"""
path = '{}/{}'.format(config.workspace_dir, idx)
if not os.path.exists(path):
os.makedirs(path)
cache_url = source[header.index('cache')]
cache_filename = re.search('/[^/]*$', cache_url).group()
fetch(cache_url, path + cache_filename)
files = rlistdir(path)
for f in files:
if re.match('.*\.(zip|obj|exe)$', f): # some files had mislabelled ext
unzip(f, path)
shapes = []
files = rlistdir(path)
for f in files:
if re.match('.*\.({})$'.format('|'.join(config.fiona_extensions)), f):
objs = import_with_fiona(f, source[0])
for obj in objs:
shapes.append(obj)
elif re.match('.*\.csv$', f):
objs = import_csv(f, source[0])
for obj in objs:
shapes.append(obj)
shutil.rmtree(path)
if not shapes:
_L.warning('failed to parse source. did not find shapes. files in archive: {}'.format(files))
return shapes
def update(self):
# Step 1: Get latest succesfull build revision
response = urllib2.urlopen(self.nightly_dir + "LAST_CHANGE")
chromium_rev = response.read()
# Step 3: Get v8 revision
response = urllib2.urlopen(self.nightly_dir + chromium_rev +
"/REVISIONS")
self.cset = re.findall('"v8_revision_git": "([a-z0-9]*)",',
response.read())[0]
# Step 3: Test if there is a new revision
utils.getOrDownload(
self.tmp_dir, "chrome", self.cset,
self.nightly_dir + chromium_rev + "/" + self.filename,
self.tmp_dir + self.filename)
# Step 4: Unzip
utils.unzip(self.tmp_dir, self.filename)
# Step 5: Install on device
if self.slaveType == "android":
print subprocess.check_output([
"adb", "install", "-r",
self.tmp_dir + "/chrome-android/apks/ChromeShell.apk"
])
# Step 6: Save info
self.updated = True
def install_Windows(self):
# ctags
ctags_zip = os.path.join(DOWNLOADS, CTAGS_VER + '.zip')
if not isfile(ctags_zip):
wget(
'http://sourceforge.net/projects/ctags/files/ctags/5.8/{}.zip'.
format(CTAGS_VER), ctags_zip)
unzip(ctags_zip, CTAGS_VER + '/ctags.exe')
def eval(ucloud=False, model_name='resnet50'):
if model_name == 'resnet50':
batch_size = 32
image_size = 224
if model_name == 'inception_resnet_v2':
batch_size = 16
image_size = 299
if ucloud:
image_dir_testA = '/data/data/ai_challenger_scene_test_a_20180103/scene_test_a_images_20180103/'
image_dir_testB = '/data/data/ai_challenger_scene_test_b_20180103/scene_test_b_images_20180103/'
weights_path = '/data/code/imagenet_models/scene-output-trained_weights_all_layers_1.h5'
submit_file_testA = '/data/output/testA_submit.json'
submit_file_testB = '/data/output/testB_submit.json'
reference_file_testA = '/data/data/ai_challenger_scene_test_a_20180103/scene_test_a_annotations_20180103.json'
reference_file_testB = '/data/data/ai_challenger_scene_test_b_20180103/scene_test_b_annotations_20180103.json'
testA_zip = '/data/data/ai_challenger_scene_test_a_20180103.zip'
testB_zip = '/data/data/ai_challenger_scene_test_b_20180103.zip'
zip_output = '/data/data/'
unzip(testA_zip, zip_output)
unzip(testB_zip, zip_output)
creat_submit(image_dir_testA, submit_file_testA, image_size,
batch_size, weights_path)
creat_submit(image_dir_testB, submit_file_testB, image_size,
batch_size, weights_path)
result_A = {'error': [], 'warning': [], 'top3': None, 'top1': None}
submit_dict_A, ref_dict_A, result_A = __load_data(
submit_file_testA, reference_file_testA, result_A)
result_A = __eval_result(submit_dict_A, ref_dict_A, result_A)
result_B = {'error': [], 'warning': [], 'top3': None, 'top1': None}
submit_dict_B, ref_dict_B, result_B = __load_data(
submit_file_testB, reference_file_testB, result_B)
result_B = __eval_result(submit_dict_B, ref_dict_B, result_B)
print 'testA result =', result_A
print 'testB result =', result_B
else:
#image_dir = '/Users/xiang/Desktop/classification/test/'
image_dir = '/Users/xiang/Desktop/classification/ai_challenger_scene_test_a_20180103/scene_test_a_images_20180103/'
weights_path = 'imagenet_models/scene-output-res50ep008-loss0.981-val_loss0.779-val_acc0.767.h5'
submit_file = 'submit/testA_res50.json'
reference_file = '/Users/xiang/Desktop/classification/ai_challenger_scene_test_a_20180103/scene_test_a_annotations_20180103.json'
creat_submit(image_dir, submit_file, image_size, batch_size,
weights_path)
result = {'error': [], 'warning': [], 'top3': None, 'top1': None}
submit_dict, ref_dict, result = __load_data(submit_file,
reference_file, result)
result = __eval_result(submit_dict, ref_dict, result)
print result
def __init__(self, root):
self.root = root
if not posixpath.exists(posixpath.join(self.root, self.ukbench_dir)):
download(self.root, self.filename, self.url)
unzip(self.root, self.filename, self.ukbench_dir)
self.uris = sorted(
list_files(root=posixpath.join(self.root, self.ukbench_dir,
'full'),
suffix=('png', 'jpg', 'jpeg', 'gif')))
def __init__(self):
super(Shumway, self).__init__('shumway', '0.1', 'shumway')
# Only update harness once a day:
from datetime import datetime
date = datetime.now().strftime("%Y-%m-%d")
utils.getOrDownload("/tmp/", "shumway", date,
"http://mozilla.github.io/shumway/shell/shumway-shell.zip",
"/tmp/shumway-shell.zip")
utils.unzip("/tmp/", "shumway-shell.zip")
def unzip_news2016zh():
path = settings.NEWS2016ZH_PATH
if os.path.exists(path):
logger.info(f'news2016zh already decompressed.')
return
zip_path = settings.NEWS2016ZH_ZIP_PATH
folder = settings.FOLDER
logger.info('news2016zh decompressing ...')
unzip(zip_path, folder)
logger.info(f'news2016zh decompressed to {folder}')
def unpack(meta):
src_path = download_to_cache(meta)
os.makedirs(WORK_DIR)
if src_path.endswith(('.tar.gz', '.tar.bz2', '.tgz', '.tar.xz', '.tar')):
tar_xf(src_path, WORK_DIR)
elif src_path.endswith('.zip'):
unzip(src_path, WORK_DIR)
else:
raise Exception("not a vaild source")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
if dest is None:
dest = os.pwd
url = self._latest_chromedriver_url(browser_binary)
self.logger.info("Downloading ChromeDriver from %s" % url)
unzip(get(url).raw, dest)
chromedriver_dir = os.path.join(dest, 'chromedriver_%s' % self.platform_string())
if os.path.isfile(os.path.join(chromedriver_dir, "chromedriver")):
shutil.move(os.path.join(chromedriver_dir, "chromedriver"), dest)
shutil.rmtree(chromedriver_dir)
return find_executable("chromedriver", dest)
def __init__(self):
super(Shumway, self).__init__('shumway', '0.1', 'shumway')
# Only update harness once a day:
from datetime import datetime
date = datetime.now().strftime("%Y-%m-%d")
utils.getOrDownload(
"/tmp/", "shumway", date,
"http://mozilla.github.io/shumway/shell/shumway-shell.zip",
"/tmp/shumway-shell.zip")
utils.unzip("/tmp/", "shumway-shell.zip")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
if dest is None:
dest = os.pwd
url = self._latest_chromedriver_url(browser_binary)
self.logger.info("Downloading ChromeDriver from %s" % url)
unzip(get(url).raw, dest)
chromedriver_dir = os.path.join(dest, 'chromedriver_%s' % self.platform_string())
if os.path.isfile(os.path.join(chromedriver_dir, "chromedriver")):
shutil.move(os.path.join(chromedriver_dir, "chromedriver"), dest)
shutil.rmtree(chromedriver_dir)
return find_executable("chromedriver", dest)
def install_prefs(self, binary, dest=None, channel=None):
if binary:
version, channel_ = self.get_version_and_channel(binary)
if channel is not None and channel != channel_:
# Beta doesn't always seem to have the b in the version string, so allow the
# manually supplied value to override the one from the binary
self.logger.warning(
"Supplied channel doesn't match binary, using supplied channel"
)
elif channel is None:
channel = channel_
else:
version = None
if dest is None:
dest = os.curdir
dest = os.path.join(dest, "profiles", channel)
if version:
dest = os.path.join(dest, version)
have_cache = False
if os.path.exists(dest) and len(os.listdir(dest)) > 0:
if channel != "nightly":
have_cache = True
else:
now = datetime.now()
have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
now - timedelta(days=1))
# If we don't have a recent download, grab and extract the latest one
if not have_cache:
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(dest)
url = self.get_profile_bundle_url(version, channel)
self.logger.info("Installing test prefs from %s" % url)
try:
extract_dir = tempfile.mkdtemp()
unzip(get(url).raw, dest=extract_dir)
profiles = os.path.join(extract_dir,
os.listdir(extract_dir)[0], 'testing',
'profiles')
for name in os.listdir(profiles):
path = os.path.join(profiles, name)
shutil.move(path, dest)
finally:
shutil.rmtree(extract_dir)
else:
self.logger.info("Using cached test prefs from %s" % dest)
return dest
def install_webdriver(self, dest=None):
if dest is None:
dest = os.pwd
latest = get("http://chromedriver.storage.googleapis.com/LATEST_RELEASE").text.strip()
url = "http://chromedriver.storage.googleapis.com/%s/chromedriver_%s.zip" % (latest,
self.platform_string())
unzip(get(url).raw, dest)
path = find_executable("chromedriver", dest)
st = os.stat(path)
os.chmod(path, st.st_mode | stat.S_IEXEC)
return path
def install_webdriver(self, dest=None):
"""Install latest Webdriver."""
if dest is None:
dest = os.pwd
latest = get("http://chromedriver.storage.googleapis.com/LATEST_RELEASE").text.strip()
url = "http://chromedriver.storage.googleapis.com/%s/chromedriver_%s.zip" % (latest,
self.platform_string())
unzip(get(url).raw, dest)
path = find_executable(dest, "chromedriver")
st = os.stat(path)
os.chmod(path, st.st_mode | stat.S_IEXEC)
return path
def install_webdriver_by_version(self, version, dest=None):
assert version, "Cannot install ChromeDriver without Chrome version"
if dest is None:
dest = os.pwd
url = self._latest_chromedriver_url(version)
self.logger.info("Downloading ChromeDriver from %s" % url)
unzip(get(url).raw, dest)
chromedriver_dir = os.path.join(
dest, 'chromedriver_%s' % self.platform_string())
if os.path.isfile(os.path.join(chromedriver_dir, "chromedriver")):
shutil.move(os.path.join(chromedriver_dir, "chromedriver"), dest)
shutil.rmtree(chromedriver_dir)
return find_executable("chromedriver", dest)
def _update(self, folder_id):
# Step 2: Find the correct file
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id)
html = response.read()
if self.slaveType == "android":
exec_file = re.findall("fennec-[a-zA-Z0-9.]*.en-US.android-arm.apk", html)[0]
json_file = re.findall("fennec-[a-zA-Z0-9.]*.en-US.android-arm.json", html)[0]
elif self.slaveType == "mac-desktop":
exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.mac.dmg", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.mac.json", html)[0]
elif self.slaveType == "linux-desktop":
exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.linux-x86_64.tar.bz2", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.linux-x86_64.json", html)[0]
else:
exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.zip", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.json", html)[0]
# Step 3: Get build information
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id+"/"+json_file)
html = response.read()
info = json.loads(html)
# Step 4: Test if there is a new revision
if self.slaveType == "android":
output = self.tmp_dir + self.folder + "/fennec.apk"
elif self.slaveType == "mac-desktop":
output = self.tmp_dir + self.folder + "/firefox.dmg"
elif self.slaveType == "linux-desktop":
output = self.tmp_dir + self.folder + "/firefox.tar.bz2"
else:
output = self.tmp_dir + self.folder + "/firefox.zip"
utils.getOrDownload(self.tmp_dir, "mozilla", info["moz_source_stamp"],
self.nightly_dir + "/" + folder_id + "/" + exec_file,
output)
# Step 5: Prepare to run
if self.slaveType == "android":
print subprocess.check_output(["adb", "install", "-r", self.tmp_dir + self.folder + "/fennec.apk"])
elif self.slaveType == "mac-desktop":
if os.path.exists("/Volumes/Nightly"):
print subprocess.check_output(["hdiutil", "detach", "/Volumes/Nightly"])
print subprocess.check_output(["hdiutil", "attach", self.tmp_dir + self.folder + "/firefox.dmg"])
elif self.slaveType == "linux-desktop":
utils.unzip(self.tmp_dir + self.folder, "firefox.tar.bz2")
else:
utils.unzip(self.tmp_dir + self.folder, "firefox.zip")
# Step 6: Save info
self.updated = True
self.cset = info["moz_source_stamp"]
def load_MovieLens_1m_dataset(target="cf"):
'''
Load MovieLens 1m dataset
:returns: features, target_values, users, movies, ratings, data
'''
dataset_name = "ml-1m"
data_path = os.path.join(movielens_path, dataset_name)
pkl_path = os.path.join(data_path, "%s.pkl" % dataset_name)
dataset = None
if not os.path.exists(pkl_path):
url = "http://files.grouplens.org/datasets/movielens/%s.zip" % dataset_name
print("Downloading MovieLens %s dataset..." % dataset_name)
utils.download(url, data_path + ".zip")
utils.unzip("%s.zip" % data_path, movielens_path)
# Read data
user_names = [user_name, "gender", "age", "occupationId", "zip-code"]
movie_names = [movie_name, "Title", "Genres"]
rating_names = [user_name, movie_name, rating_name, "timestamp"]
users = pd.read_csv(os.path.join(data_path, "users.dat"), sep="::", header=None, names=user_names, engine="python")
movies = pd.read_csv(os.path.join(data_path, "movies.dat"), sep="::", header=None, names=movie_names, engine="python")
ratings = pd.read_csv(os.path.join(data_path, "ratings.dat"), sep="::", header=None, names=rating_names, engine="python")
# Preprocessing
# Merge data
data = pd.merge(pd.merge(ratings.filter(regex='userId|movieId|rating'), users), movies)
# Separate features and rating values
features = data.drop([rating_name], axis = 1).values # drop rating column
target_values = data[rating_name].values
dataset = (features, target_values, users, movies, ratings, data)
# Save preprocessed data
with open(pkl_path, "wb") as f:
pickle.dump(dataset, f)
else:
with open(pkl_path, mode="rb") as f:
dataset = pickle.load(f)
if target is "cf":
return convert_to_cf_data(dataset[5])
else:
return dataset
def _update(self, folder_id):
# Step 2: Find the correct file
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id)
html = response.read()
if self.slaveType == "android":
exec_file = re.findall("fennec-[a-zA-Z0-9.]*.en-US.android-arm.apk", html)[0]
json_file = re.findall("fennec-[a-zA-Z0-9.]*.en-US.android-arm.json", html)[0]
elif self.slaveType == "mac-desktop":
exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.mac.dmg", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.mac.json", html)[0]
elif self.slaveType == "linux-desktop":
exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.linux-x86_64.tar.bz2", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.linux-x86_64.json", html)[0]
else:
exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.zip", html)[0]
json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.json", html)[0]
# Step 3: Get build information
response = urllib2.urlopen(self.nightly_dir+"/"+folder_id+"/"+json_file)
html = response.read()
info = json.loads(html)
# Step 4: Test if there is a new revision
if self.slaveType == "android":
output = self.tmp_dir + self.folder + "/fennec.apk"
elif self.slaveType == "mac-desktop":
output = self.tmp_dir + self.folder + "/firefox.dmg"
elif self.slaveType == "linux-desktop":
output = self.tmp_dir + self.folder + "/firefox.tar.bz2"
else:
output = self.tmp_dir + self.folder + "/firefox.zip"
utils.getOrDownload(self.tmp_dir, "mozilla", info["moz_source_stamp"],
self.nightly_dir + "/" + folder_id + "/" + exec_file,
output)
# Step 5: Prepare to run
if self.slaveType == "android":
print subprocess.check_output(["adb", "install", "-r", self.tmp_dir + self.folder + "/fennec.apk"])
elif self.slaveType == "mac-desktop":
if os.path.exists("/Volumes/Nightly"):
print subprocess.check_output(["hdiutil", "detach", "-force", "/Volumes/Nightly"])
print subprocess.check_output(["hdiutil", "attach", self.tmp_dir + self.folder + "/firefox.dmg"])
elif self.slaveType == "linux-desktop":
utils.unzip(self.tmp_dir + self.folder, "firefox.tar.bz2")
else:
utils.unzip(self.tmp_dir + self.folder, "firefox.zip")
# Step 6: Save info
self.updated = True
self.cset = info["moz_source_stamp"]
def install_webdriver(self, dest=None):
"""Install latest Geckodriver."""
if dest is None:
dest = os.getcwd()
version = self._latest_geckodriver_version()
format = "zip" if uname[0] == "Windows" else "tar.gz"
logger.debug("Latest geckodriver release %s" % version)
url = ("https://github.com/mozilla/geckodriver/releases/download/%s/geckodriver-%s-%s.%s" %
(version, version, self.platform_string_geckodriver(), format))
if format == "zip":
unzip(get(url).raw, dest=dest)
else:
untar(get(url).raw, dest=dest)
return find_executable(os.path.join(dest, "geckodriver"))
def install_webdriver(self, dest=None):
"""Install latest Geckodriver."""
if dest is None:
dest = os.getcwd()
version = self._latest_geckodriver_version()
format = "zip" if uname[0] == "Windows" else "tar.gz"
logger.debug("Latest geckodriver release %s" % version)
url = ("https://github.com/mozilla/geckodriver/releases/download/%s/geckodriver-%s-%s.%s" %
(version, version, self.platform_string_geckodriver(), format))
if format == "zip":
unzip(get(url).raw, dest=dest)
else:
untar(get(url).raw, dest=dest)
return find_executable(os.path.join(dest, "geckodriver"))
def download(_):
"""
Start checkpoints download logic.
:param _: None
:return: None
"""
Conf.log = setup_log(logging.DEBUG) if Conf.args['debug'] else setup_log()
tempdir = tempfile.mkdtemp()
cdn_url = Conf.checkpoints_cdn.format(Conf.checkpoints_version)
temp_zip = os.path.join(tempdir, "{}.zip".format(Conf.checkpoints_version))
try:
Conf.log.info("Downloading {}".format(cdn_url))
dl_file(Conf.checkpoints_cdn.format(Conf.checkpoints_version),
temp_zip)
if not os.path.exists(Conf.args['checkpoints']['checkpoints_path']):
os.mkdir(Conf.args['checkpoints']['checkpoints_path'])
Conf.log.info("Extracting {}".format(temp_zip))
unzip(temp_zip, Conf.args['checkpoints']['checkpoints_path'])
Conf.log.info("Moving Checkpoints To Final Location")
for c in ("cm.lib", "mm.lib", "mn.lib"):
if os.path.isfile(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
c)):
os.remove(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
c))
shutil.move(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
'checkpoints', c),
Conf.args['checkpoints']['checkpoints_path'])
shutil.rmtree(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
'checkpoints'))
except Exception as e:
Conf.log.error(e)
Conf.log.error(
"Something Gone Bad Download Downloading The Checkpoints")
shutil.rmtree(tempdir)
sys.exit(1)
shutil.rmtree(tempdir)
Conf.log.info("Checkpoints Downloaded Successfully")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
if dest is None:
dest = os.pwd
latest = get("https://api.github.com/repos/operasoftware/operachromiumdriver/releases/latest").json()["tag_name"]
url = "https://github.com/operasoftware/operachromiumdriver/releases/download/%s/operadriver_%s.zip" % (latest,
self.platform_string())
unzip(get(url).raw, dest)
operadriver_dir = os.path.join(dest, "operadriver_%s" % self.platform_string())
shutil.move(os.path.join(operadriver_dir, "operadriver"), dest)
shutil.rmtree(operadriver_dir)
path = find_executable("operadriver")
st = os.stat(path)
os.chmod(path, st.st_mode | stat.S_IEXEC)
return path
def fetch_sub(location, directory=None, token=None):
"""Filename is the full path of the movie file
dirname is the full path of the movie dir, fetch_sub will search for video file
diretory is the directory where sub should be stored, default to the movie dir
"""
my_open_sub = MyOpenSub(OPENSUB_URL, token)
if not token:
my_open_sub.login(USERNAME, PASSWORD, LANGUAGE, USER_AGENT)
filenames = find_video(location)
if not filenames:
print "No video file found"
return
for filename in find_video(location):
print "Fetching subtitle for %s" % filename
file_hash, file_bytes = hashfile(filename)
dl_links = my_open_sub.search_sub_links(file_hash, file_bytes)
if not dl_links:
print "Sorry, no subtitle found"
return
if not directory:
directory = os.path.dirname(filename)
for dl_link in dl_links:
try:
filename = wget.download(dl_link, out=directory)
print "Download finished: %s" % filename
filename = unzip(filename, directory)
if filename:
print "Unzipped to %s" % filename
except IOError as io_error:
print io_error
def submit(request):
team = get_team(request.user)
params = dict()
if request.method == 'POST':
form = forms.UploadSubmissionForm(request.POST, request.FILES)
params['form'] = form
if form.is_valid():
submission = Submission(
team=team,
package=request.FILES['file'],
command=request.POST['command'],
)
submission.save()
error = utils.unzip(submission.package.path)
if error:
submission.delete()
params['error'] = error
return render_submit(request, params)
try:
execute_tester(submission)
except Exception as error:
print u'ERROR: Blad wewnetrzny testerki:', error
return my_results(request,
message=_(u'Rozwiązanie zostało wysłane.'))
return render_submit(request, params)
def save_progress(options, tparams, epoch, best_perf):
sd = options.get('seed', -99)
fld = options.get('fold', -99)
with open(state_file_name(options) + '.tmp', 'a') as w:
w.write('%d,%d,%d,%f,%f\n' %
(epoch, sd, fld, best_perf[0], best_perf[1]))
pickle.dump(unzip(tparams), open(state_file_name(options) + '.pkl', 'w'))
def ordenar_vecinos(self):
if self.vecinos_ordenados:
return
for u in self.vertices():
self.lista_vecinos[u], self.pesos[u] = unzip(
sorted(list(zip(*self.vecinos(u)))))
self.vecinos_ordenados = True
def unzip_data():
if not os.path.exists(data_dir):
os.makedirs(data_dir)
fnames = [
"asian.zip",
"africanamerican.zip",
"caucasian.zip",
"hispanic.zip",
"multiracial.zip",
]
for fname in fnames:
print(".. unzipping")
unzip(os.path.join(data_dir, fname), data_dir)
def submit(request):
team = get_team(request.user)
params = dict()
if request.method == 'POST':
form = forms.UploadSubmissionForm(request.POST, request.FILES)
params['form'] = form
if form.is_valid():
submission = Submission(
team=team,
package=request.FILES['file'],
command=request.POST['command'],
)
submission.save()
error = utils.unzip(submission.package.path)
if error:
submission.delete()
params['error'] = error
return render_submit(request, params)
try:
execute_tester(submission)
except Exception as error:
print u'ERROR: Blad wewnetrzny testerki:', error
return my_results(
request, message=_(u'Rozwiązanie zostało wysłane.'))
return render_submit(request, params)
def install_webdriver(self, dest=None):
if dest is None:
dest = os.pwd
latest = get("https://api.github.com/repos/operasoftware/operachromiumdriver/releases/latest").json()["tag_name"]
url = "https://github.com/operasoftware/operachromiumdriver/releases/download/%s/operadriver_%s.zip" % (latest,
self.platform_string())
unzip(get(url).raw, dest)
operadriver_dir = os.path.join(dest, "operadriver_%s" % self.platform_string())
shutil.move(os.path.join(operadriver_dir, "operadriver"), dest)
shutil.rmtree(operadriver_dir)
path = find_executable("operadriver")
st = os.stat(path)
os.chmod(path, st.st_mode | stat.S_IEXEC)
return path
def install_agent(self, package_path):
checksum = utils.get_checksum(package_path, self.get_checksum_dir())
if not utils.check_file(package_path, checksum):
LOG.debug("check downloaded package failed, removing %s", package_path)
os.remove(package_path)
utils.remove_checksum(package_path, self.get_checksum_dir())
return None
dirname = utils.get_file_name_without_extension(package_path)
agent_dir = '%s\%s' % (self.get_install_dir(), dirname)
tmp_dir = '%s\%s' % (self.get_tmp_dir(), dirname)
try:
if utils.unzip(package_path, tmp_dir):
tmp_name = os.listdir(tmp_dir)[0]
tmp_path = '%s\%s' % (tmp_dir, tmp_name)
ren_path = '%s\%s' % (tmp_dir, dirname)
if os.system('ren "%s" "%s"' % (tmp_path, dirname)) != 0:
return None
if os.system('move "%s" "%s"' % (ren_path, self.get_install_dir())) != 0:
return None
else:
return None
finally:
if os.path.isdir(tmp_dir):
os.system('rd /s /q "%s"' % tmp_dir)
installer = Installer(agent_dir, self.recycle_queue)
installer.install()
return agent_dir
def get_shape_from_coords(coords):
""" Return shape of smallest well plate that can contain provided indices."""
well_plate_shapes = [(2,4),(8,12)] # 8 well slide, 96 well plate
rows,cols = unzip(coords)
for shape in well_plate_shapes:
if rows |are_all_in| range(shape[0]) and cols |are_all_in| range(shape[1]):
return shape
raise Exception("Given well coordinates do not fit in plate shapes:{}".format(well_plate_shapes))
def main(url=ZIP_CODE_URL):
path = utils.DATASET_HOME + ADDRESS_ZIP
utils.download(url, path)
files = utils.unzip(path)
if files and len(files) > 0:
write_address(files[0])
else:
print("failed to download or unzip the file. please see at {0}.".format(utils.DATASET_HOME))
def install_prefs(self, binary, dest=None, channel=None):
version, channel_ = self.get_version_and_channel(binary)
if channel is not None and channel != channel_:
# Beta doesn't always seem to have the b in the version string, so allow the
# manually supplied value to override the one from the binary
self.logger.warning("Supplied channel doesn't match binary, using supplied channel")
elif channel is None:
channel = channel_
if dest is None:
dest = os.pwd
dest = os.path.join(dest, "profiles", channel)
if version:
dest = os.path.join(dest, version)
have_cache = False
if os.path.exists(dest):
if channel != "nightly":
have_cache = True
else:
now = datetime.now()
have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
now - timedelta(days=1))
# If we don't have a recent download, grab and extract the latest one
if not have_cache:
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(dest)
url = self.get_profile_bundle_url(version, channel)
self.logger.info("Installing test prefs from %s" % url)
try:
extract_dir = tempfile.mkdtemp()
unzip(get(url).raw, dest=extract_dir)
profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles')
for name in os.listdir(profiles):
path = os.path.join(profiles, name)
shutil.move(path, dest)
finally:
shutil.rmtree(extract_dir)
else:
self.logger.info("Using cached test prefs from %s" % dest)
return dest
def imputResource():
flag = utils.unzip(intputLabel['text'], outputLabel['text'])
# print (intputLabel['text'] + ':' + outputLabel['text'] + ':' +xmlPathLabel['text'] )
if flag:
print("导入资源成功")
global resFlag
resFlag = True
def install_certutil(self, dest=None):
# TODO: this doesn't really work because it just gets the binary, and is missing the
# libnss3 library. Getting that means either downloading the corresponding Firefox
# and extracting the library (which is hard on mac becase DMG), or maybe downloading from
# nss's treeherder builds?
if dest is None:
dest = os.pwd
# Don't create a path like bin/bin/certutil
split = os.path.split(dest)
if split[1] == "bin":
dest = split[0]
resp = self.get_from_nightly(
"<a[^>]*>(firefox-\d+\.\d(?:\w\d)?.en-US.%s\.common\.tests.zip)</a>" % self.platform_string())
bin_path = path("bin/certutil", exe=True)
unzip(resp.raw, dest=dest, limit=[bin_path])
return os.path.join(dest, bin_path)
def mfold_result(task_id, zipname="now.zip", path="./results/mfold/", verbose=True):
"""Gets mfold result via task_id
Args:
task_id: Id of task which was given by RESTful API
zipname: Name of zip file in which client will save results.
After save this file is removed
path: Path where results should be stored
verbose: Bool which tells if function should print what she actualy does
Returns:
None
"""
req = get_request("mfold_result", task_id)
with open(zipname, "wb") as f:
for chunk in req.iter_content():
f.write(chunk)
unzip(zipname, path)
os.remove(zipname)
if verbose:
print("Result in: {}/".format(path + task_id))
def update(self):
# Step 1: Get latest succesfull build revision
response = urllib2.urlopen(self.nightly_dir+"LAST_CHANGE")
chromium_rev = response.read()
# Step 3: Get v8 revision
response = urllib2.urlopen(self.nightly_dir + chromium_rev + "/REVISIONS")
self.cset = re.findall('"v8_revision_git": "([a-z0-9]*)",', response.read())[0]
# Step 3: Test if there is a new revision
utils.getOrDownload(self.tmp_dir, "chrome", self.cset,
self.nightly_dir + chromium_rev + "/" + self.filename,
self.tmp_dir + self.filename)
# Step 4: Unzip
utils.unzip(self.tmp_dir, self.filename)
# Step 5: Install on device
if self.slaveType == "android":
print subprocess.check_output(["adb", "install", "-r", self.tmp_dir+"/chrome-android/apks/ChromeShell.apk"])
# Step 6: Save info
self.updated = True
def validate_and_save_checkpoint(model_options, dp, params, tparams, f_pred, f_pred_prob, kf_valid, save_n):
scores = prediction(f_pred, f_pred_prob, dp.prepare_valid_or_test_batch_image_data, 'val', kf_valid, dp.ix_to_word, dp.get_raw_sentences_from_imgid, model_options, save_n['prediction'])
# saving a checkpoint
save_path = os.path.join(model_options['checkpoint_save_dir'], "lstm_{0}_{1:.2f}.npz".format(save_n['checkpoint'], scores['Bleu_4'] * 100))
params = utils.unzip(tparams)
numpy.savez(save_path, checkpoint_save_n=save_n['checkpoint'], scores=scores, **params)
pickle.dump(model_options, open('%s.pkl' % save_path, 'wb'), -1)
print 'Saved checkpoint to', os.path.abspath(save_path)
save_n['checkpoint'] = save_n['checkpoint'] + 1
save_n['prediction'] = save_n['prediction'] + 1
return scores
def install_webdriver(self, dest=None, channel=None):
"""Install latest Geckodriver."""
if dest is None:
dest = os.getcwd()
if channel == "nightly":
path = self.install_geckodriver_nightly(dest)
if path is not None:
return path
else:
self.logger.warning("Nightly webdriver not found; falling back to release")
version = self._latest_geckodriver_version()
format = "zip" if uname[0] == "Windows" else "tar.gz"
self.logger.debug("Latest geckodriver release %s" % version)
url = ("https://github.com/mozilla/geckodriver/releases/download/%s/geckodriver-%s-%s.%s" %
(version, version, self.platform_string_geckodriver(), format))
if format == "zip":
unzip(get(url).raw, dest=dest)
else:
untar(get(url).raw, dest=dest)
return find_executable(os.path.join(dest, "geckodriver"))
def install_prefs(self, binary, dest=None):
version, channel = self.get_version_and_channel(binary)
if dest is None:
dest = os.pwd
dest = os.path.join(dest, "profiles", channel, version)
have_cache = False
if os.path.exists(dest):
if channel != "nightly":
have_cache = True
else:
now = datetime.now()
have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
now - timedelta(days=1))
# If we don't have a recent download, grab and extract the latest one
if not have_cache:
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(dest)
url = self.get_profile_bundle_url(version, channel)
print("Installing test prefs from %s" % url)
try:
extract_dir = tempfile.mkdtemp()
unzip(get(url).raw, dest=extract_dir)
profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles')
for name in os.listdir(profiles):
path = os.path.join(profiles, name)
shutil.move(path, dest)
finally:
shutil.rmtree(extract_dir)
else:
print("Using cached test prefs from %s" % dest)
return dest
def get_gobstones_tests(self):
tests = []
test_groups = self.get_test_groups()
for tgroup, index in zip(test_groups, range(len(test_groups))):
ops, pyfs = unzip(tgroup)
tests.append(AutoGobstonesTest(
"group %s, %s tests" % (index, len(ops)),
self,
ops,
self.program_for(ops),
pyfs,
self.gobstones_parameters()
))
return tests
def run(self):
result_to_op = []
for op in self.operations:
result_to_op.extend([op for x in range(op.nretvals)])
#print "\tRunning subtest %s" % (op,)
run_params = [
temp_test_file(self.gbscode),
os.path.dirname(__file__) + "/boards/empty.gbb",
self.gbsparams
]
results = run_gobstones(*run_params)
if results[0] == "OK":
gbsres = results[1]
pyres = []
for f in self.pyfuncs:
pyr = f()
if isinstance(pyr, tuple):
pyres += list(pyr)
else:
pyres.append(pyr)
if len(pyres) == len(gbsres):
for gbsval, pyval, index in zip(
unzip(gbsres)[1],
pyres,
range(len(pyres))
):
self.assertEqual(
gbsval,
str(pyval),
("Operation %s failed. The result %s do not match " +
"the expected value. " +
"Expected: %s. Actual: %s"
) % (
result_to_op[index],
index,
pyval,
gbsval
)
)
else:
self.fail("The execution results count do " +
"not match the expected results count")
else:
self.fail(results[1])
def _fetch_pcsapi(self, path, params=None, data=None, headers={}):
assert self._bduss is not None
url = urljoin(self._pcsapi_baseUrl, path) + "?app_id=266719"
if params:
url += "&" + urlencode(params)
headers["Cookie"] = "BDUSS=" + self._bduss
try:
r = fetch_url(url, data, headers)
except urllib2.HTTPError as e:
try:
error_content = e.read()
if e.headers.get("content-encoding") == "gzip":
error_content = unzip(error_content)
eo = json.loads(error_content)
except:
raise e
else:
raise PCSApiError(eo.get("error_code"), eo.get("error_msg"))
return json.loads(r)
def _fetch_pcsapi(self, path, params=None, data=None, headers={}):
assert self._bduss is not None
url = urljoin(self._pcsapi_baseUrl, path) + '?app_id=266719'
if params:
url += '&' + urlencode(params)
headers['Cookie'] = 'BDUSS=' + self._bduss
try:
r = fetch_url(url, data, headers)
except urllib2.HTTPError as e:
try:
error_content = e.read()
if e.headers.get('content-encoding') == 'gzip':
error_content = unzip(error_content)
eo = json.loads(error_content)
except:
raise e
else:
raise PCSApiError(eo.get('error_code'), eo.get('error_msg'))
return json.loads(r)
def submit(request):
team = request.user.team
params = dict()
if request.method == 'POST':
form = UploadSubmissionForm(request.POST, request.FILES)
params['form'] = form
if form.is_valid():
submission = Submission(
team=team,
package=request.FILES['file'],
user=request.user,
)
submission.save()
error = utils.unzip(submission.package.path)
if error:
submission.delete()
params['error'] = error
return render_submit(request, params)
submissions = team.submission_set.all()
if len(submissions) > 2:
for sub in submissions[2:]:
sub.delete()
execute_tester(submission)
messages.add_message(request, messages.INFO, _(u'Rozwiązanie zostało wysłane'))
return redirect('my_results')
#return my_results(
# request, message=_(u'Rozwiązanie zostało wysłane.'))
else:
print form.errors
return render_submit(request, params)
def setupTests(self):
zippedTests = download_url(getTestUrl(),dest=str(os.path.join(self.shellCacheDir,"tests.zip")))
unzip(self.testDir,zippedTests)
def train(dim_word=100, # word vector dimensionality
dim=1000, # the number of GRU units
encoder='gru',
patience=10, # early stopping patience
max_epochs=5000,
finish_after=10000000, # finish after this many updates
dispFreq=100,
decay_c=0., # L2 weight decay penalty
lrate=0.01,
n_words=100000, # vocabulary size
vocab_dim=100000, # Size of M, C
memory_dim=1000, # Dimension of memory
memory_size=15, # n_back to attend
maxlen=100, # maximum length of the description
optimizer='rmsprop',
batch_size=16,
valid_batch_size=16,
saveto='model.npz',
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
sampleFreq=100, # generate some samples after every sampleFreq
dataset='/data/lisatmp3/chokyun/wikipedia/extracted/wiki.tok.txt.gz',
valid_dataset='../data/dev/newstest2011.en.tok',
dictionary='/data/lisatmp3/chokyun/wikipedia/extracted/'
'wiki.tok.txt.gz.pkl',
use_dropout=False,
reload_=False):
# Model options
model_options = locals().copy()
# Theano random stream
trng = RandomStreams(1234)
# load dictionary
with open(dictionary, 'rb') as f:
worddicts = pkl.load(f)
# invert dictionary
worddicts_r = dict()
for kk, vv in worddicts.iteritems():
worddicts_r[vv] = kk
# reload options
if reload_ and os.path.exists(saveto):
with open('%s.pkl' % saveto, 'rb') as f:
model_options = pkl.load(f)
print 'Loading data'
train = TextIterator(dataset,
dictionary,
n_words_source=n_words,
batch_size=batch_size,
maxlen=maxlen)
valid = TextIterator(valid_dataset,
dictionary,
n_words_source=n_words,
batch_size=valid_batch_size,
maxlen=maxlen)
# initialize RMN
rmn_ = RMN(model_options)
print 'Building model'
rmn_.init_params()
# reload parameters
if reload_ and os.path.exists(saveto):
rmn_.load_params(saveto)
# create shared variables for parameters
tparams = rmn_.tparams
# build the symbolic computational graph
use_noise, x, x_mask, opt_ret, cost = rmn_.build_model()
inps = [x, x_mask]
print 'Buliding sampler'
f_next = rmn_.build_sampler(trng)
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, cost, profile=profile)
print 'Done'
cost = cost.mean()
# apply L2 regularization on weights
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
# after any regularizer - compile the computational graph for cost
print 'Building f_cost...',
f_cost = theano.function(inps, cost, profile=profile)
print 'Done'
print 'Computing gradient...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Done'
# compile the optimizer, the actual computational graph is compiled here
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
optimizer = getattr(importlib.import_module('optimizer'), optimizer)
f_grad_shared, f_update = optimizer(lr, tparams, grads, inps, cost)
print 'Done'
print 'Optimization'
history_errs = []
uidx = 0
estop = False
bad_counter = 0
# reload history
if reload_ and os.path.exists(saveto):
history_errs = list(numpy.load(saveto)['history_errs'])
uidx = numpy.load(saveto)['uidx']
best_p = None
if validFreq == -1:
validFreq = len(train[0])/batch_size
if saveFreq == -1:
saveFreq = len(train[0])/batch_size
if sampleFreq == -1:
sampleFreq = len(train[0])/batch_size
# Training loop
for eidx in xrange(max_epochs):
n_samples = 0
for x in train:
n_samples += len(x)
uidx += 1
use_noise.set_value(1.)
# pad batch and create mask
x, x_mask = prepare_data(x, maxlen=maxlen, n_words=n_words)
if x is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
# compute cost, grads and copy grads to shared variables
cost = f_grad_shared(x, x_mask)
# do the update on parameters
f_update(lrate)
ud = time.time() - ud_start
# check for bad numbers
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1.
# verbose
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
# save the best model so far
if numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
params = best_p
else:
params = unzip(tparams)
numpy.savez(saveto, history_errs=history_errs, uidx=uidx,
**params)
pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'))
print 'Done'
# generate some samples with the model and display them
if numpy.mod(uidx, sampleFreq) == 0:
# FIXME: random selection?
for jj in xrange(5):
sample, score = rmn_.gen_sample(tparams, f_next,
trng=trng, maxlen=30,
argmax=False)
print 'Sample ', jj, ': ',
ss = sample
for vv in ss:
if vv == 0:
break
if vv in worddicts_r:
print worddicts_r[vv],
else:
print 'UNK',
print
# validate model on validation set and early stop if necessary
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
valid_errs = rmn_.pred_probs(valid, f_log_probs, prepare_data)
valid_err = valid_errs.mean()
history_errs.append(valid_err)
if uidx == 0 or valid_err <= numpy.array(history_errs).min():
best_p = unzip(tparams)
bad_counter = 0
if len(history_errs) > patience and valid_err >= \
numpy.array(history_errs)[:-patience].min():
bad_counter += 1
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if numpy.isnan(valid_err):
ipdb.set_trace()
print 'Valid ', valid_err
# finish after this many updates
if uidx >= finish_after:
print 'Finishing after %d iterations!' % uidx
estop = True
break
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = rmn_.pred_probs(f_log_probs, prepare_data,
model_options, valid).mean()
print 'Valid ', valid_err
params = copy.copy(best_p)
numpy.savez(saveto, zipped_params=best_p,
history_errs=history_errs,
uidx=uidx,
**params)
return valid_err
def ordenar_vecinos(self):
if self.vecinos_ordenados:
return
for u in self.vertices():
self.lista_vecinos[u], self.pesos[u] = unzip(sorted(list(zip(*self.vecinos(u)))))
self.vecinos_ordenados = True