def manual_lock(self, fpath, expiration_check=True):
"""
Spins until a lock is acquired on a file. `fpath` can be absolute or relative and error checking will be done for you.
Also checks if a lock is expired (as long as `expiration_check=True`) and automatically unlocks it if so.
Doesn't return anything.
*Lock must be freed with fs.unlock() at the end of use.*
See `lock()` for a contextmanager version of this with unlocks at the end for you, and should be used the vast majority of the time.
"""
#if self.verbose: print(f"manual_lock({fpath}) called")
paths = self.get_paths(fpath)
if self.ismetafile(fpath):
return # we don't lock lock files and guard extensions but its useful to be able to call lock() on them when doing things like iterating over directories
tstart = time.time()
# spinlock with _try_lock() attempts
while not self._try_lock(paths.lock):
if expiration_check:
self._unlock_if_expired(paths)
if time.time() - tstart > 5:
print("Been waiting for a lock for over 5 seconds...")
# If we just locked a directory, we now lock all sub items
if self.isdir(fpath):
if self.verbose:
util.green(
"took a directory lock, so now taking locks on all sub items"
)
for item in self.listdir(fpath):
self.manual_lock(join(fpath, item),
expiration_check=expiration_check)
def upload_sfs_file(username, password, filename):
"""
Uploads a file to Snapchat FS.
@username User who will own the file in Snapchat FS.
@password Password of user.
@filename Path of the file to upload.
"""
print util.green('Uploading file ') + (filename)
ss = SfsSession(username, password)
ss.login()
sfs_id = ss.generate_sfs_id(filename)
ss.upload_image(filename, sfs_id)
ss.send_image_to(username, sfs_id)
def main(args):
args['exit'] = threading.Event()
args['sv'] = args['db'].server_list(args['defaults']['domain'])
args['svlink'] = linkserver
args['svlock'] = threading.RLock()
for s in args['sv'].values():
cli = linkserver(s, args['defaults']['domain'])
if cli.ping() == False:
print "%s: %s [%s]" % (s['name'], s['host'], util.red("failed"))
sys.exit(1)
print "%s: %s [%s]" % (s['name'], s['host'], util.green("OK"))
cli.close()
chksvth = threading.Thread(target=checkserver,
name='chksvth',
args=(args, ))
chksvth.daemon = True
chksvth.start()
try:
server.main(args)
except KeyboardInterrupt:
print ""
print "Caught KeyboardInterrupt, closing..."
args['exit'].set()
chksvth.join()
if args['defaults']['debug']:
print "main exited"
def exec_cmd(cmd, in_content=None, remote=None, pipe=True, print_cmd=False):
if print_cmd:
print green('[' + (remote if remote else 'localhost') +
'] [exec_cmd] ') + cmd
if remote:
# scp sh file to remote because the cmd may contain any kinds of quotes
tmp_fd, tmp_fn = tempfile.mkdtemp('.sh')
remote_tmp_fn = '/tmp/rain_tmpscript_' + os.path.basename(tmp_fn)
try:
os.write(tmp_fd, cmd)
os.close(tmp_fd)
exec_cmd('scp %s %s:%s' % (tmp_fn, remote, remote_tmp_fn))
finally:
os.unlink(tmp_fn)
if not pipe:
code = os.system(cmd)
if code != 0:
raise ExecError(code, '', '')
return '', ''
p = subprocess.Popen(cmd,
shell=True,
stdin=subprocess.PIPE if in_content else None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if in_content:
p.stdin.write(in_content)
p.stdin.close()
stdout = p.stdout.read() if p.stdout else ''
stderr = p.stderr.read() if p.stderr else ''
code = p.wait()
if code != 0:
raise ExecError(code, stdout, stderr)
if remote:
# clean the remote tmp sh file
try:
exec_cmd("ssh %s 'rm -f %s'" % (remote, remote_tmp_fn))
except ExecError, e:
print 'Failed to clean remote file %s, reason:%s' % (remote,
e.message)
def upload_sfs_file(username, password, filename):
"""
Uploads a file to Snapchat FS.
@username User who will own the file in Snapchat FS.
@password Password of user.
@filename Path of the file to upload.
"""
with open(filename) as f:
data = f.read()
basename = os.path.basename(filename)
print util.green('Uploading file ') + (basename)
ss = SfsSession(username, password)
ss.login()
sfs_id = ss.generate_sfs_id(basename, data)
ss.upload_image(data, sfs_id)
ss.send_image_to(username, sfs_id)
def show_progress(self, opt, it, loss):
time_elapsed = util.get_time(time.time() - self.time_start)
print("it {0}/{1}, lr:{3}, loss:{4}, time:{2}".format(
util.cyan("{}".format(it + 1)),
opt.to_it,
util.green("{0}:{1:02d}:{2:05.2f}".format(*time_elapsed)),
util.yellow("{:.2e}".format(opt.lr_pmo)),
util.red("{:.4e}".format(loss.all)),
))
def show_progress(self,opt,ep,loss):
[lr] = self.sched.get_lr()
time_elapsed = util.get_time(time.time()-self.time_start)
print("ep {0}/{1}, lr:{3}, loss:{4}, time:{2}"
.format(util.cyan("{}".format(ep+1)),
opt.to_epoch,
util.green("{0}:{1:02d}:{2:05.2f}".format(*time_elapsed)),
util.yellow("{:.2e}".format(lr)),
util.red("{:.4e}".format(loss.all)),
))
def login(args, data):
psk = data['pkt'].get('psk')
if psk == hashlib.md5(args['defaults']['psk']).hexdigest():
print "Auth %s From %s" % (util.green("Success"), str(data['addr']))
data['sock'].sendall(packet.Packet({}, 'OK').tostr())
return
print "Auth %s From %s (psk: %s)" % (util.red("Failed"), str(
data['addr']), str(psk))
data['sock'].sendall(packet.Packet({}, 'REJECT').tostr())
def _try_lock(self, lockpath):
"""
Makes a single attempt to take a lock, returning True if taken and False if the lock is busy.
If successful, we add the lock to self.lock_time and increment its lock_count
Works for both locks and guards
"""
lock_time = time.time()
lockmsg = f"{os.getpid()} {lock_time} {self.name}"
#if self.verbose: print(f"attempting to lock {lockpath}")
# check if we already have the lock
if lockpath in self.lock_time:
self.lock_count[lockpath] += 1
if self.verbose:
print("we already own this lock, incrementing lock_count")
return True # we let ppl take a lock multiple times if they already own it, incrementing `self.lock_count[lockpath]` each time
# check for directory locks all along the path that we're locking
dirs = lockpath[len(self.root):].split('/')
for i in range(len(dirs)):
dirlock = ''.join(dirs[:i + 1]) + LOCK_EXTENSION
if isfile(dirlock):
return False # someone locked a directory along the path that we're trying to lock, so we give up
# try to take hte lock
try:
tmpfile = f"{lockpath}.{os.getpid()}"
with open(
lockpath, 'x'
) as f: # open in 'x' mode (O_EXCL and O_CREAT) so it fails if file exists
f.write(lockmsg)
if self.verbose:
util.green(f"LOCKED {self.get_paths(lockpath).relative}")
self.lock_time[lockpath] = lock_time
self.lock_count[lockpath] = 1
return True # we got the lock!
except FileExistsError:
return False # we did not get the lock
def load(self):
"""
This is an extremely important contextmanager. If you load do any attr mutation operation you probably want to use it. It holds a lock the whole time and loads the whole shared object into local memory for the duration of the contextmanager. Any getattr/setattr calls will use this local version. This contextmanager recurses with no issues. Nobody else can access the shared object as long as you have it loaded, since load() takes a lock().
Why this is so important:
If you do sharedobj.mylist.pop() outside of this contextmanager the 'mylist' attr will be loaded from disk, then pop() will be called on it, but it won't be written back to disk.
If you have it wrapped in a load() then all of the attributes will be loaded into self.attr_dict, then the 'mylist' attr will simply be taken from this local dict, so when pop() is called it will modify the mylist copy in the local dict, and at the very end everything will get flushed back to disk when load() ends.
"""
with self.lock():
self.load_count += 1
if self.fs.verbose:
util.yellow(f'load(load_count={self.load_count})')
if self.load_count == 1: # if this is first load since being unloaded
if self.fs.verbose: util.green(f'--loaded--')
self.attr_dict = self.fs.load_nosplay(self.abspath)
try:
yield None
finally:
self.load_count -= 1
if self.fs.verbose:
util.yellow(f'unload(load_count={self.load_count})')
if self.load_count == 0: # unload for real
self.fs.save_nosplay(self.attr_dict, self.abspath)
if self.fs.verbose: util.purple(f'--unloaded--')
def download_all_sfs(username, password, target_dir):
"""
Downloads all files managed by Snapchat FS, writing them to `target_dir`
@username Username that owns the files.
@password Password for the account specified by `username`
@target_dir Where to download the files to.
"""
# get all downloadable files tracked by Snapchat FS
files = all_downloadable_sfs_files(username, password)
# download each file in sequence; if we find two files with the same
# name, we give the file a name that includes a hash of the contents
filenames_downloaded = set()
for filename, content_hash, received_id, snap in files:
try:
data = snap.download()
if filename not in filenames_downloaded:
print(util.green("Downloading snap ") + "%s" % filename)
path = os.path.join(target_dir, filename)
else:
print(util.green("Downloading snap ") + ("%s " % filename) +
(util.red("but filename is not unique; ") +
("downloading as: %s" %
(filename + "-" + content_hash))))
path = os.path.join(target_dir
, filename + "-" + content_hash)
filenames_downloaded.add(filename)
with open(os.path.join(target_dir, filename+content_hash)
, 'w') as w:
w.write(data)
except Exception as e:
print("Failed to download %s: %s" % (filename, e))
raise
def dirtest():
util.green("DIRTEST")
fs = SyncedFS('mg', 'test', verbose=True)
fs.wipe()
# directories
# populate testdir
fs.mkdir('testdir')
with fs.open('testdir/testfile', 'w') as f:
f.write('test')
with fs.open('testdir/testfile2', 'w') as f:
f.write('test')
# populate testdir/subdir
fs.mkdir('testdir/subdir')
with fs.open('testdir/subdir/subfile1', 'w') as f:
f.write('test')
with fs.open('testdir/subdir/subfile2', 'w') as f:
f.write('test')
#populate testdir/subdir/subdir
fs.mkdir('testdir/subdir/subdir')
with fs.open('testdir/subdir/subdir/ss1', 'w') as f:
f.write('test')
with fs.open('testdir/subdir/subdir/ss2', 'w') as f:
f.write('test')
# lock testdir/subdir
fs.manual_lock('testdir/subdir')
# lock testdir/subdir/subdir
fs.manual_lock('testdir/subdir/subfile1')
# lock testdir
fs.manual_lock('testdir')
fs.unlock('testdir/subdir')
fs.unlock('testdir')
fs.unlock('testdir/subdir/subfile1')
fs.no_locks()
def set():
# parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument("--problem_name",
default="Finance",
help="name for system to control")
parser.add_argument("--graph",
default="oneshot",
help="which graph to use")
parser.add_argument("--debug", action="store_true", help="")
parser.add_argument("--gpu", default="False", help="use cuda tensors")
parser.add_argument("--recurrent", action="store_true", help="recurrent")
parser.add_argument(
"--load_timestamp",
default=None,
help="timestamp of previous experiment you want to load")
parser.add_argument(
"--test_only",
default="False",
help="Run test only. For this load_timestamp is mandatory")
args = parser.parse_args()
# --- below are automatically set ---
# if args.seed is not None:
# seed = args.seed
# random.seed(seed)
# os.environ['PYTHONHASHSEED'] = str(seed)
# np.random.seed(seed)
# args.problem_name += "_seed{}".format(args.seed)
opt_dict = {name: value for (name, value) in args._get_kwargs()}
opt = argparse.Namespace(**opt_dict)
# print configurations
for o in vars(opt):
print(util.green(o), ":", util.yellow(getattr(opt, o)))
print()
return opt
def train_model(hyperparam, x_train, x_cv, y_train, y_cv, num_epochs, display=True, save=False):
print(util.green(str(hyperparam)))
if save:
saver = tf.train.Saver()
init = tf.global_variables_initializer()
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
sess.run(init)
summaries = []
for j in range(num_epochs):
train_loss, train_acc = train_epoch(x_train, y_train, hyperparam, sess)
cv_loss, cv_acc = cross_validate(x_cv, y_cv, hyperparam, sess)
summaries.append([train_loss, train_acc, cv_loss, cv_acc])
if display and j % DISPLAY_STEP == 0:
print("Train Loss: %f\nCV Loss: %f\nTrain Accuracy: %s\nCV Accuracy: %s\nEPOCH: %d\n"%(train_loss,
cv_loss,
train_acc,
cv_acc,
j))
if save and j % 50 == 0:
saver.save(sess, output_dir(hyperparam) + '/model_%d.ckpt'%(j))
if save:
saver.save(sess, output_dir(hyperparam) + '/model_%d.ckpt'%(j))
return np.array(summaries), sess
def get_or_create_labels():
labels = []
# Loading heuristics from the file system.
labels_fs = dict()
for label_type in os.scandir(HEURISTICS_DIR):
if label_type.is_dir() and not label_type.name.startswith('.'):
for label in os.scandir(label_type.path):
if label.is_file() and not label.name.startswith('.'):
with open(label.path) as file:
pattern = file.read()
label_fs = {
'name': os.path.splitext(label.name)[0],
'type': label_type.name,
'pattern': pattern,
}
labels_fs[(label_fs['type'], label_fs['name'])] = label_fs
# Loading labels from the database.
labels_db = db.query(db.Label).options(
selectinload(db.Label.heuristic).options(
selectinload(
db.Heuristic.executions).defer('output').defer('user'))).all()
status = {
'File System': len(labels_fs),
'Database': len(labels_db),
'Added': 0,
'Deleted': 0,
'Updated': 0,
}
i = 0
# Deleting labels that do not exist in the file system
for label in labels_db:
label_fs = labels_fs.pop((label.type, label.name), None)
if label_fs is not None:
# Print progress information
i += 1
progress = '{:.2%}'.format(i / status['File System'])
print(f'[{progress}] Adding label {label.type}/{label.name}:',
end=' ')
labels.append(label)
# Check if pattern has changed and, in this case, remove executions that are not accepted and verified
heuristic = label.heuristic
if heuristic.pattern != label_fs['pattern']:
heuristic.pattern = label_fs['pattern']
count = 0
for execution in list(heuristic.executions):
if not (execution.isValidated and execution.isAccepted):
# The commit does not invalidate the objects for performance reasons,
# so we need to remove the relationships before deleting the execution.
execution.heuristic = None
execution.version = None
db.delete(execution)
count += 1
print(
green(f'heuristic updated ({count} executions removed).'))
status['Updated'] += 1
else:
print(yellow('already done.'))
else:
db.delete(label)
status['Deleted'] += 1
# Adding missing labels in the database
for label_fs in labels_fs.values():
# Print progress information
i += 1
progress = '{:.2%}'.format(i / status['File System'])
print(
f'[{progress}] Adding label {label_fs["type"]}/{label_fs["name"]}:',
end=' ')
label = db.create(db.Label,
name=label_fs['name'],
type=label_fs['type'])
db.create(db.Heuristic,
pattern=label_fs['pattern'],
label=label,
executions=[])
labels.append(label)
print(green('ok.'))
status['Added'] += 1
status['Total'] = len(labels)
print_results(status)
commit()
return sorted(labels,
key=lambda item: (item.type.lower(), item.name.lower()))
def get_or_create_projects():
projects = []
# Loading projects from the Excel.
df = pd.read_excel(ANNOTATED_FILE, keep_default_na=False)
df = df[df.discardReason == ''].reset_index(drop=True)
projects_excel = dict()
for i, project_excel in df.iterrows():
projects_excel[(project_excel['owner'],
project_excel['name'])] = project_excel
# Loading projects from the database.
projects_db = db.query(db.Project).options(
load_only('id', 'owner', 'name'),
selectinload(db.Project.versions).load_only('id')).all()
status = {
'Excel': len(projects_excel),
'Database': len(projects_db),
'Added': 0,
'Deleted': 0,
'Repository not found': 0,
'Git error': 0
}
i = 0
# Deleting projects that do not exist in the Excel file
for project in projects_db:
if projects_excel.pop((project.owner, project.name), None) is not None:
# Print progress information
i += 1
progress = '{:.2%}'.format(i / status['Excel'])
print(
f'[{progress}] Adding project {project.owner}/{project.name}:',
end=' ')
projects.append(project)
print(yellow('already done.'))
else:
db.delete(project)
status['Deleted'] += 1
# Adding missing projects in the database
for project_excel in projects_excel.values():
# Print progress information
i += 1
progress = '{:.2%}'.format(i / status['Excel'])
print(
f'[{progress}] Adding project {project_excel["owner"]}/{project_excel["name"]}:',
end=' ')
project_dict = {
k: v
for k, v in project_excel.to_dict().items()
if k not in ['url', 'isSoftware', 'discardReason']
}
project_dict['createdAt'] = str(project_dict['createdAt'])
project_dict['pushedAt'] = str(project_dict['pushedAt'])
try:
os.chdir(REPOS_DIR + os.sep + project_dict['owner'] + os.sep +
project_dict['name'])
p = subprocess.run(REVPARSE_COMMAND, capture_output=True)
if p.stderr:
raise subprocess.CalledProcessError(p.returncode,
REVPARSE_COMMAND, p.stdout,
p.stderr)
project = db.create(db.Project, **project_dict)
db.create(db.Version,
sha1=p.stdout.decode().strip(),
isLast=True,
project=project)
projects.append(project)
print(green('ok.'))
status['Added'] += 1
except NotADirectoryError:
print(red('repository not found.'))
status['Repository not found'] += 1
except subprocess.CalledProcessError as ex:
print(red('Git error.'))
status['Git error'] += 1
if CODE_DEBUG:
print(ex.stderr)
status['Total'] = len(projects)
print_results(status)
commit()
return sorted(projects,
key=lambda item: (item.owner.lower(), item.name.lower()))
def main(mode):
fs = SyncedFS('mg', 'test', verbose=True)
fs.unlock_all()
# message writing test
with fs.open('test', 'w') as f:
print("START")
f.write('message1\n')
f.write('message2')
print("END")
# make sure that message was written successfully
with fs.open('test', 'r') as f:
assert f.read() == 'message1\nmessage2'
# recursive locks test
with fs.lock('test'):
with fs.lock('test'):
with fs.open('test', 'w') as f:
f.write('message')
# make sure that message was written successfully
with fs.open('test', 'r') as f:
assert f.read() == 'message'
# multiprocess tests
if mode == 1:
with fs.open('test', 'w') as f:
print("START")
print(
'Now launch this process again (preferably multiple copies) within 7 seconds!'
)
sleep(7)
f.write('message')
print("END")
elif mode == 2:
util.green("Parent trying to get lock")
fs.manual_lock('test2') # take a lock and dont release it
if self.verbose: util.green("Parent got lock")
if os.fork() != 0:
util.green("Parent sleeping with lock")
sleep(
15
) # need to sleep a while in the parent otherwise the lock will be released when the child clears the parent since its an old pid
util.green("Parent exiting")
exit(0)
else:
sleep(1)
fs = SyncedFS(
'mg', 'test',
verbose=True) # so we dont inherit fs.locks from parent
util.green("Child trying to get lock")
fs.manual_lock(
'test2'
) # take lock in child, which should succeed after 10 seconds
util.green("Child got lock")
fs.unlock('test2') # cleanup
util.green("Child released lock")
# make sure we aren't still holding any locks
fs.no_locks()
def prompt(choices, mode='*'):
if mode not in PROMPT_MODES:
raise ValueError("mode '{}' is invalid".format(mode))
if len(choices) > 26:
raise ValueError("too many choices")
if mode == '*':
header = "select zero or more:"
max, min = float('inf'), 0
elif mode == '+':
header = "select one or more:"
max, min = float('inf'), 1
elif mode in [1, '1']:
header = "select one:"
max, min = 1, 1
elif mode == '?':
header = "select zero or one:"
max, min = 1, 0
letters = list(map(lambda x: chr(ord('a') + x), range(len(choices))))
num_selections = 0
selections = [] # unique indices into choices list
while num_selections < min or num_selections < max:
util.print(util.green(header))
for i in range(len(choices)):
if i in selections:
choice = " × "
else:
choice = " "
choice += str(letters[i]) + '. ' + str(choices[i])
if i in selections:
choice = util.yellow(choice)
util.print(choice)
try:
sel = input(util.green("make a selection (or ! to commit): "))
except KeyboardInterrupt:
util.exit(util.ERR_INTERRUPTED)
if sel == '!':
if num_selections < min:
util.error("can't stop now; you must make "
"{} {}".format(min,
util.plural("selection", min)))
continue
else:
break
try:
if letters.index(sel) in selections:
selections.remove(letters.index(sel))
continue
selections.append(letters.index(sel))
num_selections += 1
except ValueError:
if sel == '':
util.print("make a selection (or ! to commit)")
else:
util.error("invalid selection: not in list")
continue
return selections
def main():
"""
This program can be useful to fix collisions (due to data migrated from case insensitive to case sensitive file
systems) and to update the workspace to the most recent version.
"""
print(f'Loading repositories from {ANNOTATED_FILE}.')
info_repositories = pd.read_excel(ANNOTATED_FILE, keep_default_na=False)
info_repositories = info_repositories[info_repositories.discardReason ==
''].reset_index(drop=True)
status = {
'Success': 0,
'Collided': 0,
'Repository not found': 0,
'Git error': 0,
'Total': len(info_repositories)
}
print(f'Resetting {status["Total"]} repositories...')
for i, row in info_repositories.iterrows():
# Print progress information
progress = '{:.2%}'.format(i / status["Total"])
print(
f'[{progress}] Processing repository {row["owner"]}/{row["name"]}:',
end=' ')
try:
target = REPOS_DIR + os.sep + row['owner'] + os.sep + row['name']
os.chdir(target)
# Removes lock file, if they exist
try:
os.remove('.git/index.lock')
except OSError:
pass
process = subprocess.run(
['git', 'config', 'core.precomposeunicode', 'false'],
capture_output=True)
if process.stderr:
raise subprocess.CalledProcessError(process.stderr)
cmd = ['git', 'reset', '--hard', '-q', 'origin/HEAD']
process = subprocess.run(cmd, capture_output=True)
if process.stderr:
raise subprocess.CalledProcessError(process.returncode, cmd,
process.stdout,
process.stderr)
cmd = ['git', 'clean', '-d', '-f', '-x', '-q']
process = subprocess.run(cmd, capture_output=True)
if process.stderr:
raise subprocess.CalledProcessError(process.returncode, cmd,
process.stdout,
process.stderr)
process = subprocess.run(['git', 'status', '-s'],
capture_output=True)
if process.stdout or process.stderr:
print(yellow('collided.'))
status['Collided'] += 1
if CODE_DEBUG:
print(process.stdout)
print(process.stderr)
else:
print(green('ok.'))
status['Success'] += 1
except NotADirectoryError:
print(red('repository not found.'))
status['Repository not found'] += 1
except subprocess.CalledProcessError as ex:
print(red('Git error.'))
status['Git error'] += 1
if CODE_DEBUG:
print(ex.stderr)
print('\n*** Processing results ***')
for k, v in status.items():
print(f'{k}: {v}')
print("\nFinished.")
def set():
# parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument("--eval", action="store_true", help="evaluation phase")
parser.add_argument("--group", default="0", help="name for group")
parser.add_argument("--name",
default="debug",
help="name for model instance")
parser.add_argument("--seed", type=int, default=0, help="fix random seed")
parser.add_argument("--gpu", type=int, default=0, help="GPU device")
parser.add_argument("--cpu", action="store_true", help="use CPU only")
parser.add_argument("--load", default=None, help="load (pre)trained model")
# dataset
parser.add_argument("--rendering-path",
default="data/rendering",
help="path to ShapeNet rendering")
parser.add_argument("--pointcloud-path",
default="data/customShapeNet",
help="path to ShapeNet 3D point cloud")
parser.add_argument("--sun360-path",
default="data/background",
help="path to SUN360 background")
parser.add_argument("--seq-path",
default="data/sequences",
help="path to RGB sequences for evaluation")
parser.add_argument("--category",
default=None,
help="train on specific category")
parser.add_argument("--num-workers",
type=int,
default=8,
help="number of data loading threads")
parser.add_argument("--size",
default="224x224",
help="rendered image size")
parser.add_argument("--sfm",
action="store_true",
help="use coordinate system mapping from SfM output")
parser.add_argument("--init-idx",
type=int,
default=27,
help="initial frame index")
parser.add_argument("--noise",
type=float,
default=None,
help="gaussian noise in coordinate system mapping")
# visualization
parser.add_argument("--log-tb",
action="store_true",
help="output loss in TensorBoard")
parser.add_argument("--log-visdom",
action="store_true",
help="visualize mesh in Visdom")
parser.add_argument("--vis-server",
default="http://localhost",
help="visdom port server")
parser.add_argument("--vis-port",
type=int,
default=8097,
help="visdom port number")
parser.add_argument("--video",
action="store_true",
help="write video sequence with optimized mesh")
# AtlasNet
parser.add_argument("--num-prim",
type=int,
default=25,
help="number of primitives")
parser.add_argument("--num-points",
type=int,
default=100,
help="number of points (per primitive)")
parser.add_argument("--num-meshgrid",
type=int,
default=5,
help="number of regular grids for mesh")
parser.add_argument("--sphere",
action="store_true",
help="use closed sphere for AtlasNet")
parser.add_argument("--sphere-densify",
type=int,
default=3,
help="densify levels")
parser.add_argument("--imagenet-enc",
action="store_true",
help="initialize with pretrained ResNet encoder")
parser.add_argument("--pretrained-dec",
default=None,
help="initialize with pretrained AtlasNet decoder")
# photometric optimization
parser.add_argument(
"--batch-size-pmo",
type=int,
default=-1,
help="batch size for photometric optimization (-1 for all)")
parser.add_argument("--lr-pmo",
type=float,
default=1e-3,
help="base learning rate for photometric optimization")
parser.add_argument("--code",
type=float,
default=None,
help="penalty on code difference")
parser.add_argument("--scale",
type=float,
default=None,
help="penalty on scale")
parser.add_argument("--to-it",
type=int,
default=100,
help="run optimization to iteration number")
parser.add_argument(
"--avg-frame",
action="store_true",
help="average photo. loss across frames instead of sampled pixels")
# AtlasNet training
parser.add_argument("--batch-size",
type=int,
default=32,
help="input batch size")
parser.add_argument(
"--aug-transl",
type=int,
default=None,
help="augment with random translation (for new dataset)")
parser.add_argument("--lr-pretrain",
type=float,
default=1e-4,
help="base learning rate")
parser.add_argument("--lr-decay",
type=float,
default=1.0,
help="learning rate decay")
parser.add_argument("--lr-step",
type=int,
default=100,
help="learning rate decay step size")
parser.add_argument("--from-epoch",
type=int,
default=0,
help="train from epoch number")
parser.add_argument("--to-epoch",
type=int,
default=500,
help="train to epoch number")
opt = parser.parse_args()
# --- below are automatically set ---
if opt.seed is not None:
np.random.seed(opt.seed)
torch.manual_seed(opt.seed)
torch.cuda.manual_seed_all(opt.seed)
opt.name += "_seed{}".format(opt.seed)
opt.device = "cpu" if opt.cpu or not torch.cuda.is_available(
) else "cuda:{}".format(opt.gpu)
opt.H, opt.W = [int(s) for s in opt.size.split("x")]
if opt.sphere:
opt.num_prim = 1
opt.num_points_all = opt.num_points * opt.num_prim
# print configurations
for o in sorted(vars(opt)):
print(util.green(o), ":", util.yellow(getattr(opt, o)))
print()
return opt
def main():
db.connect()
print(f'Loading projects from {ANNOTATED_FILE}.')
projects = get_or_create_projects()
print(f'\nLoading heuristics from {HEURISTICS_DIR}.')
labels = get_or_create_labels()
# Indexing executions by label heuristic and project version.
executions = index_executions(labels)
status = {
'Success': 0,
'Skipped': 0,
'Repository not found': 0,
'Git error': 0,
'Total': len(labels) * len(projects)
}
print(
f'\nProcessing {len(labels)} heuristics over {len(projects)} projects.'
)
for i, label in enumerate(labels):
heuristic = label.heuristic
for j, project in enumerate(projects):
version = project.versions[
0] # TODO: fix this to deal with multiple versions
# Print progress information
progress = '{:.2%}'.format(
(i * len(projects) + (j + 1)) / status['Total'])
print(
f'[{progress}] Searching for {label.name} in {project.owner}/{project.name}:',
end=' ')
# Try to get a previous execution
execution = executions.get((heuristic, version), None)
if not execution:
try:
os.chdir(REPOS_DIR + os.sep + project.owner + os.sep +
project.name)
cmd = GREP_COMMAND + [
HEURISTICS_DIR + os.sep + label.type + os.sep +
label.name + '.txt'
]
p = subprocess.run(cmd, capture_output=True)
if p.stderr:
raise subprocess.CalledProcessError(
p.returncode, cmd, p.stdout, p.stderr)
db.create(db.Execution,
output=p.stdout.decode(errors='replace').replace(
'\x00', '\uFFFD'),
version=version,
heuristic=heuristic,
isValidated=False,
isAccepted=False)
print(green('ok.'))
status['Success'] += 1
except NotADirectoryError:
print(red('repository not found.'))
status['Repository not found'] += 1
except subprocess.CalledProcessError as ex:
print(red('Git error.'))
status['Git error'] += 1
if CODE_DEBUG:
print(ex.stderr)
else: # Execution already exists
print(yellow('already done.'))
status['Skipped'] += 1
commit()
print_results(status)
db.close()
def main():
for f in get_tests():
print(util.green(f))
fp = os.path.join(qa_dir, f)
subprocess.check_call([sys.executable, fp])
def main():
for f in get_tests():
print (util.green(f))
fp = os.path.join (qa_dir, f)
subprocess.check_call([sys.executable, fp])
