def main(args):
args['exit'] = threading.Event()
args['sv'] = args['db'].server_list(args['defaults']['domain'])
args['svlink'] = linkserver
args['svlock'] = threading.RLock()
for s in args['sv'].values():
cli = linkserver(s, args['defaults']['domain'])
if cli.ping() == False:
print "%s: %s [%s]" % (s['name'], s['host'], util.red("failed"))
sys.exit(1)
print "%s: %s [%s]" % (s['name'], s['host'], util.green("OK"))
cli.close()
chksvth = threading.Thread(target=checkserver,
name='chksvth',
args=(args, ))
chksvth.daemon = True
chksvth.start()
try:
server.main(args)
except KeyboardInterrupt:
print ""
print "Caught KeyboardInterrupt, closing..."
args['exit'].set()
chksvth.join()
if args['defaults']['debug']:
print "main exited"
def predict_handler(params):
text = params['text']
word = params['word']
print(util.red(text), util.yellow(word))
response = bert_masked_lm_prob(text, word)
print(util.yellow(response))
return json.dumps(response)
def pretty_str(self):
max_x = max(map(lambda coord: coord.x, self.board.keys()))
max_y = max(map(lambda coord: coord.y, self.board.keys()))
coord_to_unit = dict()
for unit in self.units:
coord_to_unit[unit.position] = unit
pretty_list = []
for y in range(max_y + 1):
for x in range(max_x + 1):
current_coord = Coord(x, y)
if current_coord in coord_to_unit:
u = coord_to_unit[current_coord]
#print("Hello", u, self.attacks_this_round)
unit_str = coord_to_unit[current_coord].allegiance.value
if u in self.moves_this_round:
unit_str = bold(unit_str)
if u in self.attacks_this_round:
unit_str = red(unit_str)
pretty_list.append(unit_str)
elif current_coord in self.board:
pretty_list.append(self.board[current_coord].__repr__())
else:
pretty_list.append('.')
pretty_list.append('\n')
return "".join(pretty_list)
def rmdir(self, fpath, recursive=False, metafile=False):
abspath = self.get_paths(fpath).absolute
if self.verbose: util.red(f"removing {abspath}")
with self.lock(fpath):
if recursive:
for item in self.listdir(abspath):
to_delete = join(abspath, item)
assert to_delete.startswith(self.root)
assert to_delete.startswith(
os.environ['HOME']
) # im just really scared of deleting '/' somehow
if self.isfile(to_delete):
self.remove(to_delete, metafile=metafile)
if self.isdir(to_delete):
self.rmdir(to_delete,
recursive=True,
metafile=metafile)
rmdir(abspath)
def show_progress(self, opt, it, loss):
time_elapsed = util.get_time(time.time() - self.time_start)
print("it {0}/{1}, lr:{3}, loss:{4}, time:{2}".format(
util.cyan("{}".format(it + 1)),
opt.to_it,
util.green("{0}:{1:02d}:{2:05.2f}".format(*time_elapsed)),
util.yellow("{:.2e}".format(opt.lr_pmo)),
util.red("{:.4e}".format(loss.all)),
))
def show_progress(self,opt,ep,loss):
[lr] = self.sched.get_lr()
time_elapsed = util.get_time(time.time()-self.time_start)
print("ep {0}/{1}, lr:{3}, loss:{4}, time:{2}"
.format(util.cyan("{}".format(ep+1)),
opt.to_epoch,
util.green("{0}:{1:02d}:{2:05.2f}".format(*time_elapsed)),
util.yellow("{:.2e}".format(lr)),
util.red("{:.4e}".format(loss.all)),
))
def login(args, data):
psk = data['pkt'].get('psk')
if psk == hashlib.md5(args['defaults']['psk']).hexdigest():
print "Auth %s From %s" % (util.green("Success"), str(data['addr']))
data['sock'].sendall(packet.Packet({}, 'OK').tostr())
return
print "Auth %s From %s (psk: %s)" % (util.red("Failed"), str(
data['addr']), str(psk))
data['sock'].sendall(packet.Packet({}, 'REJECT').tostr())
def evaluate(self,opt,ep=None):
self.network.eval()
loss_eval = easydict.EasyDict()
count = 0
with torch.no_grad():
for it,batch in enumerate(self.test_loader):
var = self.graph_forward(opt,batch,training=False)
loss = self.compute_loss(opt,var,ep=ep)
batch_size = len(batch[0])
for k in loss:
if k not in loss_eval: loss_eval[k] = 0
loss_eval[k] += loss[k]*batch_size
count += batch_size
for k in loss_eval: loss_eval[k] /= count
print("[EVAL] loss:{0}".format(util.red("{:.4e}".format(loss_eval.all))))
if ep is not None:
self.log_losses(opt,ep,loss_eval,training=False)
def do_GET(self):
print(red(self.path))
# Send response status code
self.send_response(200)
# Send headers
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Content-type', 'application/json')
self.end_headers()
params = get_params(self.path)
response = ''
if 'predict' in params:
response = predict_handler(params)
# Write content as utf-8 data
self.wfile.write(bytes(response, "utf8"))
return
def checkserver(args):
logger.debug("checkserver thread start")
intval = int(args['defaults']['interval'])
while True:
args['svlock'].acquire()
servers = args['sv']
args['svlock'].release()
for s in servers.values():
cli = linkserver(s, args['defaults']['domain'])
if cli.ping() == False:
logger.error("%s: %s [%s]", s['name'], s['host'], util.red("failed"))
args['db'].server_failed(s['name'])
args['svlock'].acquire()
del args['sv'][s['name']]
args['svlock'].release()
cli.close()
if args['exit'].wait(intval) == True:
break
logger.debug("checkserver thread exit")
def download_all_sfs(username, password, target_dir):
"""
Downloads all files managed by Snapchat FS, writing them to `target_dir`
@username Username that owns the files.
@password Password for the account specified by `username`
@target_dir Where to download the files to.
"""
# get all downloadable files tracked by Snapchat FS
files = all_downloadable_sfs_files(username, password)
# download each file in sequence; if we find two files with the same
# name, we give the file a name that includes a hash of the contents
filenames_downloaded = set()
for filename, content_hash, received_id, snap in files:
try:
data = snap.download()
if filename not in filenames_downloaded:
print(util.green("Downloading snap ") + "%s" % filename)
path = os.path.join(target_dir, filename)
else:
print(util.green("Downloading snap ") + ("%s " % filename) +
(util.red("but filename is not unique; ") +
("downloading as: %s" %
(filename + "-" + content_hash))))
path = os.path.join(target_dir
, filename + "-" + content_hash)
filenames_downloaded.add(filename)
with open(os.path.join(target_dir, filename+content_hash)
, 'w') as w:
w.write(data)
except Exception as e:
print("Failed to download %s: %s" % (filename, e))
raise
def get_or_create_projects():
projects = []
# Loading projects from the Excel.
df = pd.read_excel(ANNOTATED_FILE, keep_default_na=False)
df = df[df.discardReason == ''].reset_index(drop=True)
projects_excel = dict()
for i, project_excel in df.iterrows():
projects_excel[(project_excel['owner'],
project_excel['name'])] = project_excel
# Loading projects from the database.
projects_db = db.query(db.Project).options(
load_only('id', 'owner', 'name'),
selectinload(db.Project.versions).load_only('id')).all()
status = {
'Excel': len(projects_excel),
'Database': len(projects_db),
'Added': 0,
'Deleted': 0,
'Repository not found': 0,
'Git error': 0
}
i = 0
# Deleting projects that do not exist in the Excel file
for project in projects_db:
if projects_excel.pop((project.owner, project.name), None) is not None:
# Print progress information
i += 1
progress = '{:.2%}'.format(i / status['Excel'])
print(
f'[{progress}] Adding project {project.owner}/{project.name}:',
end=' ')
projects.append(project)
print(yellow('already done.'))
else:
db.delete(project)
status['Deleted'] += 1
# Adding missing projects in the database
for project_excel in projects_excel.values():
# Print progress information
i += 1
progress = '{:.2%}'.format(i / status['Excel'])
print(
f'[{progress}] Adding project {project_excel["owner"]}/{project_excel["name"]}:',
end=' ')
project_dict = {
k: v
for k, v in project_excel.to_dict().items()
if k not in ['url', 'isSoftware', 'discardReason']
}
project_dict['createdAt'] = str(project_dict['createdAt'])
project_dict['pushedAt'] = str(project_dict['pushedAt'])
try:
os.chdir(REPOS_DIR + os.sep + project_dict['owner'] + os.sep +
project_dict['name'])
p = subprocess.run(REVPARSE_COMMAND, capture_output=True)
if p.stderr:
raise subprocess.CalledProcessError(p.returncode,
REVPARSE_COMMAND, p.stdout,
p.stderr)
project = db.create(db.Project, **project_dict)
db.create(db.Version,
sha1=p.stdout.decode().strip(),
isLast=True,
project=project)
projects.append(project)
print(green('ok.'))
status['Added'] += 1
except NotADirectoryError:
print(red('repository not found.'))
status['Repository not found'] += 1
except subprocess.CalledProcessError as ex:
print(red('Git error.'))
status['Git error'] += 1
if CODE_DEBUG:
print(ex.stderr)
status['Total'] = len(projects)
print_results(status)
commit()
return sorted(projects,
key=lambda item: (item.owner.lower(), item.name.lower()))
def main():
db.connect()
print(f'Loading projects from {ANNOTATED_FILE}.')
projects = get_or_create_projects()
print(f'\nLoading heuristics from {HEURISTICS_DIR}.')
labels = get_or_create_labels()
# Indexing executions by label heuristic and project version.
executions = index_executions(labels)
status = {
'Success': 0,
'Skipped': 0,
'Repository not found': 0,
'Git error': 0,
'Total': len(labels) * len(projects)
}
print(
f'\nProcessing {len(labels)} heuristics over {len(projects)} projects.'
)
for i, label in enumerate(labels):
heuristic = label.heuristic
for j, project in enumerate(projects):
version = project.versions[
0] # TODO: fix this to deal with multiple versions
# Print progress information
progress = '{:.2%}'.format(
(i * len(projects) + (j + 1)) / status['Total'])
print(
f'[{progress}] Searching for {label.name} in {project.owner}/{project.name}:',
end=' ')
# Try to get a previous execution
execution = executions.get((heuristic, version), None)
if not execution:
try:
os.chdir(REPOS_DIR + os.sep + project.owner + os.sep +
project.name)
cmd = GREP_COMMAND + [
HEURISTICS_DIR + os.sep + label.type + os.sep +
label.name + '.txt'
]
p = subprocess.run(cmd, capture_output=True)
if p.stderr:
raise subprocess.CalledProcessError(
p.returncode, cmd, p.stdout, p.stderr)
db.create(db.Execution,
output=p.stdout.decode(errors='replace').replace(
'\x00', '\uFFFD'),
version=version,
heuristic=heuristic,
isValidated=False,
isAccepted=False)
print(green('ok.'))
status['Success'] += 1
except NotADirectoryError:
print(red('repository not found.'))
status['Repository not found'] += 1
except subprocess.CalledProcessError as ex:
print(red('Git error.'))
status['Git error'] += 1
if CODE_DEBUG:
print(ex.stderr)
else: # Execution already exists
print(yellow('already done.'))
status['Skipped'] += 1
commit()
print_results(status)
db.close()
# Gets the required methods.
from util import red
from random import randint
# Get two numbers from 1 to 100.
a = randint(1, 100)
b = randint(1, 100)
# Get an answer with those randomly generated numbers.
answer = a + b
# Print the answer.
print("The answer to " + red(a) + " + " + red(b) + " = " + red(answer))
# Nothing
from os import system
go = input("Go back to main file? (Y/N)\n")
system('clear')
system('python3 main.py')
def main():
"""
This program can be useful to fix collisions (due to data migrated from case insensitive to case sensitive file
systems) and to update the workspace to the most recent version.
"""
print(f'Loading repositories from {ANNOTATED_FILE}.')
info_repositories = pd.read_excel(ANNOTATED_FILE, keep_default_na=False)
info_repositories = info_repositories[info_repositories.discardReason ==
''].reset_index(drop=True)
status = {
'Success': 0,
'Collided': 0,
'Repository not found': 0,
'Git error': 0,
'Total': len(info_repositories)
}
print(f'Resetting {status["Total"]} repositories...')
for i, row in info_repositories.iterrows():
# Print progress information
progress = '{:.2%}'.format(i / status["Total"])
print(
f'[{progress}] Processing repository {row["owner"]}/{row["name"]}:',
end=' ')
try:
target = REPOS_DIR + os.sep + row['owner'] + os.sep + row['name']
os.chdir(target)
# Removes lock file, if they exist
try:
os.remove('.git/index.lock')
except OSError:
pass
process = subprocess.run(
['git', 'config', 'core.precomposeunicode', 'false'],
capture_output=True)
if process.stderr:
raise subprocess.CalledProcessError(process.stderr)
cmd = ['git', 'reset', '--hard', '-q', 'origin/HEAD']
process = subprocess.run(cmd, capture_output=True)
if process.stderr:
raise subprocess.CalledProcessError(process.returncode, cmd,
process.stdout,
process.stderr)
cmd = ['git', 'clean', '-d', '-f', '-x', '-q']
process = subprocess.run(cmd, capture_output=True)
if process.stderr:
raise subprocess.CalledProcessError(process.returncode, cmd,
process.stdout,
process.stderr)
process = subprocess.run(['git', 'status', '-s'],
capture_output=True)
if process.stdout or process.stderr:
print(yellow('collided.'))
status['Collided'] += 1
if CODE_DEBUG:
print(process.stdout)
print(process.stderr)
else:
print(green('ok.'))
status['Success'] += 1
except NotADirectoryError:
print(red('repository not found.'))
status['Repository not found'] += 1
except subprocess.CalledProcessError as ex:
print(red('Git error.'))
status['Git error'] += 1
if CODE_DEBUG:
print(ex.stderr)
print('\n*** Processing results ***')
for k, v in status.items():
print(f'{k}: {v}')
print("\nFinished.")
def m_die(s):
u.red("Error:" + str(s))
exit(1)
# Get the necessary modules for this remix.
from util import red, white
from random import randint
# Get a random integer from 1 to 100 twice, in order to add them together.
a = randint(1, 100)
b = randint(1, 100)
# Ask what they wish to be called, used below.
c = input("What is your name? ")
# Using the name from above, say hi in white and red.
print(white("Hello, ") + red(c))
# State the calculation as they don't know it.
print(red(a) + " " + red(b))
# State it with text.
print(red(a), 'and', red(b))
# Nothing to see here
from os import system
go = input("Go back to main file? (Y/N)\n")
system('clear')
system('python3 main.py')
from os import system
from util import red, white
remixes = [
"Output using print",
"Simple Math using Pre-determined Integers and Variables",
"Simple Math using Input() and Variables", "Testing"
]
for i in remixes:
print(red(remixes.index(i) + 1) + " - " + red(i))
game = input(
white("With this in mind, pick a variable from ") + red("1") +
white(" to ") + red(str(len(remixes)) + "\n"))
if game.isdigit():
if int(game) > 0 and int(game) < len(remixes) + 1:
system('clear')
system('python3 remix' + game + '.py')
else:
print("I don't think", red(game),
"is a number from " + red("1") + " - " + red(str(len(remixes))))
goBack = input(red("Go back?") + "\n")
system('clear')
system('python3 main.py')
else:
print("I do not believe", red(game), "is a number.")