def test_adduse_empty_only():
''' Test that the adduse module method works correctly when we specify
that we want it to be specific but then don't provide a list of
entities for the only qualifier '''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine")
module.add(sub)
# Add a use statement with only=True but an empty list of entities
adduse("fred", sub.root, only=True, funcnames=[])
assert count_lines(sub.root, "USE fred") == 1
assert count_lines(sub.root, "USE fred, only") == 0
def fit(sample_file, dict_file, output_file, eps=0.1):
n_samples = utils.count_lines(sample_file)
n_features = utils.count_lines(dict_file)
print 'creating projection matrix... \r',
sys.stdout.flush()
p = _get_projection(n_samples, n_features, eps=eps)
with open(output_file, 'wb') as f:
f.write(utils.zdumps(p))
# pickle.dump(p, f)
print 'projection matrix pickled to %s.' % output_file
def __init__(self, config) -> None:
super(NMTEncoderDecoderWrapper, self).__init__(config)
self.config['metrics'] = ['dev_ppl']
self.config.metrics.extend(self._get_metrics())
for metric in self.config.metrics:
self.config['best_{}_dir'.format(metric)] = os.path.join(self.config.model_dir, 'best_{}'.format(metric))
mkdir_if_not_exists(self.config['best_{}_dir'.format(metric)])
best_metric = 'best_{}'.format(metric)
if best_metric not in self.config:
self.config[best_metric] = float('inf')
self.config['checkpoint_file'] = os.path.join(self.config.model_dir,
'{}.ckpt'.format(self._get_checkpoint_name()))
if self.config.mode == 'train':
self.config['num_train_steps'] = int(self.config.num_train_epochs * math.ceil(
count_lines(self.config.train_data) / self.config.batch_size))
self.config.vocab_file = os.path.join(self.config.model_dir,
'vocab{}.in'.format(self.config.original_vocab_size
if 'original_vocab_size' in self.config
else self.config.vocab_size))
if 'epoch_step' not in self.config:
self.config['epoch_step'] = 0
if 'epoch' not in self.config:
self.config['epoch'] = 0
self._vocab_table = None
def read_file(path, use_lower):
n_lines = count_lines(path)
bar = progressbar.ProgressBar()
with io.open(path, encoding='utf-8', errors='ignore') as f:
for line in bar(f, max_value=n_lines):
words = split_sentence(line, use_lower)
yield words
def main(inputfile, start_frame=-1, end_frame=-1, move=False, output=None):
"""
Remove or move files from paths described in a pathfile.
This function will remove/delete all files in a pathfile
in case `start_frame` and `end_frame` equal to -1. In case
of moving file, an output folder may be passed as argument.
"""
#print inputfile, start_frame, end_frame, move, output
if move and not output:
output = join(dirname(inputfile), 'removed')
if not isdir(output):
legger.info('Creating folder: {}'.format(output))
os.makedir(output)
dicfiles = {}
pb = ProgressBar(count_lines(inputfile))
with open(inputfile) as fin:
for line in fin:
path = line.strip().split()[0]
fname, _ = splitext(basename(path))
fname = int(fname)
dicfiles[fname] = path
pb.update()
for id in sorted(dicfiles):
if (start_frame <= id and id <= end_frame) \
or (start_frame == -1 and end_frame == -1):
try:
if move:
shutil.move(dicfiles[id], join(output, str(id) + '.jpg'))
else:
logger.info('Removing file {}'.format(dicfiles[id]))
os.remove(dicfiles[id])
except:
logger.warning('File {} does not exist!'.format(dicfiles[id]))
def test_subgen_implicit_none_false():
''' test that implicit none is not added to the subroutine if
not requested '''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine", implicitnone=False)
module.add(sub)
count = count_lines(sub.root, "IMPLICIT NONE")
assert count == 0, "IMPLICIT NONE SHOULD NOT EXIST"
def test_subgen_implicit_none_default():
''' test that implicit none is not added to the subroutine by
default '''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine")
module.add(sub)
count = count_lines(sub.root, "IMPLICIT NONE")
assert count == 0, "IMPLICIT NONE SHOULD NOT EXIST BY DEFAULT"
def test_subgen_implicit_none_true():
''' test that implicit none is added to the subroutine if
requested '''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine", implicitnone=True)
module.add(sub)
count = count_lines(sub.root, "IMPLICIT NONE")
assert count == 1, "IMPLICIT NONE SHOULD EXIST"
def test_imp_none_in_module_already_exists():
''' test that implicit none is not added to a module when one
already exists'''
module = ModuleGen(name="testmodule", implicitnone=True)
module.add(ImplicitNoneGen(module))
count = count_lines(module.root, "IMPLICIT NONE")
print str(module.root)
assert count == 1, \
"There should only be one instance of IMPLICIT NONE"
def test_progunitgen_multiple_use1():
'''Check that we correctly handle the case where duplicate use statements
are added but one is specific'''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine")
module.add(sub)
sub.add(UseGen(sub, name="fred"))
sub.add(UseGen(sub, name="fred", only=True, funcnames=["astaire"]))
assert count_lines(sub.root, "USE fred") == 1
def test_progunitgen_multiple_generic_use():
'''Check that we correctly handle the case where duplicate use statements
are added'''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine")
module.add(sub)
sub.add(UseGen(sub, name="fred"))
sub.add(UseGen(sub, name="fred"))
assert count_lines(sub.root, "USE fred") == 1
def test_do_loop_with_increment():
''' Test that we correctly generate code for a do loop with
non-unit increment '''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsub")
module.add(sub)
dogen = DoGen(sub, "it", "1", "10", step="2")
sub.add(dogen)
count = count_lines(sub.root, "DO it=1,10,2")
assert count == 1
def test_imp_none_in_subroutine_already_exists():
''' test that implicit none is not added to a subroutine when
one already exists'''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine", implicitnone=True)
module.add(sub)
sub.add(ImplicitNoneGen(sub))
count = count_lines(sub.root, "IMPLICIT NONE")
assert count == 1, \
"There should only be one instance of IMPLICIT NONE"
def __init__(self, colors, parent=None, sugar=True):
self.colors = colors
self.parent = parent
self.good_job = None
self.sugar = sugar
self.journal = True # set to False if we come in via main()
self.canvas = None
self.cyan_button = None
self.pattern = 1
self.num_patterns = utils.count_lines(
os.path.join('data', 'patterns.dat'))
def transform(sample_file, dict_file, projection_file, output_file):
n_samples = utils.count_lines(sample_file)
n_features = utils.count_lines(dict_file)
batch_size = 100000
print 'loading projector...'
with open(projection_file, 'rb') as f:
p = utils.zloads(f.read())
# p = pickle.load(f)
with open(output_file, 'w') as f:
for (v, idx, ptr) in utils._load_data_batch(sample_file, batch_size, np.intc, n_samples):
X = csr_matrix((v, idx, ptr), shape=(len(ptr)-1, n_features))
print 'transforming... \r',
sys.stdout.flush()
T = p.transform(X)
print 'saving... \r',
sys.stdout.flush()
utils.save_data(f, T)
print '\nTransform completed.'
def test_progunitgen_multiple_use2():
'''Check that we correctly handle the case where the same module
appears in two use statements but, because the first use is
specific, the second, generic use is included.
'''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine")
module.add(sub)
sub.add(UseGen(sub, name="fred", only=True, funcnames=["astaire"]))
sub.add(UseGen(sub, name="fred"))
assert count_lines(sub.root, "USE fred") == 2
def get_read_count_ratio(design):
'''Get the ratio of read counts for unique controls.'''
controls = design.control_tag_align.unique()
control_dict = {}
for con in controls:
no_reads = utils.count_lines(con)
control_dict[con] = no_reads
control_matrix = {c: control_dict for c in controls}
control_df = pd.DataFrame.from_dict(control_matrix)
control_ratio = control_df.divide(list(control_dict.values()), axis=0)
return control_ratio
def make_vocab(path, n_vocab, delimiter=' ', lower=True):
words = []
n_lines = count_lines(args.TRAIN)
bar = progressbar.ProgressBar()
with open(args.TRAIN) as f:
for line in bar(f, max_value=n_lines):
_, h, t, _ = line.strip().split('\t')
if lower is True:
h = str(h).lower()
t = str(t).lower()
words.extend(h.strip().split(delimiter))
words.extend(t.strip().split(delimiter))
counter = collections.Counter(words)
vocab = [w for w, c in counter.most_common(n_vocab)]
return vocab
def test_diffbind_singleend_multiple_rep():
if os.path.isfile(
os.path.join(test_output_path,
'ENCSR272GNQ_vs_ENCSR238SGC_diffbind.bed')):
assert os.path.exists(
os.path.join(test_output_path,
'ENCSR272GNQ_vs_ENCSR238SGC_diffbind.bed'))
diffbind_file = test_output_path + 'ENCSR272GNQ_vs_ENCSR238SGC_diffbind.csv'
elif os.path.isfile(
os.path.join(test_output_path,
'ENCSR238SGC_vs_ENCSR272GNQ_diffbind.bed')):
assert os.path.exists(
os.path.join(test_output_path,
'ENCSR238SGC_vs_ENCSR272GNQ_diffbind.bed'))
diffbind_file = test_output_path + 'ENCSR238SGC_vs_ENCSR272GNQ_diffbind.csv'
assert os.path.exists(diffbind_file)
assert utils.count_lines(diffbind_file) >= 197471
def test_diffbind_pairedend_single_rep():
if os.path.isfile(
os.path.join(test_output_path,
'ENCSR757EMK_vs_ENCSR729LGA_diffbind.bed')):
assert os.path.exists(
os.path.join(test_output_path,
'ENCSR757EMK_vs_ENCSR729LGA_diffbind.bed'))
diffbind_file = test_output_path + 'ENCSR757EMK_vs_ENCSR729LGA_diffbind.csv'
elif os.path.isfile(
os.path.join(test_output_path,
'ENCSR729LGA_vs_ENCSR757EMK_diffbind.bed')):
assert os.path.exists(
os.path.join(test_output_path,
'ENCSR729LGA_vs_ENCSR757EMK_diffbind.bed'))
diffbind_file = test_output_path + 'ENCSR729LGA_vs_ENCSR757EMK_diffbind.csv'
assert os.path.exists(diffbind_file)
assert utils.count_lines(diffbind_file) >= 65182
def test_module_inline_same_kernel():
'''Tests that correct results are obtained when an invoke that uses
the same kernel subroutine more than once has that kernel
inlined'''
psy, invoke = get_invoke("test14_module_inline_same_kernel.f90", 0)
schedule = invoke.schedule
kern_call = schedule.children[0].children[0].children[0]
inline_trans = KernelModuleInlineTrans()
_, _ = inline_trans.apply(kern_call)
gen = str(psy.gen)
# check that the subroutine has been inlined
assert 'SUBROUTINE time_smooth_code(' in gen
# check that the associated psy "use" does not exist
assert 'USE time_smooth_mod, ONLY: time_smooth_code' not in gen
# check that the subroutine has only been inlined once
count = count_lines(psy.gen, "SUBROUTINE time_smooth_code(")
assert count == 1, "Expecting subroutine to be inlined once"
def test_progunit_multiple_use3():
'''Check that we correctly handle the case where the same module is
specified in two UseGen objects statements both of which are
specific and they have overlapping variable names.
'''
module = ModuleGen(name="testmodule")
sub = SubroutineGen(module, name="testsubroutine")
module.add(sub)
funcnames = ["a", "b", "c"]
sub.add(UseGen(sub, name="fred", only=True, funcnames=funcnames))
funcnames = ["c", "d"]
sub.add(UseGen(sub, name="fred", only=True, funcnames=funcnames))
gen = str(sub.root)
expected = (" USE fred, ONLY: d\n" " USE fred, ONLY: a, b, c")
assert expected in gen
assert count_lines(sub.root, "USE fred") == 2
# ensure that the input list does not get modified
assert funcnames == ["c", "d"]
def softmax2npy(model_name, test_file, num_classes):
model = load_model(model_name)
model.summary()
# load test dataset
test_file_path = f"data/{test_file}"
num_tests = utils.count_lines(test_file_path)
test_generator = rna_dataset.RNA_onehot(test_file_path, num_tests, batch_size=256, dim=(200, 4),
num_channels=1, num_classes=num_classes, shuffle=False)
# prediction
print('Start predicting ... ...')
prediction = model.predict_generator(test_generator, workers=6,
use_multiprocessing=True, verbose=1)
# prediction = model.predict_generator(test_generator, verbose=1)
np.save(f"{model_name.split('.').pop(0)}/{test_file.split('.').pop(0)}.npy", prediction)
# release the memory occupied by GPU
K.clear_session()
def self_psuedoreplication(tag_file, prefix, paired):
'''Make 2 self-psuedoreplicates.'''
# Get total number of reads
no_lines = utils.count_lines(tag_file)
# Number of lines to split into
lines_per_rep = (no_lines+1)/2
# Make an array of number of psuedoreplicatesfile names
pseudoreplicate_dict = {r: prefix + '.pr' + str(r) + '.tagAlign.gz'
for r in [0, 1]}
# Shuffle and split file into equal parts
# by using the input to seed shuf we ensure multiple runs with the same
# input will produce the same output
# Produces two files named splits_prefix0n, n=1,2
splits_prefix = 'temp_split'
psuedo_command = 'bash -c "zcat {} | shuf --random-source=<(openssl enc -aes-256-ctr -pass pass:$(zcat -f {} | wc -c) -nosalt </dev/zero 2>/dev/null) | '
psuedo_command += 'split -d -l {} - {}."'
psuedo_command = psuedo_command.format(
tag_file,
tag_file,
int(lines_per_rep),
splits_prefix)
logger.info("Running psuedo with %s", psuedo_command)
subprocess.check_call(shlex.split(psuedo_command))
# Convert read pairs to reads into standard tagAlign file
for i, index in enumerate([0, 1]):
string_index = '.0' + str(index)
steps = ['cat %s' % (splits_prefix + string_index)]
if paired:
steps.extend([r"""awk 'BEGIN{OFS="\t"}{printf "%s\t%s\t%s\tN\t1000\t%s\n%s\t%s\t%s\tN\t1000\t%s\n",$1,$2,$3,$9,$4,$5,$6,$10}'"""])
steps.extend(['gzip -cn'])
out, err = utils.run_pipe(steps, outfile=pseudoreplicate_dict[i])
return pseudoreplicate_dict
def motif_search(filename, genome, experiment, peak):
'''Run motif serach on peaks.'''
file_basename = os.path.basename(
utils.strip_extensions(filename, STRIP_EXTENSIONS))
out_fa = '%s.fa' % (experiment)
out_motif = '%s_memechip' % (experiment)
# Sort Bed file and limit number of peaks
if peak == -1:
peak = utils.count_lines(filename)
peak_no = 'all'
else:
peak_no = peak
sorted_fn = '%s.%s.narrowPeak' % (file_basename, peak_no)
out, err = utils.run_pipe(
['sort -k %dgr,%dgr %s' % (5, 5, filename),
'head -n %s' % (peak)],
outfile=sorted_fn)
# Get fasta file
out, err = utils.run_pipe([
'bedtools getfasta -fi %s -bed %s -fo %s' % (genome, sorted_fn, out_fa)
])
if err:
logger.error("bedtools error: %s", err)
# Call memechip
out, err = utils.run_pipe([
'meme-chip -oc %s -meme-minw 5 -meme-maxw 15 -meme-nmotifs 10 %s -norand'
% (out_motif, out_fa)
])
if err:
logger.error("meme-chip error: %s", err)
def verify_errors(filein, fileout):
fout = open(fileout, 'w')
logger.info('Checking number of images...')
nb_imgs = count_lines(filein)
logger.info('Input folder containing %d images' % nb_imgs)
count = 0
pb = ProgressBar(nb_imgs)
with open(filein) as fin:
for line in fin:
pb.update()
line = line.strip()
if not isfile(line):
continue
img = cv2.imread(line)
if identify_green(img) or identify_red(img) or identify_topline(
img):
fout.write('%s\n' % line)
count += 1
logger.info('{} files containing errors'.format(count))
logger.info('Log of error files saved at: {}'.format(fileout))
fout.close()
def main(inputfile, output, size):
"""
Resize images from `fileinput` to `size` by `size`, saving the output
images in `output` folder.
"""
if not output:
output = join(dirname(inputfile), str(size))
if not isdir(output):
os.mkdir(output)
logger.info('Resizing images from: %s' % inputfile)
inputfile = realpath(inputfile)
#/usr/share/datasets/KSCGR_Original/data1/boild-egg/0.jpg 0
nb_lines = count_lines(inputfile)
pbar = pb.ProgressBar(nb_lines)
with open(inputfile) as fin:
for line in fin:
path, tl = line.strip().split()
newpath = create_path(path, output)
img = cv2.imread(path)
img = resize_image(img, size)
cv2.imwrite(newpath, img)
pbar.update()
logger.info('Total of images resized: %d' % nb_lines)
def test_peaks_bed_singleend():
peak_file = test_output_path + 'ENCSR238SGC/1/' + 'ENCLB144FDT.narrowPeak'
assert utils.count_lines(peak_file) == 226738
def test_limited_peaks_pairedend():
peak_file_ENCSR729LGA = test_output_path + 'ENCSR729LGA.600.narrowPeak'
assert os.path.exists(peak_file_ENCSR729LGA)
assert utils.count_lines(peak_file_ENCSR729LGA) == 600
def test_peaks_bed_pairedend():
peak_file = test_output_path + 'ENCSR729LGA/2/' + 'ENCLB568IYX.narrowPeak'
assert utils.count_lines(peak_file) == 112631
async def admin_command(cmd, ctx):
client = ctx['client']
message = ctx['message']
channel = message.channel
params = ctx['params']
params_str = ctx['params_str']
guilds = ctx['guilds']
LAST_COMMIT = ctx['LAST_COMMIT']
if cmd == 'log':
logfile = "log{}.txt".format("" if cfg.SAPPHIRE_ID is None else cfg.SAPPHIRE_ID)
if not os.path.exists(logfile):
await channel.send("No log file")
return
with open(logfile, 'r', encoding="utf8") as f:
data = f.read()
data = data[-10000:] # Drop everything but the last 10k characters to make string ops quicker
data = data.replace(' Creating channel for ', ' ✅')
data = data.replace(' Deleting ', ' ❌')
data = data.replace(' Renaming ⌛ to ', ' ⏩ ')
data = data.replace(' Renaming ', ' 🔄')
data = data.replace(' to ', ' ⏩ ')
data = data.replace(' CMD Y: ', ' C✔ ')
data = data.replace(' CMD F: ', ' C✖ ')
data = data.replace(" creating channels too quickly", " creating channels too quickly❗❗")
data = data.replace(" where I don't have permissions", " where I don't have permissions❗❗")
data = data.replace("Traceback (most recent", "❗❗Traceback (most recent")
data = data.replace("discord.errors.", "❗❗discord.errors.")
data = data.replace("Remembering channel ", "❗❗Remembering ")
data = data.replace("New tickrate is ", "🕐")
data = data.replace(", seed interval is ", " 🕐")
data = data.replace(' ', ' ') # Reduce indent to save character space
today = datetime.now(pytz.timezone(cfg.CONFIG['log_timezone'])).strftime("%Y-%m-%d")
data = data.replace(today, 'T')
character_limit = 2000 - 17 # 17 for length of ```autohotkey\n at start and ``` at end.
data = data[character_limit * -1:]
data = data.split('\n', 1)[1]
lines = data.split('\n')
for i, l in enumerate(lines):
# Fake colon (U+02D0) to prevent highlighting the line
if " ⏩" in l:
lines[i] = l.replace(':', 'ː')
elif l.startswith('T '):
if '[' in l:
s = l.split('[', 1)
lines[i] = s[0] + '[' + s[1].replace(':', 'ː')
data = '\n'.join(lines)
data = '```autohotkey\n' + data
data = data + '```'
await channel.send(data)
if cmd == 'stats':
r = await channel.send(". . .")
t1 = message.created_at
t2 = r.created_at
response_time = (t2 - t1).total_seconds()
num_users = 0
for g in guilds:
num_users += len([m for m in g.members if not m.bot])
lines_of_code = 0
for f in os.listdir(cfg.SCRIPT_DIR):
if f.lower().endswith('.py'):
lines_of_code += utils.count_lines(os.path.join(cfg.SCRIPT_DIR, f))
elif f == "commands":
for sf in os.listdir(os.path.join(cfg.SCRIPT_DIR, f)):
if sf.lower().endswith('.py'):
lines_of_code += utils.count_lines(os.path.join(cfg.SCRIPT_DIR, f, sf))
cpu = psutil.cpu_percent()
mem = psutil.virtual_memory()
disk = psutil.disk_usage('/')
await r.edit(content=(
"Servers: **{tot_servs}** (A:{active_servs} S:{shards}) \t "
"Users: **{users}** \t Channels: **{channels}** \n"
"Response time: **{rt}** \t Tick rate: **{tr}** \t Tick time: **{tt}** | **{gtt}**\n"
"CPU: **{cpu}%** \t MEM: **{memg} ({memp}%)** \t DISK: **{diskg} ({diskp}%)**\n"
"**Last commit:** {commit}\n"
"**Lines of code:** {lines}\n"
"**Timings:** \n{timings}".format(
tot_servs=len(guilds),
active_servs=utils.num_active_guilds(guilds),
shards=utils.num_shards(guilds),
users=num_users,
channels=utils.num_active_channels(guilds),
rt="{0:.2f}s".format(response_time),
tr="{0:.1f}s".format(cfg.TICK_RATE),
tt="{0:.2f}s".format(cfg.TICK_TIME),
gtt="{0:.2f}s".format(cfg.G_TICK_TIME),
cpu=cpu, memg="{0:.1f}GB".format(mem.used / 1024 / 1024 / 1024), memp=round(mem.percent),
diskg="{0:.1f}GB".format(disk.used / 1024 / 1024 / 1024), diskp=round(disk.percent),
commit=LAST_COMMIT,
lines=lines_of_code,
timings=utils.format_timings()
)
))
if cmd == 'ping':
r = await channel.send(". . .")
t1 = message.created_at
t2 = r.created_at
response_time = (t2 - t1).total_seconds()
e = '🔴🔴🔴' if response_time > 5 else ('🟠🟠' if response_time > 1 else '🟢')
await r.edit(content="**{0} {1:.1f}s**".format(e, response_time))
if cmd == 'top':
top_guilds = []
for g in guilds:
s = func.get_secondaries(g)
top_guilds.append({"name": g.name,
"size": len([m for m in g.members if not m.bot]),
"num": len(s) if s is not None else 0})
top_guilds = sorted(top_guilds, key=lambda x: x['num'], reverse=True)[:10]
r = "**Top Guilds:**"
for g in top_guilds:
r += "\n`{}` {}: \t**{}**".format(
g['size'],
func.esc_md(g['name']),
g['num']
)
r += "\n\n**{}**".format(utils.num_active_channels(guilds))
await channel.send(r)
if cmd == 'patrons':
if patreon_info is None:
await channel.send(content='❌')
return
patrons = patreon_info.fetch_patrons(force_update=True)
if not patrons:
await channel.send(content='❌')
return
fields = {}
auths = patreon_info.update_patron_servers(patrons)
for p, pv in patrons.items():
pu = client.get_user(p)
if pu is not None:
pn = pu.name
else:
pn = "Unknown"
gn = ""
if str(p) in auths:
for s in auths[str(p)]['servers']:
gn += "`{}` ".format(s)
if 'extra_gold' in auths[str(p)]:
for s in auths[str(p)]['extra_gold']:
gn += "+g`{}` ".format(s)
fields["`{}` **{}** {}".format(p, pn, cfg.TIER_ICONS[pv])] = gn
try:
for field_chunk in utils.dict_chunks(fields, 25):
e = discord.Embed(color=discord.Color.from_rgb(205, 220, 57))
e.title = "{} Patrons".format(len(field_chunk))
for f, fv in field_chunk.items():
fv = fv if fv else "None"
e.add_field(name=f, value=fv)
await channel.send(embed=e)
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'sapphiredebug':
if cfg.SAPPHIRE_ID is None:
await channel.send(content='❌ Not a sapphire')
return
if patreon_info is None:
await channel.send(content='❌ No patreon_info')
return
auths = utils.read_json(os.path.join(cfg.SCRIPT_DIR, "patron_auths.json"))
initiator_id = cfg.CONFIG["sapphires"][str(cfg.SAPPHIRE_ID)]['initiator']
msg = ("Sapphire ID: {}\n"
"User: `{}`\n"
"Actual guilds: {}\n"
"Config guilds: {}\n"
"Authenticated guilds: {}\n"
"get_guilds: {}".format(
cfg.SAPPHIRE_ID,
initiator_id,
", ".join(['`' + str(g.id) + '`' for g in client.guilds]),
", ".join(['`' + str(g) + '`' for g in cfg.CONFIG["sapphires"][str(cfg.SAPPHIRE_ID)]['servers']]),
", ".join(['`' + str(g) + '`' for g in auths[str(initiator_id)]['servers']]),
", ".join(['`' + str(g.id) + '`' for g in func.get_guilds(client)]))
)
await channel.send(msg)
if cmd == 'status':
g = utils.strip_quotes(params_str)
if not g:
await func.react(message, '❌')
return
try:
await client.change_presence(
activity=discord.Activity(name=g, type=discord.ActivityType.watching)
)
await func.react(message, '✅')
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'settings':
gid = utils.strip_quotes(params_str)
try:
int(gid)
except ValueError:
for x in guilds:
if x.name == gid:
gid = str(x.id)
break
fname = gid + '.json'
fp = os.path.join(cfg.SCRIPT_DIR, "guilds", fname)
if os.path.exists(fp):
gid = int(gid)
g = client.get_guild(gid)
head = "**{}** `{}`{}".format(g.name, gid, ("✅" if g in func.get_guilds(client) else "❌"))
head += "💎" if func.is_sapphire(gid) else ("💳" if func.is_gold(gid) else "")
s = head
s += "\n```json\n"
with open(fp, 'r') as f:
file_content = f.read()
s += file_content
s += '```'
try:
await channel.send(s)
except discord.errors.HTTPException:
# Usually because message is over character limit
haste_url = await utils.hastebin(file_content)
await channel.send("{}\n{}".format(head, haste_url))
else:
await func.react(message, '❌')
if cmd == 'refetch':
gid = utils.strip_quotes(params_str)
try:
gid = int(gid)
except ValueError:
await func.react(message, '❌')
return
g = client.get_guild(gid)
if g is None:
await func.react(message, '❓')
return
utils.get_serv_settings(g, force_refetch=True)
await func.react(message, '✅')
return
if cmd == 'disable':
try:
g = client.get_guild(int(utils.strip_quotes(params_str)))
settings = utils.get_serv_settings(g)
settings['enabled'] = False
utils.set_serv_settings(g, settings)
log("Force Disabling", g)
await func.react(message, '✅')
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'enable':
try:
g = client.get_guild(int(utils.strip_quotes(params_str)))
settings = utils.get_serv_settings(g)
settings['enabled'] = True
utils.set_serv_settings(g, settings)
log("Force Enabling", g)
await func.react(message, '✅')
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'info':
cid = utils.strip_quotes(params_str)
try:
c = client.get_channel(int(cid))
members = [m.display_name + " \t {}".format(utils.debug_unicode(m.display_name)) for m in c.members]
games = []
for m in c.members:
if m.activity:
games.append(m.activity.name + " \t {}".format(utils.debug_unicode(m.activity.name)))
s = "**__Server:__** {} `{}`\n**__Name:__** {}\n{}\n\n".format(
c.guild.name, c.guild.id, c.name, utils.debug_unicode(c.name)
)
if c.id in cfg.ATTEMPTED_CHANNEL_NAMES:
s += "**__Attempted Name:__** {}\n{}\n\n".format(
cfg.ATTEMPTED_CHANNEL_NAMES[c.id],
utils.debug_unicode(cfg.ATTEMPTED_CHANNEL_NAMES[c.id])
)
s += "**__{} Members:__**\n".format(len(members))
s += '\n'.join(members)
s += '\n\n**__{} Games:__**\n'.format(len(games))
s += '\n'.join(games)
s = s.replace('\n\n\n', '\n\n')
await channel.send(s)
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'whois':
uid = utils.strip_quotes(params_str)
try:
u = client.get_user(int(uid))
in_guilds = {}
for g in client.guilds:
if u in g.members:
m = g.get_member(int(uid))
in_guilds[g.id] = {
"guild_name": func.esc_md(g.name),
"guild_size": g.member_count,
"patron": "💎" if func.is_sapphire(g) else ("💳" if func.is_gold(g) else ""),
"user_name": func.esc_md(m.display_name),
"role": m.top_role.name,
}
if in_guilds:
s = "**{}**".format(func.user_hash(u))
s += " \t :b: :regional_indicator_o: :regional_indicator_t:" if u.bot else ""
can_dm = True
try:
await u.create_dm()
can_dm = client.user.permissions_in(u.dm_channel).send_messages
except discord.errors.Forbidden:
can_dm = False
s += " \t Can DM: {}".format('✅' if can_dm else '❌')
for gid, g in in_guilds.items():
s += "\n{}`{}` **{}** (`{}`) \t {} ({})".format(
g['patron'], gid, g['guild_name'], g['guild_size'], g['user_name'], g['role']
)
await echo(s, channel)
else:
await channel.send("¯\\_(ツ)_/¯")
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'votekicks':
try:
readable = {}
for k, kv in cfg.VOTEKICKS.items():
readable[k] = {
"initiator": kv['initiator'].display_name,
"participants": [m.display_name for m in kv['participants']],
"required_votes": kv['required_votes'],
"offender": kv['offender'].display_name,
"reason": kv['reason'],
"in_favor": [m.display_name for m in kv['in_favor']],
"voice_channel": kv['voice_channel'].id,
"message": kv['message'].id,
"end_time": datetime.fromtimestamp(kv['end_time']).strftime("%Y-%m-%d %H:%M")
}
s = "```json\n" + json.dumps(readable, indent=1, sort_keys=True) + "```"
print(s)
try:
await channel.send(s)
except discord.errors.HTTPException:
# Usually because message is over character limit
haste_url = await utils.hastebin(s)
await channel.send(haste_url)
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'exit':
attempts = 0
while attempts < 100:
attempts += 1
if not cfg.WRITES_IN_PROGRESS:
print("Exiting!")
await client.close()
sys.exit()
break
else:
print("Failed to close", cfg.WRITES_IN_PROGRESS)
await func.react(message, '❌')
if cmd == 'loop':
mode = params[0]
loop_name = params[1]
try:
loop = ctx['loops'][loop_name]
modes = { # Dict of possible loop functions/attrs as [fn, arg]
'current_loop': [loop.current_loop, None],
'next_iteration': [loop.next_iteration, None],
'next_run': [loop.next_iteration, None], # Alias
'start': [loop.start, client],
'stop': [loop.stop, None],
'cancel': [loop.cancel, None],
'restart': [loop.restart, client],
'is_being_cancelled': [loop.is_being_cancelled, None],
'last_run': [loop.last_run, None],
}
if mode not in modes:
await func.react(message, '❓')
return
fn, arg = modes[mode]
if callable(fn):
if arg is None:
r = fn()
else:
r = fn(arg)
else:
r = fn
if r is not None:
if isinstance(r, date):
r = r.astimezone(pytz.timezone(cfg.CONFIG['log_timezone']))
await channel.send(r.strftime("%Y-%m-%d %H:%M:%S"))
else:
await channel.send(str(r))
await func.react(message, '✅')
except:
await channel.send(traceback.format_exc())
await channel.send("Loops: \n{}".format('\n'.join(ctx['loops'].keys())))
await func.react(message, '❌')
if cmd == 'rename':
try:
cid = utils.strip_quotes(params[0])
c = client.get_channel(int(cid))
new_name = ' '.join(params[1:])
if not new_name:
new_name = "⌛"
await c.edit(name=new_name)
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
else:
await func.react(message, '✅')
log("{0} Force Renaming to {1}".format(cid[-4:], new_name), c.guild)
if cmd == 'forget':
try:
cid = int(utils.strip_quotes(params[0]))
c = client.get_channel(cid)
settings = utils.get_serv_settings(c.guild)
for p, pv in settings['auto_channels'].items():
tmp = settings['auto_channels'][p]['secondaries'].copy()
for s, sv in pv['secondaries'].items():
if s == cid:
del settings['auto_channels'][p]['secondaries'][s]
break
utils.set_serv_settings(c.guild, settings)
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
else:
await func.react(message, '✅')
if cmd == 'delete':
try:
cid = int(utils.strip_quotes(params[0]))
c = client.get_channel(cid)
await c.delete()
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
else:
await func.react(message, '✅')
if cmd == 'whisper':
params_str = utils.strip_quotes(params_str)
if '\n' not in params_str:
await func.react(message, '❌')
return
uid, msg = params_str.split('\n', 1)
try:
u = await client.fetch_user(uid)
except discord.errors.NotFound:
await func.react(message, '❌')
return
if u.dm_channel is None:
await u.create_dm()
try:
await u.dm_channel.send(msg)
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
else:
await func.react(message, '✅')
if cmd == 'cleanprimaries':
try:
n_primaries = 0
n_real_primaries = 0
for g in client.guilds:
settings = utils.get_serv_settings(g)
tmp = {}
n_primaries += len(settings['auto_channels'])
for p, pv in settings['auto_channels'].items():
c = g.get_channel(p)
if c:
tmp[p] = pv
n_real_primaries += len(tmp)
if len(settings['auto_channels']) != len(tmp):
settings['auto_channels'] = tmp
utils.set_serv_settings(g, settings)
await channel.send("Cleaned {} of {} primaries".format(n_real_primaries, n_primaries))
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
if cmd == 'leaveinactive':
params_str = utils.strip_quotes(params_str)
try:
total_guilds = 0
inactive_guilds = 0
cfg.CONFIG['leave_inactive'] = []
for g in client.guilds:
total_guilds += 1
if g and (not utils.guild_is_active(g) or g not in guilds):
cfg.CONFIG['leave_inactive'].append(g.id)
inactive_guilds += 1
if params_str == "go":
try:
await g.leave()
except discord.errors.NotFound:
pass
if params_str == "go":
await channel.send("Left {} of {} guilds.".format(inactive_guilds, total_guilds))
else:
await channel.send("Will leave {} of {} guilds. "
"Rerun command with 'go' at end to actually leave them.".format(
inactive_guilds, total_guilds))
cfg.CONFIG['leave_inactive'] = []
except:
await channel.send(traceback.format_exc())
await func.react(message, '❌')
f'{local_time},{log_file_name},{args.dataset},{accuracy_score(label_list, prediction_1d)},{str(args.filter_sizes).replace(","," ")},{args.num_filters},{args.batch_size},{args.num_epochs},{args.keep_prob},{args.num_hidden},{args.learning_rate},{args.loss_function},{args.optimizer}, ',
file=csv)
if __name__ == "__main__":
args = utils.cmd_onehot()
print(args)
# initialize a variable to test the existence of the validation dataset
val_bool = True
# get the paths of each dataset
# statistic the size of the dataset
train_path = os.path.join("inputdir", args.dataset,
"train.csv") # training dataset
num_trains = utils.count_lines(train_path)
try: # validation dataset
valid_path = os.path.join("inputdir", args.dataset, "validation.csv")
num_validations = utils.count_lines(valid_path)
except FileNotFoundError:
val_bool = False
test_path = os.path.join("inputdir", args.dataset,
"test.csv") # testing dataset
num_tests = utils.count_lines(test_path)
fam_dict_path = os.path.join("inputdir", args.dataset, "fam_label.csv")
# generate dataset
train_generator = rna_dataset.RNA_onehot(train_path,
