def find_all_images(input_dict):
if 'images' in input_dict and input_dict['images'] is not None:
inputFiles = []
for dir_or_image in l_eval(str(input_dict['images'])):
if os.path.isdir(dir_or_image):
inputFiles += [os.path.join(dir_or_image, f) for f in os.listdir(dir_or_image) if os.path.isfile(os.path.join(dir_or_image, f))]
else:
inputFiles += [dir_or_image]
input_dict['images'] = inputFiles
def _edit_event(self):
# select an event to edit
event_list, _ = self.logbook.get_event_timeline()
selected = self._select_list(event_list,
title="Select events to edit/view...",
multiple_select=False)
# edit event selected
dlg = EventDialog(self.logbook.get_unique_timelines(),
self.logbook.get_unique_locs(),
self.logbook.get_unique_chars(),
self.logbook.get_unique_civs())
event_data = self.logbook.get_event_data(selected)[
0] # get event data to view / edit
dlg.ui.event_title.setText(event_data[1])
dlg.ui.event_description.setPlainText(event_data[2])
dlg.ui.day.setValue(int((event_data[3] % 1) * 365))
dlg.ui.year.setValue(int(event_data[3]))
dlg.ui.timeline.setCurrentText(
event_data[7]) # set the timeline to match the event's
dlg.ui.location.setCurrentText(event_data[5])
# prep character list
char_list = event_data[4]
if isinstance(char_list, str):
char_list = l_eval(char_list)
for character in char_list: # check characters
matching_items = dlg.ui.characters.findItems(
character, Qt.MatchContains)
for item in matching_items:
item.setSelected(True)
civ_list = event_data[6]
if isinstance(civ_list, str):
civ_list = l_eval(civ_list)
for civ in civ_list:
matching_items = dlg.ui.civilizations.findItems(
civ, Qt.MatchContains)
for item in matching_items:
item.setSelected(True)
dlg.accepted.connect(lambda: self._change_event(selected, dlg))
dlg.exec_()
def parse(filename):
from ast import literal_eval as l_eval
file = open(filename, 'r')
content = file.read()
content = content.split()
contstr = ''
for i in range(len(content)):
if content[i][0] != '(':
content[i] += ','
for i in range(len(content)):
contstr += content[i]
return l_eval(contstr)
def get_unique_chars(self):
data = self._event_df['Characters Involved'].values
all_strings = list()
to_be_uniques = list()
if data.any():
for s in data:
s = l_eval(str(s))
all_strings.append(s)
for l in all_strings:
for name in l:
to_be_uniques.append(name)
return set(to_be_uniques)
def get_unique_civs(self):
all_strings = list()
to_be_uniques = list()
data = self._event_df['Civilizations Involved'].values
if data.any():
for s in data:
s = l_eval(str(s))
all_strings.append(s)
for l in all_strings:
for name in l:
to_be_uniques.append(name)
return set(to_be_uniques)
def make_list(obj):
if isinstance(obj, string_types):
try:
obj = l_eval(obj)
except:
obj = [o.strip() for o in obj.split(',')]
if not isinstance(obj, list):
if obj is None:
obj = []
else:
obj = [obj]
return obj
def eval_string(input_dict):
for key, val in list(input_dict.items()):
try:
input_dict[key] = l_eval(str(val))
except:
pass
random.seed("AMAZON")
for d in dir_to_sample:
print("Processing {}".format(d))
ori_dir = os.path.join(data_path, d)
target_dir = ori_dir + "-SAMPLE"
if not os.path.exists(target_dir):
os.makedirs(target_dir)
dev_loc = os.path.join(ori_dir, "dev.tsv")
train_loc = os.path.join(ori_dir, "train.tsv")
results = {}
statistic = []
with open(train_loc, 'r') as fin:
for line in fin:
sentence = line.split('\t')[2]
score = l_eval(line.split('\t')[1])
if sentence in results:
continue
results[sentence] = line
statistic.append(score)
if len(results) >= expected_num:
break
pprint(sorted(dict(Counter(statistic)).items()))
assert len(results) == expected_num
dev_loc = os.path.join(target_dir, "dev.tsv")
train_loc = os.path.join(target_dir, "train.tsv")
output = list(results.values())
random.shuffle(output)
with open(train_loc, 'w', newline='', encoding="UTF-8") as fout:
for l in output[:train_num]:
fout.write(l)
def huffman(ifile):
if ifile[len(ifile)-3:len(ifile)]!='huf':
text=open(ifile,'r').read()
# Frequency of each char
cf_dic = {}
for char in text:
cf_dic[char] = cf_dic.get(char, 0) + 1
# Convert cf_dict to list of tuples, flip elements and order highest freq first
value_key = sorted([(f, k) for k, f in cf_dic.items()], reverse=True)
# Convert frequency to probability
freq_total= float(sum([freq[0] for freq in value_key]))
value_key=zip([freq[0]/freq_total for freq in value_key],[char[1] for char in value_key])
bolean=True
tmp=value_key
combinacions=[]
# Huffman algorithm
while bolean:
combinacions.append([tmp[-2][1],'0'])
combinacions.append([tmp[-1][1],'1'])
tmp2=tmp
tmp=tmp[:-2]
tmp.append((tmp2[-1][0]+tmp2[-2][0],tmp2[-1][1]+tmp2[-2][1]))
tmp.sort(key=lambda tup:tup[0], reverse=True)
if len(tmp)==1:
bolean=False
taula={} # Taula a passar
for comb,b in combinacions:
for c in list(comb):
try:
taula[c]= b+ taula[c]
except:
taula[c]=b
textc=[]
taulaint={k: map(int,v) for k, v in taula.items()}
for char in text:
textc+=taulaint[char]
textb=ba.bitarray(textc)
#Taula en bitarray
tableb=ba.bitarray()
tableb.frombytes(str(taula))
# Primers bytes que ens diran la longitud de la taula i els bits que he afegit al final (que no pertanyen a la compressió, sino a acabar el byte)
lentableb=bin(len(tableb))[2:]
aux=ba.bitarray('0'*(16-len(lentableb))+lentableb)
bitsafegits=len(textb) % 8
aux+=ba.bitarray('0'*(8-len(bin(bitsafegits)[2:]))+bin(bitsafegits)[2:])
tableb=aux+tableb
# Write file
out=open(ifile+'.huf','wb')
tableb.tofile(out)
textb.tofile(out)
# Extract
else:
data=np.unpackbits(np.fromfile(ifile,dtype='uint8')).tolist()
lendic=int(''.join(map(str,data[:16])),2)
bitsafegits=int(''.join(map(str,data[16:24])),2)
tableb=ba.bitarray(data[24:(24+lendic)])
table=l_eval(tableb.tobytes())
#Invert the table
table={ v: k for k, v in table.items()}
data=data[(24+lendic):]
text=''
bits=''
for b in data:
bits+=str(b)
if bits in table:
text+= table[bits]
bits=''
out=open('HUF_'+ifile[:-4],'w')
out.write(text)
def readConfigFile():
with open("config.txt") as cfgfile:
config = cfgfile.readlines()
params = {}
hasDitto = False
# Read parents parameters
params["parents"] = {}
for gender, skip in [("Male", 3), ("Female", 17)]:
ivs = []
# Read IVs as integers 0-31
for i in range(6):
try:
iv = int(config[skip + i].split(':')[1])
except ValueError:
return None, "Invalid IVs for parent"
if iv < 0 or iv > 31:
return None, "Parent IVs out of 0-31 range"
ivs.append(iv)
# Only accepted items are d.knot and everstone
# One day there will be support for power items
item = parseInput(config[skip + 6].split(':')[1]).upper()
if item not in ["NONE", "DESTINYKNOT", "EVERSTONE"]:
return None, "Invalid item for parent"
# Ability slot should be one of 1, 2, HA
ability = parseInput(config[skip + 7].split(':')[1]).upper()
if ability not in ["1", "2", "HA"]:
return None, "Invalid ability for parent"
# Nature should be one of the valid natures
nature = parseInput(config[skip + 8].split(':')[1]).upper()
if nature not in natures:
return None, "Invalid nature for parent"
# Whether or not this parent is a ditto - boolean value
ditto = parseInput(config[skip + 9].split(':')[1]).upper()
if ditto not in ["Y", "N"]:
return None, "Invalid Ditto parameter in parent"
# Check if breeding two dittos
if ditto == "Y" and hasDitto:
return None, "Cannot breed 2 Dittos"
if ditto == "Y":
hasDitto = True
ditto = True if ditto == "Y" else False
# Store parent info
params["parents"][gender] = Parent(ivs, item, ability, nature,
gender, ditto)
# Read desired child traits parameters
skip = 31
ivs = []
# Rad IVs as range of integers 0-31
for i in range(6):
try:
iv_range = l_eval(parseInput(config[skip + i].split(':')[1]))
except Exception:
return None, "Invalid IV range for child"
if type(iv_range) != list or len(iv_range) != 2:
return None, "Invalid IV range format for child"
for iv in iv_range:
if iv < 0 or iv > 31:
return None, "Child IVs out of 0-31 range"
ivs.append(iv_range)
# Ability slot should be one of Anything, 1, 2, HA
ability = parseInput(config[skip + 6].split(':')[1]).upper()
if ability not in ["ANYTHING", "1", "2", "HA"]:
return None, "Invalid ability for child"
ability = None if ability == "ANYTHING" else ability
# Nature should be one of the valid natures or "Anything"
nature = parseInput(config[skip + 7].split(':')[1]).upper()
if nature not in ["ANYTHING"] + natures:
return None, "Invalid nature for child"
nature = None if nature == "ANYTHING" else nature
# Gender should be one of Anything, M, F, Genderless
gender = parseInput(config[skip + 8].split(':')[1]).upper()
if gender not in ["ANYTHING", "M", "F", "GENDERLESS"]:
return None, "Invalid gender for child"
# Ball should be one of Anything, M, F
gender = None if gender == "ANYTHING" else gender
ball = parseInput(config[skip + 9].split(':')[1]).upper()
if ball not in ["ANYTHING", "M", "F"]:
return None, "Invalid ball for child"
ball = None if ball == "ANYTHING" else ball
# Hidden power should be a valid type
hpower = parseInput(config[skip + 10].split(':')[1]).upper()
if hpower not in ["ANYTHING"] + types:
return None, "Invalid hidden power type"
hpower = None if hpower == "ANYTHING" else hpower
# Shiny should be one of Anything, Y, N
shiny = parseInput(config[skip + 11].split(':')[1]).upper()
if shiny not in ["ANYTHING", "Y", "N"]:
return None, "Invalid shiny parameter for child"
shiny = True if shiny == "Y" else False if shiny == "N" else None
# Store child info
params["child"] = Child(ivs, ability, nature, gender, ball, hpower,
shiny)
skip = 47
# Read RNG seed info
params["seed"] = []
for i in range(4):
try:
status = int(config[skip + i].split(':')[1], 16)
except ValueError:
return None, "Invalid seed status"
params["seed"].append(status)
# Read TSV as integer between 0 and 4096
try:
tsv = int(config[skip + 4].split(':')[1])
except ValueError:
return None, "Invalid TSV"
if tsv < 0 or tsv > 4096:
return None, "TSV must be between 0 and 4096"
params["tsv"] = tsv
# Read ESV parameter as a list of ESV values
try:
esvs = l_eval(parseInput(config[skip + 5].split(':')[1]))
except Exception:
return None, "Invalid ESV, are you usuing the right format?"
params["esvs"] = []
for esv in esvs:
try:
esv = int(esv)
except ValueError:
return None, "Invalid ESV value, not an integer"
if esv < 0 or esv > 4096:
return None, "Invalid ESV value, must e between 0 and 4096"
params["esvs"].append(esv)
skip = 57
# Read gender ratio info
ratio = parseInput(config[skip].split(':')[1]).upper()
if ratio not in ratios:
return None, "Invalid ratio for child"
params["ratio"] = ratios[ratio]
# Read masuda method info
mmethod = parseInput(config[skip + 1].split(':')[1]).upper()
if mmethod not in ["Y", "N"]:
return None, "Invalid Masuda Method parameter"
params["masuda"] = True if mmethod == "Y" else False
# Read shiny charm info
charm = parseInput(config[skip + 2].split(':')[1]).upper()
if charm not in ["Y", "N"]:
return None, "Invalid Shiny Charm parameter"
params["charm"] = True if charm == "Y" else False
# Read ballcheck info / same species info
ballcheck = parseInput(config[skip + 3].split(':')[1]).upper()
if ballcheck not in ["Y", "N"]:
return None, "Invalid same species parameter"
params["ballcheck"] = True if ballcheck == "Y" else False
# Read number of desired results
try:
params["nresults"] = int(config[skip + 4].split(':')[1])
except ValueError:
return None, "Invalid number of results to be shown"
return params, None
try:
path = u + fptr
err = open(path + ".log").read()
err = "".join(err[: err.rfind("Error termination via Lnk1e")].split("\n")[-2:]).strip()
flag = True
except:
pass
if flag:
return path, err
else:
return None, None # Return nothing if we couldn't find anything
# Get previous list of all runs
try:
old = l_eval(open(end_path + "/jlist.list").read())
except:
old = []
# Get new list
new = log.get_jlist(2)
# Find finished jobs
for a in old:
found = False
for b in new:
if a[0] == b[0]:
found = True
if not found:
fptr, err = get_err(a[0])
process_err(fptr, err, a)
# Re-write list for jobs running
'clothes': "AMAZON-C",
'men_clothing': "AMAZON-MC",
'women_clothing': "AMAZON-WC",
'movies': "AMAZON-MV",
'music': "AMAZON-MS",
'shoes': "AMAZON-S"
}
for k, v in shorthand.items():
print("begin -> {}".format(k))
def create_folder(loc):
if not os.path.exists(loc):
os.makedirs(loc)
folder = "amazon_split/{}".format(v)
create_folder(folder)
with open("amazon_split/{}-all.tsv".format(k)) as fin:
f_train = open(folder+'/train.tsv', 'w', newline='', encoding="UTF-8")
f_dev = open(folder+'/dev.tsv', 'w', newline='', encoding="UTF-8")
s_train, n_train = 0, 0
s_dev, n_dev = 0, 0
for i, l in enumerate(fin):
if i % 5 == 0:
f_dev.write(l)
s_dev += l_eval(l.split('\t')[1])
n_dev += 1
else:
f_train.write(l)
s_train += l_eval(l.split('\t')[1])
n_train += 1
print(s_train, n_train, s_dev, n_dev)
f_train.close()
f_dev.close()
document = (d.split('-[')[1].split(']-')[0].strip(),
d.split('-[')[1].split(']-')[1].strip())
else:
document = (d, "-".join(d.split('-')[1:])[6:])
if 'bert-' in d:
document = ("BERT", document[1][7:-7])
else:
document = (document[0][19:], document[1])
document = (document[0][7:-7], document[1][7:-7])
eval_result = os.path.join(loc, "eval_results.txt")
r = {}
with open(eval_result) as fin:
for l in fin:
cor = l.split(" = ")
k = cor[0].strip()
v = "%.1f" % (l_eval(cor[1].strip()) * 100)
r[k] = v
# all_results[document] = "%s/%s" % (r['pearson'], r['spearmanr'])
all_results[document] = "%s" % r['acc']
except Exception as e:
print(e)
pass
from pprint import pprint
# labels = list(set([k[1] for k in all_results.keys()]))
labels = [
'C',
'WC',
'MC',
'BC',
'S',
g09.job(job_name, ' '.join(route), queue=serv, extra_section=extras ,blurb='Re_running %s: %s' % (job_info[0],err),procs=threads, previous=job_info[0], force=True, err=True)
def get_err(fptr):
flag = False
for u in USER_PATH:
try:
path = u+fptr
err = open(path+'.log').read()
err = ''.join(err[:err.rfind('Error termination via Lnk1e')].split('\n')[-2:]).strip()
flag = True
except: pass
if flag: return path,err
else: return None,None # Return nothing if we couldn't find anything
# Get previous list of all runs
try: old=l_eval(open(end_path+'/jlist.list').read())
except: old=[]
# Get new list
new=log.get_jlist(2)
# Find finished jobs
for a in old:
found = False
for b in new:
if a[0] == b[0]:
found = True
if not found:
fptr,err = get_err(a[0])
process_err(fptr,err,a)
# Re-write list for jobs running
f = open(end_path+'/jlist.list','w')