def _read_csv(self):
if self.quotechar == '':
current_quoting = QUOTE_NONE
else:
current_quoting = QUOTE_ALL
if 'csvcontent' in self.request.FILES.keys():
csvfile = TextIOWrapper(
self.request.FILES['csvcontent'].file, encoding=self.encoding, errors='replace')
csvcontent = "".join(csvfile.readlines())
for param_idx in range(0, int(len(csvcontent) / 2048) + 2):
self.params['csvcontent%d' % param_idx] = csvcontent[
2048 * param_idx:2048 * (param_idx + 1)]
csvfile.seek(0)
else:
csvcontent = ""
for param_idx in range(0, 1000):
curent_content = self.getparam('csvcontent%d' % param_idx)
if curent_content is None:
break
else:
csvcontent += "" + curent_content
csvfile = StringIO(csvcontent)
self.spamreader = DictReader(csvfile, delimiter=self.delimiter, quotechar=self.quotechar, quoting=current_quoting)
try:
if (self.spamreader.fieldnames is None) or (len(self.spamreader.fieldnames) == 0):
raise Exception("")
except Exception:
raise LucteriosException(IMPORTANT, _('CSV file unvalid!'))
def is_csv(objects_file: TextIOWrapper, delim: str):
try:
csv.Sniffer().sniff(objects_file.read(1024), delimiters=delim)
return True
except:
return False
finally:
objects_file.seek(
0
) # need this to move back to the beginning of the file after sampling
def count_decil(file: _io.TextIOWrapper):
n = math.ceil(0.1 * sum(1 for l in file if l.startswith("open")))
file.seek(0)
dec = [0] * n
pat = re.compile(r"open .* (\d+) usec")
for l in file:
match = pat.match(l)
if match is not None:
value = float(match.group(1))
if value > dec[0]:
heapq.heappushpop(dec, value)
return dec[0]
def f30kTokenize(anno: _io.TextIOWrapper) -> (dict, Counter):
'''
tokenize all the annotations and return a dictionary containing
the results, indexed by the key specified by keyname.
'''
anno.seek(0)
j_orig = json.load(anno)
tokens = dict()
wctr = Counter()
for i, image in enumerate(j_orig['images']):
for j, sent in enumerate(image['sentences']):
raw, sid, iid = sent['raw'], sent['sentid'], sent['imgid']
tok = spk.tokenize(raw)
wctr.update(tok)
tokens[int(sid)] = (tok, int(iid))
print('.', end='')
return tokens, wctr
def cocoTokenize(anno: _io.TextIOWrapper) -> (dict, Counter):
'''
tokenize all the annotations and return a dictionary containing
the results, indexed by the key specified by keyname.
'''
anno.seek(0)
j_orig = json.load(anno)
tokens = dict()
wctr = Counter()
for i, annotation in enumerate(j_orig['annotations']):
tok = spk.tokenize(annotation['caption'])
wctr.update(tok)
tokens[int(annotation['id'])] = (tok, int(annotation['image_id']))
print('\0337\033[K>',
i,
'/',
len(j_orig['annotations']),
'-> {:.1f}%'.format(100 * i / len(j_orig['annotations'])),
end='\0338')
sys.stdout.flush()
return tokens, wctr
def process_single_log_file(file: TextIOWrapper) -> Tuple[int, List[Player]]:
"""
Parse tournament results and return list of players sorted by
Teams and kills
Return: Tuple with team size and list of parsed teams in it:
List[Tuple[team_or_name, id, kills, killscore, rankscore, total_score]]
"""
team_size = define_team_size(file)
file.seek(0)
table: List[Player] = []
team: str = ""
teams_dict: Dict[Player] = {}
teams_list: List[Player] = []
try:
for index, line in enumerate(file.readlines(), start=1):
if 'TeamName' in line:
# TeamName, Rank, KillScore, RankScore, TotalScore
# Cold Steel ['16', '10', '50', '60']
# 24 ['0', '160', '160']
if team_size == 1:
team_row = process_team_line(line, team_size)
team = f"Команда {team_row[0]}"
else:
team, *team_row = process_team_line(line, team_size)
team_id = team_row[0]
else:
# ['BustㅤSavage', '415576113', '4']
row = process_player_line(line)
if row == []:
continue
if team_size == 1:
teams_list.append(row + team_row[-3:])
else:
if team_id in teams_dict:
# Add kills
teams_dict[team_id][2] += int(row[-1])
else:
# insert kills
team_row.insert(1, int(row[-1]))
teams_dict[team_id] = [team] + team_row
except Exception as e:
# Inject line which cause Exception to render it later in Django
e.error_line = index
e.filename = file.name
raise
table = teams_list if team_size == 1 else list(teams_dict.values())
# Sort by kills
table.sort(key=lambda x: (-int(x[-1]), -int(x[2])))
return team_size, table
