def _read_csv(self): if self.quotechar == '': current_quoting = QUOTE_NONE else: current_quoting = QUOTE_ALL if 'csvcontent' in self.request.FILES.keys(): csvfile = TextIOWrapper( self.request.FILES['csvcontent'].file, encoding=self.encoding, errors='replace') csvcontent = "".join(csvfile.readlines()) for param_idx in range(0, int(len(csvcontent) / 2048) + 2): self.params['csvcontent%d' % param_idx] = csvcontent[ 2048 * param_idx:2048 * (param_idx + 1)] csvfile.seek(0) else: csvcontent = "" for param_idx in range(0, 1000): curent_content = self.getparam('csvcontent%d' % param_idx) if curent_content is None: break else: csvcontent += "" + curent_content csvfile = StringIO(csvcontent) self.spamreader = DictReader(csvfile, delimiter=self.delimiter, quotechar=self.quotechar, quoting=current_quoting) try: if (self.spamreader.fieldnames is None) or (len(self.spamreader.fieldnames) == 0): raise Exception("") except Exception: raise LucteriosException(IMPORTANT, _('CSV file unvalid!'))
def is_csv(objects_file: TextIOWrapper, delim: str): try: csv.Sniffer().sniff(objects_file.read(1024), delimiters=delim) return True except: return False finally: objects_file.seek( 0 ) # need this to move back to the beginning of the file after sampling
def count_decil(file: _io.TextIOWrapper): n = math.ceil(0.1 * sum(1 for l in file if l.startswith("open"))) file.seek(0) dec = [0] * n pat = re.compile(r"open .* (\d+) usec") for l in file: match = pat.match(l) if match is not None: value = float(match.group(1)) if value > dec[0]: heapq.heappushpop(dec, value) return dec[0]
def f30kTokenize(anno: _io.TextIOWrapper) -> (dict, Counter): ''' tokenize all the annotations and return a dictionary containing the results, indexed by the key specified by keyname. ''' anno.seek(0) j_orig = json.load(anno) tokens = dict() wctr = Counter() for i, image in enumerate(j_orig['images']): for j, sent in enumerate(image['sentences']): raw, sid, iid = sent['raw'], sent['sentid'], sent['imgid'] tok = spk.tokenize(raw) wctr.update(tok) tokens[int(sid)] = (tok, int(iid)) print('.', end='') return tokens, wctr
def cocoTokenize(anno: _io.TextIOWrapper) -> (dict, Counter): ''' tokenize all the annotations and return a dictionary containing the results, indexed by the key specified by keyname. ''' anno.seek(0) j_orig = json.load(anno) tokens = dict() wctr = Counter() for i, annotation in enumerate(j_orig['annotations']): tok = spk.tokenize(annotation['caption']) wctr.update(tok) tokens[int(annotation['id'])] = (tok, int(annotation['image_id'])) print('\0337\033[K>', i, '/', len(j_orig['annotations']), '-> {:.1f}%'.format(100 * i / len(j_orig['annotations'])), end='\0338') sys.stdout.flush() return tokens, wctr
def process_single_log_file(file: TextIOWrapper) -> Tuple[int, List[Player]]: """ Parse tournament results and return list of players sorted by Teams and kills Return: Tuple with team size and list of parsed teams in it: List[Tuple[team_or_name, id, kills, killscore, rankscore, total_score]] """ team_size = define_team_size(file) file.seek(0) table: List[Player] = [] team: str = "" teams_dict: Dict[Player] = {} teams_list: List[Player] = [] try: for index, line in enumerate(file.readlines(), start=1): if 'TeamName' in line: # TeamName, Rank, KillScore, RankScore, TotalScore # Cold Steel ['16', '10', '50', '60'] # 24 ['0', '160', '160'] if team_size == 1: team_row = process_team_line(line, team_size) team = f"Команда {team_row[0]}" else: team, *team_row = process_team_line(line, team_size) team_id = team_row[0] else: # ['BustㅤSavage', '415576113', '4'] row = process_player_line(line) if row == []: continue if team_size == 1: teams_list.append(row + team_row[-3:]) else: if team_id in teams_dict: # Add kills teams_dict[team_id][2] += int(row[-1]) else: # insert kills team_row.insert(1, int(row[-1])) teams_dict[team_id] = [team] + team_row except Exception as e: # Inject line which cause Exception to render it later in Django e.error_line = index e.filename = file.name raise table = teams_list if team_size == 1 else list(teams_dict.values()) # Sort by kills table.sort(key=lambda x: (-int(x[-1]), -int(x[2]))) return team_size, table