def device_info(self):
di = {}
lbls = Counter()
cmd = 'blkid -s LABEL -s UUID -t TYPE=btrfs'.split()
subp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
for line in subp.stdout:
dev, label, uuid = BLKID_RE.match(line).groups()
uuid = UUID(hex=uuid.decode('ascii'))
dev = fsdecode(dev)
if label is not None:
try:
label = label.decode('ascii')
except UnicodeDecodeError:
# Don't try to guess.
pass
if uuid in di:
# btrfs raid
assert di[uuid].label == label
di[uuid].devices.append(dev)
else:
lbls[label] += 1
di[uuid] = DeviceInfo(label, [dev])
rc = subp.wait()
# 2 means there is no btrfs filesystem
if rc not in (0, 2):
raise subprocess.CalledProcessError(rc, cmd)
self._label_occurs = dict(lbls)
return di
def get_next(self, timeout: int = 0) -> UUID:
'''
Remove the first item in the MEMBER_LIST and return it
'''
value = self.kvcache.get_next(MEMBERS_LIST, timeout=timeout)
if isinstance(value, bytes):
value = UUID(value.decode('utf-8'))
return value
def get_project(self):
"""Get the current project or return None if not project is set."""
# check if a project has been selected
project_id = self.get_secure_cookie("project_id")
if not project_id:
return None
project_id = UUID(project_id.decode('UTF-8'))
return self.service.projects_manager.projects[project_id]
def uuid_validator(v, field, config, **kwargs) -> UUID:
with change_exception(errors.UUIDError, ValueError):
if isinstance(v, str):
v = UUID(v)
elif isinstance(v, (bytes, bytearray)):
v = UUID(v.decode())
if not isinstance(v, UUID):
raise errors.UUIDError()
required_version = getattr(field.type_, '_required_version', None)
if required_version and v.version != required_version:
raise errors.UUIDVersionError(required_version=required_version)
return v
def uuid_validator(v, field, config, **kwargs) -> UUID:
if isinstance(v, str):
v = UUID(v)
elif isinstance(v, (bytes, bytearray)):
v = UUID(v.decode())
elif not isinstance(v, UUID):
raise ValueError(
f'str, byte or native UUID type expected not {type(v)}')
required_version = getattr(field.type_, '_required_version', None)
if required_version and v.version != required_version:
raise ValueError(
f'uuid version {required_version} expected, not {v.version}')
return v
def get_project(self):
"""Get the current project or return None if not project is set."""
# check if a project has been selected
project_id = self.get_secure_cookie("project_id")
if not project_id:
return None
project_id = UUID(project_id.decode('UTF-8'))
projects_manager = srv_or_die("projectsmanager")
if project_id not in projects_manager.projects:
self.clear_cookie("project_id")
return None
return projects_manager.projects[project_id]
async def end_game(self, partyid):
slotid = await self.redis.get(Redis.party_slotid_key.format(partyid))
slotid = UUID(slotid.decode())
groups = await self.redis.smembers(Redis.slot_groups_key.format(slotid))
for groupid in map(methodcaller("decode"), groups):
await self.redis.set(
Redis.group_state_key.format(groupid), State.GROUP_CHECK.value)
await self.redis.delete(Redis.group_partyid_key.format(groupid))
users = await self.redis.smembers(Redis.slot_players_key.format(slotid))
for userid in map(methodcaller("decode"), users):
await self.redis.delete(Redis.user_partyid_key.format(userid))
await self.redis.delete(Redis.party_gameid_key.format(partyid))
await self.redis.delete(Redis.party_slotid_key.format(partyid))
await self.redis.delete(Redis.party_host_key.format(partyid))
await self.redis.delete(Redis.party_ports_key.format(partyid))
await self.redis.delete(Redis.slot_players_key.format(slotid))
await self.redis.delete(Redis.slot_groups_key.format(slotid))
def device_info(self):
di = {}
lbls = Counter()
for line in subprocess.check_output(
'blkid -s LABEL -s UUID -t TYPE=btrfs'.split()
).splitlines():
dev, label, uuid = BLKID_RE.match(line).groups()
uuid = UUID(hex=uuid.decode('ascii'))
dev = fsdecode(dev)
if label is not None:
try:
label = label.decode('ascii')
except UnicodeDecodeError:
# Don't try to guess.
pass
if uuid in di:
# btrfs raid
assert di[uuid].label == label
di[uuid].devices.append(dev)
else:
lbls[label] += 1
di[uuid] = DeviceInfo(label, [dev])
self._label_occurs = dict(lbls)
return di
for record in tqdm(f, unit='records'):
URI = record.header.get('warc-target-uri')
content = record.payload.read()
if URI is not None and content is not None:
words = latin_sep_words.split(str(content))
words = [(k, v) for (k, v) in Counter(words).items()]
if args.uuid:
uuid = record.header.get('WARC-Record-ID')[1:-1]
uuid = UUID(uuid)
if args.compressuuid:
uuid = slugid.encode(uuid)
[
print("{word}\t{uuid} {count}".format(
word=word, uuid=uuid.decode('ascii'), count=count))
for (word, count) in words
]
else:
docID = docIdGenerator.next()
if args.urlTable:
print("{docID}\t{url}".format(docID=docID, url=URI),
file=fileURLTable)
# fileURLTable.write("{docID}\t{url}\t{lan}".format(docID=docID, url=URI, lan=lang))
if args.binary:
docID = docID.to_bytes(docIDDigits,
'little',
signed=True)
else:
docID = str(docID)
if args.binary:
