def create_json(self):
"""create json"""
file_info = FileInfo()
file_entry = file_info.get(file=self.file)
self.orig = {
"dataFileList": [file_entry],
"datasetId": self.pid,
"ownerGroup": "ess",
"accessGroups": ["loki", "brightness"],
"size": file_entry["size"]
}
print(self.orig)
def __init__(self, python_path, root="", kill_on_error=False, debug=False):
"""
:param python_path: path to file to check
can be more than one line long
"""
self.python_path = python_path
self.test_class = ((python_path.find("/tests/") > 0)
or (python_path.find("/unittests/") > 0)
or (python_path.find("\\unittests\\") > 0)
or (python_path.find("/integration_tests/") > 0)
or (python_path.find("\\integration_tests\\") > 0))
self.debug = debug
self.kill_on_error = kill_on_error
self._info = FileInfo(python_path[len(root):])
def walk_images(self, folder, destination, sort_option, mode):
""" Execute the main function, limiting to known images with exif format."""
# Relative or absolute path
exif_opt = ExifOptions()
if os.path.isdir(folder):
count_files = 0
if mode == 'r':
for folder, subs, files in os.walk(folder):
count = 0
if destination not in subs:
for file_name in files:
if count == 0:
print(folder)
count += 1
if helper.check_file_extensions(file_name):
try:
self.process_image(
FileInfo(file_name, folder,
destination), sort_option)
except:
LOGGER.error("error on file! : " +
file_name)
self.incr_errors()
count_files += 1
print('Found image files ' + str(count_files))
else:
files = [
f for f in os.listdir(folder)
if helper.check_tiff_extension(file_name)
or helper.check_jpeg_extension(file_name)
]
for file_name in files:
try:
self.process_image(
FileInfo(file_name, folder, destination),
sort_option)
except:
print("error on file! : " + file_name)
self.incr_errors()
count_files += 1
print('Processed image files ' + str(count_files))
print('Skipped files ' + str(self.skipped))
print('Error files ' + str(self.errors))
print('Written ' + str(self.writes) + ' files')
else:
print('Invalid folder given')
def test_repr(self):
fi = FileInfo(path='a',
file_hash=b64decode('b000'),
is_dir='c',
mtime='d')
assert repr(
fi) == '<FileInfo: path=\'a\' hash=b\'b000\' is_dir=c mtime=d>'
def test_is_possibly_changed__for_updated_dir_contents(self):
p = get_mock_dir_path(iterdir=[
get_mock_file_path("HAM", b"sandwich"),
get_mock_file_path("EGGS", b"with toast")
])
L = LocalFileInfoBrowser()
assert L.is_possibly_changed(p)
L._cache[p]\
= FileInfo(path=p, is_dir=True, mtime=0, file_hash=get_mock_hash())
L._cache[p._iterdir[0]]\
= FileInfo(path="HAM", is_dir=False, mtime=0, file_hash=get_mock_hash())
L._cache[p._iterdir[1]]\
= FileInfo(path="EGGS", is_dir=False, mtime=0, file_hash=get_mock_hash())
assert not L.is_possibly_changed(p)
p._iterdir[1]._stat.st_mtime_ns += 1
assert L.is_possibly_changed(p)
def unpack(file_name, beautify_xml):
"""Unpacks a .bus file.
Keyword arguments:
file -- the .bus file's name
beautify_xml -- whether XML files' output should be beautified.
"""
with open(file_name, "rb") as input:
print("Unpacking: {}".format(file_name))
header = BusFileHeader.decrypt(input)
files = []
for i in range(header.length):
offset = PKG_OFFSET + HEADER_INFO_SIZE + i * FILE_INFO_SIZE
file = FileInfo.decrypt_header(input, offset)
files.append(file)
rdptr = 0
for file in files:
print("Unpacking: {}".format(file.name))
offset = PKG_OFFSET + HEADER_INFO_SIZE + header.length * FILE_INFO_SIZE + rdptr
decrypted = file.decrypt_contents(input, offset)
path = Path(file.name)
path.parent.mkdir(parents=True, exist_ok=True)
with open(file.name, "w+b") as output_file:
output_file.write(decrypted)
rdptr += file.length
if beautify_xml and file.name.split(".")[-1] == "xml":
dom = xml.dom.minidom.parse(file.name)
with open(file.name, "w+b") as output_file:
output_file.write(dom.toprettyxml(encoding="utf-8"))
def list_files(self, folder):
if folder and folder.name == 'B Folder':
return
file_count = 4
for i in range(file_count):
name = '{}{} File'.format(self.prefix,
self._get_char(i, file_count))
yield self._intense_calculation(FileInfo(id=i, name=name))
def test_is_possibly_changed__for_updated_mtime(self):
p = get_mock_file_path()
L = LocalFileInfoBrowser()
assert L.is_possibly_changed(p)
L._cache[p]\
= FileInfo(path=p, is_dir=False, mtime=0, file_hash=get_mock_hash())
assert not L.is_possibly_changed(p)
p._stat.st_mtime_ns += 1
assert L.is_possibly_changed(p)
def list_files(walk_dir):
with DiffFileWriter(walk_dir, 'diff-name-only-list.md') as list_file:
list_file.write_header()
list_file.write_root()
for root, subdirs, files in os.walk(walk_dir):
list_file.write_newline()
list_file.write_directory(root, 0)
for filename in files:
list_file.write_file(FileInfo(root, filename), 1)
def process(audio_folder_paths, build_folder_path):
audio_file_names, audio_file_names_no_ext, audio_file_paths = list_audio_files(audio_folder_paths)
file_info_path = build_folder_path + '/_bn_audio_files_info.txt'
old_file_info = FileInfo.read(file_info_path)
new_file_info = FileInfo.build_from_files(audio_file_paths)
if old_file_info == new_file_info:
return
for audio_file_name in audio_file_names:
print(audio_file_name)
soundbank_bin_path = build_folder_path + '/_bn_audio_soundbank.bin'
soundbank_header_path = build_folder_path + '/_bn_audio_soundbank.h'
total_size = process_audio_files(audio_file_paths, soundbank_bin_path, soundbank_header_path, build_folder_path)
write_output_files(audio_file_names_no_ext, soundbank_header_path, build_folder_path)
print(' Processed audio size: ' + str(total_size) + ' bytes')
os.remove(soundbank_header_path)
new_file_info.write(file_info_path)
def list_files(self, folder):
folder_abs = os.path.join(self.path, folder.name)
return [
FileInfo(
id=i,
name=name.encode('utf-8'),
full_path=path.encode('utf-8'),
checksum=self.md5_checksum(path) if self._config.checksum else None)
for i, (name, path) in enumerate((x, os.path.join(folder_abs, x)) for x in os.listdir(folder_abs))
if self._should_include(name, self._config.include, self._config.exclude) and os.path.isfile(path)
]
def _get_file_info(self, photo):
name = photo.title.encode('utf-8') if photo.title else photo.id
checksum = None
if photo.originalformat:
name += "." + photo.originalformat
if photo.tags:
tags = photo.tags.split()
checksum = next((parts[1]
for parts in (tag.split('=') for tag in tags)
if parts[0] == CHECKSUM_PREFIX), None)
return FileInfo(id=photo.id, name=name, checksum=checksum)
def scan_files(self, folder, destination, sort_option, mode):
# Relative or absolute path
if os.path.isdir(folder):
count_files = 0
if mode == 'r':
for folder, subs, files in os.walk(folder):
count = 0
if destination not in subs:
for file_name in files:
if count == 0:
print(folder)
count += 1
if helper.check_extension(file_name, MP4):
try:
self.process_file(
FileInfo(file_name, folder,
destination), sort_option)
except:
raise
LOGGER.error("error on file! : " +
file_name)
self.incr_errors()
count_files += 1
print('Found video files ' + str(count_files))
else:
files = [
f for f in os.listdir(folder)
if helper.check_extension(file_name, MP4)
or helper.check_jpeg_extension(file_name)
]
for file_name in files:
try:
self.process_file(
FileInfo(file_name, folder, destination),
sort_option)
except:
print("error on file! : " + file_name)
self.incr_errors()
count_files += 1
def __receive_res_list(self):
file_list = []
for _ in range(self.fts.recv_int()):
path = self.fts.recv_rstring().decode()
(hashd, is_dir, mtime) = self.fts.recv_struct('!32s?Q')
file_list.append(
FileInfo(path=Path(path),
file_hash=hashd,
is_dir=is_dir,
mtime=mtime))
return file_list
def _explore_at_lv(directory: str, lv: int, max_lv: int) -> FileInfo:
if lv > max_lv:
return None
if path.isfile(directory):
return FileInfo(name=path.basename(directory),
is_file=path.isfile(directory),
children=[],
path=directory)
children_names = os.listdir(directory)
children_directories = [
path.join(directory, name) for name in children_names
]
children = [
_explore_at_lv(child_dir, lv + 1, max_lv)
for child_dir in children_directories
]
not_none_chilren = [child for child in children if child is not None]
cur_file_info = FileInfo(name=path.basename(directory),
is_file=path.isfile(directory),
children=not_none_chilren,
path=directory)
return cur_file_info
def test_depth_param_should_limit_return_value(self):
actual = file_explorer.explore(ROOT_TEST_FOLDER, depth=2)
expected = FileInfo(
name=path.basename(ROOT_TEST_FOLDER),
is_file=False,
path=ROOT_TEST_FOLDER,
children=[
FileInfo(
name='folder1',
is_file=False,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1')
),
FileInfo(
name='folder2',
is_file=False,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder2')
),
]
)
assert actual == expected
def _get_file_info(self, photo):
name = photo.title.encode('utf-8') if photo.title else photo.id
checksum = None
extension = None
if photo.tags:
# If we've just pulled the photo, tags is a string, if we've inspected any properties like 'media', it becomes a list
tags = photo.tags.split() if isinstance(
photo.tags, basestring) else [tag.text for tag in photo.tags]
checksum = next((parts[1]
for parts in (tag.split('=') for tag in tags)
if parts[0] == CHECKSUM_PREFIX), None)
extension = next((parts[1]
for parts in (tag.split('=') for tag in tags)
if parts[0] == EXTENSION_PREFIX), None)
if not extension:
extension = photo.originalformat
if extension:
name += "." + extension
return FileInfo(id=photo.id, name=name, checksum=checksum)
def list_dmg_audio_file_infos(audio_folder_paths, build_folder_path):
audio_folder_path_list = audio_folder_paths.split(' ')
audio_file_infos = []
file_names_set = set()
for audio_folder_path in audio_folder_path_list:
audio_file_names = os.listdir(audio_folder_path)
for audio_file_name in audio_file_names:
audio_file_path = audio_folder_path + '/' + audio_file_name
if os.path.isfile(audio_file_path) and FileInfo.validate(
audio_file_name):
audio_file_name_split = os.path.splitext(audio_file_name)
audio_file_name_ext = audio_file_name_split[1]
mod_extension = audio_file_name_ext == '.mod'
if mod_extension or audio_file_name_ext == '.s3m':
audio_file_name_no_ext = audio_file_name_split[0]
if audio_file_name_no_ext in file_names_set:
raise ValueError(
'There\'s two or more DMG audio files with the same name: '
+ audio_file_name_no_ext)
file_names_set.add(audio_file_name_no_ext)
file_info_path = build_folder_path + '/_bn_' + audio_file_name_no_ext + '_dmg_audio_file_info.txt'
if not os.path.exists(file_info_path):
build = True
else:
file_info_mtime = os.path.getmtime(file_info_path)
audio_file_mtime = os.path.getmtime(audio_file_path)
build = file_info_mtime < audio_file_mtime
if build:
audio_file_infos.append(
DmgAudioFileInfo(audio_file_path, audio_file_name,
audio_file_name_no_ext,
file_info_path, mod_extension))
return audio_file_infos
def test_fl_sr(self):
# Test send/recv of filelist
c1 = FTConn(MockFTSock(True))
c2 = FTConn(MockFTSock(True))
fl = [FileInfo(path=Path(), file_hash=b'12345678901234567890123456789012', \
is_dir=False, mtime=98)]
c1.send_file_list(fl)
c2.fts.sock.append_bytes(c1.fts.sock.retrieve_bytes())
t, flr = c2.receive_data()
assert t == FTProto.RES_LIST
assert len(fl) == len(flr)
for orig, recv in zip(fl, flr):
assert file_info_equals(orig, recv)
assert c1.fts.sock.ensure_esend() and c1.fts.sock.ensure_erecv()
assert c2.fts.sock.ensure_esend() and c2.fts.sock.ensure_erecv()
def list_audio_files(audio_folder_paths):
audio_folder_path_list = audio_folder_paths.split(' ')
audio_file_names = []
audio_file_names_no_ext = []
audio_file_paths = []
for audio_folder_path in audio_folder_path_list:
folder_audio_file_names = sorted(os.listdir(audio_folder_path))
for audio_file_name in folder_audio_file_names:
if FileInfo.validate(audio_file_name):
audio_file_path = audio_folder_path + '/' + audio_file_name
if os.path.isfile(audio_file_path):
audio_file_name_split = os.path.splitext(audio_file_name)
audio_file_name_no_ext = audio_file_name_split[0]
audio_file_names.append(audio_file_name)
audio_file_names_no_ext.append(audio_file_name_no_ext)
audio_file_paths.append(audio_file_path)
return audio_file_names, audio_file_names_no_ext, audio_file_paths
def list_graphics_file_infos(graphics_folder_paths, build_folder_path):
graphics_folder_path_list = graphics_folder_paths.split(' ')
graphics_file_infos = []
sprite_file_names_set = set()
regular_bg_file_names_set = set()
for graphics_folder_path in graphics_folder_path_list:
graphics_file_names = sorted(os.listdir(graphics_folder_path))
for graphics_file_name in graphics_file_names:
graphics_file_path = graphics_folder_path + '/' + graphics_file_name
if FileInfo.validate(graphics_file_name):
graphics_file_name_split = os.path.splitext(graphics_file_name)
graphics_file_name_no_ext = graphics_file_name_split[0]
graphics_file_name_ext = graphics_file_name_split[1]
if graphics_file_name_ext == '.bmp':
json_file_path = graphics_folder_path + '/' + graphics_file_name_no_ext + '.json'
if not os.path.isfile(json_file_path):
raise ValueError('Graphics json file not found: ' + json_file_path)
try:
with open(json_file_path) as json_file:
info = json.load(json_file)
except Exception as exception:
raise ValueError(json_file_path + ' graphics json file parse failed: ' + str(exception))
try:
graphics_type = str(info['type'])
except KeyError:
raise ValueError('type filed not found in graphics json file: ' + json_file_path)
if graphics_type == 'sprite':
file_names_set = sprite_file_names_set
elif graphics_type == 'regular_bg':
file_names_set = regular_bg_file_names_set
else:
raise ValueError('Unknown type (' + graphics_type + ') in graphics json file: ' +
json_file_path)
if graphics_file_name_no_ext in file_names_set:
raise ValueError('There\'s two or more ' + graphics_type +
' graphics files with the same name: ' + graphics_file_name_no_ext)
file_names_set.add(graphics_file_name_no_ext)
file_info_path_prefix = build_folder_path + '/_btn_' + graphics_file_name_no_ext + '_'
file_info_path = file_info_path_prefix + graphics_type + '_file_info.txt'
json_file_info_path = file_info_path_prefix + graphics_type + '_json_file_info.txt'
old_file_info = FileInfo.read(file_info_path)
new_file_info = FileInfo.build_from_file(graphics_file_path)
old_json_file_info = FileInfo.read(json_file_info_path)
new_json_file_info = FileInfo.build_from_file(json_file_path)
if old_file_info != new_file_info or old_json_file_info != new_json_file_info:
graphics_file_infos.append(GraphicsFileInfo(
graphics_type, info, graphics_file_path, graphics_file_name, graphics_file_name_no_ext,
new_file_info, file_info_path, new_json_file_info, json_file_info_path))
return graphics_file_infos
def test_should_explore_right_value(self):
actual = file_explorer.explore(ROOT_TEST_FOLDER, depth=10)
expected = FileInfo(
name=path.basename(ROOT_TEST_FOLDER),
is_file=False,
path=ROOT_TEST_FOLDER,
children=[
FileInfo(
name='folder1',
is_file=False,
path=path.join(ROOT_TEST_FOLDER, 'folder1'),
children=[
FileInfo(
name='folder3',
is_file=False,
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'folder3'),
children=[
FileInfo(
name='g.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'folder3', 'g.txt'),
),
FileInfo(
name='h.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'folder3', 'h.txt'),
),
FileInfo(
name='j.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'folder3', 'j.txt'),
),
FileInfo(
name='k.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'folder3', 'k.txt'),
),
]
),
FileInfo(
name='a.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'a.txt'),
),
FileInfo(
name='b.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'b.txt'),
),
FileInfo(
name='c.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder1', 'c.txt'),
),
]
),
FileInfo(
name='folder2',
is_file=False,
path=path.join(ROOT_TEST_FOLDER, 'folder2'),
children=[
FileInfo(
name='d.txt',
is_file=True,
children=[],
path=path.join(ROOT_TEST_FOLDER, 'folder2', 'd.txt'),
),
FileInfo(
name='e.txt',
is_file=True,
path=path.join(ROOT_TEST_FOLDER, 'folder2', 'e.txt'),
children=[]
),
]
),
]
)
assert actual == expected
def find_file_by_id(file_id):
sql = "select id,file_name,file_size,key_words,creator,created_at,file_path,file_doc,permission,download_count " \
"from `files` where id=%d" % file_id
global cursor
cursor.execute(sql)
values = cursor.fetchall()
for file_id, file_name, file_size, key_words, creator, created_at, file_path, file_doc, permission, download_count in values:
f = FileInfo()
f.file_id = file_id
f.file_name = file_name
f.file_size = file_size
f.key_words = key_words
f.creator = creator
f.created_at = created_at
f.file_path = file_path
f.file_doc = file_doc
f.permission = int(permission)
f.download_count = download_count
return f
max_nest_size = 3
min_nest_size = 1
nest_step = 1
total_generations = int((max_level_size-min_level_size)/level_step) \
* int((max_nest_size-min_nest_size)/nest_step)
nest_sizes = np.arange(min_nest_size, max_nest_size, nest_step)
level_sizes = np.arange(min_level_size, max_level_size, level_step)
columns = ['file_type', 'output_type', 'size', 'load_time', 'nest_level', 'records_per_level']
df = pd.DataFrame(columns=columns)
for nest_level in nest_sizes:
for records_per_level in level_sizes:
progress(count=count, total=total_generations, status='Running Data Collection')
generate_fake_data_files(nest_level=nest_level, records_per_level=records_per_level)
file_handler = FileInfo()
file_handler.get_file_info()
df2 = pd.DataFrame(file_handler.file_info)
df2['nest_level'] = nest_level
df2['records_per_level'] = records_per_level
df = df.append(df2, ignore_index=True, sort=True)
count += 1
print(len(df))
export_data(df)
def get_file_info(file_path):
return FileInfo(file_path)
def query_all_files():
sql = "select id,file_name,file_size,key_words,creator,created_at,file_path,file_doc,permission,download_count " \
"from `files`"
global cursor
cursor.execute(sql)
values = cursor.fetchall()
result = []
for file_id, file_name, file_size, key_words, creator, created_at, file_path, file_doc, permission, download_count in values:
f = FileInfo()
f.file_id = file_id
f.file_name = file_name
f.file_size = file_size
f.key_words = key_words
f.creator = creator
f.created_at = created_at
f.file_path = file_path
f.file_doc = file_doc
f.permission = int(permission)
f.download_count = download_count
result.append(f)
return result
def insert_new_file(file_io, user_name, file_permission):
if not allowed_file(file_io.filename):
return None
sec_file_name = file_io.filename
save_path = os.path.join(UPLOAD_FOLDER, sec_file_name)
file_io.save(save_path)
file_info = FileInfo()
file_info.file_name = sec_file_name
file_info.file_doc = sec_file_name.rsplit('.', 1)[1]
file_info.creator = user_name
file_info.download_count = 0
file_info.permission = file_permission
file_info.file_path = save_path
file_info.file_size = os.path.getsize(save_path)
file_info.created_at = time.time() * 1000
file_info.key_words = search_service.cal_keys_from_file(file_info)
db_helper.insert_file_info(file_info)
return file_info
class FileDocChecker(object):
""" Sets the Class up to test a file assumes to be python \
with sphinx style docstrings.
"""
_code_state = CodeState.START
_part_line = ""
_lineNum = 0
_param_indent = None
_def_string = ""
_def_type = None
_def_params = []
_doc_params = []
_doc_types = []
_def_name = None
_info = None
cl_info = None
test_class = None
_at_line = None
def __init__(self, python_path, root="", kill_on_error=False, debug=False):
"""
:param python_path: path to file to check
can be more than one line long
"""
self.python_path = python_path
self.test_class = ((python_path.find("/tests/") > 0)
or (python_path.find("/unittests/") > 0)
or (python_path.find("\\unittests\\") > 0)
or (python_path.find("/integration_tests/") > 0)
or (python_path.find("\\integration_tests\\") > 0))
self.debug = debug
self.kill_on_error = kill_on_error
self._info = FileInfo(python_path[len(root):])
def check_all_docs(self):
if self.debug:
print(self.python_path)
try:
with open(self.python_path, "r") as python_file:
for line in python_file:
if SLOTS_IMPOSSIBLE_MARKER in line:
self.cl_info.slots = SLOTS_IMPOSSIBLE
else:
self._check_line(line.rstrip().split("#")[0].rstrip())
return self._info
except Exception:
traceback.print_exc()
print("Exception call processing:")
print(self.python_path + ":" + str(self._lineNum))
sys.exit(-1)
def _check_line(self, line):
if self.debug:
print(line)
self._lineNum += 1
if line.strip().endswith("\\"):
self._part_line = self._part_line + line[:-1] + " "
return
else:
if len(self._part_line) > 0:
line = self._part_line + line
self._part_line = ""
if self._code_state == CodeState.START:
self._check_in_start(line)
elif self._code_state == CodeState.CODE:
self._check_in_code(line)
elif self._code_state == CodeState.IN_CLASS:
self._check_in_class(line)
elif self._code_state == CodeState.AFTER_CLASS:
self._check_after_class(line)
elif self._code_state == CodeState.CLASS_DOC:
self._check_in_class_doc(line)
elif self._code_state == CodeState.IN_SLOTS:
self._check_in_slots(line)
elif self._code_state == CodeState.IN_DEF:
self._check_in_def(line)
elif self._code_state == CodeState.AFTER_DEF:
self._check_after_def(line)
elif self._code_state == CodeState.DOCS_START:
self._check_in_doc_start(line)
elif self._code_state == CodeState.DOCS_START_AFTER_BLANK:
self._check_in_doc_start_after_blank(line)
elif self._code_state == CodeState.DOCS_DECLARATION:
self._check_in_doc_declaration(line)
elif self._code_state == CodeState.COMMENT:
self._check_in_comment(line)
# elif self._code_state == CodeState.DOCS_END:
# self._check_in_doc_end(line)
# elif self._code_state == CodeState.IN_PARAM:
# self._check_in_param(line)
else:
print(self._code_state)
raise NotImplementedError
if self.debug:
print(str(self._lineNum) + " " + str(self._code_state))
def _check_in_start(self, line):
if "\"\"\"" in line:
return self._verify_doc_start(line, CodeState.CLASS_DOC,
CodeState.START)
stripped = line.strip()
return self._code_check(stripped)
def _verify_doc_start(self, line, doc_state, none_doc_state):
stripped = line.strip()
if stripped.startswith("r\"\"\""):
stripped = stripped[1:]
elif not stripped.startswith("\"\"\""):
msg = "unexpected doc tags"
return self._report(line, msg, _UNEXPECTED)
parts = stripped.split("\"\"\"")
if len(parts) <= 2:
self._code_state = doc_state
return OK
elif len(parts) == 3:
if len(parts[2]) == 0:
self._code_state = none_doc_state
return OK
else:
msg = "unexpected stuff after one line doc end"
return self._report(line, msg, _UNEXPECTED)
else:
msg = "three doc tags found in one line"
return self._report(line, msg, _UNEXPECTED)
def _code_check(self, stripped):
if stripped.startswith("class "):
return self._check_in_class(stripped)
if stripped.startswith("def "):
return self._check_in_def(stripped)
if stripped.startswith("__slots__"):
return self._check_in_slots(stripped)
if stripped.startswith("@"):
return self._check_in_at(stripped)
return OK
def _check_in_code(self, line):
if "\"\"\"" in line:
return self._verify_doc_start(line, CodeState.COMMENT,
CodeState.CODE)
stripped = line.strip()
return self._code_check(stripped)
def _check_in_class(self, line):
self._def_string += line
if line.endswith(":"):
self._extract_class_def(line)
self._code_state = CodeState.AFTER_CLASS
return OK
else:
self._code_state = CodeState.IN_CLASS
if "\"\"\"" in line:
msg = "unexpected doc start in class declaration"
return self._report(line, msg, _CRITICAL)
else:
return OK
def _extract_class_def(self, line):
if self.test_class:
self.cl_info = None
else:
declaration = self._def_string.replace(" ", "")
cl_name = declaration[5:declaration.index("(")]
if cl_name == "(":
print(self.python_path + ":" + str(self._lineNum))
self.cl_info = ClassInfo.info_by_name(cl_name, self._info,
self._lineNum)
if not cl_name.startswith("_"):
self._info.add_class(self.cl_info)
supers_string = declaration[declaration.index("(") +
1:declaration.index(")")]
if supers_string.startswith("namedtuple"):
pass
elif supers_string.startswith("collections."):
pass
else:
supers = supers_string.split(",")
for super in supers:
self._extract_super(line, super)
if self._at_line is not None:
if "ABCMeta" in self._at_line:
print("{}:{} ABCMmeta!".format(self.python_path,
self._lineNum))
self._at_line = None
self._def_string = ""
def _extract_super(self, line, super):
if super.startswith("exceptions."):
super = super[11:]
if super.startswith("_"):
print(self.python_path + ":" + str(self._lineNum))
if super.startswith("logging."):
return
else:
self.cl_info.add_super_by_name(super)
def _check_after_class(self, line):
if "\"\"\"" in line:
return self._verify_doc_start(line, CodeState.CLASS_DOC,
CodeState.CODE)
stripped = line.strip()
if len(stripped) == 0:
return OK
else:
self._code_state = CodeState.CODE
return self._check_in_code(line)
def _check_in_class_doc(self, line):
stripped = line.strip()
if stripped.startswith("\"\"\""):
return self._end_docs(line)
if stripped.startswith(":return"):
msg = ":return does not make sense in class doc"
return self._report(line, msg, _UNEXPECTED)
return OK
def _check_in_slots(self, line):
self._code_state = CodeState.IN_SLOTS
stripped = line.strip()
if stripped.startswith("#"):
return OK
self._def_string += stripped
if stripped.endswith(")"):
declaration = self._def_string[self._def_string.index("(") + 1:-1]
elif stripped.endswith("]"):
declaration = self._def_string[self._def_string.index("[") + 1:-1]
else:
return OK
if not self.test_class:
declaration = declaration.replace(" ", "")
declaration = declaration.replace("\"", "")
declaration = declaration.replace("'", "")
if len(declaration) == 0:
slots = []
else:
slots = declaration.split(",")
self.cl_info.slots = slots
self._def_string = ""
self._code_state = CodeState.CODE
return OK
def _check_in_at(self, stripped):
self._at_line = stripped
self._code_state = CodeState.CODE
return OK
def _check_in_def(self, line):
self._code_state = CodeState.IN_DEF
self._def_string += line
if line.endswith(":"):
self._code_state = CodeState.AFTER_DEF
return self._extract_params(line)
if "\"\"\"" in line:
msg = "unexpected doc start in def declaration"
return self._report(line, msg, _UNEXPECTED)
return OK
def _extract_params(self, line):
declaration = self._def_string.replace("(", " ", 1)
declaration = rreplace(declaration, ")", " ", 1)
declaration = declaration.replace(",", " ")
declaration = declaration.replace('"="', "x")
declaration = re.sub(r"\(.*?\)", "junk", declaration)
declaration = re.sub(' +', ' ', declaration)
self._def_string = ""
parts = declaration.split(" ")
if (parts[0] != "def"):
msg = "unexpected start in def declaration"
return self._report(line, msg, _UNEXPECTED)
self._def_name = parts[1]
if not self.test_class and self.cl_info is not None:
try:
self.cl_info.add_method(self._def_name)
except Exception as ex:
print(self.python_path + ":" + str(self._lineNum))
raise ex
self._doc_params = []
self._doc_types = []
if (parts[1] == "__init__"):
self._def_type = INIT
elif (parts[1][0] == "_"):
self._def_type = PRIVATE
else:
self._def_type = PUBLIC
self._def_params = []
if (parts[2] == "self"):
start = 3
else:
start = 2
for part in parts[start:-1]:
if (part[0] == "=") or (part[-1] == "="):
msg = "E251 unexpected spaces around keyword/parameter equals"
return self._report(line, msg, _UNEXPECTED)
param_parts = part.split("=")
if len(param_parts) > 2:
msg = "Multiple = found in one paramter"
return self._report(line, msg, _UNEXPECTED)
name = param_parts[0]
if name[0] == "*":
if len(name) == 1:
msg = "* only parameter name"
return self._report(line, msg, _UNEXPECTED)
if name[1] == "*":
name = name[2:]
else:
name = name[1:]
self._def_params.append(name)
if (parts[-1] != ":"):
msg = "No : found at end of def declaration"
return self._report(line, msg, _UNEXPECTED)
self._at_line = None
return OK
def _check_after_def(self, line):
if "\"\"\"" in line:
self._code_state = CodeState.DOCS_START
parts = line.split("\"\"\"")
if len(parts) == 1:
# Ok just the doc start
pass
elif len(parts) == 2:
return self._check_in_doc_start(parts[1])
elif len(parts) == 3:
# Ok start and end comment on same line
self._code_state = CodeState.CODE
pass
else:
print(line)
print(parts)
raise NotImplementedError
# Oops no docs
self._code_state = CodeState.CODE
if self._def_type != PRIVATE:
msg = "No Docs found for function " + self._def_name
return self._report(line, msg, _NO_DOCS)
return OK
def _check_in_doc_start(self, line):
stripped = line.strip()
if "\"\"\"" in stripped:
return self._end_docs(line)
if len(stripped) == 0:
self._code_state = CodeState.DOCS_START_AFTER_BLANK
if stripped.startswith(":py:"):
return OK
if stripped.startswith(":"):
return self._report(line, ":param without blank line", _HIDDEN)
return OK
def _end_docs(self, line):
self._code_state = CodeState.CODE
stripped = line.strip()
if stripped == ("\"\"\""):
return OK
msg = "Closing quotes should be on their own line (See per-0257)"
return self._report(line, msg, _STYLE)
def _check_in_doc_start_after_blank(self, line):
stripped = line.strip()
if stripped.startswith("\"\"\""):
return self._end_docs(line)
if stripped.startswith(":py"):
return OK
if stripped.startswith(":"):
self._code_state = CodeState.DOCS_DECLARATION
return self._check_in_doc_declaration(line)
if len(stripped) > 0:
self._code_state = CodeState.DOCS_START
return OK
def _check_in_doc_declaration(self, line):
stripped = line.strip()
if len(stripped) == 0:
return OK
if stripped.startswith("\"\"\""):
self._previous_indent = None
return self._end_docs(line)
# Line has already had a rstrip
indent = len(line) - len(stripped)
if stripped.startswith(":param"):
self._param_indent = indent
self._previous_indent = None
return self._verify_param(line)
if stripped.startswith(":type"):
self._param_indent = indent
self._previous_indent = None
return self._verify_type(line)
if stripped.startswith(":return"):
self._param_indent = indent
self._previous_indent = None
return self._verify_return(line)
if stripped.startswith(":rtype:"):
self._param_indent = indent
self._previous_indent = None
return self._verify_rtype(line)
if stripped.startswith(":raise"):
self._param_indent = indent
self._previous_indent = None
return self._verify_raise(line)
if stripped.startswith(":"):
if stripped.startswith(":py"):
return OK
if stripped == "::":
return OK
msg = "Unxpected : line in param doc section"
return self._report(line, msg, _CRITICAL)
if indent <= self._param_indent:
msg = "Unxpected non indented line in param doc section"
return self._report(line, msg, _CRITICAL)
return OK
def _verify_param(self, line):
stripped = line.strip()
parts = stripped.split(' ')
if parts[0] != ":param":
return self._report(line, "No space after :param", _CRITICAL)
if len(parts) == 1:
return self._report(line, "singleton :param", _UNEXPECTED)
if parts[1][-1] != ":":
if len(parts) > 2:
if parts[2][-1] == ":":
check = self._verify_param_name(parts[2], line)
if check == OK:
self._doc_types.append(parts[2][:-1])
if len(parts) == 3: #: param type name:
msg = "parameter " + parts[2] + " in " + \
self._def_name + " has no text"
return self._report(line, msg, _MISSING)
return check
return self._report(line, "paramater name must end with :",
_CRITICAL)
else:
return self._verify_param_name(parts[1], line)
def _verify_param_name(self, name, line):
name = name[:-1]
if name not in self._def_params:
msg = "param " + name + " not in parameter list " + \
str(self._def_params)
return self._report(line, msg, _CRITICAL)
self._doc_params.append(name)
return OK
def _verify_type(self, line):
stripped = line.strip()
parts = stripped.split(' ')
if parts[0] != ":type":
return self._report(line, "No space after :type", _CRITICAL)
if len(parts) == 1:
return self._report(line, ":type requires a param_name: type",
_UNEXPECTED)
if len(parts[1]) == 0:
return self._report(line, "Double space after :type", _CRITICAL)
if parts[1][-1] != ":":
return self._report(line, "in type paramater_name must end with :",
_CRITICAL)
else:
report = self._verify_param_name(parts[1], line)
if report != OK:
return report
if len(parts) == 2:
# return self._report(line, ":type requires a param_name: type",
# _MISSING)
pass
else:
if parts[2].lower() == "bytestring":
msg = "bytestring not a python type use str"
return self._report(line, msg, _CRITICAL)
return OK
def _verify_rtype(self, line):
stripped = line.strip()
parts = stripped.split(' ')
if parts[0] != ":rtype:":
return self._report(line, "No space after :rtype:", _CRITICAL)
if len(parts) == 1:
return self._report(line, ":rtype: requires a type", _UNEXPECTED)
return OK
def _verify_raise(self, line):
stripped = line.strip()
parts = stripped.split(' ')
if not parts[0] in [":raise", ":raise:", ":raises", ":raises:"]:
return self._report(line, "No space after :raise", _CRITICAL)
if len(parts) == 1:
return self._report(line, ":raise requires a type", _UNEXPECTED)
# if parts[1].lower().startswith("none"):
# return self._report(line, "Do not use :raise None", _UNEXPECTED)
return OK
def _verify_return(self, line):
stripped = line.strip()
parts = stripped.split(' ')
if len(parts) == 1:
msg = "singleton :return possible :rtype: None"
return self._report(line, msg, _UNEXPECTED)
if parts[0] != ":return":
if parts[0] == ":return:":
if parts[1].lower() == "none":
if len(parts) == 2:
msg = "replace :return: None with :rtype: None"
return self._report(line, msg, _UNEXPECTED)
return OK
else:
return self._report(line, "No space after :return", _WRONG)
if parts[1][-1] != ":":
if len(parts) > 2:
if len(parts[2]) > 0 and parts[2][-1] == ":":
return OK
return self._report(line, "return must end with :", _MISSING)
return OK
def _check_in_comment(self, line):
stripped = line.strip()
if "\"\"\"" in stripped:
return self._end_docs(line)
return OK
def _report(self, line, msg, level):
error = self.python_path + ":" + str(self._lineNum) + " " + msg
if not self.test_class:
# print error
pass
if level < _MISSING:
if self.kill_on_error:
raise DocException(self.python_path, msg, self._lineNum)
else:
self._info.add_error(error)
return error