def test_exception_reserved_name(self, value, platform, expected):
with pytest.raises(expected) as e:
validate_filename(value, platform=platform)
assert e.value.reserved_name
assert e.value.reusable_name is False
assert not is_valid_filename(value, platform=platform)
def test_minmax_len(self, value, min_len, max_len, expected):
if expected is None:
validate_filename(value, min_len=min_len, max_len=max_len)
assert is_valid_filename(value, min_len=min_len, max_len=max_len)
else:
with pytest.raises(expected):
validate_filename(value, min_len=min_len, max_len=max_len)
def test_normal_pathlike(self, value, replace_text, expected):
sanitized_name = sanitize_filename(value, replace_text)
assert sanitized_name == expected
assert is_pathlike_obj(sanitized_name)
validate_filename(sanitized_name)
assert is_valid_filename(sanitized_name)
def test_locale_ja(self, locale):
from faker import Factory
fake = Factory.create(locale=locale, seed=1)
for _ in range(100):
filename = fake.file_name()
validate_filename(filename)
assert is_valid_filename(filename)
def test_locale_ja(self, locale):
from faker import Factory
fake = Factory.create(locale=locale, seed=1)
for _ in range(100):
filename = fake.file_name()
validate_filename(filename)
assert is_valid_filename(filename)
def test_max_len(self, value, platform, max_len, expected):
if expected is None:
validate_filename(value, platform=platform, max_len=max_len)
assert is_valid_filename(value, platform=platform, max_len=max_len)
return
with pytest.raises(ValidationError) as e:
validate_filename(value, platform=platform, max_len=max_len)
assert e.value.reason == expected
def get_arg():
if len(argv) > 1:
filename = argv[1]
if not is_valid_filename(filename):
print("Invalid filename.")
elif not isfile(filename):
print("File {} not found.".format(filename))
else:
return filename
return ""
def test_exception_reserved_name(self, value, platform, expected):
if expected is None:
validate_filename(value, platform=platform)
else:
with pytest.raises(expected) as e:
validate_filename(value, platform=platform)
assert e.value.reason == ErrorReason.RESERVED_NAME
assert e.value.reserved_name
assert e.value.reusable_name is False
assert not is_valid_filename(value, platform=platform)
def loadSaveGame(self, saveData):
if not self.AstroRCON.connected:
return False
self.setStatus("loadsave")
self.busy = "LoadSave"
name = saveData['name']
if pathvalidate.is_valid_filename(name):
# time.sleep(1)
AstroLogging.logPrint(f"Loading save: {name}")
self.AstroRCON.DSLoadGame(name)
self.getSaves()
self.busy = False
def deleteSaveGame(self, saveData):
if not self.AstroRCON.connected:
return False
name = saveData['name']
if pathvalidate.is_valid_filename(name):
self.setStatus("delsave")
self.busy = "DelSave"
saveGamePath = r"Astro\Saved\SaveGames"
AstroLogging.logPrint(f"Deleting save: {saveData['fileName']}")
sfPath = os.path.join(self.astroPath, saveGamePath,
saveData['fileName'])
if os.path.exists(sfPath):
os.remove(sfPath)
self.getSaves()
self.busy = False
def CreateFile():
filename = e.get()
filepath = str(z.get()).replace("/", "\\\\")
if (filepath == ''):
Errormessage('Enter File Path')
elif (filename == ''):
Errormessage('File name cannot be blank')
elif (is_valid_filename(filename) != True):
Errormessage('Invalid file name')
elif (combo.get() == 'Excel'):
createExcel()
elif (combo.get() == 'Word'):
createWord()
else:
Errormessage('Select File Type')
def renameSaveGame(self, oldSave, newName):
if not self.AstroRCON.connected:
return False
self.setStatus("renamesave")
self.busy = "RenameSave"
if pathvalidate.is_valid_filename(
oldSave['name']) and pathvalidate.is_valid_filename(newName):
saveGamePath = r"Astro\Saved\SaveGames"
saveGamePath = os.path.join(self.astroPath, saveGamePath)
AstroLogging.logPrint(
f"Renaming save: {oldSave['name']} to {newName}")
if oldSave['active']:
self.saveGame(newName)
sfPath = os.path.join(saveGamePath, oldSave['fileName'])
self.getSaves()
newSave = [
x for x in self.DSListGames['gameList']
if x['name'] == newName
]
if newSave:
newSave = newSave[0]
sfNPath = os.path.join(saveGamePath, newSave['fileName'])
if os.path.exists(sfNPath) and os.path.exists(sfPath):
os.remove(sfPath)
else:
saveFileName = oldSave['fileName']
sfPath = os.path.join(saveGamePath, saveFileName)
newSaveFileName = saveFileName.replace(oldSave['name'],
newName)
sfNPath = os.path.join(saveGamePath, newSaveFileName)
# time.sleep(1)
if os.path.exists(sfPath) and not os.path.exists(sfNPath):
os.rename(sfPath, sfNPath)
self.getSaves()
self.busy = False
def find_type(value):
"""This function infers the data type of the input argument.
Parameters
----------
value :
value for which we want to find data type
Returns
----------
datatype:
data type of the given input argumnet
"""
value = value.replace("'", "")
value = value.replace('"', "")
value = value.replace(" ", "")
if (value.lower() == 'nan') or (value.lower()
== 'none') or len(value) == 0:
datatype = None
elif value.lower() in bool_arr:
datatype = 'bool'
elif value == '0' or value == '1':
datatype = '0||1'
elif is_ip(value):
datatype = 'ip_address'
elif is_url(value):
datatype = 'uri'
elif is_number(value):
datatype = 'int'
elif is_valid_filename(
ntpath.basename(value)) and ntpath.basename(value).find('.') != -1:
datatype = 'filename||filepath+filename'
elif is_filepath(value):
datatype = 'filepath'
elif re.search(r'^[\d ]*(k|m|g|kb|mb|gb)$', value.lower()):
datatype = 'size'
else:
datatype = 'string'
return datatype
def __init__(self, path=""):
if path == "":
self.dbcon = sqlite3.connect(':memory:')
elif is_valid_filename(path) and path.endswith(".world.db"):
source = sqlite3.connect(path)
self.dbcon = sqlite3.connect(':memory:')
source.backup(self.dbcon)
else:
#TODO: Implement errors and error handling for the database initialization
print("error detected")
self.dbcon.execute(
"CREATE TABLE tile (tile_id INTEGER PRIMARY KEY, tile_x \
INTEGER NOT NULL, tile_y INTEGER NOT NULL, tile_elevation DOUBLE NOT NULL,\
tile_precipitation INTEGER, tile_temp INTEGER, tile_type TEXT, \
tile_mv_cost FLOAT, UNIQUE(tile_x, tile_y));")
self.dbcon.execute(
"CREATE TABLE subtile (subtile_id INTEGER PRIMARY KEY, subtile_x\
INTEGER NOT NULL, subtile_y INTEGER NOT NULL, subtile_elevation DOUBLE NOT NULL,\
subtile_type TEXT NOT NULL, subtile_mv_mod DOUBLE, tile_id INTEGER, FOREIGN KEY (tile_id) REFERENCES tile (tile_id), UNIQUE(subtile_x, subtile_y));"
)
def entrada_str(self):
'''
Método em que usuário efetua uma entrada cujo retorno será uma string.
Possui consistência ao verificar se foi utilizado algum caracter inválido
utilizando-se da biblioteca pathvalidate.
'''
while True:
print("\n" + "-" * 100 + "\n")
resposta = input("\n>>> ")
if not resposta:
print("\nDigite algo!!\n")
continue
if not is_valid_filename(resposta):
print(
"\nNão é possível usar os seguites caracteres: :, *, /, \", ?, >, |, <, \\. Tente novamente!\n"
)
continue
break
print("\n" + "-" * 100 + "\n")
return resposta + ".db"
def verify_attachment(name, data, dtype='text/tab-separated-values'):
types = mimetypes.types_map
# check for extension
# if found, check its dtype matches
ext = path.splitext(name)[1]
valid_fname = is_valid_filename(name)
if ext:
output_dtype = types.get(ext, None)
if dtype == output_dtype:
valid_dtype = True
else:
valid_dtype = False
else:
# no extension, just check dtype
valid_dtype = dtype in list(mimetypes.types_map.values())
valid_data = isinstance(data, str)
return valid_fname, valid_data, valid_dtype
def grabClipboard():
filename = e.get()
filepath = str(z.get()).replace("/", "\\\\")
if (filepath == ''):
Errormessage('Enter File Path')
elif (filename == ''):
Errormessage('File name cannot be blank')
elif (is_valid_filename(filename) != True):
Errormessage('Invalid file name')
elif (combo.get() == 'Excel'):
root.wm_state('iconic')
time.sleep(0.5)
im = ImageGrab.grabclipboard()
root.wm_state('normal')
saveToexcel(im, filepath + '\\lastcopiedimg.png')
elif (combo.get() == 'Word'):
root.wm_state('iconic')
time.sleep(0.5)
im = ImageGrab.grabclipboard()
root.wm_state('normal')
saveToWord(im, filepath + '\\lastcopiedimg.png')
else:
Errormessage('Select File Type')
def test_normal_str(self, platform, value, replace_text, expected):
sanitized_name = sanitize_filename(value, platform=platform, replacement_text=replace_text)
assert sanitized_name == expected
assert isinstance(sanitized_name, str)
validate_filename(sanitized_name, platform=platform)
assert is_valid_filename(sanitized_name, platform=platform)
def test_exception(self, value, expected):
with pytest.raises(expected):
validate_filename(value)
assert not is_valid_filename(value)
def test_exception_filepath(self, value, platform):
with pytest.raises(ValidationError) as e:
validate_filename(value, platform=platform)
assert e.value.reason in [ErrorReason.FOUND_ABS_PATH, ErrorReason.INVALID_CHARACTER]
assert not is_valid_filename(value, platform=platform)
def test_exception_win_invalid_char(self, value, platform):
with pytest.raises(InvalidCharError):
validate_filename(value, platform=platform)
assert not is_valid_filename(value, platform=platform)
def test_normal_multibyte(self, value, replace_text, expected):
sanitized_name = sanitize_filename(value, replace_text)
assert sanitized_name == expected
validate_filename(sanitized_name)
assert is_valid_filename(sanitized_name)
def test_exception_type(self, value, expected):
with pytest.raises(expected):
sanitize_filename(value)
assert not is_valid_filename(value)
def test_normal_reserved_name(self, value, test_platform, expected):
filename = sanitize_filename(value, platform=test_platform)
assert filename == expected
assert is_valid_filename(filename, platform=test_platform)
def test_exception_win_invalid_char(self, value, platform):
with pytest.raises(InvalidCharError):
validate_filename(value, platform=platform)
assert not is_valid_filename(value, platform=platform)
def test_normal_max_len(self, value, max_len, expected):
filename = sanitize_filename(value, max_len=max_len)
assert len(filename) == expected
assert is_valid_filename(filename, max_len=max_len)
def test_exception(self, value, expected):
with pytest.raises(expected):
validate_filename(value)
assert not is_valid_filename(value)
def on_post(self, req, resp):
#pylint: disable=too-many-locals
"""on post request
return either sucessful transfer or error message
"""
msg_error = "Generic Error"
sftp_info = self.get_sftp_info(req)
remote_path = req.params['remotepath']
file_name = req.params['filename']
local_dir = os.environ['LOCAL_DIR']
local_file_path = os.path.join(local_dir, file_name)
if (not file_name or file_name[0] == '.'
or not is_valid_filename(file_name)):
return self.resp_error("Invalid Filename", resp)
data = req.bounded_stream.read()
file = open(local_file_path, "wb+")
file_written = file.write(data)
file.close()
if not isinstance(file_written, int):
return self.resp_error("Cannot write file", resp)
local_file_host_path = os.path.join(local_dir, 'host_' + file_name)
file_host = open(local_file_host_path, "w+")
file_host_written = file_host.write(sftp_info['HOST-KEY'])
file_host.close()
if not file_host_written:
if file_written:
try:
os.remove(local_file_path)
#pylint: disable=broad-except
except Exception as exception:
with sentry_sdk.configure_scope() as scope:
scope.set_extra('remote_local_exception',
str(exception))
return self.resp_error("Cannot write host file", resp)
sftp_put = None
if file_written:
try:
sftp_put = self.transfer_file(sftp_info, local_file_path,
remote_path,
local_file_host_path)
#pylint: disable=broad-except
except Exception as exception:
msg_error = "Cannot transfer file: " + str(exception)
return self.resp_error(msg_error, resp)
finally:
os.remove(local_file_path)
os.remove(local_file_host_path)
if sftp_put:
msg = {'message': "File transferred successfully"}
resp.body = json.dumps(jsend.success(msg))
resp.status = falcon.HTTP_200
return True
return self.resp_error(msg_error, resp)
def validate_filename(filename):
if is_valid_filename(filename):
return filename, None
else:
mapping_name = sanitize_filename(filename)
return mapping_name, f"Log Folder Name: {filename} is invalid using {mapping_name}"
def test_normal_multibyte(self, value, replace_text, expected):
sanitized_name = sanitize_filename(value, replace_text)
assert sanitized_name == expected
validate_filename(sanitized_name)
assert is_valid_filename(sanitized_name)
def test_exception_type(self, value, expected):
with pytest.raises(expected):
sanitize_filename(value)
assert not is_valid_filename(value)
def test_normal_max_len(self, value, max_len, expected):
filename = sanitize_filename(value, max_len=max_len)
assert len(filename) == expected
assert is_valid_filename(filename, max_len=max_len)
def test_normal_reserved_name(self, value, test_platform, expected):
filename = sanitize_filename(value, platform=test_platform)
assert filename == expected
assert is_valid_filename(filename, platform=test_platform)
def test_normal_space_or_period_at_tail(self, platform, value, expected):
filename = sanitize_filename(value, platform=platform)
assert filename == expected
assert is_valid_filename(filename, platform=platform)
def validate_filename(self, metadata):
filename = metadata.get("filename", "")
if not is_valid_filename(filename):
filename = FilenameGenerator.random_string(16)
return filename
def get_filename():
while True:
filename = input("Enter filename: ")
if is_valid_filename(filename):
return filename + ".heap"
print("Invalid filename.")
def test_normal_space_or_period_at_tail(self, platform, value, expected):
filename = sanitize_filename(value, platform=platform)
assert filename == expected
assert is_valid_filename(filename, platform=platform)
def __set__(self, instance, value):
if (value is not None) and (not is_valid_filename(value)):
raise ValueError(f"{self.name} has an invalid fileName of {value}")
super().__set__(instance, value)
def checkFileName(name):
reserved = {'CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 'COM4',
'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 'LPT1', 'LPT2',
'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'}
return is_valid_filename(name) and name.upper() not in reserved
def test_normal_multibyte(self, value, platform):
validate_filename(value, platform)
assert is_valid_filename(value, platform=platform)
def test_normal_multibyte(self, value, platform):
validate_filename(value, platform)
assert is_valid_filename(value, platform=platform)
def upload_subjects(
subject_set_id,
manifest_files,
allow_missing,
remote_location,
mime_type,
file_column,
):
"""
Uploads subjects from each of the given MANIFEST_FILES.
Example with only local files:
$ panoptes subject-set upload-subjects 4667 manifest.csv
Local filenames will be automatically detected in the manifest and
uploaded, or filename columns can be specified with --file-column.
If you are hosting your media yourself, you can put the URLs in the
manifest and specify the column number(s):
$ panoptes subject-set upload-subjects -r 1 4667 manifest.csv
$ panoptes subject-set upload-subjects -r 1 -r 2 4667 manifest.csv
Any local files will still be detected and uploaded.
"""
if (
len(manifest_files) > 1
and any(map(lambda m: m.endswith('.yaml'), manifest_files))
):
click.echo(
'Error: YAML manifests must be processed one at a time.',
err=True,
)
return -1
elif manifest_files[0].endswith('.yaml'):
with open(manifest_files[0], 'r') as yaml_manifest:
upload_state = yaml.load(yaml_manifest, Loader=yaml.FullLoader)
if upload_state['state_version'] > CURRENT_STATE_VERSION:
click.echo(
'Error: {} was generated by a newer version of the Panoptes '
'CLI and is not compatible with this version.'.format(
manifest_files[0],
),
err=True,
)
return -1
if upload_state['subject_set_id'] != subject_set_id:
click.echo(
'Warning: You specified subject set {} but this YAML '
'manifest is for subject set {}.'.format(
subject_set_id,
upload_state['subject_set_id'],
),
err=True,
)
click.confirm(
'Upload {} to subject set {} ({})?'.format(
manifest_files[0],
subject_set_id,
SubjectSet.find(subject_set_id).display_name,
),
abort=True
)
upload_state['subject_set_id'] = subject_set_id
resumed_upload = True
else:
upload_state = {
'state_version': CURRENT_STATE_VERSION,
'subject_set_id': subject_set_id,
'manifest_files': manifest_files,
'allow_missing': allow_missing,
'remote_location': remote_location,
'mime_type': mime_type,
'file_column': file_column,
'waiting_to_upload': [],
'waiting_to_link': {},
}
resumed_upload = False
remote_location_count = len(upload_state['remote_location'])
mime_type_count = len(upload_state['mime_type'])
if remote_location_count > 1 and mime_type_count == 1:
upload_state['mime_type'] = (
upload_state['mime_type'] * remote_location_count
)
elif remote_location_count > 0 and mime_type_count != remote_location_count:
click.echo(
'Error: The number of MIME types given must be either 1 or equal '
'to the number of remote locations.',
err=True,
)
return -1
def validate_file(file_path):
if not os.path.isfile(file_path):
click.echo(
'Error: File "{}" could not be found.'.format(
file_path,
),
err=True,
)
return False
file_size = os.path.getsize(file_path)
if file_size == 0:
click.echo(
'Error: File "{}" is empty.'.format(
file_path,
),
err=True,
)
return False
elif file_size > MAX_UPLOAD_FILE_SIZE:
click.echo(
'Error: File "{}" is {}, larger than the maximum {}.'.format(
file_path,
humanize.naturalsize(file_size),
humanize.naturalsize(MAX_UPLOAD_FILE_SIZE),
),
err=True,
)
return False
return True
subject_set = SubjectSet.find(upload_state['subject_set_id'])
if not resumed_upload:
subject_rows = []
for manifest_file in upload_state['manifest_files']:
with open(manifest_file, 'U') as manifest_f:
file_root = os.path.dirname(manifest_file)
r = csv.reader(manifest_f, skipinitialspace=True)
headers = next(r)
for row in r:
metadata = dict(zip(headers, row))
files = []
if not upload_state['file_column']:
upload_state['file_column'] = []
for field_number, col in enumerate(row, start=1):
file_path = os.path.join(file_root, col)
if os.path.exists(file_path):
upload_state['file_column'].append(
field_number,
)
if not validate_file(file_path):
return -1
files.append(file_path)
else:
for field_number in upload_state['file_column']:
file_path = os.path.join(
file_root,
row[field_number - 1]
)
if not validate_file(file_path):
return -1
files.append(file_path)
for field_number, _mime_type in zip(
upload_state['remote_location'],
upload_state['mime_type'],
):
files.append({_mime_type: row[field_number - 1]})
if len(files) == 0:
click.echo(
'Could not find any files in row:',
err=True,
)
click.echo(','.join(row), err=True)
if not upload_state['allow_missing']:
return -1
else:
continue
subject_rows.append((files, metadata))
if not subject_rows:
click.echo(
'File {} did not contain any rows.'.format(
manifest_file,
),
err=True,
)
return -1
subject_rows = list(enumerate(subject_rows))
upload_state['waiting_to_upload'] = copy.deepcopy(subject_rows)
else:
for subject_id, subject_row in upload_state['waiting_to_link'].items():
try:
subject = Subject.find(subject_id)
except PanoptesAPIException:
upload_state['waiting_to_upload'].append(subject_row)
del upload_state['waiting_to_link'][subject_id]
subject_rows = copy.deepcopy(upload_state['waiting_to_upload'])
pending_subjects = []
def move_created(limit):
while len(pending_subjects) > limit:
for subject, subject_row in pending_subjects:
if subject.async_save_result:
pending_subjects.remove((subject, subject_row))
upload_state['waiting_to_upload'].remove(subject_row)
upload_state['waiting_to_link'][subject.id] = subject_row
time.sleep(0.5)
def link_subjects(limit):
if len(upload_state['waiting_to_link']) > limit:
subject_set.add(list(upload_state['waiting_to_link'].keys()))
upload_state['waiting_to_link'].clear()
with click.progressbar(
subject_rows,
length=len(subject_rows),
label='Uploading subjects',
) as _subject_rows:
try:
with Subject.async_saves():
for subject_row in _subject_rows:
count, (files, metadata) = subject_row
subject = Subject()
subject.links.project = subject_set.links.project
for media_file in files:
subject.add_location(media_file)
subject.metadata.update(metadata)
subject.save()
pending_subjects.append((subject, subject_row))
move_created(MAX_PENDING_SUBJECTS)
link_subjects(LINK_BATCH_SIZE)
move_created(0)
link_subjects(0)
finally:
if (
len(pending_subjects) > 0
or len(upload_state['waiting_to_link']) > 0
):
click.echo('Error: Upload failed.', err=True)
if click.confirm(
'Would you like to save the upload state to resume the '
'upload later?',
default=True,
):
while True:
state_file_name = 'panoptes-upload-{}.yaml'.format(
subject_set_id,
)
state_file_name = click.prompt(
'Enter filename to save to',
default=state_file_name,
)
if not state_file_name.endswith('.yaml'):
click.echo(
'Error: File name must end in ".yaml".',
err=True,
)
if click.confirm(
'Save to {}.yaml?'.format(state_file_name),
default=True,
):
state_file_name += '.yaml'
else:
continue
if not is_valid_filename(state_file_name):
click.echo(
'Error: {} is not a valid file name'.format(
state_file_name,
),
err=True,
)
sanitized_filename = sanitize_filename(
state_file_name,
)
if click.confirm(
'Save to {}?'.format(
sanitized_filename,
),
default=True,
):
state_file_name = sanitized_filename
else:
continue
if os.path.exists(state_file_name):
if not click.confirm(
'File {} already exists. Overwrite?'.format(
state_file_name,
),
default=False,
):
continue
break
with open(state_file_name, 'w') as state_file:
yaml.dump(upload_state, state_file)
def test_normal_str(self, platform, value, replace_text, expected):
sanitized_name = sanitize_filename(value, platform=platform, replacement_text=replace_text)
assert sanitized_name == expected
assert isinstance(sanitized_name, six.text_type)
validate_filename(sanitized_name, platform=platform)
assert is_valid_filename(sanitized_name, platform=platform)
