def test_exception_reserved_name(self, value, platform, expected): with pytest.raises(expected) as e: validate_filename(value, platform=platform) assert e.value.reserved_name assert e.value.reusable_name is False assert not is_valid_filename(value, platform=platform)
def test_minmax_len(self, value, min_len, max_len, expected): if expected is None: validate_filename(value, min_len=min_len, max_len=max_len) assert is_valid_filename(value, min_len=min_len, max_len=max_len) else: with pytest.raises(expected): validate_filename(value, min_len=min_len, max_len=max_len)
def test_normal_pathlike(self, value, replace_text, expected): sanitized_name = sanitize_filename(value, replace_text) assert sanitized_name == expected assert is_pathlike_obj(sanitized_name) validate_filename(sanitized_name) assert is_valid_filename(sanitized_name)
def test_locale_ja(self, locale): from faker import Factory fake = Factory.create(locale=locale, seed=1) for _ in range(100): filename = fake.file_name() validate_filename(filename) assert is_valid_filename(filename)
def test_max_len(self, value, platform, max_len, expected): if expected is None: validate_filename(value, platform=platform, max_len=max_len) assert is_valid_filename(value, platform=platform, max_len=max_len) return with pytest.raises(ValidationError) as e: validate_filename(value, platform=platform, max_len=max_len) assert e.value.reason == expected
def get_arg(): if len(argv) > 1: filename = argv[1] if not is_valid_filename(filename): print("Invalid filename.") elif not isfile(filename): print("File {} not found.".format(filename)) else: return filename return ""
def test_exception_reserved_name(self, value, platform, expected): if expected is None: validate_filename(value, platform=platform) else: with pytest.raises(expected) as e: validate_filename(value, platform=platform) assert e.value.reason == ErrorReason.RESERVED_NAME assert e.value.reserved_name assert e.value.reusable_name is False assert not is_valid_filename(value, platform=platform)
def loadSaveGame(self, saveData): if not self.AstroRCON.connected: return False self.setStatus("loadsave") self.busy = "LoadSave" name = saveData['name'] if pathvalidate.is_valid_filename(name): # time.sleep(1) AstroLogging.logPrint(f"Loading save: {name}") self.AstroRCON.DSLoadGame(name) self.getSaves() self.busy = False
def deleteSaveGame(self, saveData): if not self.AstroRCON.connected: return False name = saveData['name'] if pathvalidate.is_valid_filename(name): self.setStatus("delsave") self.busy = "DelSave" saveGamePath = r"Astro\Saved\SaveGames" AstroLogging.logPrint(f"Deleting save: {saveData['fileName']}") sfPath = os.path.join(self.astroPath, saveGamePath, saveData['fileName']) if os.path.exists(sfPath): os.remove(sfPath) self.getSaves() self.busy = False
def CreateFile(): filename = e.get() filepath = str(z.get()).replace("/", "\\\\") if (filepath == ''): Errormessage('Enter File Path') elif (filename == ''): Errormessage('File name cannot be blank') elif (is_valid_filename(filename) != True): Errormessage('Invalid file name') elif (combo.get() == 'Excel'): createExcel() elif (combo.get() == 'Word'): createWord() else: Errormessage('Select File Type')
def renameSaveGame(self, oldSave, newName): if not self.AstroRCON.connected: return False self.setStatus("renamesave") self.busy = "RenameSave" if pathvalidate.is_valid_filename( oldSave['name']) and pathvalidate.is_valid_filename(newName): saveGamePath = r"Astro\Saved\SaveGames" saveGamePath = os.path.join(self.astroPath, saveGamePath) AstroLogging.logPrint( f"Renaming save: {oldSave['name']} to {newName}") if oldSave['active']: self.saveGame(newName) sfPath = os.path.join(saveGamePath, oldSave['fileName']) self.getSaves() newSave = [ x for x in self.DSListGames['gameList'] if x['name'] == newName ] if newSave: newSave = newSave[0] sfNPath = os.path.join(saveGamePath, newSave['fileName']) if os.path.exists(sfNPath) and os.path.exists(sfPath): os.remove(sfPath) else: saveFileName = oldSave['fileName'] sfPath = os.path.join(saveGamePath, saveFileName) newSaveFileName = saveFileName.replace(oldSave['name'], newName) sfNPath = os.path.join(saveGamePath, newSaveFileName) # time.sleep(1) if os.path.exists(sfPath) and not os.path.exists(sfNPath): os.rename(sfPath, sfNPath) self.getSaves() self.busy = False
def find_type(value): """This function infers the data type of the input argument. Parameters ---------- value : value for which we want to find data type Returns ---------- datatype: data type of the given input argumnet """ value = value.replace("'", "") value = value.replace('"', "") value = value.replace(" ", "") if (value.lower() == 'nan') or (value.lower() == 'none') or len(value) == 0: datatype = None elif value.lower() in bool_arr: datatype = 'bool' elif value == '0' or value == '1': datatype = '0||1' elif is_ip(value): datatype = 'ip_address' elif is_url(value): datatype = 'uri' elif is_number(value): datatype = 'int' elif is_valid_filename( ntpath.basename(value)) and ntpath.basename(value).find('.') != -1: datatype = 'filename||filepath+filename' elif is_filepath(value): datatype = 'filepath' elif re.search(r'^[\d ]*(k|m|g|kb|mb|gb)$', value.lower()): datatype = 'size' else: datatype = 'string' return datatype
def __init__(self, path=""): if path == "": self.dbcon = sqlite3.connect(':memory:') elif is_valid_filename(path) and path.endswith(".world.db"): source = sqlite3.connect(path) self.dbcon = sqlite3.connect(':memory:') source.backup(self.dbcon) else: #TODO: Implement errors and error handling for the database initialization print("error detected") self.dbcon.execute( "CREATE TABLE tile (tile_id INTEGER PRIMARY KEY, tile_x \ INTEGER NOT NULL, tile_y INTEGER NOT NULL, tile_elevation DOUBLE NOT NULL,\ tile_precipitation INTEGER, tile_temp INTEGER, tile_type TEXT, \ tile_mv_cost FLOAT, UNIQUE(tile_x, tile_y));") self.dbcon.execute( "CREATE TABLE subtile (subtile_id INTEGER PRIMARY KEY, subtile_x\ INTEGER NOT NULL, subtile_y INTEGER NOT NULL, subtile_elevation DOUBLE NOT NULL,\ subtile_type TEXT NOT NULL, subtile_mv_mod DOUBLE, tile_id INTEGER, FOREIGN KEY (tile_id) REFERENCES tile (tile_id), UNIQUE(subtile_x, subtile_y));" )
def entrada_str(self): ''' Método em que usuário efetua uma entrada cujo retorno será uma string. Possui consistência ao verificar se foi utilizado algum caracter inválido utilizando-se da biblioteca pathvalidate. ''' while True: print("\n" + "-" * 100 + "\n") resposta = input("\n>>> ") if not resposta: print("\nDigite algo!!\n") continue if not is_valid_filename(resposta): print( "\nNão é possível usar os seguites caracteres: :, *, /, \", ?, >, |, <, \\. Tente novamente!\n" ) continue break print("\n" + "-" * 100 + "\n") return resposta + ".db"
def verify_attachment(name, data, dtype='text/tab-separated-values'): types = mimetypes.types_map # check for extension # if found, check its dtype matches ext = path.splitext(name)[1] valid_fname = is_valid_filename(name) if ext: output_dtype = types.get(ext, None) if dtype == output_dtype: valid_dtype = True else: valid_dtype = False else: # no extension, just check dtype valid_dtype = dtype in list(mimetypes.types_map.values()) valid_data = isinstance(data, str) return valid_fname, valid_data, valid_dtype
def grabClipboard(): filename = e.get() filepath = str(z.get()).replace("/", "\\\\") if (filepath == ''): Errormessage('Enter File Path') elif (filename == ''): Errormessage('File name cannot be blank') elif (is_valid_filename(filename) != True): Errormessage('Invalid file name') elif (combo.get() == 'Excel'): root.wm_state('iconic') time.sleep(0.5) im = ImageGrab.grabclipboard() root.wm_state('normal') saveToexcel(im, filepath + '\\lastcopiedimg.png') elif (combo.get() == 'Word'): root.wm_state('iconic') time.sleep(0.5) im = ImageGrab.grabclipboard() root.wm_state('normal') saveToWord(im, filepath + '\\lastcopiedimg.png') else: Errormessage('Select File Type')
def test_normal_str(self, platform, value, replace_text, expected): sanitized_name = sanitize_filename(value, platform=platform, replacement_text=replace_text) assert sanitized_name == expected assert isinstance(sanitized_name, str) validate_filename(sanitized_name, platform=platform) assert is_valid_filename(sanitized_name, platform=platform)
def test_exception(self, value, expected): with pytest.raises(expected): validate_filename(value) assert not is_valid_filename(value)
def test_exception_filepath(self, value, platform): with pytest.raises(ValidationError) as e: validate_filename(value, platform=platform) assert e.value.reason in [ErrorReason.FOUND_ABS_PATH, ErrorReason.INVALID_CHARACTER] assert not is_valid_filename(value, platform=platform)
def test_exception_win_invalid_char(self, value, platform): with pytest.raises(InvalidCharError): validate_filename(value, platform=platform) assert not is_valid_filename(value, platform=platform)
def test_normal_multibyte(self, value, replace_text, expected): sanitized_name = sanitize_filename(value, replace_text) assert sanitized_name == expected validate_filename(sanitized_name) assert is_valid_filename(sanitized_name)
def test_exception_type(self, value, expected): with pytest.raises(expected): sanitize_filename(value) assert not is_valid_filename(value)
def test_normal_reserved_name(self, value, test_platform, expected): filename = sanitize_filename(value, platform=test_platform) assert filename == expected assert is_valid_filename(filename, platform=test_platform)
def test_normal_max_len(self, value, max_len, expected): filename = sanitize_filename(value, max_len=max_len) assert len(filename) == expected assert is_valid_filename(filename, max_len=max_len)
def on_post(self, req, resp): #pylint: disable=too-many-locals """on post request return either sucessful transfer or error message """ msg_error = "Generic Error" sftp_info = self.get_sftp_info(req) remote_path = req.params['remotepath'] file_name = req.params['filename'] local_dir = os.environ['LOCAL_DIR'] local_file_path = os.path.join(local_dir, file_name) if (not file_name or file_name[0] == '.' or not is_valid_filename(file_name)): return self.resp_error("Invalid Filename", resp) data = req.bounded_stream.read() file = open(local_file_path, "wb+") file_written = file.write(data) file.close() if not isinstance(file_written, int): return self.resp_error("Cannot write file", resp) local_file_host_path = os.path.join(local_dir, 'host_' + file_name) file_host = open(local_file_host_path, "w+") file_host_written = file_host.write(sftp_info['HOST-KEY']) file_host.close() if not file_host_written: if file_written: try: os.remove(local_file_path) #pylint: disable=broad-except except Exception as exception: with sentry_sdk.configure_scope() as scope: scope.set_extra('remote_local_exception', str(exception)) return self.resp_error("Cannot write host file", resp) sftp_put = None if file_written: try: sftp_put = self.transfer_file(sftp_info, local_file_path, remote_path, local_file_host_path) #pylint: disable=broad-except except Exception as exception: msg_error = "Cannot transfer file: " + str(exception) return self.resp_error(msg_error, resp) finally: os.remove(local_file_path) os.remove(local_file_host_path) if sftp_put: msg = {'message': "File transferred successfully"} resp.body = json.dumps(jsend.success(msg)) resp.status = falcon.HTTP_200 return True return self.resp_error(msg_error, resp)
def validate_filename(filename): if is_valid_filename(filename): return filename, None else: mapping_name = sanitize_filename(filename) return mapping_name, f"Log Folder Name: {filename} is invalid using {mapping_name}"
def test_normal_space_or_period_at_tail(self, platform, value, expected): filename = sanitize_filename(value, platform=platform) assert filename == expected assert is_valid_filename(filename, platform=platform)
def validate_filename(self, metadata): filename = metadata.get("filename", "") if not is_valid_filename(filename): filename = FilenameGenerator.random_string(16) return filename
def get_filename(): while True: filename = input("Enter filename: ") if is_valid_filename(filename): return filename + ".heap" print("Invalid filename.")
def __set__(self, instance, value): if (value is not None) and (not is_valid_filename(value)): raise ValueError(f"{self.name} has an invalid fileName of {value}") super().__set__(instance, value)
def checkFileName(name): reserved = {'CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'} return is_valid_filename(name) and name.upper() not in reserved
def test_normal_multibyte(self, value, platform): validate_filename(value, platform) assert is_valid_filename(value, platform=platform)
def upload_subjects( subject_set_id, manifest_files, allow_missing, remote_location, mime_type, file_column, ): """ Uploads subjects from each of the given MANIFEST_FILES. Example with only local files: $ panoptes subject-set upload-subjects 4667 manifest.csv Local filenames will be automatically detected in the manifest and uploaded, or filename columns can be specified with --file-column. If you are hosting your media yourself, you can put the URLs in the manifest and specify the column number(s): $ panoptes subject-set upload-subjects -r 1 4667 manifest.csv $ panoptes subject-set upload-subjects -r 1 -r 2 4667 manifest.csv Any local files will still be detected and uploaded. """ if ( len(manifest_files) > 1 and any(map(lambda m: m.endswith('.yaml'), manifest_files)) ): click.echo( 'Error: YAML manifests must be processed one at a time.', err=True, ) return -1 elif manifest_files[0].endswith('.yaml'): with open(manifest_files[0], 'r') as yaml_manifest: upload_state = yaml.load(yaml_manifest, Loader=yaml.FullLoader) if upload_state['state_version'] > CURRENT_STATE_VERSION: click.echo( 'Error: {} was generated by a newer version of the Panoptes ' 'CLI and is not compatible with this version.'.format( manifest_files[0], ), err=True, ) return -1 if upload_state['subject_set_id'] != subject_set_id: click.echo( 'Warning: You specified subject set {} but this YAML ' 'manifest is for subject set {}.'.format( subject_set_id, upload_state['subject_set_id'], ), err=True, ) click.confirm( 'Upload {} to subject set {} ({})?'.format( manifest_files[0], subject_set_id, SubjectSet.find(subject_set_id).display_name, ), abort=True ) upload_state['subject_set_id'] = subject_set_id resumed_upload = True else: upload_state = { 'state_version': CURRENT_STATE_VERSION, 'subject_set_id': subject_set_id, 'manifest_files': manifest_files, 'allow_missing': allow_missing, 'remote_location': remote_location, 'mime_type': mime_type, 'file_column': file_column, 'waiting_to_upload': [], 'waiting_to_link': {}, } resumed_upload = False remote_location_count = len(upload_state['remote_location']) mime_type_count = len(upload_state['mime_type']) if remote_location_count > 1 and mime_type_count == 1: upload_state['mime_type'] = ( upload_state['mime_type'] * remote_location_count ) elif remote_location_count > 0 and mime_type_count != remote_location_count: click.echo( 'Error: The number of MIME types given must be either 1 or equal ' 'to the number of remote locations.', err=True, ) return -1 def validate_file(file_path): if not os.path.isfile(file_path): click.echo( 'Error: File "{}" could not be found.'.format( file_path, ), err=True, ) return False file_size = os.path.getsize(file_path) if file_size == 0: click.echo( 'Error: File "{}" is empty.'.format( file_path, ), err=True, ) return False elif file_size > MAX_UPLOAD_FILE_SIZE: click.echo( 'Error: File "{}" is {}, larger than the maximum {}.'.format( file_path, humanize.naturalsize(file_size), humanize.naturalsize(MAX_UPLOAD_FILE_SIZE), ), err=True, ) return False return True subject_set = SubjectSet.find(upload_state['subject_set_id']) if not resumed_upload: subject_rows = [] for manifest_file in upload_state['manifest_files']: with open(manifest_file, 'U') as manifest_f: file_root = os.path.dirname(manifest_file) r = csv.reader(manifest_f, skipinitialspace=True) headers = next(r) for row in r: metadata = dict(zip(headers, row)) files = [] if not upload_state['file_column']: upload_state['file_column'] = [] for field_number, col in enumerate(row, start=1): file_path = os.path.join(file_root, col) if os.path.exists(file_path): upload_state['file_column'].append( field_number, ) if not validate_file(file_path): return -1 files.append(file_path) else: for field_number in upload_state['file_column']: file_path = os.path.join( file_root, row[field_number - 1] ) if not validate_file(file_path): return -1 files.append(file_path) for field_number, _mime_type in zip( upload_state['remote_location'], upload_state['mime_type'], ): files.append({_mime_type: row[field_number - 1]}) if len(files) == 0: click.echo( 'Could not find any files in row:', err=True, ) click.echo(','.join(row), err=True) if not upload_state['allow_missing']: return -1 else: continue subject_rows.append((files, metadata)) if not subject_rows: click.echo( 'File {} did not contain any rows.'.format( manifest_file, ), err=True, ) return -1 subject_rows = list(enumerate(subject_rows)) upload_state['waiting_to_upload'] = copy.deepcopy(subject_rows) else: for subject_id, subject_row in upload_state['waiting_to_link'].items(): try: subject = Subject.find(subject_id) except PanoptesAPIException: upload_state['waiting_to_upload'].append(subject_row) del upload_state['waiting_to_link'][subject_id] subject_rows = copy.deepcopy(upload_state['waiting_to_upload']) pending_subjects = [] def move_created(limit): while len(pending_subjects) > limit: for subject, subject_row in pending_subjects: if subject.async_save_result: pending_subjects.remove((subject, subject_row)) upload_state['waiting_to_upload'].remove(subject_row) upload_state['waiting_to_link'][subject.id] = subject_row time.sleep(0.5) def link_subjects(limit): if len(upload_state['waiting_to_link']) > limit: subject_set.add(list(upload_state['waiting_to_link'].keys())) upload_state['waiting_to_link'].clear() with click.progressbar( subject_rows, length=len(subject_rows), label='Uploading subjects', ) as _subject_rows: try: with Subject.async_saves(): for subject_row in _subject_rows: count, (files, metadata) = subject_row subject = Subject() subject.links.project = subject_set.links.project for media_file in files: subject.add_location(media_file) subject.metadata.update(metadata) subject.save() pending_subjects.append((subject, subject_row)) move_created(MAX_PENDING_SUBJECTS) link_subjects(LINK_BATCH_SIZE) move_created(0) link_subjects(0) finally: if ( len(pending_subjects) > 0 or len(upload_state['waiting_to_link']) > 0 ): click.echo('Error: Upload failed.', err=True) if click.confirm( 'Would you like to save the upload state to resume the ' 'upload later?', default=True, ): while True: state_file_name = 'panoptes-upload-{}.yaml'.format( subject_set_id, ) state_file_name = click.prompt( 'Enter filename to save to', default=state_file_name, ) if not state_file_name.endswith('.yaml'): click.echo( 'Error: File name must end in ".yaml".', err=True, ) if click.confirm( 'Save to {}.yaml?'.format(state_file_name), default=True, ): state_file_name += '.yaml' else: continue if not is_valid_filename(state_file_name): click.echo( 'Error: {} is not a valid file name'.format( state_file_name, ), err=True, ) sanitized_filename = sanitize_filename( state_file_name, ) if click.confirm( 'Save to {}?'.format( sanitized_filename, ), default=True, ): state_file_name = sanitized_filename else: continue if os.path.exists(state_file_name): if not click.confirm( 'File {} already exists. Overwrite?'.format( state_file_name, ), default=False, ): continue break with open(state_file_name, 'w') as state_file: yaml.dump(upload_state, state_file)
def test_normal_str(self, platform, value, replace_text, expected): sanitized_name = sanitize_filename(value, platform=platform, replacement_text=replace_text) assert sanitized_name == expected assert isinstance(sanitized_name, six.text_type) validate_filename(sanitized_name, platform=platform) assert is_valid_filename(sanitized_name, platform=platform)