def _test_simple_makedirs_with_perms(self, quiet=0): test_perms = (0770, 0755, 0644) if self.VERBOSE and not quiet: cur_umask = os.umask(0) os.umask(cur_umask) print 'cur_umask is %x.' % cur_umask for p in test_perms: if self.VERBOSE and not quiet: self.write("_test_simple_makedirs_with_perms():") paths = ("one", ) if self.VERBOSE and not quiet: self.write(" Creating") for path in paths: if self.VERBOSE and not quiet: self.write(" %s,", path) makedirs(path, p, verbosity=0) assert os.path.isdir(path), ("Failed to create %s" % (path, )) stmode = S_IMODE(os.stat(path).st_mode) if p != stmode: if self.VERBOSE and not quiet: print os.system('ls -al .') raise 'Requested mode was 0x%X - Actual mode is 0x%X.' % ( p, stmode) self.writeln(" done.") os.system('chmod -R a+wxr one') os.system('rm -rf one') return
def _test_simple_makedirs_with_perms(self, quiet=0): test_perms = (0770, 0755, 0644) if self.VERBOSE and not quiet: cur_umask = os.umask(0) os.umask(cur_umask) print 'cur_umask is %x.' % cur_umask for p in test_perms: if self.VERBOSE and not quiet: self.write("_test_simple_makedirs_with_perms():") paths = ( "one", ) if self.VERBOSE and not quiet: self.write(" Creating") for path in paths: if self.VERBOSE and not quiet: self.write(" %s,", path) makedirs(path, p, verbosity=0) assert os.path.isdir(path), ( "Failed to create %s" % (path,)) stmode = S_IMODE(os.stat(path).st_mode) if p != stmode: if self.VERBOSE and not quiet: print os.system('ls -al .') raise 'Requested mode was 0x%X - Actual mode is 0x%X.' % (p, stmode) self.writeln(" done.") os.system('chmod -R a+wxr one') os.system('rm -rf one') return
def test_brokenlink_makedirs(self, quiet=0): if self.VERBOSE and not quiet: self.writeln("test_brokenlink_makedirs():") map = { "one":"one", "one/two":"one", "one/two/three":"one/two", "one/two/three/four":"one", "one/two/three/four/five":"one/two/three/four/five", } for dirpath, badlinkpath in map.items(): dir,link = os.path.split(badlinkpath) if dir: # Create up to (but not including) the broken link. makedirs(dir, verbosity=0) assert not os.path.exists(badlinkpath), ( "Internal test failure, %r appears to exist.") assert not os.path.islink(badlinkpath), ( "Internal test failure, %r appears NOT to be a link.") # Now try to create the complete path which should replace the # broken link. if self.VERBOSE and not quiet: self.writeln(" Creating %s, replacing %s", dirpath, badlinkpath) # Create the broken link. os.symlink("broken_link_points_here_which_does_not_exist", badlinkpath) makedirs(dirpath, verbosity=0) os.removedirs(dirpath) return
def test_brokenlink_makedirs(self, quiet=0): if self.VERBOSE and not quiet: self.writeln("test_brokenlink_makedirs():") map = { "one": "one", "one/two": "one", "one/two/three": "one/two", "one/two/three/four": "one", "one/two/three/four/five": "one/two/three/four/five", } for dirpath, badlinkpath in map.items(): dir, link = os.path.split(badlinkpath) if dir: # Create up to (but not including) the broken link. makedirs(dir, verbosity=0) assert not os.path.exists(badlinkpath), ( "Internal test failure, %r appears to exist.") assert not os.path.islink(badlinkpath), ( "Internal test failure, %r appears NOT to be a link.") # Now try to create the complete path which should replace the # broken link. if self.VERBOSE and not quiet: self.writeln(" Creating %s, replacing %s", dirpath, badlinkpath) # Create the broken link. os.symlink("broken_link_points_here_which_does_not_exist", badlinkpath) makedirs(dirpath, verbosity=0) os.removedirs(dirpath) return
def download_schema(schema_file, url): # TODO refactor into more general download methods if not file_exists(schema_file): makedirs(schema_file.split("/")[0]) response = requests.get(url, stream=True) # shamelessly stolen from https://stackoverflow.com/a/10744565 with open(schema_file, "wb") as handle: for data in tqdm(response.iter_content()): handle.write(data)
def move_file_to(self, path): import shutil src = self.get_filename() dirname = filesystem.dirname(path) if not filesystem.exists(dirname): filesystem.makedirs(dirname) shutil.copy2(src, path) os.remove(src)
def save_download_link(parser, settings, link): try: path_store = filesystem.join(settings.torrents_path(), 'nnmclub') if not filesystem.exists(path_store): filesystem.makedirs(path_store) source = parser.link() match = re.search(r'\.php.+?t=(\d+)', source) if match: with filesystem.fopen(filesystem.join(path_store, match.group(1)), 'w') as f: f.write(link) except: pass
def save_download_link(parser, settings, link): #try: if True: path_store = filesystem.join(settings.torrents_path(), 'nnmclub') if not filesystem.exists(path_store): filesystem.makedirs(path_store) source = parser.link() match = re.search(r'\.php.+?t=(\d+)', source) if match: with filesystem.fopen(filesystem.join(path_store, match.group(1)), 'w') as f: f.write(link)
def move_file_to(self, path): src = self.get_filename() dirname = filesystem.dirname(path) if not filesystem.exists(dirname): filesystem.makedirs(dirname) filesystem.copyfile(src, path) filesystem.remove(src) self.saved_to = path self.log('{} was moved to {}'.format(src, path))
def __init__(self, **kwargs): from remotesettings import Settings self.settings = Settings() if 'resume_file' in kwargs: resume_path = filesystem.join(self.settings.storage_path, '.resume') if not filesystem.exists(resume_path): filesystem.makedirs(resume_path) resume_name = filesystem.basename(kwargs['resume_file']) resume_name = resume_name.split('\\')[-1] kwargs['resume_file'] = filesystem.join(resume_path, resume_name) log.debug('resume_file is: ' + kwargs['resume_file']) kwargs['bind_host'] = self.settings.remote_host Engine.__init__(self, **kwargs)
def __init__(self, name: str, confidence_query_threshold: float = 0.8, seed_if_empty = True): """ Chatbot for semantic reasoning :param name: Display name for the bot :param confidence_query_threshold: If a response to an input to is below this threshold we may need instruction :param seed_if_empty: If the bot doesn't have anything in it's database should be import some test data? """ self.name = name self.confidence_query_threshold = confidence_query_threshold makedirs("./database") self.bot = ChatBot( name, storage_adapter='chatterbot.storage.SQLStorageAdapter', preprocessors=[ 'chatterbot.preprocessors.clean_whitespace', 'chatterbot.preprocessors.unescape_html', 'chatterbot.preprocessors.convert_to_ascii', ], logic_adapters=[ { 'import_path': 'onto_adapter.OntoAdapter' }, { "import_path": "chatterbot.logic.BestMatch", "statement_comparison_function": comparisons.levenshtein_distance, "response_selection_method": response_selection.get_first_response, "default_response": "", } ], database_uri='sqlite:///database/database.db' ) pub.sendMessage(Event.info, message = "Loading...") if seed_if_empty: if self.bot.storage.count() == 0: self.train() self.bot.read_only = True self.set_ready(True) pub.sendMessage(Event.info, message=f"Hello, my name is {self.name}")
def train(self): """ Import seed data :return: """ pub.sendMessage(Event.info, message = "Training, may take a while...") trainer = ChatterBotCorpusTrainer(self.bot) trainer.train( "chatterbot.corpus.english" ) pub.sendMessage(Event.info, message = "exporting training data") makedirs("./output") trainer.export_for_training('./output/training_export.json') pub.sendMessage(Event.info, message = "Training complete.")
def move_video_files(self): debug('Runner: move video') for file in self.get_relative_torrent_files_list(): dest_path = filesystem.join(self.settings.copy_video_path, file) if not filesystem.exists(filesystem.dirname(dest_path)): filesystem.makedirs(filesystem.dirname(dest_path)) src_path = filesystem.join(self.storage_path, file) if not filesystem.exists(src_path): continue if not filesystem.exists(dest_path): # Move file if no exists filesystem.movefile(src_path, dest_path) else: filesystem.remove(src_path) self.change_resume_file(self.settings.copy_video_path)
def test_simple_makedirs(self, quiet=0): if self.VERBOSE and not quiet: self.write("_test_simple_makedirs():") paths = ( "one", "one/two", "one/two/three", "one/two/three/four", "one/two/three/four/five", ) if self.VERBOSE and not quiet: self.write(" Creating") for path in paths: if self.VERBOSE and not quiet: self.write(" %s,", path) makedirs(path, verbosity=0) assert os.path.isdir(path), ("Failed to create %s" % (path, )) if self.VERBOSE and not quiet: self.writeln(" done.") return
def test_simple_makedirs(self, quiet=0): if self.VERBOSE and not quiet: self.write("_test_simple_makedirs():") paths = ( "one", "one/two", "one/two/three", "one/two/three/four", "one/two/three/four/five", ) if self.VERBOSE and not quiet: self.write(" Creating") for path in paths: if self.VERBOSE and not quiet: self.write(" %s,", path) makedirs(path, verbosity=0) assert os.path.isdir(path), ( "Failed to create %s" % (path,)) if self.VERBOSE and not quiet: self.writeln(" done.") return
def get_filename(self): path = filesystem.join(self.saveDir, self.get_subdir_name()) if not filesystem.exists(path): filesystem.makedirs(path) return filesystem.join(self.saveDir, self.get_subdir_name(), self.get_post_index() + self.extension)
return args ##################################################################################### # Download and extract the datasets into ``path`` def download_me(path, overwrite=False): _DOWNLOAD_URLS = [ 'https://people.eecs.berkeley.edu/~hendrycks/imagenet-a.tar', 'https://people.eecs.berkeley.edu/~hendrycks/imagenet-o.tar' ] for url in _DOWNLOAD_URLS: filename = download(url, path=path, overwrite=overwrite) # extract with tarfile.open(filename) as tar: tar.extractall(path=path) if __name__ == '__main__': args = parse_args() path = os.path.expanduser(args.download_dir) makedirs(path) download_me(path, overwrite=args.overwrite) # make symlink os.symlink(os.path.join(path, 'imagenet-a'), os.path.join(_TARGET_DIR, 'imagenet-adv-a')) os.symlink(os.path.join(path, 'imagenet-o'), os.path.join(_TARGET_DIR, 'imagenet-adv-o'))
def create(settings): #import vsdbg #vsdbg._bp() need_restart = False sources = Sources() with filesystem.save_make_chdir_context(settings.base_path()): if settings.anime_save: path = settings.anime_tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Аниме', 'tvshows') need_restart = True if settings.animation_save: path = settings.animation_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Мультфильмы', 'movies') need_restart = True if settings.animation_tvshows_save: path = settings.animation_tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Мультсериалы', 'tvshows') need_restart = True if settings.tvshows_save: path = settings.tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Сериалы', 'tvshows') need_restart = True if settings.documentary_save: path = settings.documentary_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Документальное', 'movies') need_restart = True if settings.movies_save: path = settings.movies_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Фильмы', 'movies') need_restart = True if settings.documentary_tvshows_save: path = settings.documentary_tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Документальные сериалы', 'tvshows') need_restart = True if settings.kids_save: path = settings.kids_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Детские фильмы', 'movies') need_restart = True if settings.concert_save: path = settings.concert_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Концерты', 'movies') need_restart = True if settings.theater_save: path = settings.theater_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Спектакли', 'movies') need_restart = True return need_restart
def create(settings): need_restart = False sources = Sources() with filesystem.save_make_chdir_context(settings.base_path()): if settings.anime_save: path = settings.anime_tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Аниме', 'tvshows') need_restart = True if settings.animation_save: path = settings.animation_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Мультфильмы', 'movies') need_restart = True if settings.animation_tvshows_save: path = settings.animation_tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Мультсериалы', 'tvshows') need_restart = True if settings.tvshows_save: path = settings.tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Сериалы', 'tvshows') need_restart = True if settings.documentary_save: path = settings.documentary_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Документальное', 'movies') need_restart = True if settings.movies_save: path = settings.movies_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Фильмы', 'movies') need_restart = True return need_restart