def _test_simple_makedirs_with_perms(self, quiet=0):
test_perms = (0770, 0755, 0644)
if self.VERBOSE and not quiet:
cur_umask = os.umask(0)
os.umask(cur_umask)
print 'cur_umask is %x.' % cur_umask
for p in test_perms:
if self.VERBOSE and not quiet:
self.write("_test_simple_makedirs_with_perms():")
paths = ("one", )
if self.VERBOSE and not quiet:
self.write(" Creating")
for path in paths:
if self.VERBOSE and not quiet:
self.write(" %s,", path)
makedirs(path, p, verbosity=0)
assert os.path.isdir(path), ("Failed to create %s" % (path, ))
stmode = S_IMODE(os.stat(path).st_mode)
if p != stmode:
if self.VERBOSE and not quiet:
print os.system('ls -al .')
raise 'Requested mode was 0x%X - Actual mode is 0x%X.' % (
p, stmode)
self.writeln(" done.")
os.system('chmod -R a+wxr one')
os.system('rm -rf one')
return
def _test_simple_makedirs_with_perms(self, quiet=0):
test_perms = (0770, 0755, 0644)
if self.VERBOSE and not quiet:
cur_umask = os.umask(0)
os.umask(cur_umask)
print 'cur_umask is %x.' % cur_umask
for p in test_perms:
if self.VERBOSE and not quiet:
self.write("_test_simple_makedirs_with_perms():")
paths = (
"one",
)
if self.VERBOSE and not quiet:
self.write(" Creating")
for path in paths:
if self.VERBOSE and not quiet:
self.write(" %s,", path)
makedirs(path, p, verbosity=0)
assert os.path.isdir(path), (
"Failed to create %s" % (path,))
stmode = S_IMODE(os.stat(path).st_mode)
if p != stmode:
if self.VERBOSE and not quiet:
print os.system('ls -al .')
raise 'Requested mode was 0x%X - Actual mode is 0x%X.' % (p, stmode)
self.writeln(" done.")
os.system('chmod -R a+wxr one')
os.system('rm -rf one')
return
def test_brokenlink_makedirs(self, quiet=0):
if self.VERBOSE and not quiet:
self.writeln("test_brokenlink_makedirs():")
map = {
"one":"one",
"one/two":"one",
"one/two/three":"one/two",
"one/two/three/four":"one",
"one/two/three/four/five":"one/two/three/four/five",
}
for dirpath, badlinkpath in map.items():
dir,link = os.path.split(badlinkpath)
if dir:
# Create up to (but not including) the broken link.
makedirs(dir, verbosity=0)
assert not os.path.exists(badlinkpath), (
"Internal test failure, %r appears to exist.")
assert not os.path.islink(badlinkpath), (
"Internal test failure, %r appears NOT to be a link.")
# Now try to create the complete path which should replace the
# broken link.
if self.VERBOSE and not quiet:
self.writeln(" Creating %s, replacing %s",
dirpath, badlinkpath)
# Create the broken link.
os.symlink("broken_link_points_here_which_does_not_exist",
badlinkpath)
makedirs(dirpath, verbosity=0)
os.removedirs(dirpath)
return
def test_brokenlink_makedirs(self, quiet=0):
if self.VERBOSE and not quiet:
self.writeln("test_brokenlink_makedirs():")
map = {
"one": "one",
"one/two": "one",
"one/two/three": "one/two",
"one/two/three/four": "one",
"one/two/three/four/five": "one/two/three/four/five",
}
for dirpath, badlinkpath in map.items():
dir, link = os.path.split(badlinkpath)
if dir:
# Create up to (but not including) the broken link.
makedirs(dir, verbosity=0)
assert not os.path.exists(badlinkpath), (
"Internal test failure, %r appears to exist.")
assert not os.path.islink(badlinkpath), (
"Internal test failure, %r appears NOT to be a link.")
# Now try to create the complete path which should replace the
# broken link.
if self.VERBOSE and not quiet:
self.writeln(" Creating %s, replacing %s", dirpath,
badlinkpath)
# Create the broken link.
os.symlink("broken_link_points_here_which_does_not_exist",
badlinkpath)
makedirs(dirpath, verbosity=0)
os.removedirs(dirpath)
return
def download_schema(schema_file, url):
# TODO refactor into more general download methods
if not file_exists(schema_file):
makedirs(schema_file.split("/")[0])
response = requests.get(url, stream=True)
# shamelessly stolen from https://stackoverflow.com/a/10744565
with open(schema_file, "wb") as handle:
for data in tqdm(response.iter_content()):
handle.write(data)
def move_file_to(self, path):
import shutil
src = self.get_filename()
dirname = filesystem.dirname(path)
if not filesystem.exists(dirname):
filesystem.makedirs(dirname)
shutil.copy2(src, path)
os.remove(src)
def save_download_link(parser, settings, link): try: path_store = filesystem.join(settings.torrents_path(), 'nnmclub') if not filesystem.exists(path_store): filesystem.makedirs(path_store) source = parser.link() match = re.search(r'\.php.+?t=(\d+)', source) if match: with filesystem.fopen(filesystem.join(path_store, match.group(1)), 'w') as f: f.write(link) except: pass
def save_download_link(parser, settings, link):
#try:
if True:
path_store = filesystem.join(settings.torrents_path(), 'nnmclub')
if not filesystem.exists(path_store):
filesystem.makedirs(path_store)
source = parser.link()
match = re.search(r'\.php.+?t=(\d+)', source)
if match:
with filesystem.fopen(filesystem.join(path_store, match.group(1)),
'w') as f:
f.write(link)
def move_file_to(self, path):
src = self.get_filename()
dirname = filesystem.dirname(path)
if not filesystem.exists(dirname):
filesystem.makedirs(dirname)
filesystem.copyfile(src, path)
filesystem.remove(src)
self.saved_to = path
self.log('{} was moved to {}'.format(src, path))
def __init__(self, **kwargs):
from remotesettings import Settings
self.settings = Settings()
if 'resume_file' in kwargs:
resume_path = filesystem.join(self.settings.storage_path, '.resume')
if not filesystem.exists(resume_path):
filesystem.makedirs(resume_path)
resume_name = filesystem.basename(kwargs['resume_file'])
resume_name = resume_name.split('\\')[-1]
kwargs['resume_file'] = filesystem.join(resume_path, resume_name)
log.debug('resume_file is: ' + kwargs['resume_file'])
kwargs['bind_host'] = self.settings.remote_host
Engine.__init__(self, **kwargs)
def __init__(self, **kwargs):
from remotesettings import Settings
self.settings = Settings()
if 'resume_file' in kwargs:
resume_path = filesystem.join(self.settings.storage_path,
'.resume')
if not filesystem.exists(resume_path):
filesystem.makedirs(resume_path)
resume_name = filesystem.basename(kwargs['resume_file'])
resume_name = resume_name.split('\\')[-1]
kwargs['resume_file'] = filesystem.join(resume_path, resume_name)
log.debug('resume_file is: ' + kwargs['resume_file'])
kwargs['bind_host'] = self.settings.remote_host
Engine.__init__(self, **kwargs)
def __init__(self, name: str, confidence_query_threshold: float = 0.8, seed_if_empty = True):
"""
Chatbot for semantic reasoning
:param name: Display name for the bot
:param confidence_query_threshold: If a response to an input to is below this threshold we may need instruction
:param seed_if_empty: If the bot doesn't have anything in it's database should be import some test data?
"""
self.name = name
self.confidence_query_threshold = confidence_query_threshold
makedirs("./database")
self.bot = ChatBot(
name,
storage_adapter='chatterbot.storage.SQLStorageAdapter',
preprocessors=[
'chatterbot.preprocessors.clean_whitespace',
'chatterbot.preprocessors.unescape_html',
'chatterbot.preprocessors.convert_to_ascii',
],
logic_adapters=[
{
'import_path': 'onto_adapter.OntoAdapter'
},
{
"import_path": "chatterbot.logic.BestMatch",
"statement_comparison_function": comparisons.levenshtein_distance,
"response_selection_method": response_selection.get_first_response,
"default_response": "",
}
],
database_uri='sqlite:///database/database.db'
)
pub.sendMessage(Event.info, message = "Loading...")
if seed_if_empty:
if self.bot.storage.count() == 0:
self.train()
self.bot.read_only = True
self.set_ready(True)
pub.sendMessage(Event.info, message=f"Hello, my name is {self.name}")
def train(self):
"""
Import seed data
:return:
"""
pub.sendMessage(Event.info, message = "Training, may take a while...")
trainer = ChatterBotCorpusTrainer(self.bot)
trainer.train(
"chatterbot.corpus.english"
)
pub.sendMessage(Event.info, message = "exporting training data")
makedirs("./output")
trainer.export_for_training('./output/training_export.json')
pub.sendMessage(Event.info, message = "Training complete.")
def move_video_files(self):
debug('Runner: move video')
for file in self.get_relative_torrent_files_list():
dest_path = filesystem.join(self.settings.copy_video_path, file)
if not filesystem.exists(filesystem.dirname(dest_path)):
filesystem.makedirs(filesystem.dirname(dest_path))
src_path = filesystem.join(self.storage_path, file)
if not filesystem.exists(src_path):
continue
if not filesystem.exists(dest_path):
# Move file if no exists
filesystem.movefile(src_path, dest_path)
else:
filesystem.remove(src_path)
self.change_resume_file(self.settings.copy_video_path)
def move_video_files(self):
debug('Runner: move video')
for file in self.get_relative_torrent_files_list():
dest_path = filesystem.join(self.settings.copy_video_path, file)
if not filesystem.exists(filesystem.dirname(dest_path)):
filesystem.makedirs(filesystem.dirname(dest_path))
src_path = filesystem.join(self.storage_path, file)
if not filesystem.exists(src_path):
continue
if not filesystem.exists(dest_path):
# Move file if no exists
filesystem.movefile(src_path, dest_path)
else:
filesystem.remove(src_path)
self.change_resume_file(self.settings.copy_video_path)
def test_simple_makedirs(self, quiet=0):
if self.VERBOSE and not quiet:
self.write("_test_simple_makedirs():")
paths = (
"one",
"one/two",
"one/two/three",
"one/two/three/four",
"one/two/three/four/five",
)
if self.VERBOSE and not quiet:
self.write(" Creating")
for path in paths:
if self.VERBOSE and not quiet:
self.write(" %s,", path)
makedirs(path, verbosity=0)
assert os.path.isdir(path), ("Failed to create %s" % (path, ))
if self.VERBOSE and not quiet:
self.writeln(" done.")
return
def test_simple_makedirs(self, quiet=0):
if self.VERBOSE and not quiet:
self.write("_test_simple_makedirs():")
paths = (
"one",
"one/two",
"one/two/three",
"one/two/three/four",
"one/two/three/four/five",
)
if self.VERBOSE and not quiet:
self.write(" Creating")
for path in paths:
if self.VERBOSE and not quiet:
self.write(" %s,", path)
makedirs(path, verbosity=0)
assert os.path.isdir(path), (
"Failed to create %s" % (path,))
if self.VERBOSE and not quiet:
self.writeln(" done.")
return
def get_filename(self):
path = filesystem.join(self.saveDir, self.get_subdir_name())
if not filesystem.exists(path):
filesystem.makedirs(path)
return filesystem.join(self.saveDir, self.get_subdir_name(),
self.get_post_index() + self.extension)
return args
#####################################################################################
# Download and extract the datasets into ``path``
def download_me(path, overwrite=False):
_DOWNLOAD_URLS = [
'https://people.eecs.berkeley.edu/~hendrycks/imagenet-a.tar',
'https://people.eecs.berkeley.edu/~hendrycks/imagenet-o.tar'
]
for url in _DOWNLOAD_URLS:
filename = download(url, path=path, overwrite=overwrite)
# extract
with tarfile.open(filename) as tar:
tar.extractall(path=path)
if __name__ == '__main__':
args = parse_args()
path = os.path.expanduser(args.download_dir)
makedirs(path)
download_me(path, overwrite=args.overwrite)
# make symlink
os.symlink(os.path.join(path, 'imagenet-a'),
os.path.join(_TARGET_DIR, 'imagenet-adv-a'))
os.symlink(os.path.join(path, 'imagenet-o'),
os.path.join(_TARGET_DIR, 'imagenet-adv-o'))
def create(settings):
#import vsdbg
#vsdbg._bp()
need_restart = False
sources = Sources()
with filesystem.save_make_chdir_context(settings.base_path()):
if settings.anime_save:
path = settings.anime_tvshow_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Аниме', 'tvshows')
need_restart = True
if settings.animation_save:
path = settings.animation_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Мультфильмы', 'movies')
need_restart = True
if settings.animation_tvshows_save:
path = settings.animation_tvshow_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Мультсериалы', 'tvshows')
need_restart = True
if settings.tvshows_save:
path = settings.tvshow_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Сериалы', 'tvshows')
need_restart = True
if settings.documentary_save:
path = settings.documentary_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Документальное', 'movies')
need_restart = True
if settings.movies_save:
path = settings.movies_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Фильмы', 'movies')
need_restart = True
if settings.documentary_tvshows_save:
path = settings.documentary_tvshow_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Документальные сериалы', 'tvshows')
need_restart = True
if settings.kids_save:
path = settings.kids_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Детские фильмы', 'movies')
need_restart = True
if settings.concert_save:
path = settings.concert_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Концерты', 'movies')
need_restart = True
if settings.theater_save:
path = settings.theater_path()
if not filesystem.exists(path):
filesystem.makedirs(path)
sources.add_video(path, u'Спектакли', 'movies')
need_restart = True
return need_restart
def get_filename(self):
path = filesystem.join(self.saveDir, self.get_subdir_name())
if not filesystem.exists(path):
filesystem.makedirs(path)
return filesystem.join(self.saveDir, self.get_subdir_name(), self.get_post_index() + self.extension)
def create(settings): need_restart = False sources = Sources() with filesystem.save_make_chdir_context(settings.base_path()): if settings.anime_save: path = settings.anime_tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Аниме', 'tvshows') need_restart = True if settings.animation_save: path = settings.animation_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Мультфильмы', 'movies') need_restart = True if settings.animation_tvshows_save: path = settings.animation_tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Мультсериалы', 'tvshows') need_restart = True if settings.tvshows_save: path = settings.tvshow_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Сериалы', 'tvshows') need_restart = True if settings.documentary_save: path = settings.documentary_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Документальное', 'movies') need_restart = True if settings.movies_save: path = settings.movies_path() if not filesystem.exists(path): filesystem.makedirs(path) sources.add_video(path, u'Фильмы', 'movies') need_restart = True return need_restart
