def download_qualities(force=False, i_have_enough_space=False, keep=False):
url = 'https://ndownloader.figshare.com/files/6059502'
bz2 = Path(DATA_DIR, 'article_qualities.tsv.bz2')
tsv = Path(DATA_DIR, 'article_qualities.tsv')
# Try to prevent accidentally downloading too big a file.
if not i_have_enough_space:
try:
gb_available = int(run("df -g . | awk '/\//{ print $4 }'").stdout)
if gb_available < 100:
raise NotEnoughGBAvailable
except:
raise NotEnoughGBAvailable
else:
logger.info('Rest easy, you have enough space.')
else:
logger.info('Skipping space check. Good luck soldier!')
if SQLITE_PATH.exists() and not force:
raise DBAlreadyExists
logger.info('Downloading and decompressing.')
run('wget {url} > {bz2} && bunzip2 {bz2}'.format(url=url, bz2=bz2))
logger.info('Importing into sqlite.')
conn = connect_to_sqlite_db()
for chunk in pandas.read_table(tsv, chunksize=100000):
chunk.to_sql('qualities', conn, if_exists='append', index=False)
conn.close()
if not keep:
tsv.remove()
def delete_file(self, name):
"""Delete a specific file"""
if self.ftp:
self.ftp.delete(name)
else:
path = Path(self.path).child(name)
path.remove()
def delete(self, name):
"""
Given a name, delete the group. Idempotent.
:param name: the group name to delete.
:type name: str
:returns: The deletion status.
:rtype: bool
"""
#1. Remove the conf file
path = Path(
os.path.join(self.path, 'conf', 'groups', '{}.conf'.format(name)))
if not path.exists(): #Already exist
return False
path.remove()
#2. Remove it from the repos file.
for repo in Path(self.path, 'conf', 'repos').walk():
if repo.isdir():
continue
with open(str(repo)) as f:
if name in f.read():
Repository.get(
os.path.splitext(os.path.basename(repo))[0], self.path,
self.git).replace(r'.*= *@%s\n' % name, '')
#3. Commit
self.git.commit([str(path)], 'Deleted group {}.'.format(name))
return True
def delete_file(self, name):
"""Delete a specific file"""
if self.ftp:
self.ftp.delete(name)
else:
path = Path(self.path).child(name)
path.remove()
def delete(self, lookup_repo_name):
repo = Repository(lookup_repo_name, self.path, self.git)
if not repo:
return
dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
if dest.exists():
dest.remove()
self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)
def delete(self, lookup_repo_name):
repo = Repository(lookup_repo_name, self.path, self.git)
dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
if not dest.exists():
raise ValueError('Repository %s not existing.' % lookup_repo_name)
dest.remove()
self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)
return repo
def delete(self, lookup_repo_name):
repo = Repository(lookup_repo_name, self.path, self.git)
dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
if not dest.exists():
raise ValueError('Repository %s not existing.' % lookup_repo_name)
dest.remove()
self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)
return repo
def remove(self, key):
directory = Path(self.user.path, 'keydir', self.user.name,
hashlib.md5(key.strip().split()[1]).hexdigest())
key_file = Path(directory, "%s.pub" % self.user.name)
if key_file.exists():
key_file.remove()
key_file.parent.rmdir()
self.user.git.commit(['keydir'],
'Removed key for user %s' % self.user.name)
def remove(self, key):
directory = Path(self.user.path, 'keydir', self.user.name,
hashlib.md5(key.strip().split()[1]).hexdigest())
key_file = Path(directory, "%s.pub" % self.user.name)
if not key_file.exists():
raise ValueError("Invalid key")
key_file.remove()
key_file.parent.rmdir()
self.user.git.commit(['keydir'],
'Removed key for user %s' % self.user.name)
def yolo_detect_image_task(analysed_image_id):
from src.core.models import AnalysedImage
try:
db_image = AnalysedImage.objects.get(id=analysed_image_id)
except AnalysedImage.DoesNotExist:
return None
filename = db_image.image.name.split('/')[-1]
clean_filename = filename.split('.')[0]
temp_file = settings.TEMP_DIR.child(filename)
with open(temp_file, 'bw') as fd:
fd.write(db_image.image.read())
pred_file = Path(settings.DARKNET_DIR,
'pred-{}.png'.format(clean_filename))
command = ' '.join([
settings.DARKNET_BIN,
'detect',
settings.YOLO_CONF,
settings.YOLO_WEIGHTS,
temp_file,
'-out',
pred_file.name.split('.')[0],
])
print('Exec --> {}'.format(command))
detect = subprocess.Popen(
shlex.split(command),
#stdout=subprocess.DEVNULL,
#stderr=subprocess.DEVNULL,
cwd=settings.DARKNET_DIR,
)
detect.wait()
if not pred_file.exists():
pred_file = Path(settings.DARKNET_DIR,
'pred-{}.jpg'.format(clean_filename))
print("Finshed yolo analysis!")
print("Prediction file is located at: " + pred_file.absolute())
db_image.write_yolo_file(pred_file)
db_image.save()
temp_file.remove()
pred_file.remove()
print('Success!')
def remove(self, key):
if not isinstance(key, bytes):
key = key.encode('utf-8')
directory = Path(self.user.path, 'keydir', self.user.name,
hashlib.md5(key.strip().split()[1]).hexdigest())
key_file = Path(directory, "%s.pub" % self.user.name)
if not key_file.exists():
raise ValueError("Invalid key")
key_file.remove()
key_file.parent.rmdir()
self.user.git.commit(['keydir'],
'Removed key for user %s' % self.user.name)
def move_to_venv(self, which_one):
"""
Moves the created config_files into the bin folder to be executed.
Does this by first pasting all the contents of the temporary file
into the new or existing target file and then deleting the temp file.
"""
target = Path(self.venv_folder, self.project_name, 'bin', which_one)
source = Path(self.install_path, which_one)
logger.info('target: %s, move_orig: %s' % (target, source))
if source.exists():
logger.info('Moving %s into place ...' % which_one)
content = source.read_file()
#make sure the directory exists
if not target.parent.exists():
target.parent.mkdir(parents=True)
target.write_file(content, 'w+')
source.remove()
logger.info('...done')
def test_remove_broken_symlink(self):
symlink = Path(self.d, "symlink")
symlink.write_link("broken")
assert symlink.lexists()
symlink.remove()
assert not symlink.lexists()
class Base(object):
"""
set and make directory
Parameters
----------
home : str
set a directory as home
"""
def __init__(self, home='.'):
"""
set home directory
Parameters
----------
home : str
set a directory as home
Returns
-------
"""
self._home = Path(home).absolute()
def __str__(self):
return self.home
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self.home)
# def __abs(self, string):
# return os.path.abspath(string)
@property
def home(self):
return self._home.__str__()
@home.setter
def home(self, path):
self._home = Path(path).absolute()
def make_home(self, force=False):
"""
make home directory
Parameters
----------
force : bool
if True, if home exists and is a dir that
containing contents, then delete contents
in it, if exists and not a dir, remove it
and make dir
Returns
-------
"""
self.__mkdir(force)
def __mkdir(self, force=False):
if self._home.exists():
if not self._home.isdir():
if not force:
raise Exception('%s exists but is not a dir' % self.home)
self._home.remove()
self._home.mkdir()
if force:
self._home.rmtree()
self._home.mkdir()
else:
self._home.mkdir(parents=True)
def __rmdir(self, force=False):
if self._home.exists():
if not self._home.isdir():
if not force:
raise Exception('%s exists but is not a dir' % self.home)
self._home.remove()
if force:
self._home.rmtree()
else:
self._home.rmdir()
def rm_home(self, force=False):
"""
remove home directory
Parameters
----------
force : bool
if True, if home exists and is a dir that
containing contents, then delete it and
it's contents, if exists and not a dir,
remove then
Returns
-------
"""
self.__rmdir(force)
def test_remove_broken_symlink(self):
symlink = Path(self.d, "symlink")
symlink.write_link("broken")
assert symlink.lexists()
symlink.remove()
assert not symlink.lexists()
def delete_avatar(player_model):
avatar = Path(settings.MEDIA_ROOT, str(player_model.avatar))
avatar_sm = Path(settings.MEDIA_ROOT, str(player_model.avatar_sm))
avatar.remove()
avatar_sm.remove()