Пример #1
0
def download_qualities(force=False, i_have_enough_space=False, keep=False):
    url = 'https://ndownloader.figshare.com/files/6059502'
    bz2 = Path(DATA_DIR, 'article_qualities.tsv.bz2')
    tsv = Path(DATA_DIR, 'article_qualities.tsv')

    # Try to prevent accidentally downloading too big a file.
    if not i_have_enough_space:
        try:
            gb_available = int(run("df -g . | awk '/\//{ print $4 }'").stdout)
            if gb_available < 100:
                raise NotEnoughGBAvailable
        except:
            raise NotEnoughGBAvailable
        else:
            logger.info('Rest easy, you have enough space.')
    else:
        logger.info('Skipping space check. Good luck soldier!')

    if SQLITE_PATH.exists() and not force:
        raise DBAlreadyExists

    logger.info('Downloading and decompressing.')
    run('wget {url} > {bz2} && bunzip2 {bz2}'.format(url=url, bz2=bz2))

    logger.info('Importing into sqlite.')
    conn = connect_to_sqlite_db()
    for chunk in pandas.read_table(tsv, chunksize=100000):
        chunk.to_sql('qualities', conn, if_exists='append', index=False)
    conn.close()

    if not keep:
        tsv.remove()
Пример #2
0
 def delete_file(self, name):
     """Delete a specific file"""
     if self.ftp:
         self.ftp.delete(name)
     else:
         path = Path(self.path).child(name)
         path.remove()
Пример #3
0
 def delete(self, name):
     """
     Given a name, delete the group. Idempotent.
     :param name: the group name to delete.
     :type name: str
     :returns: The deletion status.
     :rtype: bool
     """
     #1. Remove the conf file
     path = Path(
         os.path.join(self.path, 'conf', 'groups', '{}.conf'.format(name)))
     if not path.exists():  #Already exist
         return False
     path.remove()
     #2. Remove it from the repos file.
     for repo in Path(self.path, 'conf', 'repos').walk():
         if repo.isdir():
             continue
         with open(str(repo)) as f:
             if name in f.read():
                 Repository.get(
                     os.path.splitext(os.path.basename(repo))[0], self.path,
                     self.git).replace(r'.*= *@%s\n' % name, '')
     #3. Commit
     self.git.commit([str(path)], 'Deleted group {}.'.format(name))
     return True
Пример #4
0
 def delete_file(self, name):
     """Delete a specific file"""
     if self.ftp:
         self.ftp.delete(name)
     else:
         path = Path(self.path).child(name)
         path.remove()
Пример #5
0
 def delete(self, lookup_repo_name):
     repo = Repository(lookup_repo_name, self.path, self.git)
     if not repo:
         return
     dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
     if dest.exists():
         dest.remove()
         self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)
Пример #6
0
  def delete(self, lookup_repo_name):
    repo = Repository(lookup_repo_name, self.path, self.git)
    dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
    if not dest.exists():
      raise ValueError('Repository %s not existing.' % lookup_repo_name)
    dest.remove()
    self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)

    return repo
Пример #7
0
    def delete(self, lookup_repo_name):
        repo = Repository(lookup_repo_name, self.path, self.git)
        dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
        if not dest.exists():
            raise ValueError('Repository %s not existing.' % lookup_repo_name)
        dest.remove()
        self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)

        return repo
Пример #8
0
    def remove(self, key):
        directory = Path(self.user.path, 'keydir', self.user.name,
                         hashlib.md5(key.strip().split()[1]).hexdigest())
        key_file = Path(directory, "%s.pub" % self.user.name)

        if key_file.exists():
            key_file.remove()
            key_file.parent.rmdir()
            self.user.git.commit(['keydir'],
                                 'Removed key for user %s' % self.user.name)
Пример #9
0
    def remove(self, key):
        directory = Path(self.user.path, 'keydir', self.user.name,
                                         hashlib.md5(key.strip().split()[1]).hexdigest())
        key_file = Path(directory, "%s.pub" % self.user.name)

        if not key_file.exists():
            raise ValueError("Invalid key")

        key_file.remove()
        key_file.parent.rmdir()

        self.user.git.commit(['keydir'],
                                                 'Removed key for user %s' % self.user.name)
Пример #10
0
def yolo_detect_image_task(analysed_image_id):
    from src.core.models import AnalysedImage
    try:
        db_image = AnalysedImage.objects.get(id=analysed_image_id)
    except AnalysedImage.DoesNotExist:
        return None

    filename = db_image.image.name.split('/')[-1]
    clean_filename = filename.split('.')[0]
    temp_file = settings.TEMP_DIR.child(filename)

    with open(temp_file, 'bw') as fd:
        fd.write(db_image.image.read())

    pred_file = Path(settings.DARKNET_DIR,
                     'pred-{}.png'.format(clean_filename))
    command = ' '.join([
        settings.DARKNET_BIN,
        'detect',
        settings.YOLO_CONF,
        settings.YOLO_WEIGHTS,
        temp_file,
        '-out',
        pred_file.name.split('.')[0],
    ])
    print('Exec --> {}'.format(command))

    detect = subprocess.Popen(
        shlex.split(command),
        #stdout=subprocess.DEVNULL,
        #stderr=subprocess.DEVNULL,
        cwd=settings.DARKNET_DIR,
    )
    detect.wait()

    if not pred_file.exists():
        pred_file = Path(settings.DARKNET_DIR,
                         'pred-{}.jpg'.format(clean_filename))

    print("Finshed yolo analysis!")
    print("Prediction file is located at: " + pred_file.absolute())

    db_image.write_yolo_file(pred_file)
    db_image.save()

    temp_file.remove()
    pred_file.remove()
    print('Success!')
Пример #11
0
    def remove(self, key):
        if not isinstance(key, bytes):
            key = key.encode('utf-8')

        directory = Path(self.user.path, 'keydir', self.user.name,
                         hashlib.md5(key.strip().split()[1]).hexdigest())
        key_file = Path(directory, "%s.pub" % self.user.name)

        if not key_file.exists():
            raise ValueError("Invalid key")

        key_file.remove()
        key_file.parent.rmdir()

        self.user.git.commit(['keydir'],
                             'Removed key for user %s' % self.user.name)
Пример #12
0
    def move_to_venv(self, which_one):
        """
        Moves the created config_files into the bin folder to be executed.
        Does this by first pasting all the contents of the temporary file
        into the new or existing target file and then deleting the temp file.
        """
        target = Path(self.venv_folder, self.project_name, 'bin', which_one)
        source = Path(self.install_path, which_one)
        logger.info('target: %s, move_orig: %s' % (target, source))

        if source.exists():
            logger.info('Moving %s into place ...' % which_one)
            content = source.read_file()

            #make sure the directory exists
            if not target.parent.exists():
                target.parent.mkdir(parents=True)
            target.write_file(content, 'w+')

            source.remove()

        logger.info('...done')
Пример #13
0
 def test_remove_broken_symlink(self):
     symlink = Path(self.d, "symlink")
     symlink.write_link("broken")
     assert symlink.lexists()
     symlink.remove()
     assert not symlink.lexists()
Пример #14
0
class Base(object):
    """
    set and make directory
    
    Parameters
    ----------
    home : str
        set a directory as home
    """

    def __init__(self, home='.'):
        """
        set home directory

        Parameters
        ----------
        home : str
            set a directory as home

        Returns
        -------
        """
        self._home = Path(home).absolute()

    def __str__(self):
        return self.home

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self.home)

    # def __abs(self, string):
    #     return os.path.abspath(string)

    @property
    def home(self):
        return self._home.__str__()

    @home.setter
    def home(self, path):
        self._home = Path(path).absolute()

    def make_home(self, force=False):
        """
        make home directory

        Parameters
        ----------
        force : bool
            if True, if home exists and is a dir that
            containing contents, then delete contents
            in it, if exists and not a dir, remove it
            and make dir

        Returns
        -------

        """
        self.__mkdir(force)

    def __mkdir(self, force=False):
        if self._home.exists():
            if not self._home.isdir():
                if not force:
                    raise Exception('%s exists but is not a dir' % self.home)
                self._home.remove()
                self._home.mkdir()
            if force:
                self._home.rmtree()
                self._home.mkdir()
        else:
            self._home.mkdir(parents=True)

    def __rmdir(self, force=False):
        if self._home.exists():
            if not self._home.isdir():
                if not force:
                    raise Exception('%s exists but is not a dir' % self.home)
                self._home.remove()

            if force:
                self._home.rmtree()
            else:
                self._home.rmdir()

    def rm_home(self, force=False):
        """
        remove home directory

        Parameters
        ----------
        force : bool
            if True, if home exists and is a dir that
            containing contents, then delete it and
            it's contents, if exists and not a dir,
            remove then

        Returns
        -------

        """
        self.__rmdir(force)
Пример #15
0
 def test_remove_broken_symlink(self):
     symlink = Path(self.d, "symlink")
     symlink.write_link("broken")
     assert symlink.lexists()
     symlink.remove()
     assert not symlink.lexists()
Пример #16
0
def delete_avatar(player_model):
    avatar = Path(settings.MEDIA_ROOT, str(player_model.avatar))
    avatar_sm = Path(settings.MEDIA_ROOT, str(player_model.avatar_sm))
    avatar.remove()
    avatar_sm.remove()