Esempio n. 1
0
def double_dump_test():
    """
    Perform a "double dump test" on every demo database.
    TODO: convert this to a Lino management command.
    """
    raise Exception("Not yet converted after 20150129")
    if len(env.demo_databases) == 0:
        return
    a = Path(env.temp_dir, 'a')
    b = Path(env.temp_dir, 'b')
    rmtree_after_confirm(a)
    rmtree_after_confirm(b)
    #~ if not confirm("This will possibly break the demo databases. Are you sure?"):
        #~ return
    #~ a.mkdir()
    with lcd(env.temp_dir):
        for db in env.demo_databases:
            if a.exists():
                a.rmtree()
            if b.exists():
                b.rmtree()
            local("django-admin.py dump2py --settings=%s --traceback a" % db)
            local(
                "django-admin.py run --settings=%s --traceback a/restore.py" %
                db)
            local("django-admin.py dump2py --settings=%s --traceback b" % db)
            local("diff a b")
Esempio n. 2
0
    def check_avatar(self):
        avatar_path = Path("/var/www/loka3/static/media/" + str(self.avatar))
        avatar_sm_path = Path("/var/www/loka3/static/media/" + str(self.avatar_sm))

        print "{} exists: {}".format(avatar_path, avatar_path.exists())

        if not avatar_path.exists() or not avatar_sm_path.exists():
            print "Wiping avatars for", self.name
            self.avatar = None
            self.avatar_sm = None
            self.save()
    def handle(self, *args, **options):
        directionals_path = Path(self.BASE_DIR, 'directionals.txt')
        if directionals_path.exists():
            Directional.load_directionals(directionals_path)

        states_path = Path(self.BASE_DIR, 'states.txt')
        if states_path.exists():
            State.load_states(states_path)

        street_types_path = Path(self.BASE_DIR, 'street_types.txt')
        if street_types_path.exists():
            StreetType.load_street_types(street_types_path)
Esempio n. 4
0
def py_clean(ctx, batch=False):
    """
    Delete :xfile:`.pyc` files, :xfile:`.eggs` and :xfile:`__cache__`
    directories under the project's root direcotory.
    """
    paths = []
    for root, dirs, files in os.walk(ctx.root_dir):
        p = Path(root).child('__pycache__')
        if p.exists():
            paths.append(p)
    if len(paths):
        if batch or confirm("Remove {0} __pycache__ directories".format(
                len(paths))):
            for p in paths:
                rmtree_after_confirm(p, True)

    for root, dirs, files in os.walk(ctx.root_dir):
        for fn in files:
            if fn.endswith(".pyc"):
                full_path = os.path.join(root, fn)
                if batch or confirm("Remove file %s:" % full_path):
                    os.remove(full_path)

    # cleanup_pyc(ctx.root_dir, batch)

    # if atelier.current_project.main_package is not None:
    #     try:
    #         p = Path(atelier.current_project.main_package.__file__).parent
    #         cleanup_pyc(atelier.current_project.root_dir, batch)
    #     except AttributeError:
    #         # happened 20170310 in namespace package:
    #         # $ pywhich commondata
    #         # Traceback (most recent call last):
    #         #   File "<string>", line 1, in <module>
    #         # AttributeError: 'module' object has no attribute '__file__'
    #         pass

    p = ctx.root_dir.child('.eggs')
    if p.exists():
        rmtree_after_confirm(p, batch)

    files = []
    for pat in ctx.cleanable_files:
        for p in glob.glob(os.path.join(ctx.root_dir, pat)):
            files.append(p)
    if len(files):
        if batch or confirm("Remove {0} cleanable files".format(len(files))):
            for p in files:
                os.remove(p)
Esempio n. 5
0
def download_frames(service_frames, output_dir, pdfs_only):
    output_dir = Path(output_dir)
    if not output_dir.exists():
        os.makedirs(output_dir)

    y1, y2 = output_dir.child('y1'), output_dir.child('y2')
    if not y1.exists():
        os.makedirs(y1)
    if not y2.exists():
        os.makedirs(y2)

    args = []
    if service_frames in FRAMES_BY_SERVICE:
        frames_ranges = FRAMES_BY_SERVICE[service_frames]
    else:
        print('Service "{}" does not have frames ranges. Options are:'.format(
            service_frames))
        for k in FRAMES_BY_SERVICE:
            print('\t- {}'.format(k))
        return

    if pdfs_only:
        for pdf in get_detectron_pdfs_frames_paths(*frames_ranges):
            args.append((pdf, output_dir))
    else:
        for frame in get_video_frames_path(frames_ranges[0], []):
            if not check_if_not_downlowaded(frame, y1):
                args.append((frame, y1))
        for frame in get_video_frames_path([], frames_ranges[1]):
            if not check_if_not_downlowaded(frame, y2):
                args.append((frame, y2))

    print('Donwloading {} frames...'.format(len(args)))
    with Pool(NUM_OF_PROCESSES) as pool:
        pool.starmap(download_s3_file, args)
Esempio n. 6
0
def superuser(pubkey=None, username=None):
    """
    fab env superuser
    """
    env.user = '******'

    keyfile = Path(pubkey or Path('~', '.ssh', 'id_rsa.pub')).expand()

    if not keyfile.exists():
        abort('Public key file does not exist: %s' % keyfile)

    username = username or prompt('Username: '******'Password: '******'perl -e \'print crypt(\"%s\", \"password\")\'' % (password),
                     capture=True)

    with open(keyfile, 'r') as f:
        pubkey = f.read(65535)

    commands = (
        'useradd -m -s /bin/bash -p {password} {username}',
        'mkdir ~{username}/.ssh -m 700',
        'echo "{pubkey}" >> ~{username}/.ssh/authorized_keys',
        'chmod 644 ~{username}/.ssh/authorized_keys',
        'chown -R {username}:{username} ~{username}/.ssh',
        'usermod -a -G sudo {username}',
    )

    for command in commands:
        run(command.format(**locals()))
Esempio n. 7
0
    def handle(self, *args, **options):

        from ....media.models import MediaFile
        from ...models import Transcript

        if len(args) != 1:
            raise CommandError('Provide media URL.')

        (url,) = args

        local_path = Path(url)
        if local_path.exists():
            url = 'file://{}'.format(local_path.absolute())

        media_file = MediaFile.objects.create(
            data_url=url,
        )
        if options['verbosity']:
            self.stdout.write('Created media file: {}'.format(media_file))

        transcript = Transcript.objects.create(
            name=url,
        )
        if options['verbosity']:
            self.stdout.write('Created transcript: {}'.format(transcript))
Esempio n. 8
0
def test_parameters(image_path, test_parameter, out_dir):
    image_path = Path(image_path)
    out_dir = Path(out_dir)

    if not out_dir.exists():
        os.mkdir(out_dir)

    config = NightmareConfig(out_dir=out_dir)
    if test_parameter == 'layers':
        for i in range(1, 21):
            config.layers = i
            deep_dream(image_path, config)
    elif test_parameter == 'rounds':
        for i in range(1, 21):
            config.rounds = i
            deep_dream(image_path, config)
    elif test_parameter == 'iters':
        for i in range(1, 31):
            config.iters = i
            deep_dream(image_path, config)
    elif test_parameter == 'range':
        for i in range(1, 11):
            config.range = i
            deep_dream(image_path, config)
    elif test_parameter == 'octaves':
        for i in range(1, 21):
            config.octaves = i
            deep_dream(image_path, config)
    elif test_parameter == 'rate':
        for i in range(1, 11):
            config.rate = i / 10
            deep_dream(image_path, config)
    else:
        print('Parameter "{}" is invalid.'.format(test_parameter))
        return
Esempio n. 9
0
def migrar_dados(interativo=True):

    # restaura dump
    arq_dump = Path(
        DIR_DADOS_MIGRACAO.child('dumps_mysql',
                                 '{}.sql'.format(NOME_BANCO_LEGADO)))
    assert arq_dump.exists(), 'Dump do mysql faltando: {}'.format(arq_dump)
    info('Restaurando dump mysql de [{}]'.format(arq_dump))
    normaliza_dump_mysql(arq_dump)
    roda_comando_shell('mysql -uroot < {}'.format(arq_dump))

    # executa ajustes pré-migração, se existirem
    arq_ajustes_pre_migracao = DIR_DADOS_MIGRACAO.child(
        'ajustes_pre_migracao', '{}.sql'.format(sigla_casa))
    if arq_ajustes_pre_migracao.exists():
        exec_legado(arq_ajustes_pre_migracao.read_file())

    uniformiza_banco()

    # excluindo database antigo.
    if interativo:
        info('Todos os dados do banco serão excluidos. '
             'Recomendamos que faça backup do banco sapl '
             'antes de continuar.')
        info('Deseja continuar? [s/n]')
        resposta = input()
        if resposta.lower() in ['s', 'sim', 'y', 'yes']:
            pass
        else:
            info('Migração cancelada.')
            return 0
    info('Excluindo entradas antigas do banco destino.')
    call([
        PROJECT_DIR.child('manage.py'), 'flush', '--database=default',
        '--no-input'
    ],
         stdout=PIPE)

    # apaga tipos de autor padrão (criados no flush acima)
    TipoAutor.objects.all().delete()

    fill_vinculo_norma_juridica()
    fill_dados_basicos()
    info('Começando migração: ...')
    try:
        ocorrencias.clear()
        migrar_todos_os_models()
    except Exception as e:
        ocorrencias['traceback'] = str(traceback.format_exc())
        raise e
    finally:
        # grava ocorrências
        arq_ocorrencias = Path(REPO.working_dir, 'ocorrencias.yaml')
        with open(arq_ocorrencias, 'w') as arq:
            pyaml.dump(ocorrencias, arq, vspacing=1)
        REPO.git.add([arq_ocorrencias.name])
        info('Ocorrências salvas em\n  {}'.format(arq_ocorrencias))

    # recria tipos de autor padrão que não foram criados pela migração
    cria_models_tipo_autor()
Esempio n. 10
0
def read_look_dream(out_dir):
    out_dir = Path(out_dir)
    download_dir = out_dir.child('download')

    if not out_dir.exists():
        os.mkdir(out_dir)

    if not download_dir.exists():
        os.mkdir(download_dir)

    for i, text in enumerate(FIRST_PAGE):
        print('Reading "{}"...'.format(text))
        image = search_random_image(text, download_dir)

        print('Dreaming "{}"...'.format(text))
        dirname = '000{} {}'.format(i, text).replace(' ', '_')
        config = NightmareConfig(out_dir.child(dirname))
        os.mkdir(config.out_dir)
        config.force_all_random()
        deep_dream(image, config)

        for d in config.list_output_for_image(image):
            print("  New dream: {}".format(d))

        sleep(10)
        print()
Esempio n. 11
0
def content(request=None):
    base = Path(current_app.config.get('INUPYPI_REPO', Path('.', 'packages')))

    if request:
        repo = Path(base, request)
    else:
        repo = base

    try:
        repo = repo.absolute()
        base = base.absolute()

        if not repo.exists():
            if base == repo:
                raise InuPyPIMissingRepoPath

            # sets the request to lowercase and compares it with
            # the existing items in the repository in lowercase
            repo = search_path(repo, base)

            if not repo:
                raise InuPyPI404Exception

        if repo.isdir():
            return Dirs(repo)
        if repo.isfile():
            return repo

    except InuPyPIMissingRepoPath:
        abort(500, 'Missing repository or package path!')
    except InuPyPI404Exception:
        abort(404, 'Path or File could not be found!')
    except:
        abort(500, 'Internal Server Error!')
    return repo
Esempio n. 12
0
def boostrap_nltk_data():
    nltk.data.path.append('./data/')
    nltkdata_exists = Path('./data/tokenizers/punkt/english.pickle')

    if not nltkdata_exists.exists():
        logging.info("Downloading NLTK Data")
        nltk.download('punkt', './data')
Esempio n. 13
0
def run(ctx, experiment):
    """Run an experiment in "experiments/" or from a config file.

    Run an experiment by passing in a path to a config file. Experiments will
    be searched for in the experiments directory. To list available
    experiments, pass 'list' as the first argument. Experiments
    in this directory can be run by the stem name of the file. To run all the
    available experiments, pass 'all'.

        $ inv run path/to/my/exp-1.yaml  # run a single experiment
        $ inv run list                   # list experiments in "experiments/"
        $ inv run exp-1                  # run "experiments/exp-1.yaml"
        $ inv run all                    # run all available experiments
    """
    if experiment == 'list':
        print('Experiments:')
        for experiment in EXPERIMENTS.listdir('*.yaml'):
            print(' - ' + experiment.stem)
        return
    elif experiment == 'all':
        experiments = EXPERIMENTS.listdir('*.yaml')
    elif Path(experiment).exists():
        experiments = [Path(experiment)]
    else:
        experiment = Path(EXPERIMENTS, experiment + '.yaml')
        assert experiment.exists(), 'experiment %s not found' % experiment
        experiments = [experiment]

    for experiment in experiments:
        output = Path(R_PKG, 'data-raw', experiment.stem + '.csv')
        print('Running experiment { %s }' % experiment.stem)
        peaks.run_experiment(experiment, output=output)
Esempio n. 14
0
def setup_babel_userdocs(babelcmd):
    """Create userdocs .po files if necessary."""
    userdocs = env.root_dir.child('userdocs')
    if not userdocs.isdir():
        return
    locale_dir = userdocs.child('translations')
    for domain in locale_dir.listdir('*.pot', names_only=True):
        domain = domain[:-4]
        for loc in env.languages:
            if loc != env.languages[0]:
                po_file = Path(locale_dir, loc, 'LC_MESSAGES', '%s.po' %
                               domain)
                mo_file = Path(locale_dir, loc, 'LC_MESSAGES', '%s.mo' %
                               domain)
                pot_file = Path(locale_dir, '%s.pot' % domain)
                if babelcmd == 'init_catalog' and po_file.exists():
                    print("Skip %s because file exists." % po_file)
                #~ elif babelcmd == 'compile_catalog' and not mo_file.needs_update(po_file):
                    #~ print "Skip %s because newer than .po" % mo_file
                else:
                    args = ["python", "setup.py"]
                    args += [babelcmd]
                    args += ["-l", loc]
                    args += ["--domain", domain]
                    args += ["-d", locale_dir]
                    #~ args += [ "-o" , po_file ]
                    #~ if babelcmd == 'init_catalog':
                    if babelcmd == 'compile_catalog':
                        args += ["-i", po_file]
                    else:
                        args += ["-i", pot_file]
                    cmd = ' '.join(args)
                    #~ must_confirm(cmd)
                    local(cmd)
Esempio n. 15
0
def transcrypt_sketch(sketch_name, sketch_dir, pyp5js):
    """
    Command to generate the P5.js code for a python sketch

    Params:
    - sketch_name: name of the sketch (will create a {sketch_name}.py)

    Opitionals
    - sketch_dir: sketch's directory (defaults to ./{sketch_name})
    - pyp5hs: path to the pyp5js main file (defaults to local install)
    """
    SKETCH_DIR = Path(sketch_dir or f'./{sketch_name}')
    if not SKETCH_DIR.exists():
        cprint.warn(f"Couldn't find the sketch.")
        cprint.err(f"The directory {SKETCH_DIR} doesn't exist.",
                   interrupt=True)

    sketch = SKETCH_DIR.child(f"{sketch_name}.py")
    pyp5js = Path(pyp5js or PYP5_DIR)

    command = ' '.join([
        str(c)
        for c in ['transcrypt', '-xp', pyp5js, '-b', '-m', '-n', sketch]
    ])
    cprint.info(f"Command:\n\t {command}")

    transcrypt = subprocess.Popen(shlex.split(command))
    transcrypt.wait()
Esempio n. 16
0
    def create_file(self, name, contents):
        """
        Creates a gzip file
        :param name: (str) name of the file to be created
        :param contents: (str) contents to be written in the file
        :return: (str or False) path of the created file
        """

        # write a tmp file
        tmp = mkstemp()[1]
        with gzip.open(tmp, 'wb') as handler:
            handler.write(contents)

        # send it to the FTP server
        if self.ftp:
            self.ftp.storbinary('STOR {}'.format(name), open(tmp, 'rb'))
            return '{}{}'.format(self.path, name)

        # or save it locally
        else:
            new_path = Path(self.path).child(name)
            tmp_path = Path(tmp)
            tmp_path.copy(new_path)
            if new_path.exists():
                return new_path.absolute()
            return False
def descargarArchivosUrl(urlDescargar=None, urlGuardar=None, accion=0):
    """Descarga un archivo de la web mediante la url y lo guarda en disco.

    Devuelve True si se ha descargado y guardado el archivo correctamente, sino devolvera False.

    Parametros:
    urlDescargar -- url del archivo a descargar
    urlGuardar -- ruta donde se desea guardar el dataset (incluido el nombre del fichero)
    accion -- 0 = reemplazar el archivo si existe, 1 = guardar con otro nombre sin reemplazar (nombre (n), n=1,2,3...), 2 = no sustituir ni guardar el archivo si existe. 

    Excepciones:
    ValueError -- Si (urlDescargar = None) o (urlGuardar = None) 
    
    """

    if (urlDescargar == None) or (urlGuardar == None):
        raise ValueError('Error en las rutas')
    else:
        f = Path(urlGuardar)
        existe = f.exists()

        if (accion == 2) and (existe == True):
            return False
        else:
            if (accion == 0) and (existe == True):
                os.remove(urlGuardar)
            wget.download(urlDescargar, urlGuardar)
            return True
Esempio n. 18
0
    def run(self,
            username=None,
            pubkey=None,
            as_root=False,
            with_password=True):
        if as_root:
            remote_user = '******'
            execute = run
        else:
            remote_user = env.local_user
            execute = sudo

        with settings(user=remote_user):
            keyfile = Path(pubkey or Path('~', '.ssh', 'id_rsa.pub')).expand()

            if not keyfile.exists():
                abort('Public key file does not exist: %s' % keyfile)

            pubkey = keyfile.read_file().strip()

            username = username or prompt('Username: '******'s password: "******"%s\", \"password\")\'' %
                        (password),
                        capture=True)
            else:
                password = '******'  # empty means disabled

            for command in self.commands:
                execute(command.format(**locals()))
Esempio n. 19
0
def enviar_email_con_cupon(modeladmin, request, queryset):
    leads_incorrectos = 0
    leads_correctos = 0
    for lead in queryset:
        if lead.enviado_en_csv is True and lead.enviado_cupon is False and lead.colectivo_validado is True:
            for fichero in os.listdir(settings.COUPONS_ROOT):
                if fnmatch.fnmatch(fichero, str(lead.id)+'_*.pdf'):
                    cupon_fichero = Path(settings.COUPONS_ROOT, fichero)
                    if cupon_fichero.exists():
                        codigo = fichero.split("_")[1].split(".")[0]
                        url_cupon = settings.BASE_URL+'/static/coupons/'+fichero
                        mail = EmailMultiAlternatives(
                            subject="Mi cupón de 10€ de Juguetes Blancos",
                            body='Descarga tu cupon aqui: '+url_cupon+' </p>',
                            from_email="Rocio, JueguetesBlancos <*****@*****.**>",
                            to=[lead.email]
                        )
                        mail.attach_alternative(render_to_string('leads/email_cupon.html', {'lead': lead, 'url_cupon': url_cupon}), "text/html")
                        mail.send()
                        lead.enviado_cupon = True
                        lead.codigo_cupon = codigo
                        lead.save()
                        leads_correctos = leads_correctos+1
        else:
            leads_incorrectos = leads_incorrectos+1
    messages.success(request, str(leads_correctos)+' Email/s enviado Correctamente')
    messages.error(request, str(leads_incorrectos)+' Leads no cumplian las condiciones.')
Esempio n. 20
0
    def append(self, key):
        key_path = Path(str(key))

        if key_path.isfile():
            with open(str(key_path)) as f:
                key = f.read()

        if not isinstance(key, bytes):
            key = key.encode('utf-8')

        if key in self:
            return

        directory = Path(self.user.path, 'keydir', self.user.name,
                         hashlib.md5(key.strip().split()[1]).hexdigest())
        directory.mkdir(parents=True)

        key_file = Path(directory, "%s.pub" % self.user.name)
        if key_file.exists() and key_file.read_file() == key:
            return

        key_file.write_file(key, mode='wb')

        self.user.git.commit(['keydir'],
                             'Added new key for user %s' % self.user.name)

        super(ListKeys, self).append(key)
Esempio n. 21
0
def boostrap_nltk_data():
    nltk.data.path.append('./data/')
    nltkdata_exists = Path('./data/tokenizers/punkt/english.pickle')

    if not nltkdata_exists.exists():
        logging.info("Downloading NLTK Data")
        nltk.download('punkt', './data')
Esempio n. 22
0
    def create_file(self, name, contents):
        """
        Creates a gzip file
        :param name: (str) name of the file to be created
        :param contents: (str) contents to be written in the file
        :return: (str or False) path of the created file
        """

        # write a tmp file
        tmp = mkstemp()[1]
        with gzip.open(tmp, 'wb') as handler:
            handler.write(contents)

        # send it to the FTP server
        if self.ftp:
            self.ftp.storbinary('STOR {}'.format(name), open(tmp, 'rb'))
            return '{}{}'.format(self.path, name)

        # or save it locally
        else:
            new_path = Path(self.path).child(name)
            tmp_path = Path(tmp)
            tmp_path.copy(new_path)
            if new_path.exists():
                return new_path.absolute()
            return False
Esempio n. 23
0
    def run(self, username=None, pubkey=None, as_root=False):
        if as_root:
            remote_user = '******'
            execute = run
        else:
            remote_user = env.local_user
            execute = sudo

        with settings(user=remote_user):
            keyfile = Path(pubkey or Path('~', '.ssh', 'id_rsa.pub')).expand()

            if not keyfile.exists():
                abort('Public key file does not exist: %s' % keyfile)

            with open(keyfile, 'r') as f:
                pubkey = f.read(65535)

            username = username or prompt('Username: '******'s password: "******"%s\", \"password\")\'' % (password),
                             capture=True)

            for command in self.commands:
                execute(command.format(**locals()))
Esempio n. 24
0
 def delete(self, name):
     """
     Given a name, delete the group. Idempotent.
     :param name: the group name to delete.
     :type name: str
     :returns: The deletion status.
     :rtype: bool
     """
     #1. Remove the conf file
     path = Path(
         os.path.join(self.path, 'conf', 'groups', '{}.conf'.format(name)))
     if not path.exists():  #Already exist
         return False
     path.remove()
     #2. Remove it from the repos file.
     for repo in Path(self.path, 'conf', 'repos').walk():
         if repo.isdir():
             continue
         with open(str(repo)) as f:
             if name in f.read():
                 Repository.get(
                     os.path.splitext(os.path.basename(repo))[0], self.path,
                     self.git).replace(r'.*= *@%s\n' % name, '')
     #3. Commit
     self.git.commit([str(path)], 'Deleted group {}.'.format(name))
     return True
Esempio n. 25
0
class PickleStorage(QueueStorage):
    """docstring for PickleStorage"""
    def __init__(self):
        super(PickleStorage, self).__init__()
        
        self._prepareStorage()
    
    def _prepareStorage(self):
        default = Path(ingo.active_project.base_path, '/queue')
        self._storage_path = Path(self.config.get('path', default))
        self._storage_name = 'items.pckl'        
        
        if not self._storage_path.exists():
            raise Exception("PickleStorage storage path (%s) doesn't exist!" % self._storage_path)
        
    def load(self):
        storage_file = open(Path(self._storage_path, self._storage_name), 'rb')
        results = pickle.load(storage_file)
        storage_file.close()
        
        return results

    def store(self, items):
        storage_file = open(Path(self._storage_path, self._storage_name), 'wb')
        results = pickle.dump(items, storage_file)
        storage_file.close()
        
        return results

    def clear(self):
        self.store([])
Esempio n. 26
0
def setup_from_tasks(globals_dict,
                     main_package=None,
                     settings_module_name=None,
                     **kwargs):
    """
    This is the function you must call from your :xfile:`tasks.py` file
    in order to activate the tasks defined by atelier.

    Arguments:

    - `globals_dict` must be the `globals()` of the calling script.

    - Optional `main_package` is the name of the main Python package provided by
      this project.

    - Optional `settings_module_name` will be stored in the
      :envvar:`DJANGO_SETTINGS_MODULE`, and certain project configuration
      options will get their default value from that module.

    - All remaining keyword arguments are project configuration parameters and
      stored to the :ref:`project configuration options <atelier.prjconf>`.

    """
    if '__file__' not in globals_dict:
        raise Exception(
            "No '__file__' in %r. "
            "First argument to setup_from_tasks() must be `globals()`." %
            globals_dict)

    tasks_file = Path(globals_dict['__file__'])
    if not tasks_file.exists():
        raise Exception("No such file: %s" % tasks_file)
    # print("20180428 setup_from_tasks() : {}".format(root_dir))

    from atelier.invlib import tasks
    from atelier.projects import get_project_from_tasks
    prj = get_project_from_tasks(tasks_file.parent)

    if atelier.current_project is None:
        atelier.current_project = prj

    if kwargs:
        prj.config.update(kwargs)

    if settings_module_name is not None:
        os.environ['DJANGO_SETTINGS_MODULE'] = settings_module_name
        from django.conf import settings
        prj.config.update(
            languages=[lng.name for lng in settings.SITE.languages])

    if isinstance(main_package, str):
        main_package = import_module(main_package)
    if main_package:
        prj.set_main_package(main_package)

    self = Collection.from_module(tasks)
    prj.set_namespace(self)

    return self
 def test_pdf_to_png(self):
     testdir = Path(r"C:\tmp\pdfprocessing\test")
     testdir.chdir()
     input_file = "testpdf.pdf"
     output_file = Path(r"C:\tmp\pdfprocessing\test\test_gs_pdf_to_png.png")
     gs = GhostScript()
     gs.pdf_to_png(input_file,output_file)
     self.assertTrue(output_file.exists(),"File")
Esempio n. 28
0
def read_json_data(json_filename):
    path = Path(DATA_DIR.child(f'{json_filename}.json'))

    if not path.exists():
        cprint.err(f"JSON file {path} does not exists.", True)

    with open(path) as fd:
        return json.load(fd)
Esempio n. 29
0
 def delete(self, lookup_repo_name):
     repo = Repository(lookup_repo_name, self.path, self.git)
     if not repo:
         return
     dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
     if dest.exists():
         dest.remove()
         self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)
Esempio n. 30
0
class ReleaseUnpackerRarFile(object):
    """Release unpacker RAR file."""
    def __init__(self, rar_file_path):
        """Initialize and validate rar file path."""
        self.rar_file_path = Path(rar_file_path)

        if (not self.rar_file_path.exists() or not self.rar_file_path.isfile()
                or not self.rar_file_path.ext == ".rar"):
            raise ReleaseUnpackerRarFileError("Invalid RAR file {}".format(
                self.rar_file_path))

        self.rar_file_path_abs = self.rar_file_path.absolute()
        self.rar_file = rarfile.RarFile(self.rar_file_path)

    def __repr__(self):
        """Return object string representation."""
        return "<ReleaseUnpackerRarFile: {}>".format(self.rar_file_path_abs)

    @lazy
    def name(self):
        """Return name of release folder."""
        if self.subs_dir:
            name = self.rar_file_path.parent.parent.name
        else:
            name = self.rar_file_path.parent.name

        return str(name)

    @lazy
    def subs_dir(self):
        """Return True if RAR file is located in a Subs folder."""
        if self.rar_file_path.parent.name.lower() in ("subs", "sub"):
            return True
        else:
            return False

    @lazy
    def file_list(self):
        """Return file list of RAR file."""
        files = []
        for file in self.rar_file.infolist():
            files.append({"name": Path(file.filename), "size": file.file_size})

        return files

    def extract_file(self, file_name, unpack_dir):
        """Extract file_name and return extracted file path."""
        self.rar_file.extract(file_name, path=unpack_dir)
        self.extracted_file_path = Path(unpack_dir, file_name)

        # Set the mtime to current time
        self.set_mtime()

        return self.extracted_file_path

    def set_mtime(self):
        """Set mtime of extracted file path to current time."""
        os.utime(self.extracted_file_path, None)
Esempio n. 31
0
    def delete(self, lookup_repo_name):
        repo = Repository(lookup_repo_name, self.path, self.git)
        dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
        if not dest.exists():
            raise ValueError('Repository %s not existing.' % lookup_repo_name)
        dest.remove()
        self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)

        return repo
Esempio n. 32
0
  def delete(self, lookup_repo_name):
    repo = Repository(lookup_repo_name, self.path, self.git)
    dest = Path(self.path, 'conf/repos/%s.conf' % lookup_repo_name)
    if not dest.exists():
      raise ValueError('Repository %s not existing.' % lookup_repo_name)
    dest.remove()
    self.git.commit([str(dest)], 'Deleted repo %s.' % lookup_repo_name)

    return repo
Esempio n. 33
0
    def get(self, id):
        upload_directory = os.path.join(
            current_app.config.get("UPLOAD_FOLDER", "uploads"), "secciones")
        f = Path(os.path.join(upload_directory, "%s.jpg" % str(id)))
        if (f.exists() == False):
            return send_file('uploads/secciones/default.jpg')

        image_path = os.path.join(upload_directory, "%s.jpg" % str(id))
        return send_file(image_path)
Esempio n. 34
0
 def delete(self, name):
   user = User(self.path, self.git, name)
   dest = Path(self.path, 'keydir/%s' % name)
   if not dest.exists():
     raise ValueError('Repository %s not existing.' % name)
   dest.rmtree()
   self.git.commit([str(dest)], 'Deleted user %s.' % name)
   
   return user
Esempio n. 35
0
 def test_resolve(self):
     p1 = Path(self.link_to_images_dir, "image3.png")
     p2 = p1.resolve()
     assert p1.components()[-2:] == ["link_to_images_dir", "image3.png"]
     assert p2.components()[-2:] == ["images", "image3.png"]
     assert p1.exists()
     assert p2.exists()
     assert p1.same_file(p2)
     assert p2.same_file(p1)
Esempio n. 36
0
 def delete(self, name):
     user = User(self.path, self.git, name)
     dest = Path(self.path, 'keydir/%s' % name)
     if not dest.exists():
         raise ValueError('Repository %s not existing.' % name)
     dest.rmtree()
     self.git.commit([str(dest)], 'Deleted user %s.' % name)
     
     return user
Esempio n. 37
0
 def test_resolve(self):
     p1 = Path(self.link_to_images_dir, "image3.png")
     p2 = p1.resolve()
     assert p1.components()[-2:] == ["link_to_images_dir", "image3.png"]
     assert p2.components()[-2:] == ["images", "image3.png"]
     assert p1.exists()
     assert p2.exists()
     assert p1.same_file(p2)
     assert p2.same_file(p1)
Esempio n. 38
0
def boostrap_crawled_files():
    # If we don't have the right files, grab them.
    crawled_page = Path('./data/1.html')

    if not crawled_page.exists():
        logging.info("Crawling Exeter Book from en.wikisource.org")
        run_spider()
    else:
        logging.info("Crawled files in place")
Esempio n. 39
0
def convert_wav_to_mp3(src_dir=None, dst_dir=None):
    """Convert wav sounds to mp3."""
    if src_dir is None:
        src_dir = seeds_dir
    if dst_dir is None:
        dst_dir = src_dir

    src_dir = Path(src_dir)
    assert src_dir.exists()

    dst_dir = Path(dst_dir)
    if not dst_dir.exists():
        dst_dir.mkdir(True)

    wav_seeds = Path(src_dir).listdir('*.wav', names_only=True)
    for wav in wav_seeds:
        mp3 = Path(wav).stem + '.mp3'
        cmd = 'ffmpeg -i {} -codec:a libmp3lame -qscale:a 2 {}'
        local_run(cmd.format(Path(src_dir, wav), Path(dst_dir, mp3)))
Esempio n. 40
0
    def get_backups_folder(self) -> Path:
        """Get the path to the local backup folder. Automatically creates folder if does not exist.

        Returns:
            path: List of Paths 
        """
        backup_folder = Path(settings.BACKUP['folder'])
        if not backup_folder.exists():
            backup_folder.mkdir()
        return backup_folder
Esempio n. 41
0
def convert_wav_to_mp3(src_dir=None, dst_dir=None):
    """Convert wav sounds to mp3."""
    if src_dir is None:
        src_dir = seeds_dir
    if dst_dir is None:
        dst_dir = src_dir

    src_dir = Path(src_dir)
    assert src_dir.exists()

    dst_dir = Path(dst_dir)
    if not dst_dir.exists():
        dst_dir.mkdir(True)

    wav_seeds = Path(src_dir).listdir('*.wav', names_only=True)
    for wav in wav_seeds:
        mp3 = Path(wav).stem + '.mp3'
        cmd = 'ffmpeg -i {} -codec:a libmp3lame -qscale:a 2 {}'
        local_run(cmd.format(Path(src_dir, wav), Path(dst_dir, mp3)))
Esempio n. 42
0
    def remove(self, key):
        directory = Path(self.user.path, 'keydir', self.user.name,
                         hashlib.md5(key.strip().split()[1]).hexdigest())
        key_file = Path(directory, "%s.pub" % self.user.name)

        if key_file.exists():
            key_file.remove()
            key_file.parent.rmdir()
            self.user.git.commit(['keydir'],
                                 'Removed key for user %s' % self.user.name)
Esempio n. 43
0
def files_iter(path):
    origin_file = path.name.replace('.part1', '')
    file_name_base = '{file_name}.part%d'.format(file_name=origin_file)
    i = 1
    while True:
        p = Path(file_name_base % i)
        if p.exists():
            yield p
        else:
            break
        i += 1
Esempio n. 44
0
def get_faces(output_dir):
    output_dir = Path(output_dir)
    if not output_dir.exists():
        os.makedirs(output_dir)

    bienal = BienalClient()
    for col in bienal.get_all_collections():
        print('Processando imagens da coleção "{}"'.format(col['title']))
        for image in tqdm(col.images):
            extract_faces(image, output_dir)
        print()
Esempio n. 45
0
    def get(self, id):
        upload_directory = os.path.join(
            current_app.config.get("UPLOAD_FOLDER", "uploads"), "categorias")

        f = Path(os.path.join(upload_directory, "%s_thumbnail.jpg" % str(id)))
        if (f.exists() == False):
            return send_file(os.path.join(upload_directory, 'default.jpg'))

        image_path = os.path.join(upload_directory,
                                  "%s_thumbnail.jpg" % str(id))
        return send_file(image_path)
Esempio n. 46
0
def files_iter(path):
    origin_file = path.name.replace('.part1', '')
    file_name_base = '{file_name}.part%d'.format(file_name=origin_file)
    i = 1
    while True:
        p = Path(file_name_base % i)
        if p.exists():
            yield p
        else:
            break
        i += 1
Esempio n. 47
0
    def get(self, id):
        import os.path
        upload_directory = os.path.join(
            current_app.config.get("UPLOAD_FOLDER", "uploads"), "alumnos")
        f = Path(os.path.join(upload_directory, "%s_thumbnail.jpg" % str(id)))
        if (f.exists() == False):
            return send_file('uploads/alumnos/default_thumbnail.jpg')

        image_path = os.path.join(upload_directory,
                                  "%s_thumbnail.jpg" % str(id))
        return send_file(image_path)
Esempio n. 48
0
File: fs.py Progetto: turicas/pyp5js
    def sketch_dir(self):
        sketch_dir = Path(self._sketch_dir)

        if not sketch_dir:
            return sketch_dir.child(f'{self.sketch_name}')

        if self.check_sketch_dir and not sketch_dir.exists():
            cprint.err(f"The directory {sketch_dir} does not exists.",
                       interrupt=True)

        return sketch_dir
Esempio n. 49
0
def find_labels():
    files = {}
    work_dir = Path("tmp/segments")
    if not work_dir.exists():
        raise Exception("You must first create the labels")

    files = {}
    labels = work_dir.listdir(filter=DIRS_NO_LINKS)
    for label in labels:
        files[str(label.name)] = label.listdir(pattern='*.ppm')
    return files
Esempio n. 50
0
 def __check_directory(self):
     """
     Check if the entered directory exists
     :return: (unipath.Path or False) the path to the existing directory
     """
     directory = Path(self.arguments['<directory>'])
     if not directory.exists() or not directory.isdir():
         msg = '{} is not a valid directory'.format(directory.absolute())
         self.__output(msg, error=True)
         return False
     return directory
Esempio n. 51
0
def addkey(pub_file):
    '''
    fab env setup.addkey:id_rsa.pub
    '''
    f = Path(pub_file)

    if not f.exists():
        abort('Public key file not found: %s' % keyfile)

    pub_key = f.read_file().strip()

    append('~/.ssh/authorized_keys', pub_key)
Esempio n. 52
0
def dict2dir(dir, dic, mode="w"):
    dir = Path(dir)
    if not dir.exists():
        dir.mkdir()
    for filename, content in dic.items():
        p = Path(dir, filename)
        if isinstance(content, dict):
            dict2dir(p, content)
            continue
        f = open(p, mode)
        f.write(content)
        f.close()
Esempio n. 53
0
    def remove(self, key):
        directory = Path(self.user.path, 'keydir', self.user.name,
                                         hashlib.md5(key.strip().split()[1]).hexdigest())
        key_file = Path(directory, "%s.pub" % self.user.name)

        if not key_file.exists():
            raise ValueError("Invalid key")

        key_file.remove()
        key_file.parent.rmdir()

        self.user.git.commit(['keydir'],
                                                 'Removed key for user %s' % self.user.name)
    def cleanup(self):
        """Remove subtitle files left over where the media file is removed"""

        log.debug('Running subtitle cleanup on dir {}'.format(self.search_dir))

        subtitle_extensions = ('.srt', '.sub', '.idx')

        for file_path in self.search_dir.walk(filter=FILES, top_down=False):
            if not file_path.ext in subtitle_extensions:
                continue

            # Remove the subtitle file if no media file exists in the same dir
            media_file_path_mkv = Path(file_path.parent, '{}.mkv'.format(
                file_path.stem))
            media_file_path_avi = Path(file_path.parent, '{}.avi'.format(
                file_path.stem))
            if (not media_file_path_mkv.exists() and
                    not media_file_path_avi.exists()):
                log.info('Removing leftover subtitle file {}'.format(
                    self.relative_path(file_path, self.search_dir)))

                file_path.remove()
Esempio n. 55
0
class ReleaseUnpackerRarFile(object):
    def __init__(self, rar_file_path):
        self.rar_file_path = Path(rar_file_path)

        if (not self.rar_file_path.exists() or not self.rar_file_path.isfile()
           or not self.rar_file_path.ext == '.rar'):
                raise ReleaseUnpackerRarFileError('Invalid RAR file {}'.format(
                    self. rar_file_path))

        self.rar_file_path_abs = self.rar_file_path.absolute()
        self.rar_file = rarfile.RarFile(self.rar_file_path)

    def __repr__(self):
        return '<ReleaseUnpackerRarFile: {}>'.format(self.rar_file_path_abs)

    @lazy
    def name(self):
        if self.subs_dir:
            name = self.rar_file_path.parent.parent.name
        else:
            name = self.rar_file_path.parent.name

        return str(name)

    @lazy
    def subs_dir(self):
        if self.rar_file_path.parent.name.lower() in ('subs', 'sub'):
            return True
        else:
            return False

    @lazy
    def file_list(self):
        files = []
        for file in self.rar_file.infolist():
            files.append({'name': Path(file.filename), 'size': file.file_size})

        return files

    def set_mtime(self):
        with file(self.extracted_file_path, 'a'):
            os.utime(self.extracted_file_path, None)

    def extract_file(self, file_name, unpack_dir):
        self.rar_file.extract(file_name, path=unpack_dir)
        self.extracted_file_path = Path(unpack_dir, file_name)

        # Set the mtime to current time
        self.set_mtime()

        return self.extracted_file_path
Esempio n. 56
0
def download_file(downloadable_url):
    """ File is only downloaded if it doesn't exist in DOWNLOADS_DIR

    This function does not rename the file. It only downloads the file
    if the expected name is not present in the DOWNLOADS_DIR.

    Returns a django.core.files.File object that can be stored in a FileField.
    """
    download_dir = Path(settings.DOWNLOADS_DIR)
    if not download_dir.exists():
        download_dir.mkdir()

    name_in_url = Path(downloadable_url).name
    expected_loc = Path(download_dir, name_in_url)

    # only download if necessary
    if not expected_loc.exists():
        response = requests.get(downloadable_url, stream = True)
        with open(expected_loc, 'wb') as expected_loc_handle:
            for chunk in response.iter_content(chunk_size = 1024):
                expected_loc_handle.write(chunk)

    return File(open(expected_loc, 'rb'))
Esempio n. 57
0
def download_survey_responses(survey_name):
    """Download the survey data.

    Args:
        survey_name: 'sound_similarity_6' or 'sound_similarity_4'
    """
    qualtrics = Qualtrics(**get_creds())
    responses = qualtrics.get_survey_responses(survey_name)
    survey_dir = Path(exp_dir, survey_name)
    if not survey_dir.exists():
        survey_dir.mkdir()

    output = Path(survey_dir, survey_name + '.csv')
    responses.to_csv(output, index=False)
Esempio n. 58
0
    def create(self, entity):
        repo_file = Path(self.path, 'conf/repos/%s.conf' % entity)
        if repo_file.exists():
            raise ValueError('Repository %s already exists' % entity)
        # If there are missing parent paths in the repo path, create them so we don't get IOErrors
        # In the case of a repo having names with slashes (e.g. "username/reponame")
        elif repo_file.parent != Path(""):
            repo_file.parent.mkdir(parents=True)

        repo_file.write_file("repo %s\n" % entity)

        self.git.commit([str(repo_file)], 'Created repo %s' % entity)

        return Repository(entity, self.path, self.git)
Esempio n. 59
0
class SheerSite(object):

    def __init__(self, slug):
        self.slug = slug
        if slug in settings.SHEER_SITES:
            self.path = Path(settings.SHEER_SITES[slug])
        else:
            self.path = None

    @property
    def urls(self):
        return self.urls_for_prefix()

    def urls_for_prefix(self, prefix='.'):
        url_patterns = []

        if self.path is None or not self.path.exists():
            return url_patterns

        prefixed_path = Path(self.path, prefix)
        for html_path in prefixed_path.walk():
            # skip files that don't end in .html
            if not html_path.endswith('.html'):
                continue
            rel_path = self.path.rel_path_to(html_path)
            prefix_rel_path = prefixed_path.rel_path_to(html_path)
            # skip files in underscore directories
            if rel_path.startswith('_'):
                continue
            view = SheerTemplateView.as_view(
                template_engine=self.slug,
                template_name=str(rel_path))
            regex_template = r'^%s$'
            index_template = r'^%s/$'
            if rel_path.name == 'index.html':
                if prefix_rel_path.parent:
                    slash_regex = index_template % prefix_rel_path.parent
                else:
                    slash_regex = r'^$'
                pattern = url(slash_regex, view)
                redirect_regex = regex_template % prefix_rel_path
                index_redirect = RedirectView.as_view(url='./', permanent=True)
                redirect_pattern = url(redirect_regex, index_redirect)
                url_patterns += [pattern, redirect_pattern]
            else:
                regex = regex_template % prefix_rel_path
                pattern = url(regex, view)
                url_patterns.append(pattern)
        return url_patterns
Esempio n. 60
0
def create_file_name(path):
    output_file_path = Path(path.name.replace('.part1', ''))
    output_file_ext = output_file_path.ext or '.dat'
    c = 0
    while output_file_path.exists():
        aux = output_file_path.name.replace(
            ' (%d)%s' % ((c - 1), output_file_ext),
            output_file_ext
        ).replace(
            output_file_ext,
            " (%d)%s" % (c, output_file_ext)
        )
        output_file_path = Path(aux)
        c += 1
    return output_file_path