Example #1
0
    def __init__(self,
                 executor_uri,
                 cpu_limit,
                 mem_limit,
                 push_registry,
                 staging_uri,
                 stream=False,
                 verbose=False,
                 repository=None,
                 pull_registry=None,
                 docker_host=None,
                 container_image=None,
                 insecure_registries=False,
                 max_retries=3):

        self.executor_uri = executor_uri
        self.cpu = float(cpu_limit)
        self.mem = int(mem_limit)
        self.push_registry = push_registry
        self.pull_registry = pull_registry
        self.staging_uri = staging_uri
        self.stream = stream
        self.verbose = verbose
        self.repository = repository
        self.docker_host = docker_host
        self.container_image = container_image
        self.insecure_registries = insecure_registries
        self.max_retries = max_retries

        self.pending = 0
        self.running = 0
        self.finished = 0
        self.failed = 0
        self.queued_tasks = []
        self.task_status = defaultdict(lambda: None)
        self.task_history = {}
        self.task_retries = defaultdict(int)
        self.blacklist = set()

        self._processing_offers = threading.Lock()
        self._processing_queue = threading.Lock()

        # Ensure the staging directory exists
        self.filesystem = None
        if self.staging_uri:
            staging_uri = urlparse(self.staging_uri)
            staging_fs = opener.opendir(
                "%s://%s/" % (staging_uri.scheme, staging_uri.netloc))

            staging_fs.makedir(staging_uri.path.lstrip("/"),
                               recursive=True,
                               allow_recreate=True)

            self.filesystem = opener.opendir(self.staging_uri)

        self.cleanup = TaskCleanupThread(self.filesystem)
        self.cleanup.start()
Example #2
0
    def put(self):
        filename = parse_options_header(
            request.headers.get('Content-Disposition', ''))[1].get('filename')

        d = Document.create({'deleted': False})
        opener.opendir(directory_name(d), create_dir=True)
        d.setcontents(
            request.stream,
            name=lambda s: os.path.join(directory_name(s),
                                        secure_filename(filename))
        )
        return d.dumps()
Example #3
0
    def __init__(self,
                 tasks,
                 executor_uri,
                 cpu_limit,
                 mem_limit,
                 push_registry,
                 staging_uri,
                 stream=False,
                 verbose=False,
                 repository=None,
                 pull_registry=None,
                 docker_host=None,
                 container_image=None,
                 insecure_registries=False):

        self.executor_uri = executor_uri
        self.cpu = float(cpu_limit)
        self.mem = int(mem_limit)
        self.push_registry = push_registry
        self.pull_registry = pull_registry
        self.staging_uri = staging_uri
        self.stream = stream
        self.verbose = verbose
        self.repository = repository
        self.docker_host = docker_host
        self.container_image = container_image
        self.insecure_registries = insecure_registries

        self.queued_tasks = []
        for path, tags in tasks:
            dockerfile = parse_dockerfile(path, registry=pull_registry)
            self.queued_tasks.append((path, dockerfile, tags))

        self.pending = len(self.queued_tasks)
        self.running = 0
        self.finished = 0
        self.failed = 0
        self.task_ids = {}

        self.processing_offers = threading.Lock()

        # Ensure the staging directory exists
        self.filesystem = None
        if self.staging_uri:
            staging_uri = urlparse(self.staging_uri)
            staging_fs = opener.opendir(
                "%s://%s/" % (staging_uri.scheme, staging_uri.netloc))

            staging_fs.makedir(staging_uri.path.lstrip("/"),
                               recursive=True,
                               allow_recreate=True)

            self.filesystem = opener.opendir(self.staging_uri)
Example #4
0
    def put(self, oauth):
        filename = parse_options_header(
            request.headers.get('Content-Disposition', ''))[1].get('filename')

        d = Document.create({'deleted': False})
        opener.opendir(directory_name(d), create_dir=True)
        d.setcontents(
            request.stream,
            name=lambda s: os.path.join(directory_name(s),
                                        secure_filename(filename))
        )
        return d.dumps()
Example #5
0
    def __init__(self, executor_uri, cpu_limit, mem_limit, push_registry,
                 staging_uri, stream=False, verbose=False, repository=None,
                 pull_registry=None, docker_host=None, container_image=None,
                 insecure_registries=False, max_retries=3):

        self.executor_uri = executor_uri
        self.cpu = float(cpu_limit)
        self.mem = int(mem_limit)
        self.push_registry = push_registry
        self.pull_registry = pull_registry
        self.staging_uri = staging_uri
        self.stream = stream
        self.verbose = verbose
        self.repository = repository
        self.docker_host = docker_host
        self.container_image = container_image
        self.insecure_registries = insecure_registries
        self.max_retries = max_retries

        self.pending = 0
        self.running = 0
        self.finished = 0
        self.failed = 0
        self.queued_tasks = []
        self.task_status = defaultdict(lambda: None)
        self.task_history = {}
        self.task_retries = defaultdict(int)
        self.blacklist = set()

        self._processing_offers = threading.Lock()
        self._processing_queue = threading.Lock()

        # Ensure the staging directory exists
        self.filesystem = None
        if self.staging_uri:
            staging_uri = urlparse(self.staging_uri)
            staging_fs = opener.opendir(
                "%s://%s/" % (staging_uri.scheme, staging_uri.netloc)
            )

            staging_fs.makedir(
                staging_uri.path.lstrip("/"),
                recursive=True,
                allow_recreate=True
            )

            self.filesystem = opener.opendir(self.staging_uri)

        self.cleanup = TaskCleanupThread(self.filesystem)
        self.cleanup.start()
Example #6
0
    def __init__(self, tasks, executor_uri, cpu_limit, mem_limit, push_registry,
                 staging_uri, stream=False, verbose=False, repository=None,
                 pull_registry=None, docker_host=None, container_image=None,
                 insecure_registries=False):

        self.executor_uri = executor_uri
        self.cpu = float(cpu_limit)
        self.mem = int(mem_limit)
        self.push_registry = push_registry
        self.pull_registry = pull_registry
        self.staging_uri = staging_uri
        self.stream = stream
        self.verbose = verbose
        self.repository = repository
        self.docker_host = docker_host
        self.container_image = container_image
        self.insecure_registries = insecure_registries

        self.queued_tasks = []
        for path, tags in tasks:
            dockerfile = parse_dockerfile(path, registry=pull_registry)
            self.queued_tasks.append((path, dockerfile, tags))

        self.pending = len(self.queued_tasks)
        self.running = 0
        self.finished = 0
        self.failed = 0
        self.task_ids = {}

        self._processing_offers = threading.Lock()

        # Ensure the staging directory exists
        self.filesystem = None
        if self.staging_uri:
            staging_uri = urlparse(self.staging_uri)
            staging_fs = opener.opendir(
                "%s://%s/" % (staging_uri.scheme, staging_uri.netloc)
            )

            staging_fs.makedir(
                staging_uri.path.lstrip("/"),
                recursive=True,
                allow_recreate=True
            )

            self.filesystem = opener.opendir(self.staging_uri)

        self.cleanup = TaskCleanupThread(self.filesystem)
        self.cleanup.start()
    def _get_fs(self, create_dir=True):
        """."""
        filedir = dirname(self.fileurl)
        filename = basename(self.fileurl)

        return (opener.opendir(filedir, writeable=True,
                               create_dir=create_dir), filename)
Example #8
0
    def test_verify_remote_scroll(self):
        pyfs = opener.opendir(settings.config['fs_path'])

        signing_key = crypto.generate_signing_key()
        scroll_id = sha1_hexdigest
        s = scroll.RemoteScroll(pyfs, 
                               scroll_id, 
                               signing_key.verify_key.encode(crypto.HexEncoder))
        s.get_verify_key = lambda y: signing_key.verify_key
        
        items = []
        for i in range(10):
            item_hash = HASH(i)
            s.add(item_hash)
            items.append(item_hash)

        four, five = s.slice(4, 2)
        self.assertEqual(four, HASH(4))
        self.assertEqual(five, HASH(5))

        new_item_hash = HASH(i)
        new_state = HASH(s.state + new_item_hash)
        update = new_item_hash + new_state
        signed_update = signing_key.sign(update)

        self.assertEqual(s.verify_update(signed_update), update)
Example #9
0
 def test_put_get_pool(self):
     my_data = "B"*20
     scroll_id = sha1_hexdigest
     pyfs = opener.opendir(settings.config['fs_path'])
     p = pool.Pool(pyfs)
     hash_bytes = p.put(my_data)
     self.assertEqual(p.get(hash_bytes), my_data)
Example #10
0
def archive_fs(locations):
    """Fixture to check the BagIt file generation."""
    archive_path = locations['archive'].uri
    fs = opener.opendir(archive_path, writeable=False, create_dir=True)
    yield fs
    for d in fs.listdir():
        fs.removedir(d, force=True)
Example #11
0
 def test_put_get_pool(self):
     my_data = "B" * 20
     scroll_id = sha1_hexdigest
     pyfs = opener.opendir(settings.config['fs_path'])
     p = pool.Pool(pyfs)
     hash_bytes = p.put(my_data)
     self.assertEqual(p.get(hash_bytes), my_data)
Example #12
0
    def _get_fs(self, create_dir=True):
        """Return tuple with filesystem and filename."""
        filedir = dirname(self.fileurl)
        filename = basename(self.fileurl)

        return (opener.opendir(filedir, writeable=True,
                               create_dir=create_dir), filename)
def archive_fs(locations):
    """Fixture to check the BagIt file generation."""
    archive_path = locations['archive'].uri
    fs = opener.opendir(archive_path, writeable=False, create_dir=True)
    yield fs
    for d in fs.listdir():
        fs.removedir(d, force=True)
Example #14
0
    def test_verify_remote_scroll(self):
        pyfs = opener.opendir(settings.config['fs_path'])

        signing_key = crypto.generate_signing_key()
        scroll_id = sha1_hexdigest
        s = scroll.RemoteScroll(
            pyfs, scroll_id, signing_key.verify_key.encode(crypto.HexEncoder))
        s.get_verify_key = lambda y: signing_key.verify_key

        items = []
        for i in range(10):
            item_hash = bytes_hash_int(i)
            s.add(item_hash)
            items.append(item_hash)

        four, five = s.slice(4, 2)
        self.assertEqual(four, bytes_hash_int(4))
        self.assertEqual(five, bytes_hash_int(5))

        new_item_hash = bytes_hash_int(i)
        new_state = settings.HASH(s.state + new_item_hash)
        update = new_item_hash + new_state
        signed_update = signing_key.sign(update)

        self.assertEqual(s.verify_update(signed_update), update)
Example #15
0
 def test_add_one(self):
     scroll_id = 'test_add'
     data1 = HASH( '1' )
     pyfs = opener.opendir(settings.config['fs_path'])
     s = scroll.Scroll(pyfs, scroll_id)
     s.add(data1)
     contents = pyfs.getcontents(scroll_id + '.scroll')
     self.assertEqual(contents, data1)
Example #16
0
 def test_save_scroll(self):
     item_hash = twenty_bytes
     scroll_id = sha1_hexdigest
     pyfs = opener.opendir(settings.config['fs_path'])
     s = scroll.Scroll(pyfs, scroll_id)
     s.add(item_hash)
     self.assertTrue(s.has(item_hash))
     contents = pyfs.getcontents(scroll_id + '.scroll')
     self.assertEqual(contents, item_hash)
Example #17
0
    def _get_fs(self, create_dir=True):
        """."""
        filedir = dirname(self.fileurl)
        filename = basename(self.fileurl)

        return (
            opener.opendir(filedir, writeable=True, create_dir=create_dir),
            filename
        )
Example #18
0
    def _get_fs(self, create_dir=True):
        """Return tuple with filesystem and filename."""
        filedir = dirname(self.fileurl)
        filename = basename(self.fileurl)

        return (
            opener.opendir(filedir, writeable=True, create_dir=create_dir),
            filename
        )
Example #19
0
    def test_sign(self):
        my_message = 'Do it Like La La La!'
        pyfs = opener.opendir(settings.config['fs_path'])
        fingerprint = notary.Notary.generate(pyfs)
        publisher = None
        n = notary.Notary(publisher, pyfs, fingerprint)

        signed_message = n.signing_key.sign(my_message)
        verify_key = n.signing_key.verify_key
        self.assertEqual(verify_key.verify(signed_message), my_message)
Example #20
0
    def test_sign(self):
        my_message = 'Do it Like La La La!'
        pyfs = opener.opendir(settings.config['fs_path'])
        fingerprint = notary.Notary.generate(pyfs)
        publisher = None
        n = notary.Notary(publisher, pyfs, fingerprint)

        signed_message = n.signing_key.sign(my_message)
        verify_key = n.signing_key.verify_key
        self.assertEqual(verify_key.verify(signed_message), my_message)
Example #21
0
    def _process_files(self, metadata):
        """Transfer files in a list from one ``fs`` object to another.

        All tranferer files will maintain the same filename.
        """
        # FS object created at the folder of where the record specific bag is
        fs_dest = opener.opendir(self.folder, "files", create_dir=True)

        files_to_upload = []
        for file_to_upload in metadata.get("files", []):
            dirname_ = os.path.dirname(file_to_upload["path"])
            basename_ = os.path.basename(file_to_upload["path"])
            fs_src = opener.opendir(dirname_)
            copyfile(fs_src, basename_, fs_dest, basename_)
            file_to_upload["path"] = os.path.join("files", basename_)
            del file_to_upload["url"]
            files_to_upload.append(file_to_upload)
        metadata["files_to_upload"] = files_to_upload
        return metadata
Example #22
0
    def test_load_scroll(self):
        item_hash = twenty_bytes
        scroll_id = sha1_hexdigest
        pyfs = opener.opendir(settings.config['fs_path'])

        with open(scroll_id + '.scroll', 'w+') as f:
            f.write(item_hash)

        s = scroll.Scroll(pyfs, scroll_id)
        s.load()
        self.assertTrue(s.has(item_hash))
Example #23
0
    def save(self, incoming_stream, size=None, chunk_size=None):
        """Save file in the file system."""
        uri = self.obj.bucket.location.uri
        path = self.make_path()

        with opener.opendir(uri) as fs:
            dest_file = fs.makeopendir(path, recursive=True).open('data', 'wb')
            bytes_written, checksum = self._save_stream(
                incoming_stream, dest_file, chunk_size=chunk_size)
            dest_file.close()

        return join(uri, path, 'data'), bytes_written, checksum
Example #24
0
    def transfer_data_files(self):
        """Transfer files in a list from one ``fs`` object to another.

        All tranferer files will maintain the same filename.
        """
        # FS object created at the folder of where the record specific bag is
        fs_dest = opener.opendir(self.bagit_folder_path, create_dir=True)

        try:

            for docfile in self.docfiles:
                fs_src = opener.opendir(os.path.dirname(docfile.get_path()))
                copyfile(fs_src, os.path.basename(docfile.get_path()),
                         fs_dest, os.path.basename(docfile.get_path()))

        except Exception:
            raise ArchiverError("Error while copying %s to %s"
                                % (docfile.get_path(),
                                    os.path.join(self.bagit_folder_path,
                                                 docfile.name +
                                                 docfile.format)))
        return True
Example #25
0
 def test_slice(self):
     pyfs = opener.opendir(settings.config['fs_path'])
     scroll_id = sha1_hexdigest
     s = scroll.Scroll(pyfs, scroll_id)
     
     items = []
     for i in range(10):
         item_hash = HASH(i)
         s.add(item_hash)
         items.append(item_hash)
     
     four, five = s.slice(4, 2)
     self.assertEqual(four, HASH(4))
     self.assertEqual(five, HASH(5))
Example #26
0
    def test_slice(self):
        pyfs = opener.opendir(settings.config['fs_path'])
        scroll_id = sha1_hexdigest
        s = scroll.Scroll(pyfs, scroll_id)

        items = []
        for i in range(10):
            item_hash = bytes_hash_int(i)
            s.add(item_hash)
            items.append(item_hash)

        four, five = s.slice(4, 2)
        self.assertEqual(four, bytes_hash_int(4))
        self.assertEqual(five, bytes_hash_int(5))
def test_put_error(client, bucket):
    """Test upload - cancelled by user."""
    object_url = url_for('invenio_files_rest.object_api',
                         bucket_id=bucket.id,
                         key='test.txt')

    pytest.raises(ValueError,
                  client.put,
                  object_url,
                  input_stream=BadBytesIO(b'a' * 128))
    assert FileInstance.query.count() == 0
    assert ObjectVersion.query.count() == 0
    # Ensure that the file was removed.
    assert len(list(opener.opendir(bucket.location.uri).walk('.'))) == 3
Example #28
0
 def test_load(self):
     scroll_id = 'test_load'
     pyfs = opener.opendir(settings.config['fs_path'])
     data1 = HASH( '2' )
     data2 = HASH( '3' )
     with open(scroll_id + '.scroll', 'w+') as f:
         f.write( data1 + data2 )
     s = scroll.Scroll(pyfs, scroll_id)
     self.assertTrue(s.has(data1))
     self.assertTrue(s.has(data2))
     self.assertFalse(s.has('1'))
     self.assertFalse(s.has(''))
     expected_state = HASH( HASH( HASH( scroll_id ) + data1 ) + data2 )
     self.assertEqual(s.state, expected_state)
Example #29
0
    def save(self, incoming_stream, chunk_size=None, progress_callback=None):
        """Save file in the file system."""
        fs = opener.opendir(self.file.uri or self.make_path(), create_dir=True)
        fp = fs.open(self.filename, 'wb')
        try:
            bytes_written, checksum = self._write_stream(
                    incoming_stream, fp, chunk_size=chunk_size,
                    progress_callback=progress_callback)
        finally:
            fp.close()

        uri = fs.getpathurl(self.filename, allow_none=True) or \
            fs.getsyspath(self.filename, allow_none=True)

        return uri, bytes_written, checksum
def test_put_error(client, bucket):
    """Test upload - cancelled by user."""
    object_url = url_for(
        'invenio_files_rest.object_api', bucket_id=bucket.id, key='test.txt')

    pytest.raises(
        ValueError,
        client.put,
        object_url,
        input_stream=BadBytesIO(b'a' * 128)
    )
    assert FileInstance.query.count() == 0
    assert ObjectVersion.query.count() == 0
    # Ensure that the file was removed.
    assert len(list(opener.opendir(bucket.location.uri).walk('.'))) == 3
Example #31
0
    def test_save_big_scroll(self):
        pyfs = opener.opendir(settings.config['fs_path'])
        scroll_id = sha1_hexdigest
        s = scroll.Scroll(pyfs, scroll_id)

        items = []
        for i in range(100):
            item_hash = bytes_hash_int(i)
            s.add(item_hash)
            items.append(item_hash)

        for item_hash in items:
            self.assertTrue(s.has(item_hash))

        contents = pyfs.getcontents(scroll_id + '.scroll')
        self.assertEqual(contents, ''.join(items))
Example #32
0
    def test_add(self):
        my_data = "B"*20
        pyfs = opener.opendir(settings.config['fs_path'])
        scroll_id = sha1_hexdigest

        chissel_set_id = 'spam'

        fingerprint = notary.Notary.generate(pyfs)
        
        chisel_set = notary.ChiselSet(pyfs, chissel_set_id, fingerprint)
        hash_bytes = chisel_set.add(my_data)

        self.assertEqual(chisel_set.pool.get(hash_bytes), my_data)
        self.assertTrue(chisel_set.scroll.has(hash_bytes))

        self.assertTrue(chisel_set.has(hash_bytes))
Example #33
0
    def test_add_many(self):
        pyfs = opener.opendir(settings.config['fs_path'])
        scroll_id = sha1_hexdigest
        s = scroll.Scroll(pyfs, scroll_id)

        items = []
        for i in range(100):
            item_hash = HASH(i)
            s.add(item_hash)
            items.append(item_hash)

        for item_hash in items:
            self.assertTrue(s.has(item_hash))

        contents = pyfs.getcontents(scroll_id + '.scroll')
        self.assertEqual(contents, ''.join(items))
Example #34
0
    def test_add(self):
        my_data = "B" * 20
        pyfs = opener.opendir(settings.config['fs_path'])
        scroll_id = sha1_hexdigest

        chissel_set_id = 'spam'

        fingerprint = notary.Notary.generate(pyfs)

        chisel_set = notary.ChiselSet(pyfs, chissel_set_id, fingerprint)
        hash_bytes = chisel_set.add(my_data)

        self.assertEqual(chisel_set.pool.get(hash_bytes), my_data)
        self.assertTrue(chisel_set.scroll.has(hash_bytes))

        self.assertTrue(chisel_set.has(hash_bytes))
Example #35
0
 def test_load_artificial(self):
     """
     This test writes its own scroll to test scroll loading
     """
     scroll_id = 'test_load'
     data1 = HASH( '2' )
     data2 = HASH( '3' )
     expected_state = HASH( HASH( HASH( scroll_id ) + data1 ) + data2 )
     pyfs = opener.opendir(settings.config['fs_path'])
     with open(scroll_id + '.scroll', 'w+') as f:
         f.write( data1 + data2 )
     with open(scroll_id + '.state', 'w') as f:
         f.write( expected_state )
     s = scroll.Scroll(pyfs, scroll_id)
     self.assertTrue(s.has(data1))
     self.assertTrue(s.has(data2))
     self.assertFalse(s.has('1'))
     self.assertFalse(s.has(''))
     self.assertEqual(s.state, expected_state)
Example #36
0
def path_to_fs(path):
    """Convert path to an FS object."""
    return opener.opendir(path)
Example #37
0
def archive_fs(locations):
    """File system for the archive location."""
    archive_path = locations['archive']
    fs = opener.opendir(archive_path, writeable=False, create_dir=True)
    return fs
Example #38
0
def archive_fs(locations):
    """File system for the archive location."""
    archive_path = locations['archive']
    fs = opener.opendir(archive_path, writeable=False, create_dir=True)
    return fs
Example #39
0
def loadNotary():
    pyfs = opener.opendir(settings.config['fs_path'])
    chissel_set_id = 'spam'
    notary_fingerprint = notary.Notary.generate(pyfs)
    return notary.Notary(SubscribeHandler, pyfs, notary_fingerprint)
Example #40
0
 def test_generate(self):
     pyfs = opener.opendir(settings.config['fs_path'])
     fingerprint = notary.Notary.generate(pyfs)
     publisher = None
     n = notary.Notary(publisher, pyfs, fingerprint)
Example #41
0
def get_sample_project_mapping(basepath,
                               samplesheet=None,
                               suffix='.fastq.gz',
                               absolute_paths=False,
                               catch_undetermined=True):
    """
    Given a path containing fastq.gz files, possibily nested in Project/Sample
    directories, return a data structure mapping fastq-samples to projects.

    TODO: The SampleSheet.csv may be used as a hint but is not required.

    :param basepath: Path to directory tree of fastq.gz files - eg, bcl2fastq
                     output directory
    :type basepath: str
    :return: Dictionary lists, {project_id : [relative fastq.gz paths]}
    :rtype: OrderedDict
    """

    from fs.opener import opener

    fq_files = []
    with opener.opendir(basepath) as vfs:
        for fn in vfs.walkfiles():
            if suffix in fn:
                fq_files.append(fn.lstrip('/').lstrip('\\'))

    fq_files = sorted(fq_files)

    project_mapping = OrderedDict()
    for fqpath in fq_files:
        project = ''
        fqfile = fqpath
        parts = Path(fqpath).parts
        if len(parts) == 3:
            project, sample_id, fqfile = map(str, parts)
        if len(parts) == 2:
            project, fqfile = map(str, parts)
        if len(parts) == 1:
            fqfile = str(parts[0])

        if catch_undetermined and 'Undetermined' in fqfile:
            project = u'Undetermined_indices'

        # TODO: we currently don't deal with Project_ prefixes, really
        #       the project ID doesn't include Project_. If we strip
        #       this here, maybe we need to include the project directory
        #       in the fastq paths so we can know the path and project id
        #       - will require fixes to downstream code that
        #       does join(bcl2fastq_output_dir, project_id, fastq_file)

        # TODO: also incorporate sample_id in this datastructure
        if project not in project_mapping:
            project_mapping[project] = []
        if absolute_paths:
            fqpath = join(basepath, fqpath)
        project_mapping[project].append(fqpath)

    # TODO: Use the SampleSheet.csv to validate or hint
    # TODO: Also assign sample_id, sample_name, lane, read, etc
    #       we could use parse_sample_info_from_filename for this,
    #       and/or use the FASTQ header(s)

    return project_mapping
Example #42
0
 def _make_bag(self, **kwargs):
     """Turn a folder into bagit form and compress it."""
     opener.opendir(self.folder, create_dir=True)
     info = {'Bagging-Date': datetime.now().strftime("%Y-%m-%d")}
     info.update(kwargs)
     return make_bag(self.folder, info)
Example #43
0
def loadNotary():
    pyfs = opener.opendir(settings.config['fs_path'])
    chissel_set_id = 'spam'
    notary_fingerprint = notary.Notary.generate(pyfs)
    return notary.Notary(SubscribeHandler, pyfs, notary_fingerprint)
Example #44
0
 def test_generate(self):
     pyfs = opener.opendir(settings.config['fs_path'])
     fingerprint = notary.Notary.generate(pyfs)
     publisher = None
     n = notary.Notary(publisher, pyfs, fingerprint)
Example #45
0
def test_opendir(tmppath):
    """Test opendir."""
    rooturl = mkurl(tmppath)
    fs = opener.opendir(rooturl + "/data")
    assert fs.listdir()
Example #46
0
        count, new = 0,0
        for dirPath, filename in remote.walkFiles():
            count += 1
            if not self.has( filename, 0 ):
                new += 1
                self.put( remote._fs.getcontents( "%s/%s" % (dirPath, filename) ) )
        print "Saw %s files; copied %s that are new" % (count, new)

if __name__ == "__main__":
    
    if len(sys.argv) < 2:
        print "usage: %s [put|ls|cat|test]" % os.path.basename( sys.argv[0] )
    else:
        cmd = sys.argv[1]
        if cmd == 'test':
            print doctest.testmod( )
        else:
            from fs.opener import opener
            chisel = Chisel( opener.opendir( sys.argv[2] ) )
            if cmd == 'put':
                chisel.put( sys.stdin.read(), *map(int, sys.argv[3:]) )
            elif cmd == 'ls':
                print "\n".join( "%s/%s" % (a,b) for a,b in chisel.walkFiles( *map(int, sys.argv[3:]) ) ),
            elif cmd == 'cat':
                print "".join( chisel.get( *map(int, sys.argv[3:]) ) ),
            elif cmd == 'sync':
                chisel.sync( Chisel( opener.opendir( sys.argv[3] ) ), *map(int, sys.argv[4:]) )
            else:
                print "unknown command %s" % (cmd,)