示例#1
0
def deploy_register():
    """Put the staging version to production hosted at 
    register.geostandaarden.nl
    """

    ## TODO: feed this function absolute paths

    print "Deploying production..."
    logging.info("Deploying production...")

    production = OSFS(production_path)

    # NOTE: only build paths within script_dir are currently supported
    call ('cp -r %s %s' % (ospath.join(build_path, register_path), ospath.join(production_path, register_path + '-new')), shell=True)

    if production.exists(register_path):
        backup_dir = time.strftime('%Y-%m-%d-%H-%M-%S')

        production.copydir(register_path, '%s/%s' % (backups_path, backup_dir), overwrite=True)
        
        try:
            production.movedir(register_path, register_path + '-old', overwrite=True)
        except ResourceNotFoundError:
            pass

    production.movedir(register_path + '-new', register_path, overwrite=True)

    try:
        production.removedir(register_path + '-old', force=True)
    except ResourceNotFoundError:
        pass

    call('chmod -R a+rx %s/%s' % (production_path, register_path), shell=True)

    logging.info("Production built successfully!")
示例#2
0
    def test_should_download_zipped_csv(self):
        os = OSFS("./tests/test_integration/resources/")
        file_name = "test_csv_zipped"
        test_zip_file = 'http://localhost:8001/local_data/base_train.zip'

        test_ds_zip = DataSet(os, file_name, "test_id", test_zip_file,
                              "test dataset", "zip")
        test_ds_zip.download()
        test_ds_zip.unzip_file()
        df = pd.read_csv(test_ds_zip.uri)
        self.assertEqual((2, 2), df.shape)
        os.remove(file_name + "/train.csv")
        os.removedir(file_name)

        ## only download
        os = OSFS("./tests/test_integration/resources/")
        file_name = "train.csv"
        test_file = 'http://localhost:8001/local_data/train.csv'

        test_ds = DataSet(os, file_name, "test_id", test_file, "test dataset")
        test_ds.download()
        test_ds.unzip_file()
        df = pd.read_csv(test_ds.uri)
        self.assertEqual((2, 2), df.shape)
        os.remove(file_name)
示例#3
0
def create_production(build_dir, backups, script_dir):
    """Put the staging version to production hosted at 
    register.geostandaarden.nl
    """

    print "Building production..."
    logging.info("Building production...")

    deploy = OSFS('..')
    
    if deploy.exists(backups) == False:
        deploy.makedir(backups)

    deploy.copydir('%s/%s' % (script_dir, build_dir), 'register-new', overwrite=True)

    if deploy.exists('register') == True:
        # server refuses to recursively remove register/staging
        # hence we excplicitly remove symbolic link to staging
        try:
            deploy.remove('register/staging/staging')
        except ResourceNotFoundError:
            print "Warning, register/staging/staging not found..."

        try:
            deploy.removedir('register/staging')
        except ResourceNotFoundError:
            print "Warning, register/staging not found..."
        
        backup_dir = time.strftime('%Y-%m-%d-%H-%M-%S')

        # if deploy.exists('backups/%s' % backup_dir): 
        #     deploy.removedir('backups/%s' % backup_dir, force=True)
        
        deploy.copydir('register', 'backups/%s' % backup_dir, overwrite=True)
        
        try:
            deploy.movedir('register', 'register-old', overwrite=True)
        except ResourceNotFoundError:
            pass

    deploy.movedir('register-new', 'register', overwrite=True)

    # create symbolic link to standalone staging directory
    # fails if production is built first...
    deploy.makedir('register/staging')
    call('cd ../register/staging; ln -s ../../staging', shell=True)
    call('cd ../register; ln -s ../%s/log.txt' % script_dir  , shell=True)
    
    try:
        deploy.removedir('register-old', force=True)
    except ResourceNotFoundError:
        pass

    call('chmod -R a+rx ../register', shell=True)

    print "Done building production..."
    logging.info("Production built successfully!")
示例#4
0
文件: bagit.py 项目: mhellmic/b2share
 def remove_dir(self, path):
     """Remove a folder at a given path."""
     try:
         file_to_delete = os.path.basename(path)
         to_delete_from = OSFS(os.path.dirname(path))
         to_delete_from.removedir(file_to_delete,
                                  recursive=True,
                                  force=True)
     except ResourceNotFoundError:
         raise ArchiverError("Folder %s not found" % path)
示例#5
0
 def test_unzip_local_data(self):
     os = OSFS(".")
     os_remove = os.remove
     os.remove = mock.Mock(return_value=None)
     os.copy("./tests/resources/local_data/base_train.zip",
             "./tests/resources/local_data/train.zip")
     test_local = DataSet(os, "/local/path", "train",
                          "./tests/resources/local_data/train.zip",
                          "test dataset", "zip")
     test_local.unzip_file()
     result = os.exists("./tests/resources/local_data/train/train.csv")
     os.remove = os_remove
     os.remove("./tests/resources/local_data/train/train.csv")
     os.remove("./tests/resources/local_data/train.zip")
     os.removedir("./tests/resources/local_data/train")
     self.assertTrue(result)
示例#6
0
def destroy_project(username, project_name):

    user_path = config.project_path + "/" + username
    home_dir = OSFS(user_path)
    if home_dir.exists(project_name):
        home_dir.removedir(project_name, force=True)

    project = Project.get_project_by_name(username, project_name)
    if project is None:
        return 'project not found'
    project.stop()
    project.remove()

    database_update.delete_project(username, project_name)
    database_update.delete_service_for_scale(username, project_name)

    return 'success'
示例#7
0
    def _zip(self, destination=None):
        """Compresse a bagit file."""
        # Removes the final forwardslash if there is one
        destination = destination or cfg["ARCHIVER_TMPDIR"]
        if destination.endswith(os.path.sep):
            destination = destination[:-len(os.path.sep)]
        filename = os.path.join(destination, "{0}.zip".format(self.name))

        # Create and FS object
        with OSFS(self.folder) as to_zip_fs:
            with ZipFS(filename, mode='w') as zip_fs:
                copydir(to_zip_fs, zip_fs, overwrite=True)
            file_to_delete = os.path.basename(self.folder)
            to_delete_from = OSFS(os.path.dirname(self.folder))
            to_delete_from.removedir(file_to_delete, recursive=True,
                                     force=True)
        return filename
示例#8
0
def cleanup(source, destination_temp, standard):
	"""Remove the source and temporary destination folders."""

	try:
		source_fs = OSFS('%s/%s' % (source, standard))
	except ResourceNotFoundError:
		return None

	destination_fs = OSFS(destination_temp)

	artifacts = source_fs.listdir(dirs_only=True)
	if '.git' in artifacts: artifacts.remove('.git')

	for artifact in artifacts:
		path = '%s/%s' % (artifact, standard)
		if destination_fs.exists(path):
			destination_fs.removedir(path, force=True)

	if destination_fs.exists(standard): destination_fs.removedir(standard, force=True)
示例#9
0
def cleanup(build_path, source, destination_temp, standard):
    """Remove the source and temporary destination folders."""

    try:
        source_fs = OSFS(ospath.join(build_path, source, standard))
    except ResourceNotFoundError:
        return None

    destination_fs = OSFS(ospath.join(build_path, destination_temp))

    artifacts = source_fs.listdir(dirs_only=True)
    if '.git' in artifacts: artifacts.remove('.git')

    for artifact in artifacts:
        path = ospath.join(artifact, standard)
        if destination_fs.exists(path):
            destination_fs.removedir(path, force=True)

    if destination_fs.exists(standard):
        destination_fs.removedir(standard, force=True)
示例#10
0
def create_staging(staging_path, production_path, build_path):
    """Create a staging version of the register hosted at
    register.geostandaarden.nl/staging
    """

    logging.info("Building staging...")

    production = OSFS(production_path)

    print "Removing current staging..."
    if production.exists(staging_path):
        production.removedir(staging_path, force=True)

    print 'Moving new register to staging...'

    # OSFS cannot copy to arbitrary locations
    call('cp -r %s %s' % (ospath.join(build_path, staging_path), production_path), shell=True)
    
    call('chmod -R a+rx %s' % (ospath.join(production_path, staging_path)), shell=True)

    logging.info("Staging built successfully!")
示例#11
0
def create_production(destination, backups, script_entry_path, production_path):
    """Put the staging version to production hosted at 
    register.geostandaarden.nl
    """

    ## TODO: feed this function absolute paths

    print "Building production..."
    logging.info("Building production...")

    production = OSFS(production_path)
    
    # if production.exists(backups) == False:
    #     production.makedir(backups)

    # copy newly baked register/staging to production directory
    # NOTE: only build paths within script_dir are currently supported
    call ('cp -r %s %s' % (ospath.join(build_path, destination), ospath.join(production_path, destination + '-new')), shell=True)
    # production.copydir('%s/%s/%s' % (script_dir, build_path, destination), destination + '-new', overwrite=True)

    if production.exists(destination) == True:
        # server refuses to recursively remove register/staging
        # hence we excplicitly remove symbolic link to staging
        try:
            production.remove('%s/staging/staging' % destination)
        except ResourceNotFoundError:
            print "Warning, %s/staging/staging not found..." % destination

        try:
            production.removedir('%s/staging' % destination)
        except ResourceNotFoundError:
            print "Warning, %s/staging not found..." % destination
        
        backup_dir = time.strftime('%Y-%m-%d-%H-%M-%S')

        # if production.exists('backups/%s' % backup_dir): 
        #     production.removedir('backups/%s' % backup_dir, force=True)
        
        production.copydir(destination, '%s/%s' % (backups, backup_dir), overwrite=True)
        
        try:
            production.movedir(destination, destination + '-old', overwrite=True)
        except ResourceNotFoundError:
            pass

    production.movedir(destination + '-new', destination, overwrite=True)

    # create symbolic link to standalone staging directory
    # fails if production is built first...
    production.makedir('%s/staging' % destination)
    
    call('cd %s; ln -s %s' % (ospath.join(production_path, destination, 'staging'), ospath.join(production_path, 'staging')), shell=True)
    call('cd %s; ln -s %s' % (ospath.join(production_path, destination), ospath.join(script_entry_path, 'log.txt')), shell=True)
    
    try:
        production.removedir(destination + '-old', force=True)
    except ResourceNotFoundError:
        pass

    call('chmod -R a+rx %s/%s' % (production_path, destination), shell=True)

    print "Done building production..."
    logging.info("Production built successfully!")
示例#12
0
from fs.memoryfs import MemoryFS
from fs.expose import fuse
fs = MemoryFS()  # create an in memory file system
fs.createfile('filename.txt')  # creating an empty file
fs.setcontents('filename.txt',
               'contents of file')  # putting content into the file.
from fs.osfs import OSFS
home_fs = OSFS('/')  #
home_fs.makedir(
    '/home/dave/scratch/ramdrive', allow_recreate=True
)  # have to make a directory for us to mount our memory file system on.
mp = fuse.mount(
    fs, '/home/dave/scratch/ramdrive'
)  # exposes fs to everything else on machine. (ie: other system calls can see these files)
mp.path  # in case you need the path to the files created.
mp.unmount()  # files are no longer being exposed via fuse
home_fs.removedir('/home/dave/scratch/ramdrive/'
                  )  #remove the real file system directory when done.

fs.remove('filename.txt')

home_fs.close()
fs.close()

# creating a ramdrive like this wont work for my desired task, as other external applications cannot write to the directory. They only have read access.