def deploy_register():
"""Put the staging version to production hosted at
register.geostandaarden.nl
"""
## TODO: feed this function absolute paths
print "Deploying production..."
logging.info("Deploying production...")
production = OSFS(production_path)
# NOTE: only build paths within script_dir are currently supported
call ('cp -r %s %s' % (ospath.join(build_path, register_path), ospath.join(production_path, register_path + '-new')), shell=True)
if production.exists(register_path):
backup_dir = time.strftime('%Y-%m-%d-%H-%M-%S')
production.copydir(register_path, '%s/%s' % (backups_path, backup_dir), overwrite=True)
try:
production.movedir(register_path, register_path + '-old', overwrite=True)
except ResourceNotFoundError:
pass
production.movedir(register_path + '-new', register_path, overwrite=True)
try:
production.removedir(register_path + '-old', force=True)
except ResourceNotFoundError:
pass
call('chmod -R a+rx %s/%s' % (production_path, register_path), shell=True)
logging.info("Production built successfully!")
def test_should_download_zipped_csv(self):
os = OSFS("./tests/test_integration/resources/")
file_name = "test_csv_zipped"
test_zip_file = 'http://localhost:8001/local_data/base_train.zip'
test_ds_zip = DataSet(os, file_name, "test_id", test_zip_file,
"test dataset", "zip")
test_ds_zip.download()
test_ds_zip.unzip_file()
df = pd.read_csv(test_ds_zip.uri)
self.assertEqual((2, 2), df.shape)
os.remove(file_name + "/train.csv")
os.removedir(file_name)
## only download
os = OSFS("./tests/test_integration/resources/")
file_name = "train.csv"
test_file = 'http://localhost:8001/local_data/train.csv'
test_ds = DataSet(os, file_name, "test_id", test_file, "test dataset")
test_ds.download()
test_ds.unzip_file()
df = pd.read_csv(test_ds.uri)
self.assertEqual((2, 2), df.shape)
os.remove(file_name)
def create_production(build_dir, backups, script_dir):
"""Put the staging version to production hosted at
register.geostandaarden.nl
"""
print "Building production..."
logging.info("Building production...")
deploy = OSFS('..')
if deploy.exists(backups) == False:
deploy.makedir(backups)
deploy.copydir('%s/%s' % (script_dir, build_dir), 'register-new', overwrite=True)
if deploy.exists('register') == True:
# server refuses to recursively remove register/staging
# hence we excplicitly remove symbolic link to staging
try:
deploy.remove('register/staging/staging')
except ResourceNotFoundError:
print "Warning, register/staging/staging not found..."
try:
deploy.removedir('register/staging')
except ResourceNotFoundError:
print "Warning, register/staging not found..."
backup_dir = time.strftime('%Y-%m-%d-%H-%M-%S')
# if deploy.exists('backups/%s' % backup_dir):
# deploy.removedir('backups/%s' % backup_dir, force=True)
deploy.copydir('register', 'backups/%s' % backup_dir, overwrite=True)
try:
deploy.movedir('register', 'register-old', overwrite=True)
except ResourceNotFoundError:
pass
deploy.movedir('register-new', 'register', overwrite=True)
# create symbolic link to standalone staging directory
# fails if production is built first...
deploy.makedir('register/staging')
call('cd ../register/staging; ln -s ../../staging', shell=True)
call('cd ../register; ln -s ../%s/log.txt' % script_dir , shell=True)
try:
deploy.removedir('register-old', force=True)
except ResourceNotFoundError:
pass
call('chmod -R a+rx ../register', shell=True)
print "Done building production..."
logging.info("Production built successfully!")
def remove_dir(self, path):
"""Remove a folder at a given path."""
try:
file_to_delete = os.path.basename(path)
to_delete_from = OSFS(os.path.dirname(path))
to_delete_from.removedir(file_to_delete,
recursive=True,
force=True)
except ResourceNotFoundError:
raise ArchiverError("Folder %s not found" % path)
def test_unzip_local_data(self):
os = OSFS(".")
os_remove = os.remove
os.remove = mock.Mock(return_value=None)
os.copy("./tests/resources/local_data/base_train.zip",
"./tests/resources/local_data/train.zip")
test_local = DataSet(os, "/local/path", "train",
"./tests/resources/local_data/train.zip",
"test dataset", "zip")
test_local.unzip_file()
result = os.exists("./tests/resources/local_data/train/train.csv")
os.remove = os_remove
os.remove("./tests/resources/local_data/train/train.csv")
os.remove("./tests/resources/local_data/train.zip")
os.removedir("./tests/resources/local_data/train")
self.assertTrue(result)
def destroy_project(username, project_name):
user_path = config.project_path + "/" + username
home_dir = OSFS(user_path)
if home_dir.exists(project_name):
home_dir.removedir(project_name, force=True)
project = Project.get_project_by_name(username, project_name)
if project is None:
return 'project not found'
project.stop()
project.remove()
database_update.delete_project(username, project_name)
database_update.delete_service_for_scale(username, project_name)
return 'success'
def _zip(self, destination=None):
"""Compresse a bagit file."""
# Removes the final forwardslash if there is one
destination = destination or cfg["ARCHIVER_TMPDIR"]
if destination.endswith(os.path.sep):
destination = destination[:-len(os.path.sep)]
filename = os.path.join(destination, "{0}.zip".format(self.name))
# Create and FS object
with OSFS(self.folder) as to_zip_fs:
with ZipFS(filename, mode='w') as zip_fs:
copydir(to_zip_fs, zip_fs, overwrite=True)
file_to_delete = os.path.basename(self.folder)
to_delete_from = OSFS(os.path.dirname(self.folder))
to_delete_from.removedir(file_to_delete, recursive=True,
force=True)
return filename
def cleanup(source, destination_temp, standard):
"""Remove the source and temporary destination folders."""
try:
source_fs = OSFS('%s/%s' % (source, standard))
except ResourceNotFoundError:
return None
destination_fs = OSFS(destination_temp)
artifacts = source_fs.listdir(dirs_only=True)
if '.git' in artifacts: artifacts.remove('.git')
for artifact in artifacts:
path = '%s/%s' % (artifact, standard)
if destination_fs.exists(path):
destination_fs.removedir(path, force=True)
if destination_fs.exists(standard): destination_fs.removedir(standard, force=True)
def cleanup(build_path, source, destination_temp, standard):
"""Remove the source and temporary destination folders."""
try:
source_fs = OSFS(ospath.join(build_path, source, standard))
except ResourceNotFoundError:
return None
destination_fs = OSFS(ospath.join(build_path, destination_temp))
artifacts = source_fs.listdir(dirs_only=True)
if '.git' in artifacts: artifacts.remove('.git')
for artifact in artifacts:
path = ospath.join(artifact, standard)
if destination_fs.exists(path):
destination_fs.removedir(path, force=True)
if destination_fs.exists(standard):
destination_fs.removedir(standard, force=True)
def create_staging(staging_path, production_path, build_path):
"""Create a staging version of the register hosted at
register.geostandaarden.nl/staging
"""
logging.info("Building staging...")
production = OSFS(production_path)
print "Removing current staging..."
if production.exists(staging_path):
production.removedir(staging_path, force=True)
print 'Moving new register to staging...'
# OSFS cannot copy to arbitrary locations
call('cp -r %s %s' % (ospath.join(build_path, staging_path), production_path), shell=True)
call('chmod -R a+rx %s' % (ospath.join(production_path, staging_path)), shell=True)
logging.info("Staging built successfully!")
def create_production(destination, backups, script_entry_path, production_path):
"""Put the staging version to production hosted at
register.geostandaarden.nl
"""
## TODO: feed this function absolute paths
print "Building production..."
logging.info("Building production...")
production = OSFS(production_path)
# if production.exists(backups) == False:
# production.makedir(backups)
# copy newly baked register/staging to production directory
# NOTE: only build paths within script_dir are currently supported
call ('cp -r %s %s' % (ospath.join(build_path, destination), ospath.join(production_path, destination + '-new')), shell=True)
# production.copydir('%s/%s/%s' % (script_dir, build_path, destination), destination + '-new', overwrite=True)
if production.exists(destination) == True:
# server refuses to recursively remove register/staging
# hence we excplicitly remove symbolic link to staging
try:
production.remove('%s/staging/staging' % destination)
except ResourceNotFoundError:
print "Warning, %s/staging/staging not found..." % destination
try:
production.removedir('%s/staging' % destination)
except ResourceNotFoundError:
print "Warning, %s/staging not found..." % destination
backup_dir = time.strftime('%Y-%m-%d-%H-%M-%S')
# if production.exists('backups/%s' % backup_dir):
# production.removedir('backups/%s' % backup_dir, force=True)
production.copydir(destination, '%s/%s' % (backups, backup_dir), overwrite=True)
try:
production.movedir(destination, destination + '-old', overwrite=True)
except ResourceNotFoundError:
pass
production.movedir(destination + '-new', destination, overwrite=True)
# create symbolic link to standalone staging directory
# fails if production is built first...
production.makedir('%s/staging' % destination)
call('cd %s; ln -s %s' % (ospath.join(production_path, destination, 'staging'), ospath.join(production_path, 'staging')), shell=True)
call('cd %s; ln -s %s' % (ospath.join(production_path, destination), ospath.join(script_entry_path, 'log.txt')), shell=True)
try:
production.removedir(destination + '-old', force=True)
except ResourceNotFoundError:
pass
call('chmod -R a+rx %s/%s' % (production_path, destination), shell=True)
print "Done building production..."
logging.info("Production built successfully!")
from fs.memoryfs import MemoryFS
from fs.expose import fuse
fs = MemoryFS() # create an in memory file system
fs.createfile('filename.txt') # creating an empty file
fs.setcontents('filename.txt',
'contents of file') # putting content into the file.
from fs.osfs import OSFS
home_fs = OSFS('/') #
home_fs.makedir(
'/home/dave/scratch/ramdrive', allow_recreate=True
) # have to make a directory for us to mount our memory file system on.
mp = fuse.mount(
fs, '/home/dave/scratch/ramdrive'
) # exposes fs to everything else on machine. (ie: other system calls can see these files)
mp.path # in case you need the path to the files created.
mp.unmount() # files are no longer being exposed via fuse
home_fs.removedir('/home/dave/scratch/ramdrive/'
) #remove the real file system directory when done.
fs.remove('filename.txt')
home_fs.close()
fs.close()
# creating a ramdrive like this wont work for my desired task, as other external applications cannot write to the directory. They only have read access.