def remove_recursive(store,
dataset,
path,
name=None,
email=None,
cookies=None):
"""Remove a path within a dataset recursively."""
ds = store.get_dataset(dataset)
with CommitInfo(ds, name, email):
ds.remove(path, recursive=True, check=False)
update_head(store, dataset, cookies=cookies)
def move_files_into_repo(dataset_id, dataset_path, upload_path, name, email,
cookies):
repo = pygit2.Repository(dataset_path)
unlock_files = [
os.path.relpath(filename, start=upload_path)
for filename in pathlib.Path(upload_path).glob('**/*')
if os.path.islink(
os.path.join(dataset_path,
os.path.relpath(filename, start=upload_path)))
]
gevent.sleep()
move_files(upload_path, dataset_path)
author = pygit2.Signature(name, email)
hexsha = git_commit(repo, unlock_files, author).hex
update_head(dataset_id, dataset_path, hexsha, cookies)
def _finish_upload(self, dataset_id, upload, name, email, cookies):
try:
ds = self.store.get_dataset(dataset_id)
with CommitInfo(ds, name, email):
upload_path = self.store.get_upload_path(dataset_id, upload)
unlock_files = [
os.path.relpath(filename, start=upload_path)
for filename in pathlib.Path(upload_path).glob('**/*')
if os.path.islink(
os.path.join(
ds.path,
os.path.relpath(filename, start=upload_path)))
]
gevent.sleep()
move_files(upload_path, ds.path)
ds.save(unlock_files)
update_head(ds, dataset_id, cookies)
gevent.sleep()
shutil.rmtree(upload_path)
except:
self.logger.exception('Dataset upload could not be finalized')
sentry_sdk.capture_exception()
def _finish_upload(self, dataset_id, upload, name, email, cookies):
try:
ds = self.store.get_dataset(dataset_id)
with CommitInfo(ds, name, email):
upload_path = self.store.get_upload_path(dataset_id, upload)
unlock_files = [
os.path.relpath(filename, start=upload_path)
for filename in glob.iglob(upload_path + '**/**',
recursive=True)
if os.path.islink(
os.path.join(
ds.path,
os.path.relpath(filename, start=upload_path)))
]
ds.unlock(unlock_files)
shutil.copytree(upload_path, ds.path, dirs_exist_ok=True)
shutil.rmtree(upload_path)
ds.save(unlock_files)
update_head(ds, dataset_id, cookies)
except:
self.logger.exception('Dataset upload could not be finalized')
sentry_sdk.capture_exception()
def remove_files(store, dataset, files, name=None, email=None, cookies=None):
ds = store.get_dataset(dataset)
with CommitInfo(ds, name, email):
for filename in files:
ds.remove(filename, check=False)
update_head(store, dataset, cookies)