def push(self, spec, retry=2, clear_on_fail=False):
repo_type = self.__repo_type
try:
objects_path = get_objects_path(self.__config, repo_type)
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
met = Metadata(spec, metadata_path, self.__config, repo_type)
fields = met.git_user_config()
if None in fields.values():
log.error(
'Your name and email address need to be configured in git. '
'Please see the commands below:',
class_name=REPOSITORY_CLASS_NAME)
log.error('git config --global user.name \'Your Name\'',
class_name=REPOSITORY_CLASS_NAME)
log.error('git config --global user.email [email protected]',
class_name=REPOSITORY_CLASS_NAME)
return
if met.fetch() is False:
return
ref = Refs(refs_path, spec, repo_type)
tag, sha = ref.branch()
categories_path = get_path_with_categories(tag)
spec_path, spec_file = None, None
try:
spec_path, spec_file = search_spec_file(self.__repo_type, spec,
categories_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
if spec_path is None:
return
full_spec_path = os.path.join(spec_path, spec_file)
repo = LocalRepository(self.__config, objects_path, repo_type)
ret = repo.push(objects_path, full_spec_path, retry, clear_on_fail)
# ensure first we're on master !
met.checkout()
if ret == 0:
# push metadata spec to LocalRepository git repository
try:
met.push()
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
MultihashFS(objects_path).reset_log()
def test_push(self):
mlgit_dir = os.path.join(self.tmp_dir, '.ml-git')
indexpath = os.path.join(mlgit_dir, 'index-test')
mdpath = os.path.join(mlgit_dir, 'metadata-test')
objectpath = os.path.join(mlgit_dir, 'objects-test')
specpath = os.path.join(mdpath, 'vision-computing/images/dataset-ex')
ensure_path_exists(specpath)
ensure_path_exists(indexpath)
shutil.copy('hdata/dataset-ex.spec', specpath + '/dataset-ex.spec')
shutil.copy('hdata/config.yaml', mlgit_dir + '/config.yaml')
manifestpath = os.path.join(specpath, 'MANIFEST.yaml')
yaml_save(
{
'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh':
{'imghires.jpg'}
}, manifestpath)
# adds chunks to ml-git Index
idx = MultihashIndex(specpath, indexpath, objectpath)
idx.add('data-test-push/', manifestpath)
fi = yaml_load(os.path.join(specpath, 'INDEX.yaml'))
self.assertTrue(len(fi) > 0)
self.assertTrue(os.path.exists(indexpath))
o = Objects(specpath, objectpath)
o.commit_index(indexpath, self.tmp_dir)
self.assertTrue(os.path.exists(objectpath))
c = yaml_load('hdata/config.yaml')
r = LocalRepository(c, objectpath)
r.push(objectpath, specpath + '/dataset-ex.spec')
s3 = boto3.resource(
's3',
region_name='eu-west-1',
aws_access_key_id='fake_access_key',
aws_secret_access_key='fake_secret_key',
)
for key in idx.get_index():
self.assertIsNotNone(s3.Object(testbucketname, key))
def test_push(self):
indexpath = os.path.join(self.tmp_dir, 'index-test')
mdpath = os.path.join(self.tmp_dir, 'metadata-test')
objectpath = os.path.join(self.tmp_dir, 'objects-test')
specpath = os.path.join(mdpath, 'vision-computing/images/dataset-ex')
ensure_path_exists(indexpath)
ensure_path_exists(specpath)
shutil.copy('hdata/dataset-ex.spec', specpath + '/dataset-ex.spec')
manifestpath = os.path.join(specpath, 'MANIFEST.yaml')
yaml_save(files_mock, manifestpath)
idx = MultihashIndex(specpath, indexpath, objectpath)
idx.add('data-test-push-1/', manifestpath)
fidx = FullIndex(specpath, indexpath)
self.assertTrue(os.path.exists(indexpath))
c = yaml_load('hdata/config.yaml')
o = Objects(specpath, objectpath)
o.commit_index(indexpath, self.tmp_dir)
self.assertTrue(os.path.exists(objectpath))
r = LocalRepository(c, objectpath)
self.assertTrue(r.push(objectpath, specpath + '/dataset-ex.spec') == 0)
self.assertTrue(len(fidx.get_index()) == 1)