def push(self, spec, retry=2, clear_on_fail=False): repo_type = self.__repo_type try: objects_path = get_objects_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return met = Metadata(spec, metadata_path, self.__config, repo_type) fields = met.git_user_config() if None in fields.values(): log.error( 'Your name and email address need to be configured in git. ' 'Please see the commands below:', class_name=REPOSITORY_CLASS_NAME) log.error('git config --global user.name \'Your Name\'', class_name=REPOSITORY_CLASS_NAME) log.error('git config --global user.email [email protected]', class_name=REPOSITORY_CLASS_NAME) return if met.fetch() is False: return ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) spec_path, spec_file = None, None try: spec_path, spec_file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) if spec_path is None: return full_spec_path = os.path.join(spec_path, spec_file) repo = LocalRepository(self.__config, objects_path, repo_type) ret = repo.push(objects_path, full_spec_path, retry, clear_on_fail) # ensure first we're on master ! met.checkout() if ret == 0: # push metadata spec to LocalRepository git repository try: met.push() except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return MultihashFS(objects_path).reset_log()
def test_push(self): mlgit_dir = os.path.join(self.tmp_dir, '.ml-git') indexpath = os.path.join(mlgit_dir, 'index-test') mdpath = os.path.join(mlgit_dir, 'metadata-test') objectpath = os.path.join(mlgit_dir, 'objects-test') specpath = os.path.join(mdpath, 'vision-computing/images/dataset-ex') ensure_path_exists(specpath) ensure_path_exists(indexpath) shutil.copy('hdata/dataset-ex.spec', specpath + '/dataset-ex.spec') shutil.copy('hdata/config.yaml', mlgit_dir + '/config.yaml') manifestpath = os.path.join(specpath, 'MANIFEST.yaml') yaml_save( { 'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh': {'imghires.jpg'} }, manifestpath) # adds chunks to ml-git Index idx = MultihashIndex(specpath, indexpath, objectpath) idx.add('data-test-push/', manifestpath) fi = yaml_load(os.path.join(specpath, 'INDEX.yaml')) self.assertTrue(len(fi) > 0) self.assertTrue(os.path.exists(indexpath)) o = Objects(specpath, objectpath) o.commit_index(indexpath, self.tmp_dir) self.assertTrue(os.path.exists(objectpath)) c = yaml_load('hdata/config.yaml') r = LocalRepository(c, objectpath) r.push(objectpath, specpath + '/dataset-ex.spec') s3 = boto3.resource( 's3', region_name='eu-west-1', aws_access_key_id='fake_access_key', aws_secret_access_key='fake_secret_key', ) for key in idx.get_index(): self.assertIsNotNone(s3.Object(testbucketname, key))
def test_push(self): indexpath = os.path.join(self.tmp_dir, 'index-test') mdpath = os.path.join(self.tmp_dir, 'metadata-test') objectpath = os.path.join(self.tmp_dir, 'objects-test') specpath = os.path.join(mdpath, 'vision-computing/images/dataset-ex') ensure_path_exists(indexpath) ensure_path_exists(specpath) shutil.copy('hdata/dataset-ex.spec', specpath + '/dataset-ex.spec') manifestpath = os.path.join(specpath, 'MANIFEST.yaml') yaml_save(files_mock, manifestpath) idx = MultihashIndex(specpath, indexpath, objectpath) idx.add('data-test-push-1/', manifestpath) fidx = FullIndex(specpath, indexpath) self.assertTrue(os.path.exists(indexpath)) c = yaml_load('hdata/config.yaml') o = Objects(specpath, objectpath) o.commit_index(indexpath, self.tmp_dir) self.assertTrue(os.path.exists(objectpath)) r = LocalRepository(c, objectpath) self.assertTrue(r.push(objectpath, specpath + '/dataset-ex.spec') == 0) self.assertTrue(len(fidx.get_index()) == 1)