def build(self, memory_limit=None):
# Checkout to the correct commit
git.checkout_commit(repo_path=self.repo_path, commit=self.image_tag)
limits = {
# Always disable memory swap for building, since mostly
# nothing good can come of that.
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
# Create DockerFile
with open(self.dockerfile_path, 'w') as dockerfile:
dockerfile.write(self.render())
self.connect()
check_pulse = 0
for log_line in self.docker.build(
path=self.build_path,
tag='{}:{}'.format(self.image_name, self.image_tag),
buildargs={},
decode=True,
forcerm=True,
rm=True,
pull=True,
nocache=False,
container_limits=limits,
stream=True,
):
check_pulse += 1
publisher.publish_log(
log_line=log_line,
status=ExperimentLifeCycle.BUILDING,
experiment_uuid=self.experiment_uuid,
experiment_name=self.experiment_name,
job_uuid='all',
persist=False # TODO: ADD log persistence
)
# Check if experiment is not stopped in the meanwhile
if check_pulse > self.CHECK_INTERVAL:
if not experiment_still_running(self.experiment_uuid):
logger.info(
'Experiment `{}` is not running, stopping build'.
format(self.experiment_uuid))
return False
else:
check_pulse = 0
# Checkout back to master
git.checkout_commit(repo_path=self.repo_path)
return True
def handle_new_files(user_id, repo_id, tar_file_name):
if not tarfile.is_tarfile(tar_file_name):
raise ValueError('Received wrong file format.')
User = get_user_model()
try:
user = User.objects.get(id=user_id)
except User.DoesNotExist:
logger.warning(
'User with id `{}` does not exist anymore.'.format(user_id))
return
try:
repo = Repo.objects.get(id=repo_id)
# Checkout to master
git.checkout_commit(repo.path)
except User.DoesNotExist:
logger.warning(
'Repo with id `{}` does not exist anymore.'.format(repo_id))
return
# Destination files
new_repo_path = repo.get_tmp_tar_path()
# clean the current path from all files
path_files = os.listdir(repo.path)
for member in path_files:
if member == '.git':
continue
member = os.path.join(repo.path, member)
if os.path.isfile(member):
os.remove(member)
else:
delete_path(member)
# Move the tar inside the repo path
shutil.move(tar_file_name, new_repo_path)
# Untar the file
with tarfile.open(new_repo_path) as tar:
tar.extractall(repo.path)
# Delete the current tar
os.remove(new_repo_path)
# Get the git repo
if not git.get_status(repo.path):
return
# commit changes
git.commit(repo.path, user.email, user.username)
def build(self, memory_limit=None):
logger.debug('Starting build in `{}`'.format(self.build_repo_path))
# Checkout to the correct commit
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.build_repo_path,
commit=self.image_tag)
limits = {
# Always disable memory swap for building, since mostly
# nothing good can come of that.
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
# Create DockerFile
with open(self.dockerfile_path, 'w') as dockerfile:
dockerfile.write(self.render())
self.connect()
check_pulse = 0
for log_line in self.docker.build(
path=self.build_path,
tag='{}:{}'.format(self.image_name, self.image_tag),
buildargs={},
decode=True,
forcerm=True,
rm=True,
pull=True,
nocache=False,
container_limits=limits,
stream=True,
):
self._handle_logs(log_line)
# Check if we need to stop this process
check_pulse, should_stop = self._check_pulse(check_pulse)
if should_stop:
return False
# Checkout back to master
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.build_repo_path)
return True
def test_checkout_commit_and_master(self):
git_url = 'https://github.com/polyaxon/empty.git'
# Create repo
repo = ExternalRepo(project=self.project, git_url=git_url)
repo.save()
# Check last commit
assert repo.last_commit is None
# Add new file
file_path = os.path.join(repo.path, 'file1.dat')
open(file_path, 'w+')
assert git.get_status(repo.path) is not None
git.commit(repo.path, '*****@*****.**', 'username')
# Check last commit
commit1 = repo.last_commit[0]
assert commit1 is not None
# Add new file
file_path = os.path.join(repo.path, 'file2.dat')
open(file_path, 'w+')
assert git.get_status(repo.path) is not None
git.commit(repo.path, '*****@*****.**', 'username')
# Check last commit
commit2 = repo.last_commit[0]
assert commit2 is not None
# Commits are different
assert commit1 != commit2
# Checkout to commit1
git.checkout_commit(repo_path=repo.path, commit=commit1)
assert repo.last_commit[0] == commit1
# Checkout to master
git.checkout_commit(repo_path=repo.path)
assert repo.last_commit[0] == commit2
