def remove_attachments(odoo_config):
"""Remove attachments folder
Args:
odoo_config (dict): Odoo configuration
"""
if 'odoo_container' in odoo_config:
env_vars = get_docker_env(odoo_config.get('odoo_container'))
odoo_config_file = env_vars.get('ODOO_CONFIG_FILE')
cli = Client()
try:
res = cli.copy(odoo_config.get('odoo_container'), odoo_config_file)
except docker.errors.APIError as error:
if "Could not find the file" in error.message:
logger.error("Odoo config file is not in the path '%s'", odoo_config_file)
else:
logger.error("Could not get the config file '%s'", error.message)
return None
for i in res.data.split('\n'):
if i.strip().startswith("data_dir"):
data_dir = i.split("=")[1].strip()
break
fs_name = os.path.join(data_dir, "filestore", odoo_config.get('database'))
exec_id = cli.exec_create(odoo_config.get('odoo_container'), "rm -r {}".format(fs_name))
res = cli.exec_start(exec_id.get('Id'))
if res:
logger.info("Removing previous filestore returned '%s'", res)
else:
fs_name = os.path.join(odoo_config.get('data_dir'),
'filestore',
odoo_config.get('database'))
shutil.rmtree(fs_name)
def fetch_results_using_cp(self, **kwargs):
self.containerName = kwargs['container_name']
self.resource = kwargs['container_path']
print('container_name: , container_path:', self.containerName,
self.resource)
invoke_clientAPI = Client(base_url='unix://var/run/docker.sock',
version='1.12')
try:
print('try')
_fileObtained = invoke_clientAPI.copy(self.containerName,
self.resource)
filelike = StringIO.StringIO(_fileObtained.read())
tar = tarfile.open(fileobj=filelike)
file1 = tar.extractfile(os.path.basename(self.resource))
x = file1.read()
filename = os.path.join(self._folderName)
print('filename: ', filename)
with open(self._folderName, "w") as text_file:
text_file.write(x)
print('response: ', _fileObtained)
except HTTPError:
print('exception')
def copy(container, path):
client = Client()
response = client.copy(container, path)
buffer = io.BytesIO()
buffer.write(response.data)
buffer.seek(0)
tar = tarfile.open(fileobj=buffer, mode='r')
for member in tar.getmembers():
return tar.extractfile(member).read()
class DockerPatch(object):
dockerClient = None
containerId = None
patches = {}
def __init__(self, containerName):
if (containerName is None):
raise IllegalArgumentError("containerName must be not None")
self.dockerClient = Client(
base_url='unix://var/run/docker.sock', version='auto')
for i in self.dockerClient.containers():
if (i['Names'][0] == containerName):
self.containerId = i['Id']
break
if (self.containerId is None):
raise IllegalContainerName(
"Container " + containerName + " not found")
def addPatch(self, patchFile, prefix='/'):
self.patches[patchFile] = {
'prefix': prefix,
'files': extractFilesFromPatch(patchFile),
'patch': patchFile
}
def copy_from_docker(self, src, dest):
reply = self.dockerClient.copy(self.containerId, src)
filelike = io.BytesIO(reply.read())
tar = tarfile.open(fileobj=filelike)
file = tar.extractfile(os.path.basename(src))
with open(dest, 'wb') as f:
f.write(file.read())
reply.release_conn()
def apply(self):
tempdir = tempfile.mkdtemp('patching')
for p in self.patches.values():
for f in p['files']:
filePath = tempdir + '/' + f
fileDir = os.path.dirname(filePath)
if not os.path.exists(fileDir):
os.makedirs(fileDir)
self.copy_from_docker(p['prefix'] + '/' + f, filePath)
self.patchIt(p['patch'], tempdir)
def patchIt(self, patchFile, tempdir):
with open(patchFile, "r") as f:
subprocess.Popen(shlex.split('patch -p0 -d ' + tempdir), stdin=f)
class DockerPatch(object):
dockerClient = None
containerId = None
patches = {}
def __init__(self, containerName):
if (containerName is None):
raise IllegalArgumentError("containerName must be not None")
self.dockerClient = Client(base_url='unix://var/run/docker.sock',
version='auto')
for i in self.dockerClient.containers():
if (i['Names'][0] == containerName):
self.containerId = i['Id']
break
if (self.containerId is None):
raise IllegalContainerName("Container " + containerName +
" not found")
def addPatch(self, patchFile, prefix='/'):
self.patches[patchFile] = {
'prefix': prefix,
'files': extractFilesFromPatch(patchFile),
'patch': patchFile
}
def copy_from_docker(self, src, dest):
reply = self.dockerClient.copy(self.containerId, src)
filelike = io.BytesIO(reply.read())
tar = tarfile.open(fileobj=filelike)
file = tar.extractfile(os.path.basename(src))
with open(dest, 'wb') as f:
f.write(file.read())
reply.release_conn()
def apply(self):
tempdir = tempfile.mkdtemp('patching')
for p in self.patches.values():
for f in p['files']:
filePath = tempdir + '/' + f
fileDir = os.path.dirname(filePath)
if not os.path.exists(fileDir):
os.makedirs(fileDir)
self.copy_from_docker(p['prefix'] + '/' + f, filePath)
self.patchIt(p['patch'], tempdir)
def patchIt(self, patchFile, tempdir):
with open(patchFile, "r") as f:
subprocess.Popen(shlex.split('patch -p0 -d ' + tempdir), stdin=f)
def fetch_results_using_cp(self, **kwargs):
self.containerName = kwargs['container']
self.resource = kwargs['container_path']
print('container_name: , container_path:', self.containerName,
self.resource)
invoke_clientAPI = Client(base_url='unix://var/run/docker.sock',
version='1.12')
(self._appName, self._extension) = os.path.splitext(self.resource)
print('folderName: extension: ', self._appName, self._extension)
self.resource = '/home/' + os.path.join(self._appName, self._output)
print(self.resource)
try:
print('try')
_fileObtained = invoke_clientAPI.copy(self.containerName,
self.resource)
filelike = StringIO.StringIO(
_fileObtained.read()
) # Sends contains of file as string. Converting it back to a file.
tar = tarfile.open(fileobj=filelike)
file1 = tar.extractfile(os.path.basename(self.resource))
x = file1.read()
filename = self._appName
print('filename: ', filename)
with open(filename, "w") as text_file:
text_file.write(x)
print('response: ', _fileObtained)
result_path = os.path.dirname(os.path.realpath(__file__))
result_path = result_path + '/../../'
result_path = os.path.join(result_path, filename)
print result_path
return result_path
except HTTPError:
print('exception\None')
return None
def restore_docker_filestore(src_folder, odoo_config,
docker_url="unix://var/run/docker.sock"):
""" Restore a filestore folder into a docker container that is already running
and has the /tmp folder mounted as a volume un the host
Args:
src_folder (str): Full path to the folder thar contains the filestore you want to restore
odoo_config (dict): configuration
docker_url (str): url to use in docker cli client
"""
container_name = odoo_config.get('odoo_container')
cli = Client(base_url=docker_url, timeout=3000)
try:
inspected = cli.inspect_container(container_name)
except docker.errors.APIError as error:
if "no such id" not in error.explanation:
logger.error("No such container: %s", container_name)
else:
logger.error(error.explanation)
return None
ports = inspected.get('NetworkSettings').get('Ports')
if not ports:
logger.error('Container "%s" is not running', container_name)
return None
mounts = inspected.get('Mounts')
for mount in mounts:
if '/tmp' in mount['Destination']:
dest_folder = mount['Source']
break
else:
logger.error("Could not restore filestore into %s container", container_name)
logger.error("You should run the docker with a volume in /tmp")
return None
try:
shutil.move(src_folder, dest_folder)
except shutil.Error as error:
if "already exists" in error.message:
folder = os.path.basename(src_folder)
logger.info("%s folder already exists, replacing", folder)
clean_files(os.path.join(dest_folder, folder))
shutil.move(src_folder, dest_folder)
else:
raise
except IOError as error:
if "No such file or directory" in error.strerror:
logger.warn("No filestore in the backup file")
else:
raise
env_vars = get_docker_env(container_name)
odoo_config_file = env_vars.get('ODOO_CONFIG_FILE')
try:
res = cli.copy(container_name, odoo_config_file)
except docker.errors.APIError as error:
if "Could not find the file" in error.message:
logger.error("Odoo config file is not in the path '%s'", odoo_config_file)
else:
logger.error("Could not get the config file '%s'", error.message)
return None
for line in res.data.split('\n'):
if line.strip().startswith("data_dir"):
data_dir = line.split("=")[1].strip()
break
fs_name = os.path.join(data_dir, "filestore", odoo_config.get('database'))
exec_id = cli.exec_create(container_name, "rm -rf {0}".format(fs_name))
res = cli.exec_start(exec_id.get('Id'))
if res:
logger.info("Removing previous filestore returned '%s'", res)
exec_id = cli.exec_create(container_name, "mv /tmp/filestore {0}".format(fs_name))
res = cli.exec_start(exec_id.get('Id'))
if res:
logger.info("Moving filestore returned '%s'", res)
exec_id = cli.exec_create(container_name, "chown -R {0}:{0} {1}"
.format(env_vars.get('ODOO_USER'), fs_name))
res = cli.exec_start(exec_id.get('Id'))
if res:
logger.info("Changing filestore owner returned '%s'", res)
class GenericSandbox(object):
def __init__(self, sandbox_dir='/sandbox', sandbox_user="******", client_args={}):
self.client = Client(**client_args)
self.dockerdir = tempfile.mkdtemp('codebox_docker')
self.sandbox_dir = sandbox_dir
self.sandbox_user = sandbox_user
self.image_tag = 'codebox/generic'
self.built = False
self.containers = []
self.dockerfile = [
'FROM ubuntu',
'RUN apt-get update -y',
'RUN apt-get install -y unzip',
'RUN mkdir %s' % sandbox_dir,
'RUN useradd %s' % sandbox_user,
'RUN chown %s %s' % (sandbox_user, sandbox_dir),
'WORKDIR %s' % sandbox_dir,
]
self.dockerfile_finalize = [
'USER %s' % sandbox_user,
] # Instructions to run after whatever the baseclass does
def build(self):
dockerpath = os.path.join(self.dockerdir, 'Dockerfile')
dockercontents = '\n'.join(self.dockerfile + self.dockerfile_finalize)
with open(dockerpath, 'w') as f:
#print(dockercontents)
f.write(dockercontents)
buildresult = self.client.build(path=self.dockerdir,
tag=self.image_tag,
)
rowbuf = ""
for row in buildresult:
if type(row) == bytes:
row = str(row, 'utf-8')
if type(row) != str:
raise TypeError("Non-string result row!", row)
print(row.strip())
rowbuf += row.strip() + "\n"
row = json.loads(row)
if row.get('error'):
raise Exception('Image building error %s \n\n %s' % (row, rowbuf))
self.built = True
def include_file(self, fname, source_fname=None, source_contents=None):
if not (source_fname or source_contents):
raise ValueError("No source specified!")
if source_fname:
sourcecopy_path = os.path.join(self.dockerdir, fname)
shutil.copyfile(source_fname, sourcecopy_path)
destination_path = os.path.join(self.sandbox_dir, fname)
command = "ADD {source} {dest}".format(source=fname,
dest=destination_path)
self.dockerfile.append(command)
else:
if type(source_contents)==bytes:
data = source_contents
elif type(source_contents)==str:
data = source_contents.encode()
else:
raise TypeError('Source contents are not string or bytes.')
source_path = os.path.join(self.dockerdir, fname)
destination_path = os.path.join(self.sandbox_dir, fname)
with open(source_path, 'wb') as f:
f.write(data)
command = "ADD {source} {dest}".format(source=fname,
dest=destination_path)
self.dockerfile.append(command)
def include_zip(self, source_fname=None, source_contents=None):
if not (source_fname or source_contents):
raise ValueError("No source specified!")
ZIPNAME = "__codebox_zip_upload.zip"
self.include_file(ZIPNAME, source_fname, source_contents)
command = "RUN unzip {name}".format(name=ZIPNAME)
self.dockerfile.append(command)
command = "RUN rm {name}".format(name=ZIPNAME)
self.dockerfile.append(command)
def run_cmd(self, command, cleanup=True):
assert self.built
container = self.client.create_container(image=self.image_tag,
command=command,
hostname="codebox",
mem_limit="100m",
#memswap_limit=1,
working_dir=self.sandbox_dir,
network_disabled=True)
cid = container['Id']
self.client.start(container=cid)
exitcode = self.client.wait(container=cid)
print("Exit code: %s" % exitcode)
logs = self.client.logs(container=cid)
self.containers.append(cid)
if cleanup:
self.cleanup_container(cid)
return {'logs': logs,
'container_id': cid,
}
def cleanup_container(self, container_id):
self.containers.remove(container_id)
self.client.remove_container(container=container_id)
def get_file(self, container_id, fname):
fpath = os.path.join('/sandbox', fname)
data = self.client.copy(container=container_id,
resource=fpath).read()
data = data[512:] # XXX: WTF, Docker?
data = data.rstrip(b'\x00') # XXX: WTF, Docker?
return data
def destroy(self):
for container in self.containers:
self.cleanup_container(container)
shutil.rmtree(self.dockerdir)
