def postbuild(version, version_module, builddir_path, debian_root):
"""
Summary.
Post-build clean up
Returns:
Success | Failure, TYPE: bool
"""
root = git_root()
project_dirname = root.split('/')[-1]
build_root = os.path.split(builddir_path)[0]
package = locate_deb(build_root)
try:
if package:
copyfile(package, debian_root)
package_path = debian_root + '/' + os.path.split(package)[1]
# remove build directory, residual artifacts
if os.path.exists(builddir_path):
rmtree(builddir_path)
if display_package_contents(BUILD_ROOT, VERSION):
return package_path
except OSError as e:
logger.exception('{}: Postbuild clean up failure'.format(
inspect.stack()[0][3]))
return False
return package_path
def tar_archive(archive, source_dir):
"""
Summary.
- Creates .tar.gz compressed archive
- Checks that file was created before exit
Returns:
Success | Failure, TYPE: bool
"""
try:
with tarfile.open(archive, "w:gz") as tar:
tar.add(source_dir, arcname=os.path.basename(source_dir))
if os.path.exists(archive):
return True
except OSError:
logger.exception('{}: Unable to create tar archive {}'.format(
inspect.stack()[0][3], archive))
except Exception as e:
logger.exception(
'%s: Unknown problem while creating tar archive %s:\n%s' %
(inspect.stack()[0][3], archive, str(e)))
return False
def ospackages(pkg_list):
"""Summary
Install OS Package Prerequisites
Returns:
Success | Failure, TYPE: bool
"""
try:
for pkg in pkg_list:
if is_installed(pkg):
logger.info(f'{pkg} binary is already installed - skip')
continue
elif which('yum'):
cmd = 'sudo yum install ' + pkg + ' 2>/dev/null'
print(subprocess.getoutput(cmd))
elif which('dnf'):
cmd = 'sudo dnf install ' + pkg + ' 2>/dev/null'
print(subprocess.getoutput(cmd))
else:
logger.warning(
'%s: Dependent OS binaries not installed - package manager not identified'
% inspect.stack()[0][3])
except OSError as e:
logger.exception('{}: Problem installing os package {}'.format(
inspect.stack()[0][3], pkg))
return False
return True
def current_branch(path):
"""
Returns.
git repository source url, TYPE: str
"""
pwd = os.getcwd()
os.chdir(path)
try:
if '.git' in os.listdir('.'):
branch = subprocess.getoutput('git branch').split('*')[1].split(
'\n')[0][1:]
else:
ex = Exception(
'%s: Unable to identify current branch - path not a git repository: %s'
% (inspect.stack()[0][3], path))
raise ex
os.chdir(pwd) # return cursor
except IndexError:
logger.exception('%s: problem retrieving git branch for %s' %
(inspect.stack()[0][3], path))
return ''
return branch
def list_dirs(relative_path):
images = []
dirs = []
try:
path = join(static_dir, relative_path)
all_files = listdir(path)
relative_path = "/" if relative_path == "" else "/" + relative_path + "/"
areas = []
points = []
dirs = []
images = []
for f in all_files:
if f.endswith("_areas.txt"):
img_areas = []
with open(join(path, f)) as f_areas:
for line in f_areas:
annotations = line.rstrip('\n').split(' ')
img_areas.append({
'x': annotations[0],
'y': annotations[1],
'width': annotations[2],
'height': annotations[3]
})
areas.append((relative_path + f.rsplit('_areas.txt')[0], json.dumps(img_areas)))
if f.endswith("_points.txt"):
img_points = []
with open(join(path, f)) as f_areas:
for line in f_areas:
annotations = line.rstrip('\n').split(' ')
img_points.append({
'x': annotations[0],
'y': annotations[1]
})
points.append((relative_path + f.rsplit('_points.txt')[0], json.dumps(img_points)))
if f.endswith(".jpg") or f.endswith(".JPEG"):
images.append(relative_path + f)
if os.path.isdir(join(path, f)):
dirs.append(unicode(f, "utf-8") if type(f) != unicode else f)
areas = dict(areas)
points = dict(points)
except Exception as e:
logger.info("Exception occured {}", e.message)
logger.exception(e)
return render_template("browse.html",
title='Browse',
dirs=sorted(dirs),
images=sorted(images),
areas=areas,
points=points,
total=len(images) + len(dirs))
def preclean(dir):
""" Cleans residual build artifacts """
try:
if os.path.exists(dir):
rmtree(dir)
except OSError as e:
logger.exception(
'%s: Error while cleaning residual build artifacts: %s' %
(inspect.stack()[0][3], str(e)))
return False
return True
def build_package(build_root, builddir):
"""
Summary.
Creates final os installable package for current build, build version
Returns:
Success | Failure, TYPE: bool
"""
try:
pwd = os.getcwd()
os.chdir(build_root)
if os.path.exists(builddir):
cmd = 'dpkg-deb --build ' + builddir + ' 2>/dev/null'
stdout_message('Building {}... '.format(bn + builddir + rst))
stdout_message(subprocess.getoutput(cmd))
os.chdir(pwd)
else:
logger.warning(
'Build directory {} not found. Failed to create .deb package'.
format(builddir))
os.chdir(pwd)
return False
except OSError as e:
logger.exception('{}: Error during os package creation: {}'.format(
inspect.stack()[0][3], e))
return False
except Exception as e:
logger.exception(
'{}: Unknown Error during os package creation: {}'.format(
inspect.stack()[0][3], e))
return False
return True
def query_page(relative_path):
q = request.form['query']
search_engine = request.form['engine']
max = int(request.form['max'])
skip = int(request.form['skip'])
if search_engine == 'google':
searcher = google_searcher
elif search_engine == 'flickr':
global flickr_searcher
if flickr_searcher is None:
flickr_searcher = searchtools.query.FlickrAPISearch()
searcher = flickr_searcher
elif search_engine == 'bing':
global bing_searcher
if bing_searcher is None:
bing_searcher = searchtools.query.BingAPISearch()
searcher = bing_searcher
elif search_engine == 'instagram':
global instagram_searcher
if instagram_searcher is None:
instagram_searcher = specific_engines.InstagramSearcher()
searcher = instagram_searcher
elif search_engine == 'yandex':
searcher = yandex_searcher
else:
searcher = imagenet_searcher
try:
images = searcher.query(q, num_results=max)[skip:]
except Exception as e:
logger.info("Exception occurred {}", e.message)
logger.exception(e)
images = []
return render_template("query.html",
title='Home',
images=images,
total=len(images))
def create_builddirectory(param_dict, path, version, force):
"""
- Creates the deb package binary working directory
- Checks if build artifacts preexist; if so, halts
- If force is True, continues even if artifacts exist (overwrites)
Returns:
builddir, TYPE: str
"""
try:
PROJECT = param_dict['Project']
builddir = PROJECT + '_' + version + '_amd64'
stdout_message(message='BUILDDIR IS: {}'.format(builddir))
# rm builddir when force if exists
if force is True and builddir in os.listdir(path):
rmtree(os.path.join(path, builddir))
elif force is False and builddir in os.listdir(path):
stdout_message(
'Cannot create build directory {} - preexists. Use --force option to overwrite'
.format(builddir),
prefix='WARN',
severity='WARNING')
return None
# create build directory
os.mkdir(os.path.join(path, builddir))
except OSError as e:
logger.exception('{}: Unable to create build directory {}'.format(
inspect.stack()[0][3], builddir))
return builddir
def query_page(relative_path):
q = request.form['query']
search_engine = request.form['engine']
max = int(request.form['max'])
skip = int(request.form['skip'])
if search_engine == 'google':
searcher = google_searcher
elif search_engine == 'flickr':
global flickr_searcher
if flickr_searcher is None:
flickr_searcher = searchtools.query.FlickrAPISearch()
searcher = flickr_searcher
elif search_engine == 'bing':
global bing_searcher
if bing_searcher is None:
bing_searcher = searchtools.query.BingAPISearch()
searcher = bing_searcher
elif search_engine == 'instagram':
global instagram_searcher
if instagram_searcher is None:
instagram_searcher = specific_engines.InstagramSearcher()
searcher = instagram_searcher
elif search_engine == 'yandex':
searcher = yandex_searcher
else:
searcher = imagenet_searcher
try:
images = searcher.query(q, num_results=max)[skip:]
except Exception as e:
logger.info("Exception occurred {}", e.message)
logger.exception(e)
images = []
return render_template("query.html",
title='Home',
images=images,
total=len(images))
def list_dirs(relative_path):
images = []
dirs = []
try:
path = join(static_dir, relative_path)
all_files = listdir(path)
relative_path = "/" if relative_path == "" else "/" + relative_path + "/"
areas = []
points = []
labels = []
dirs = []
images = {}
for f in all_files:
if f.endswith("_areas.txt"):
img_areas = []
with open(join(path, f)) as f_areas:
for line in f_areas:
annotations = line.rstrip('\n').split(' ')
img_areas.append({
'x': annotations[0],
'y': annotations[1],
'width': annotations[2],
'height': annotations[3]
})
areas.append((relative_path + f.rsplit('_areas.txt')[0],
json.dumps(img_areas)))
if f.endswith("_points.txt"):
img_points = []
with open(join(path, f)) as f_areas:
for line in f_areas:
annotations = line.rstrip('\n').split(' ')
img_points.append({
'x': annotations[0],
'y': annotations[1]
})
points.append((relative_path + f.rsplit('_points.txt')[0],
json.dumps(img_points)))
if f.endswith("_labels.txt"):
with open(join(path, f)) as f_label:
img_label = f_label.read()
img_label = img_label[:-1]
labels.append((relative_path + f.rsplit('_labels.txt')[0],
json.dumps(img_label)))
if f.endswith(".jpg") or f.endswith(".JPEG"):
url = relative_path + f
images[f] = {'url': url, 'name': f, 'mark': False}
if os.path.isdir(join(path, f)):
dirs.append(unicode(f, "utf-8") if type(f) != unicode else f)
marks_path = os.path.join(path, '_marks.txt')
if os.path.exists(marks_path):
for line in open(marks_path):
photo_name = line.rstrip('\n')
if photo_name in images:
images[photo_name]['mark'] = True
images = sorted(images.values(), key=lambda x: x['name'])
mark_now = False
for idx, img in enumerate(images):
if img['mark']:
mark_now = False if mark_now else True
if mark_now:
img['mark_class'] = 'mark_border' if img['mark'] else 'mark'
else:
img['mark_class'] = 'mark_border' if img['mark'] else ''
areas = dict(areas)
points = dict(points)
labels = dict(labels)
except Exception as e:
logger.info("Exception occured {}", e.message)
logger.exception(e)
return render_template("browse.html",
title='Browse',
dirs=sorted(dirs),
images=images,
areas=areas,
points=points,
labels=labels,
total=len(images) + len(dirs))
def prebuild(builddir, volmnt, parameter_file):
"""
Summary.
Prerequisites and dependencies for build execution
"""
def preclean(dir):
""" Cleans residual build artifacts """
try:
if os.path.exists(dir):
rmtree(dir)
except OSError as e:
logger.exception(
'%s: Error while cleaning residual build artifacts: %s' %
(inspect.stack()[0][3], str(e)))
return False
return True
version_module = json.loads(read(parameter_file))['VersionModule']
if preclean(builddir) and preclean(volmnt):
stdout_message(
f'Removed pre-existing build artifacts ({builddir}, {volmnt})')
os.makedirs(builddir)
os.makedirs(volmnt)
root = git_root()
lib_relpath = PROJECT_BIN
lib_path = root + '/' + lib_relpath
sources = [lib_path]
illegal = ['__pycache__']
module = inspect.stack()[0][3]
fx = inspect.stack()[0][3]
try:
global __version__
sys.path.insert(0, os.path.abspath(git_root() + '/' + lib_relpath))
from _version import __version__
# normalize path
sys.path.pop(0)
except ImportError as e:
logger.exception(
'{}: Problem importing program version module (%s). Error: %s' %
(fx, __file__, str(e)))
except Exception as e:
logger.exception(
'{}: Failure to import _version module _version'.format(fx))
return False
## clean up source ##
try:
for directory in sources:
for artifact in os.listdir(directory):
if artifact in illegal:
rmtree(directory + '/' + artifact)
except OSError:
logger.exception(
'{}: Illegal file object detected, but unable to remove {}'.format(
module, archive))
return False
return True
def builddir_content_updates(param_dict, osimage, builddir, version):
"""
Summary.
Updates builddir contents:
- main exectuable has path to libraries updated
- builddir DEBIAN/control file version is updated to current
- updates the version.py file if version != to __version__
contained in the file. This occurs if user invokes the -S /
--set-version option
Args:
:root (str): project root full fs path
:builddir (str): dirname of the current build directory
:binary (str): name of the main exectuable
:version (str): version label provided with --set-version parameter. None otherwise
Returns:
Success | Failure, TYPE: bool
"""
root = git_root()
project_dirname = root.split('/')[-1]
build_root = TMPDIR
debian_dir = 'DEBIAN'
control_filename = param_dict['ControlFile']['Name']
deb_src = root + '/packaging/deb'
major = '.'.join(version.split('.')[:2])
minor = version.split('.')[-1]
# files
version_module = param_dict['VersionModule']
issues_url = param_dict['IssuesUrl']
project_url = param_dict['ProjectUrl']
buildarch = param_dict['ControlFile']['BuildArch']
# full paths
builddir_path = build_root + '/' + builddir
debian_path = builddir_path + '/' + debian_dir
control_filepath = debian_path + '/' + control_filename
lib_dst = builddir_path + '/usr/lib/python3/dist-packages/' + PROJECT_BIN
# assemble dependencies
deplist = None
for dep in param_dict['DependencyList']:
if deplist is None:
deplist = str(dep)
else:
deplist = deplist + ', ' + str(dep)
try:
# debian control files
with open(control_filepath) as f1:
f2 = f1.readlines()
for index, line in enumerate(f2):
if line.startswith('Version:'):
newline = 'Version: ' + version + '\n'
f2[index] = newline
f1.close()
# rewrite file
with open(control_filepath, 'w') as f3:
f3.writelines(f2)
path = project_dirname + (control_filepath)[len(root):]
stdout_message('Control file {} version updated: {}.'.format(
yl + control_filepath + rst, version))
## rewrite version file with current build version in case delta ##
# orig source version module
with open(LIB_SRC + '/' + version_module, 'w') as f3:
f2 = ['__version__ = \"' + version + '\"\n']
f3.writelines(f2)
path = os.path.join(root, PROJECT_BIN, version_module)
stdout_message('Module {} successfully updated: {}.'.format(
yl + path + rst, version))
# package version module
with open(lib_dst + '/' + version_module, 'w') as f3:
f2 = ['__version__ = \"' + version + '\"\n']
f3.writelines(f2)
path = os.path.join(lib_dst, version_module)
stdout_message('Module {} successfully updated: {}.'.format(
yl + path + rst, version))
except OSError as e:
logger.exception('%s: Problem while updating builddir contents: %s' %
(inspect.stack()[0][3], str(e)))
return False
return True
def builddir_structure(param_dict, builddir, version):
"""
Summary.
- Updates paths in binary exectuable
- Updates
Args:
:root (str): full path to root directory of the git project
:builddir (str): name of current build directory which we need to populate
Vars:
:lib_path (str): src path to library modules in project root
:builddir_path (str): dst path to root of the current build directory
(/<path>/xlines-1.X.X dir)
Returns:
Success | Failure, TYPE: bool
"""
def _mapper(venv_dir):
"""Identifies path to python modules in virtual env"""
for i in (6, 7, 8, 9):
path = venv_dir + '/lib/python3.' + str(i) + '/site-packages/'
if os.path.exists(path):
return path
def module_search(module, packages_path):
t = []
for i in os.listdir(packages_path):
if re.search(module, i, re.IGNORECASE):
t.append(i)
return t
root = git_root()
project_dirname = os.path.split(git_root())[1]
build_root = TMPDIR
# files
binary = param_dict['Executable']
control_file = param_dict['ControlFile']['Name']
compfile = param_dict['BashCompletion']
# LIB source files
env = os.environ.get('VIRTUAL_ENV') or root
#lib_src = _mapper(env)
lib_src = os.path.join(root, PROJECT_BIN)
# full paths
builddir_path = build_root + '/' + builddir
deb_src = root + '/packaging/deb'
debian_dir = 'DEBIAN'
debian_path = deb_src + '/' + debian_dir
binary_path = builddir_path + '/usr/bin'
lib_dst = builddir_path + '/usr/lib/python3/dist-packages'
comp_src = os.path.join(root, 'bash')
comp_dst = builddir_path + '/etc/bash_completion.d'
try:
# create build directory
if os.path.exists(builddir_path):
rmtree(builddir_path)
os.makedirs(builddir_path)
stdout_message(message='Created:\t{}'.format(yl + builddir_path + rst),
prefix='OK')
stdout_message(
f'Copying DEBIAN package control files to {bn + builddir + rst}')
_src = os.path.join(deb_src, debian_dir)
_dst = os.path.join(builddir_path, debian_dir)
copytree(_src, _dst)
operation_status(_src, _dst)
stdout_message(
f'Creating build directory subdirectories in {bn + builddir + rst}'
)
# binary exec
if not os.path.exists(binary_path):
os.makedirs(binary_path)
_src_path = os.path.join(deb_src, 'bin', 'xlines')
_dst_path = os.path.join(binary_path, 'xlines')
copyfile(_src_path, _dst_path)
# status msg
stdout_message(message='Copied:\t{} {} {}'.format(
lk + _src_path + rst, arrow, lk + _dst_path + rst),
prefix='OK')
# library components
if not os.path.exists(lib_dst):
os.makedirs(lib_dst)
_src = lib_src
_dst = os.path.join(lib_dst, PROJECT_BIN)
copytree(_src, _dst)
stdout_message(message='Copied:\t{} {} {}'.format(
lk + _src + rst, arrow, lk + _dst + rst),
prefix='OK')
if not os.path.exists(comp_dst):
# create path
os.makedirs(comp_dst)
# copy
for artifact in list(
filter(lambda x: x.endswith('.bash'),
os.listdir(comp_src))):
_src = comp_src + '/' + artifact
_dst = comp_dst + '/' + artifact
copyfile(_src, _dst)
stdout_message(message='Copied:\t{} {} {}'.format(
lk + _src + rst, arrow, lk + _dst + rst),
prefix='OK')
stdout_message(f'Creating config subdirectory in {lk + lib_dst + rst}')
os.makedirs(os.path.join(lib_dst, PROJECT_BIN, 'config'))
source = os.path.join(root, 'config')
for file in list(
filter(lambda x: x.endswith('.list'), os.listdir(source))):
_src = os.path.join(source, file)
_dst = os.path.join(lib_dst, PROJECT_BIN, 'config', file)
copyfile(_src, _dst)
except OSError as e:
logger.exception('{}: Problem creating dirs on local fs'.format(
inspect.stack()[0][3]))
return False
return True
