def postbuild(version, version_module, builddir_path, debian_root): """ Summary. Post-build clean up Returns: Success | Failure, TYPE: bool """ root = git_root() project_dirname = root.split('/')[-1] build_root = os.path.split(builddir_path)[0] package = locate_deb(build_root) try: if package: copyfile(package, debian_root) package_path = debian_root + '/' + os.path.split(package)[1] # remove build directory, residual artifacts if os.path.exists(builddir_path): rmtree(builddir_path) if display_package_contents(BUILD_ROOT, VERSION): return package_path except OSError as e: logger.exception('{}: Postbuild clean up failure'.format( inspect.stack()[0][3])) return False return package_path
def tar_archive(archive, source_dir): """ Summary. - Creates .tar.gz compressed archive - Checks that file was created before exit Returns: Success | Failure, TYPE: bool """ try: with tarfile.open(archive, "w:gz") as tar: tar.add(source_dir, arcname=os.path.basename(source_dir)) if os.path.exists(archive): return True except OSError: logger.exception('{}: Unable to create tar archive {}'.format( inspect.stack()[0][3], archive)) except Exception as e: logger.exception( '%s: Unknown problem while creating tar archive %s:\n%s' % (inspect.stack()[0][3], archive, str(e))) return False
def ospackages(pkg_list): """Summary Install OS Package Prerequisites Returns: Success | Failure, TYPE: bool """ try: for pkg in pkg_list: if is_installed(pkg): logger.info(f'{pkg} binary is already installed - skip') continue elif which('yum'): cmd = 'sudo yum install ' + pkg + ' 2>/dev/null' print(subprocess.getoutput(cmd)) elif which('dnf'): cmd = 'sudo dnf install ' + pkg + ' 2>/dev/null' print(subprocess.getoutput(cmd)) else: logger.warning( '%s: Dependent OS binaries not installed - package manager not identified' % inspect.stack()[0][3]) except OSError as e: logger.exception('{}: Problem installing os package {}'.format( inspect.stack()[0][3], pkg)) return False return True
def current_branch(path): """ Returns. git repository source url, TYPE: str """ pwd = os.getcwd() os.chdir(path) try: if '.git' in os.listdir('.'): branch = subprocess.getoutput('git branch').split('*')[1].split( '\n')[0][1:] else: ex = Exception( '%s: Unable to identify current branch - path not a git repository: %s' % (inspect.stack()[0][3], path)) raise ex os.chdir(pwd) # return cursor except IndexError: logger.exception('%s: problem retrieving git branch for %s' % (inspect.stack()[0][3], path)) return '' return branch
def list_dirs(relative_path): images = [] dirs = [] try: path = join(static_dir, relative_path) all_files = listdir(path) relative_path = "/" if relative_path == "" else "/" + relative_path + "/" areas = [] points = [] dirs = [] images = [] for f in all_files: if f.endswith("_areas.txt"): img_areas = [] with open(join(path, f)) as f_areas: for line in f_areas: annotations = line.rstrip('\n').split(' ') img_areas.append({ 'x': annotations[0], 'y': annotations[1], 'width': annotations[2], 'height': annotations[3] }) areas.append((relative_path + f.rsplit('_areas.txt')[0], json.dumps(img_areas))) if f.endswith("_points.txt"): img_points = [] with open(join(path, f)) as f_areas: for line in f_areas: annotations = line.rstrip('\n').split(' ') img_points.append({ 'x': annotations[0], 'y': annotations[1] }) points.append((relative_path + f.rsplit('_points.txt')[0], json.dumps(img_points))) if f.endswith(".jpg") or f.endswith(".JPEG"): images.append(relative_path + f) if os.path.isdir(join(path, f)): dirs.append(unicode(f, "utf-8") if type(f) != unicode else f) areas = dict(areas) points = dict(points) except Exception as e: logger.info("Exception occured {}", e.message) logger.exception(e) return render_template("browse.html", title='Browse', dirs=sorted(dirs), images=sorted(images), areas=areas, points=points, total=len(images) + len(dirs))
def preclean(dir): """ Cleans residual build artifacts """ try: if os.path.exists(dir): rmtree(dir) except OSError as e: logger.exception( '%s: Error while cleaning residual build artifacts: %s' % (inspect.stack()[0][3], str(e))) return False return True
def build_package(build_root, builddir): """ Summary. Creates final os installable package for current build, build version Returns: Success | Failure, TYPE: bool """ try: pwd = os.getcwd() os.chdir(build_root) if os.path.exists(builddir): cmd = 'dpkg-deb --build ' + builddir + ' 2>/dev/null' stdout_message('Building {}... '.format(bn + builddir + rst)) stdout_message(subprocess.getoutput(cmd)) os.chdir(pwd) else: logger.warning( 'Build directory {} not found. Failed to create .deb package'. format(builddir)) os.chdir(pwd) return False except OSError as e: logger.exception('{}: Error during os package creation: {}'.format( inspect.stack()[0][3], e)) return False except Exception as e: logger.exception( '{}: Unknown Error during os package creation: {}'.format( inspect.stack()[0][3], e)) return False return True
def query_page(relative_path): q = request.form['query'] search_engine = request.form['engine'] max = int(request.form['max']) skip = int(request.form['skip']) if search_engine == 'google': searcher = google_searcher elif search_engine == 'flickr': global flickr_searcher if flickr_searcher is None: flickr_searcher = searchtools.query.FlickrAPISearch() searcher = flickr_searcher elif search_engine == 'bing': global bing_searcher if bing_searcher is None: bing_searcher = searchtools.query.BingAPISearch() searcher = bing_searcher elif search_engine == 'instagram': global instagram_searcher if instagram_searcher is None: instagram_searcher = specific_engines.InstagramSearcher() searcher = instagram_searcher elif search_engine == 'yandex': searcher = yandex_searcher else: searcher = imagenet_searcher try: images = searcher.query(q, num_results=max)[skip:] except Exception as e: logger.info("Exception occurred {}", e.message) logger.exception(e) images = [] return render_template("query.html", title='Home', images=images, total=len(images))
def create_builddirectory(param_dict, path, version, force): """ - Creates the deb package binary working directory - Checks if build artifacts preexist; if so, halts - If force is True, continues even if artifacts exist (overwrites) Returns: builddir, TYPE: str """ try: PROJECT = param_dict['Project'] builddir = PROJECT + '_' + version + '_amd64' stdout_message(message='BUILDDIR IS: {}'.format(builddir)) # rm builddir when force if exists if force is True and builddir in os.listdir(path): rmtree(os.path.join(path, builddir)) elif force is False and builddir in os.listdir(path): stdout_message( 'Cannot create build directory {} - preexists. Use --force option to overwrite' .format(builddir), prefix='WARN', severity='WARNING') return None # create build directory os.mkdir(os.path.join(path, builddir)) except OSError as e: logger.exception('{}: Unable to create build directory {}'.format( inspect.stack()[0][3], builddir)) return builddir
def list_dirs(relative_path): images = [] dirs = [] try: path = join(static_dir, relative_path) all_files = listdir(path) relative_path = "/" if relative_path == "" else "/" + relative_path + "/" areas = [] points = [] labels = [] dirs = [] images = {} for f in all_files: if f.endswith("_areas.txt"): img_areas = [] with open(join(path, f)) as f_areas: for line in f_areas: annotations = line.rstrip('\n').split(' ') img_areas.append({ 'x': annotations[0], 'y': annotations[1], 'width': annotations[2], 'height': annotations[3] }) areas.append((relative_path + f.rsplit('_areas.txt')[0], json.dumps(img_areas))) if f.endswith("_points.txt"): img_points = [] with open(join(path, f)) as f_areas: for line in f_areas: annotations = line.rstrip('\n').split(' ') img_points.append({ 'x': annotations[0], 'y': annotations[1] }) points.append((relative_path + f.rsplit('_points.txt')[0], json.dumps(img_points))) if f.endswith("_labels.txt"): with open(join(path, f)) as f_label: img_label = f_label.read() img_label = img_label[:-1] labels.append((relative_path + f.rsplit('_labels.txt')[0], json.dumps(img_label))) if f.endswith(".jpg") or f.endswith(".JPEG"): url = relative_path + f images[f] = {'url': url, 'name': f, 'mark': False} if os.path.isdir(join(path, f)): dirs.append(unicode(f, "utf-8") if type(f) != unicode else f) marks_path = os.path.join(path, '_marks.txt') if os.path.exists(marks_path): for line in open(marks_path): photo_name = line.rstrip('\n') if photo_name in images: images[photo_name]['mark'] = True images = sorted(images.values(), key=lambda x: x['name']) mark_now = False for idx, img in enumerate(images): if img['mark']: mark_now = False if mark_now else True if mark_now: img['mark_class'] = 'mark_border' if img['mark'] else 'mark' else: img['mark_class'] = 'mark_border' if img['mark'] else '' areas = dict(areas) points = dict(points) labels = dict(labels) except Exception as e: logger.info("Exception occured {}", e.message) logger.exception(e) return render_template("browse.html", title='Browse', dirs=sorted(dirs), images=images, areas=areas, points=points, labels=labels, total=len(images) + len(dirs))
def prebuild(builddir, volmnt, parameter_file): """ Summary. Prerequisites and dependencies for build execution """ def preclean(dir): """ Cleans residual build artifacts """ try: if os.path.exists(dir): rmtree(dir) except OSError as e: logger.exception( '%s: Error while cleaning residual build artifacts: %s' % (inspect.stack()[0][3], str(e))) return False return True version_module = json.loads(read(parameter_file))['VersionModule'] if preclean(builddir) and preclean(volmnt): stdout_message( f'Removed pre-existing build artifacts ({builddir}, {volmnt})') os.makedirs(builddir) os.makedirs(volmnt) root = git_root() lib_relpath = PROJECT_BIN lib_path = root + '/' + lib_relpath sources = [lib_path] illegal = ['__pycache__'] module = inspect.stack()[0][3] fx = inspect.stack()[0][3] try: global __version__ sys.path.insert(0, os.path.abspath(git_root() + '/' + lib_relpath)) from _version import __version__ # normalize path sys.path.pop(0) except ImportError as e: logger.exception( '{}: Problem importing program version module (%s). Error: %s' % (fx, __file__, str(e))) except Exception as e: logger.exception( '{}: Failure to import _version module _version'.format(fx)) return False ## clean up source ## try: for directory in sources: for artifact in os.listdir(directory): if artifact in illegal: rmtree(directory + '/' + artifact) except OSError: logger.exception( '{}: Illegal file object detected, but unable to remove {}'.format( module, archive)) return False return True
def builddir_content_updates(param_dict, osimage, builddir, version): """ Summary. Updates builddir contents: - main exectuable has path to libraries updated - builddir DEBIAN/control file version is updated to current - updates the version.py file if version != to __version__ contained in the file. This occurs if user invokes the -S / --set-version option Args: :root (str): project root full fs path :builddir (str): dirname of the current build directory :binary (str): name of the main exectuable :version (str): version label provided with --set-version parameter. None otherwise Returns: Success | Failure, TYPE: bool """ root = git_root() project_dirname = root.split('/')[-1] build_root = TMPDIR debian_dir = 'DEBIAN' control_filename = param_dict['ControlFile']['Name'] deb_src = root + '/packaging/deb' major = '.'.join(version.split('.')[:2]) minor = version.split('.')[-1] # files version_module = param_dict['VersionModule'] issues_url = param_dict['IssuesUrl'] project_url = param_dict['ProjectUrl'] buildarch = param_dict['ControlFile']['BuildArch'] # full paths builddir_path = build_root + '/' + builddir debian_path = builddir_path + '/' + debian_dir control_filepath = debian_path + '/' + control_filename lib_dst = builddir_path + '/usr/lib/python3/dist-packages/' + PROJECT_BIN # assemble dependencies deplist = None for dep in param_dict['DependencyList']: if deplist is None: deplist = str(dep) else: deplist = deplist + ', ' + str(dep) try: # debian control files with open(control_filepath) as f1: f2 = f1.readlines() for index, line in enumerate(f2): if line.startswith('Version:'): newline = 'Version: ' + version + '\n' f2[index] = newline f1.close() # rewrite file with open(control_filepath, 'w') as f3: f3.writelines(f2) path = project_dirname + (control_filepath)[len(root):] stdout_message('Control file {} version updated: {}.'.format( yl + control_filepath + rst, version)) ## rewrite version file with current build version in case delta ## # orig source version module with open(LIB_SRC + '/' + version_module, 'w') as f3: f2 = ['__version__ = \"' + version + '\"\n'] f3.writelines(f2) path = os.path.join(root, PROJECT_BIN, version_module) stdout_message('Module {} successfully updated: {}.'.format( yl + path + rst, version)) # package version module with open(lib_dst + '/' + version_module, 'w') as f3: f2 = ['__version__ = \"' + version + '\"\n'] f3.writelines(f2) path = os.path.join(lib_dst, version_module) stdout_message('Module {} successfully updated: {}.'.format( yl + path + rst, version)) except OSError as e: logger.exception('%s: Problem while updating builddir contents: %s' % (inspect.stack()[0][3], str(e))) return False return True
def builddir_structure(param_dict, builddir, version): """ Summary. - Updates paths in binary exectuable - Updates Args: :root (str): full path to root directory of the git project :builddir (str): name of current build directory which we need to populate Vars: :lib_path (str): src path to library modules in project root :builddir_path (str): dst path to root of the current build directory (/<path>/xlines-1.X.X dir) Returns: Success | Failure, TYPE: bool """ def _mapper(venv_dir): """Identifies path to python modules in virtual env""" for i in (6, 7, 8, 9): path = venv_dir + '/lib/python3.' + str(i) + '/site-packages/' if os.path.exists(path): return path def module_search(module, packages_path): t = [] for i in os.listdir(packages_path): if re.search(module, i, re.IGNORECASE): t.append(i) return t root = git_root() project_dirname = os.path.split(git_root())[1] build_root = TMPDIR # files binary = param_dict['Executable'] control_file = param_dict['ControlFile']['Name'] compfile = param_dict['BashCompletion'] # LIB source files env = os.environ.get('VIRTUAL_ENV') or root #lib_src = _mapper(env) lib_src = os.path.join(root, PROJECT_BIN) # full paths builddir_path = build_root + '/' + builddir deb_src = root + '/packaging/deb' debian_dir = 'DEBIAN' debian_path = deb_src + '/' + debian_dir binary_path = builddir_path + '/usr/bin' lib_dst = builddir_path + '/usr/lib/python3/dist-packages' comp_src = os.path.join(root, 'bash') comp_dst = builddir_path + '/etc/bash_completion.d' try: # create build directory if os.path.exists(builddir_path): rmtree(builddir_path) os.makedirs(builddir_path) stdout_message(message='Created:\t{}'.format(yl + builddir_path + rst), prefix='OK') stdout_message( f'Copying DEBIAN package control files to {bn + builddir + rst}') _src = os.path.join(deb_src, debian_dir) _dst = os.path.join(builddir_path, debian_dir) copytree(_src, _dst) operation_status(_src, _dst) stdout_message( f'Creating build directory subdirectories in {bn + builddir + rst}' ) # binary exec if not os.path.exists(binary_path): os.makedirs(binary_path) _src_path = os.path.join(deb_src, 'bin', 'xlines') _dst_path = os.path.join(binary_path, 'xlines') copyfile(_src_path, _dst_path) # status msg stdout_message(message='Copied:\t{} {} {}'.format( lk + _src_path + rst, arrow, lk + _dst_path + rst), prefix='OK') # library components if not os.path.exists(lib_dst): os.makedirs(lib_dst) _src = lib_src _dst = os.path.join(lib_dst, PROJECT_BIN) copytree(_src, _dst) stdout_message(message='Copied:\t{} {} {}'.format( lk + _src + rst, arrow, lk + _dst + rst), prefix='OK') if not os.path.exists(comp_dst): # create path os.makedirs(comp_dst) # copy for artifact in list( filter(lambda x: x.endswith('.bash'), os.listdir(comp_src))): _src = comp_src + '/' + artifact _dst = comp_dst + '/' + artifact copyfile(_src, _dst) stdout_message(message='Copied:\t{} {} {}'.format( lk + _src + rst, arrow, lk + _dst + rst), prefix='OK') stdout_message(f'Creating config subdirectory in {lk + lib_dst + rst}') os.makedirs(os.path.join(lib_dst, PROJECT_BIN, 'config')) source = os.path.join(root, 'config') for file in list( filter(lambda x: x.endswith('.list'), os.listdir(source))): _src = os.path.join(source, file) _dst = os.path.join(lib_dst, PROJECT_BIN, 'config', file) copyfile(_src, _dst) except OSError as e: logger.exception('{}: Problem creating dirs on local fs'.format( inspect.stack()[0][3])) return False return True