def pathRead(fpaths): ''' Try to read a file path or or lst of file paths. :params: fpath[s] [str][list] - path[s] to check :return: path[s] read [str] or [list] ''' fig = None _fpaths = _fmtResFiles(fpaths) if _fpaths and isinstance(_fpaths, list): fig = [] for resfile in _fpaths: try: filepath = getTrueFilePath(resfile) fig.append(filepath) except IOError as exi: msg = "\n add file to archive manually:\n" + \ "bad file | path or can't parse {}".format(resfile) partialError(exi, msg) elif _fpaths and isinstance(_fpaths, (str, unicode)): fig = None fig = getTrueFilePath(_fpaths) return fig
def uniqueZips(zips, zipsExisting): zipex = list(zipsExisting) zips = list(set(zips)) #unique addZips = list(zips) for zfile in zips: for zex in zipex: if zex: zf = oprel(zfile).replace('\\','/').replace('../','') zx = zex if zf == zx: try: addZips.remove(zfile) except ValueError as ex: log.warn('\nUnable to trim duplicate zip additions' + \ '\nzip: {}\narchive {}'.format(zf, zx)) msg = '\nUnable to trim duplicate zip additions' + \ '\nzip: {}\narchive {}'.format(zf, zx) partialError(ex,msg) continue continue if addZips and gsBuild.Verbose or not gsBuild.INFO: log.info(('\n addZips:\n' + '{}\n'*len(addZips)).format(*addZips)) return addZips
def _subprocMove(moveCmd, source, dest, dotExt): rpath = dest if dotExt: rpath = rpath + dotExt if os.path.isfile(rpath): try: os.remove(rpath) except IOError as ex: pass po = None try: po = subprocess.check_output(moveCmd) except subprocess.CalledProcessError as ex: try: msg = str(ex) raise IOError(msg) except IOError as exc: msg = 'Move failed File | Filepath \n{} ' \ '- access denied or does not exist:\n\t{}' \ .format(source, ex.message) partialError(exc, msg) if 'moved' not in str(po): log.info('\nRelative path resolution Error:\n {} \nmoving to {}' \ .format(source, rpath)) else: if op.isfile(rpath): log.FILE('Build Moved: {}'.format(rpath))
def openResListFiles(rpath): ''' #TODO always partial error in final recursive read as #this last try read the text content and not a valid path. #Even a regx match errors on "someClass.run" because it #looks like a path Check if path is existing file or partial error :filters: .dll and .exe files to not read :param: rpath [str] - a file path :return: [str] - path if isfile :raise: [IOError] sent to partialError ''' rpath = pathRead(rpath) resfiles = None resFiles = None ignoreExt = ['.dll', '.py', '.sh', '.exe', '.bat', '.config'] # arg exists as file if all(ext not in rpath.lower() for ext in ignoreExt): with open(rpath, 'r') as txtr: # errors on bad file | path resFiles = txtr.readlines() # chk list or str try: # None if not path if resFiles: resfiles = pathRead(resFiles) except IOError as ex: rname = os.path.basename(rpath) msg = '\nPossible err(s)' + \ '\n\tCheck if Bad File | Path *or* - \n' + \ '\n\t"OK" if err shows content text and not file path.' + \ '\n\t - Tried to read recursively into file.' if resFiles: msg = ('\n Possible err(s) in:\n "{}": ' + \ '\n - retrieved:\n' + '"\n\t{}"'*len(resFiles[:4]) + \ '\n\tCheck if Bad File | Path *or* - \n' + \ '\n\t"OK" if err shows "{rn:}"'+ \ ' content text and not file path.' \ '\n\t - Tried to read recursively into '+ \ ' "{rn:}"\n\t - searching' + \ ' for files.').format(rpath, *resFiles[:4], rn=rname) partialError(ex, msg) if resfiles: return resfiles return pathRead(rpath)
def checkRequiredIP(): if gsBuild.IPATH: return gsBuild.HAVELIB gsBuild.HAVELIBDLL ironPythonPath = FindIronPython() if opex(ironPythonPath): gsBuild.IPATH = ironPythonPath userReqBaseLst = os.listdir(ironPythonPath) userReqLst = [ opn(opj(ironPythonPath, urf)) for urf in os.listdir(ironPythonPath) ] if opex(opj(ironPythonPath, 'Lib')): log.FILE('Exists: {}'.format(opj(ironPythonPath, 'Lib'))) gsBuild.HAVELIB = True # TODO check for downloaded StdLib if op.isfile(opn(opj(ironPythonPath, 'StdLib.dll'))): gsBuild.HAVELIBDLL = True if not all(rf in userReqBaseLst for rf in DefaultReqList): try: raise NotImplementedError except NotImplementedError as ex: msg = 'Failed to find all required IronPython Distribution files' partialError(ex, msg) log.FILE('\n Exists required path {}'.format( opj(os.getcwd(), gsBuild.requiredPath))) WD = os.getcwd() if 'Tests' in WD: WD = opd(WD) with open(opj(os.getcwd(), gsBuild.requiredPath), 'w') as tw: for f in userReqLst: relp = opn(op.relpath(f)) # skip dirs if op.isfile(relp): tw.write(opn(os.path.relpath(f)) + '\n') log.FILE('Exists {}'.format(gsBuild.requiredPath)) return
def checkRequired(path): ''' Read file list of required files and check they if exist or partial error. :param: - path [str] - of file to read list - tag= [opt][str] - writelog tag for filtering :return: list of files **not** found - should be empty [list] :raise: partialError - sink errors for stdout in PartialErrors [list] ''' tag = 'Exists' reqpath = getTrueFilePath(path) amsg = 'need {} file'.format(reqpath) assert os.path.isfile(reqpath), amsg with open(reqpath, 'r') as txtr: reqfiles = txtr.readlines() log.FILE('{} {}'.format(tag, path)) reqfs = list(reqfiles) if gsBuild.Verbose: log.info('\ngsBuild.IPATH:\n {}'.format(gsBuild.IPATH)) if gsBuild.IPATH != 'clr': for f in reqfiles: if gsBuild.IPATH and gsBuild.IPATH not in f: fr = opn(opj(gsBuild.IPATH, opb(f))) else: fr = f try: fp = getTrueFilePath(fr.strip()) log.FILE('{} {}'.format(tag, fp)) reqfs.remove(f) except IOError as ex: log.debug('raise partial') partialError(ex, ex.message) continue return reqfs elif gsBuild.IPATH == 'clr': return []
def checkRequired(path): ''' Read file list of required files and check they if exist or partial error. :param: - path [str] - of file to read list - tag= [opt][str] - writelog tag for filtering :return: list of files **not** found - should be empty [list] :raise: partialError - sink errors for stdout in PartialErrors [list] ''' tag = 'Exists' reqpath = getTrueFilePath(path) amsg = 'need {} file'.format(reqpath) assert os.path.isfile(reqpath), amsg with open(reqpath, 'r') as txtr: reqfiles = txtr.readlines() log.FILE('{} {}'.format(tag, path)) reqfs = list(reqfiles) for f in reqfiles: try: fp = getTrueFilePath(f.strip()) log.FILE('{} {}'.format(tag, fp)) reqfs.remove(f) except IOError as ex: log.debug('raise partial') partialError(ex, ex.message) continue return reqfs
def CheckConfig(uconfig): ''' Check for valid config entries after parse/loading :param: config Dict from LoadConfig [OrderedDict] :return: userconfig [config Dict] - as written to file path shown in writefiles.log :raise: FatalError or partialError [Exception] or [error sink list] *user can re-run ipybuild on a partialError fix* *why FatalError - a user messed with defaults - error vs program error* ''' userfig = ndcDict(uconfig) if not len([val for val in userfig if val or val != '']) >= 1: raise ValueError('need at least one parameter for config') if not userfig['MAKEDIR'] or not opex(opd(userfig['MAKEDIR'])): raise NameError('bad path to config file {}' \ .format(opd(userfig['MAKEDIR']))) if not all(k in userfig.keys() for k in defaultconfig.keys()): log.error('\n keys {}\n\t{}'.format(str(userfig.keys()), \ str(defaultconfig.keys()))) raise FatalError('KeyError', 'bad userconfig key set') if not userfig['MAINFILE'] or userfig['MAINFILE'] == '': raise FatalError('NameError', 'need main name.py') if not '.py' in userfig['MAINFILE'] or not opex(userfig['MAINFILE']): try: raise NameError except NameError as ex: msg = ' main name "{}" must be .py file and exist' \ .format(userfig['MAINFILE']) partialError(ex, msg) if not isinstance(userfig['MAKEEXE'], bool) and \ str(userfig['MAKEEXE']).upper() not in ['TRUE', 'FALSE']: try: raise ValueError except ValueError as ex: msg = 'makeEXE {}: need type bool in params (no key needed)' \ .format(userfig['MAKEEXE']) partialError(ex, msg) if not opex(userfig['JSONPATH']): try: raise NameError except NameError as ex: msg = 'bad path to assembly.json file "{}"' \ .format(userfig['JSONPATH']) partialError(ex, msg) if not opex(userfig['CONFIGPATH']): try: raise NameError except NameError as ex: msg = 'bad path to assembly.json file "{}"' \ .format(userfig['CONFIGPATH']) partialError(ex, msg)
def AssignMakeZip(uconfig, args): config = ndcDict(uconfig) outzipPath = None cfz = config['ZIPPATH'] if args['listzip']: zfiles = [] # missing zipfiles in listzip.txt or input zips = loadRes(getTrueFilePath(args['listzip'])) zpl = None uzips = None # unique zips if zips: zpl = len(zips) # zip exists with cur name if have zipped before isautozip = False try: import zlib mode = zipfile.ZIP_DEFLATED except ImportError: mode = zipfile.ZIP_STORED config['LISTFILES']['zip'] = [] if isinstance(zips, list): config['LISTFILES']['zip'].extend(zips) else: config['LISTFILES']['zip'].append(zips) # set outzip path manf = 'default.zip' if config['MAINFILE']: manf = ('.').join(opb(config['MAINFILE']).split('.')[:-1]) + '.zip' outzipPath = opj(config['OUTDIR'], manf) # stop trying to overwrite same file # current zip path cfzp = None if cfz: try: cfzp = getTrueFilePath(cfz) if opex(cfzp): isautozip = True except IOError: pass # auto zip path elif outzipPath: try: cfzp = getTrueFilePath(outzipPath) if opex(cfzp): isautozip = True except IOError: pass # update zip path config['ZIPPATH'] = cfzp cfz = cfzp # -- zipping --- # uzips if isautozip: infoExistZip(cfz, outzipPath) log.FILE('Confirmed: {}'.format(cfzp)) zipsRelative = [] for zipname in zips: relname = None relname = _getRelativeZipName(zipname) if relname: zipsRelative.append(relname) else: zipsRelative.append(zipname) with zipfile.ZipFile(cfzp, 'a', mode) as ziprd: uzips = list(uniqueZips(zipsRelative, list(ziprd.namelist()))) for zfile in uzips: if not os.path.isfile(zfile): zfiles.append(zfile) continue arcname = _getRelativeZipName(zfile) if not arcname: arcname = oprel(zfile) ziprd.write(zfile, arcname) ziprd.close() # changed if uzips if uzips: if gsBuild.Verbose or not gsBuild.INFO: log.info(('\nSome Files already zipped in:\n{}\n\t' + \ '- delete to replace existing' + \ '\nadding zip files to existing archive:\n' + \ '{}\n'*len(uzips)) \ .format(cfz, *uzips)) # Need new zip with ZIPPATH/outzipPath as name elif not isautozip: warnZip(outzipPath) if isinstance(zips, list): with zipfile.ZipFile(cfz, 'a', mode) as zipr: for zfile in list(set(zips)): if not os.path.isfile(zfile): zfiles.append(zfile) continue arcname = _getRelativeZipName(zfile) if not arcname: arcname = oprel(zfile) zipr.write(zfile, arcname) log.FILE('{}'.format(config['ZIPPATH'])) zipr.close() if isinstance(zips, (str, unicode)): with zipfile.ZipFile(cfz, 'w', mode) as zipr: arcname = oprel(zips) zipr.write(zips, arcname) zipr.close() log.FILE('{}'.format(cfz)) if zfiles: log.warn(('\nFile | path does not exist - ' +\ 'skipped adding zip files:\n\t' + \ '{} \n\t'*len(zfiles)).format(*zfiles)) log.FILE(('*Missing zip: {}\n'*len(zfiles)).format(*zfiles)) partialError('ValueError', ('*Missing zip: {}\n'*len(zfiles)).format(*zfiles)) return ndcDict(config)
def _setCompilerClass(rconfig): config = ndcDict(rconfig) f_standalone = None f_embed = None f_libembed = None f_parerr = None # f_parerr - Stop build on partialError that is Fatal to Compile # Let makeBuild finish so that user can fix partialErrs with open(config['CONFIGPATH'], 'r') as jbcr: config = ndcDict(json.load(jbcr)) if gsBuild.Verbose or not gsBuild.INFO: log.info('\n Building from CONFIG:\n {}\n'.format( json.dumps(config, indent=4))) if not opex(config['MAINFILE']): try: raise IOError except IOError as ex: msg = 'File Filepath does Not exist:\n "{}"' \ .format(config['MAINFILE']) partialError(ex, msg) f_parerr = True if not f_parerr: log.FILE('Build Loaded: {}'.format(config['MAINFILE'])) assemInfo = config['ASSEMBLY'] if isinstance(assemInfo['standalone'], bool) or \ str(assemInfo['standalone']).upper() in ['TRUE', 'FALSE']: f_standalone = True if str(assemInfo['standalone']).upper() == \ 'TRUE' else False if isinstance(assemInfo['embed'], bool) or \ str(assemInfo['embed']).upper() in ['TRUE', 'FALSE']: f_embed = True if str(assemInfo['embed']).upper() == 'TRUE' else False if isinstance(assemInfo['libembed'], bool) or \ str(assemInfo['libembed']).upper() in ['TRUE', 'FALSE']: f_libembed = True if str(assemInfo['libembed']).upper() \ == 'TRUE' else False ext = '.dll' if config['MAKEEXE'] == True or \ str(config['MAKEEXE']).upper() == 'TRUE': ext = '.exe' if f_standalone and not config['MAKEEXE']: log.warn('\n** Switching to exe /stanalone == true in Assembly:' + \ '\n {}\n Overrides default or makeEXE input arg == False' \ .format(config['JSONPATH'])) MAINOUT = opn(opj(config['OUTDIR'], ('.').join(opb(config['MAINFILE']) \ .split('.')[:-1])) + ext) IPATH = gsBuild.IPATH STDLIBSOURCE = opabs(opj(IPATH, 'StdLib.dll')) LIBPATH = opabs(opj(IPATH, 'Lib')) compiler.pycpath = (opn(opd(opabs(gsBuild.IPYBLDPATH)))) + '\pyc.py' compiler.stdlibsource = STDLIBSOURCE compiler.ipath = IPATH compiler.libpath = LIBPATH if not op.isfile(STDLIBSOURCE): _createStdLib() MAINOUTDIR = ('.').join(MAINOUT.split('.')[:-1]) PYCDIR = opn(opj(os.getcwd(), opb(MAINOUTDIR)) + ext) STDLIBRELEASE = opj(opd(MAINOUTDIR), 'StdLib.dll') MAINFILE = config['MAINFILE'] isLib = opex(LIBPATH) isStdLib = op.isfile(STDLIBSOURCE) haveStdLib = op.isfile(opj(os.getcwd(), 'StdLib.dll')) isReleasedStdLib = op.isfile(STDLIBRELEASE) lstdll = [] if config['LISTFILES']['dll']: if isinstance(config['LISTFILES']['dll'], list): for lfile in config['LISTFILES']['dll']: if lfile and '__init__' not in lfile: lstdll.append(lfile) else: lstdll.append(config['LISTFILES']['dll']) lstexe = [] if config['LISTFILES']['exe']: if isinstance(config['LISTFILES']['exe'], list): for xfile in config['LISTFILES']['exe']: if xfile and '__init__' not in xfile: lstexe.append(xfile) else: lstexe.append(config['LISTFILES']['exe']) lstexe = nullList(lstexe) compiler.f_standalone = f_standalone compiler.f_embed = f_embed compiler.f_libembed = f_libembed compiler.f_parerr = f_parerr compiler.mainout = MAINOUT compiler.ipath = IPATH compiler.mainoutdir = MAINOUTDIR compiler.pycdir = PYCDIR compiler.stdlibrelease = STDLIBRELEASE compiler.stdlibsource = STDLIBSOURCE compiler.libpath = LIBPATH compiler.mainfile = MAINFILE compiler.isLib = isLib compiler.isStdLib = isStdLib compiler.haveStdLib = haveStdLib compiler.isReleasedStdLib = isReleasedStdLib compiler.lstdll = lstdll compiler.lstexe = lstexe compiler.ext = ext compiler.lstexedlls = None if not opex(opd(compiler.pycdir)): raise IOError('FilePath {}:\t Use absolute or relative to:\n\t {}' \ .format(opd(compiler.pycdir), os.getcwd())) if compiler.f_standalone: if gsBuild.Verbose or not gsBuild.INFO: log.info('\nNew {} compile standalone from:\n {}' \ .format(ext.upper().replace('.', ''), config['MAINFILE'])) else: mfn = 'application/lib' if config['MAINFILE']: mfn = opb(config['MAINFILE']) if gsBuild.Verbose or not gsBuild.INFO: log.info(("\nNew {} compile from: \n {}" + \ "\n\tAs Required: add your {}, project, and ironpython"+ \ "\n\tdll(s) to path:\n\t{}\n\n") .format(ext.upper().replace('.', ''), config['MAINFILE'], mfn, config['OUTDIR'])) if gsBuild.Verbose or not gsBuild.INFO: log.info('\n Lib source path {}'.format(LIBPATH)) log.info('\n "IF" set "True", f_libembed adds ~23mb to file:' + \ 'now set as {}'.format(compiler.f_libembed)) if compiler.f_libembed and compiler.isStdLib: if gsBuild.Verbose or not gsBuild.INFO: if compiler.isReleasedStdLib: log.info('\nOK - "StdLib.dll" exists delete'+ \ ' or move to update:\n{}'.format(STDLIBRELEASE)) else: log.info('\nOK - "StdLib.dll" exists delete'+ \ ' or move to update:\n{}'.format(STDLIBSOURCE)) elif not compiler.isStdLib and compiler.f_libembed and \ not compiler.isReleasedStdLib and compiler.isLib: _createStdLib() if not compiler.isStdLib: raise NotImplementedError('StdLib: Need ironpython2.7 distribution' + \ ' in something like ../ironpython path')
def _subprocMove(moveCmd, source, dest, dotExt): po = None rpath = dest if dotExt: rpath = rpath + dotExt clr = None # keep out global clr try: import clr except Exception as ex: pass if clr: try: clr.AddReference("System") import System except Exception as ex: pass if System.IO.File.Exists(rpath): if System.IO.File.Exists(rpath): System.IO.File.Delete(rpath) if System.IO.File.Exists(source): try: # log.error('\n Try noving:\n\t{}\n To:\n\t\n\t{}' \ # .format(source,rpath)) System.IO.File.Move(source,rpath) except Exception as ex: print('Failed to Move:\n\t{}'.format(source)) print(ex) elif not clr: if os.path.isfile(rpath): try: os.remove(rpath) except IOError as ex: pass if os.path.isfile(source): try: # log.error('\n Try noving:\n\t{}\nTo:\n\t\n{}' \ # .format(source,dest)) po = subprocess.check_output(moveCmd) except subprocess.CalledProcessError as ex: try: msg = str(ex) raise IOError(msg) except IOError as exc: msg = 'Move failed File | Filepath \n{} ' \ '- access denied or does not exist:\n\t{}' \ .format(source, ex.message) partialError(exc, msg) if po and 'moved' not in str(po): log.info('\nRelative path resolution Error:\n {} \nmoving to {}' \ .format(source, rpath)) elif (clr or po) and op.isfile(rpath): if gsBuild.Verbose: log.info('Build Moved: {}'.format(rpath)) log.FILE('Build Moved: {}'.format(rpath))