def uninstall(): """ uninstall BEE2.4 """ path = config.load('beePath').replace('/BEE2.exe', '') logger.info('removing BEE2.4!') logger.info('deleting app files..') utilities.removeDir(path) config.save(None, 'beePath') logger.info('app files deleted!')
def main(path, outPath, module): # create lock curr_py_path = os.path.realpath( __file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split( curr_py_path) # current folder and file - abs path curr_py_dir_arr = curr_py_dir.split(os.path.sep) try: os.makedirs(os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock')) except OSError: return 1 recDate = utilities.getNowDateTimeString() recDate = recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8] for dirName, subdirList, fileList in os.walk(curr_py_dir): myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - curr_py_dir.count(os.path.sep) if depth == 0: for f in fileList: if string.find(f, '.edw') > -1: tokenFile = os.path.join(dirName, f) #retToken = utilities.getTokenInfo( tokenFile, False ) #if retToken['mktToken'] == 'PEN' and retToken['dbStatusTable'] == 'umts_process_status': ''' if len( retToken['mktToken'] ) == 2 or len( retToken['mktToken'] ) == 3: val = utilities.dbStatusUpdater( 'new', retToken, 'NODATA_' + recDate + '.zip', recDate, '' ) val = utilities.dbStatusUpdater( 'insert', retToken, 'NODATA_' + recDate + '.zip', recDate, 'NODATA' ) ''' print ( '[{}] LINE FILE -- [{}] [{}] '\ .format(curr_py_filename, str(depth), f ) ) #print ( cmd = ('python {} {} {} {}'.format( os.path.join( curr_py_dir, 'ttssinfointerimPackager-SpecificGroup.py'), path, outPath, f[:f.find('.')])) ret = utilities.subprocessShellExecute(cmd) os.remove(os.path.join(dirName, f)) break # remove lock utilities.removeDir(os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock')) return 0
def main(path, module): # create lock curr_py_path = os.path.realpath( __file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split( curr_py_path) # current folder and file - abs path try: os.makedirs(os.path.sep + os.path.join('tmp', curr_py_filename + '.lock')) except OSError: return 1 recDate = utilities.getNowDateTimeString() recDate = recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8] for dirName, subdirList, fileList in os.walk(path): myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) if depth == 0: for f in fileList: if string.find(f, module) > -1: tokenFile = os.path.join(dirName, f) retToken = utilities.getTokenInfo(tokenFile, False) #if retToken['mktToken'] == 'PEN' and retToken['dbStatusTable'] == 'umts_process_status': if len(retToken['mktToken']) == 2 or len( retToken['mktToken']) == 3: val = utilities.dbStatusUpdater( 'new', retToken, 'NODATA_' + recDate + '.zip', recDate, '') val = utilities.dbStatusUpdater( 'insert', retToken, 'NODATA_' + recDate + '.zip', recDate, 'NODATA') print ( '[{}] LINE TOKEN -- [{}] [{}] [{}] [{}]'\ .format(curr_py_filename, str(depth), retToken['dbSchema'], retToken['mktToken'] , retToken['dbStatusTable']) ) break # remove lock utilities.removeDir(os.path.sep + os.path.join('tmp', curr_py_filename + '.lock')) return 0
def main(path): # create lock curr_py_path = os.path.realpath( __file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split( curr_py_path) # current folder and file - abs path try: os.makedirs(os.path.sep + 'tmp' + os.path.sep + curr_py_filename + '.lock') except OSError: return 1 utilities.copyFile(curr_py_dir + os.path.sep + 'MarketAbbr.cfg', path) ''' cfgParser = ConfigParser.ConfigParser() # The following will tell cfgParserParser to keep case-sensitive names cfgParser.optionxform = str ''' tech = '' region = '' market = '' vendor = '' oFile = '' mktCnt = 1 # NEED TO BE DELETED & MOVED' + os.path.sep + 'UNCOMMENT BELOW mAbbrFile = ConfigParser.ConfigParser() mAbbrFile.read(path + os.path.sep + 'MarketAbbr.cfg') # Vendor & Tech specifics here for dirName, subdirList, fileList in os.walk(path): myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) #print ( 'LINE 165 -- ' + '[' + str(depth) + ']' + '[' + dirName + ']' ) if depth == 1: if (len(myArr[len(myArr) - 1].split('-')) > 1): tech = myArr[len(myArr) - 1].split('-')[1] else: tech = myArr[len(myArr) - 1] #mktCnt = 1 ''' if depth == 1: # RAWFILES vs SITEFILES folders ''' if depth == 3: region = myArr[len(myArr) - 1] if depth == 4: market = myArr[len(myArr) - 1] try: marketAbbr = mAbbrFile.get('DEFAULT', market) except (ConfigParser.NoOptionError): marketAbbr = market if mktCnt != 1: cfgParser.write(cfgFile) cfgFile.close() ziph.write(path + os.path.sep + 'Market.ini', 'Market.ini') ziph.close() #cfgFile = open(path + os.path.sep + 'Market' + str(mktCnt) + '.ini', 'w') cfgFile = open(path + os.path.sep + 'Market.ini', 'w') mktCnt += 1 cfgParser = ConfigParser.ConfigParser() # The following will tell cfgParserParser to keep case-sensitive names cfgParser.optionxform = str print('Found directory: %s %s' % (tech + '-' + region + '-' + market + '-' + myArr[len(myArr) - 1], str(depth))) #print( str( datetime.datetime.today() ) + ' ' + str( datetime.datetime.now().time() ) ) try: # add the settings to the structure of the file, and lets write it out... # Note to self: for DEFAULT section name, no need to add the section, it is by default there cfgParser.set('DEFAULT', 'Num', 0) cfgParser.set('DEFAULT', "Count", 0) cfgParser.set('DEFAULT', 'Tech', '') cfgParser.set('DEFAULT', 'Region', '') cfgParser.set('DEFAULT', 'Market', '') cfgParser.set('DEFAULT', 'VendorCount', 0) cfgParser.set('DEFAULT', 'VendorN', '') cfgParser.add_section('main') cfgParser.set('main', 'Company', 'TMO') cfgParser.set('main', 'Tech', tech) cfgParser.set('main', 'Region', region) cfgParser.set('main', 'Market', market) cfgParser.set('main', 'VendorCount', '') except (ConfigParser.DuplicateSectionError): dummy = 1 i = 0 for myDir in subdirList: i += 1 vendor = myDir try: cfgParser.add_section('checklist_' + vendor) cfgParser.add_section('filelist_' + vendor) cfgParser.add_section('bsc_' + vendor) except (ConfigParser.DuplicateSectionError): dummy = 1 cfgParser.set('bsc_' + vendor, 'Num', '') cfgParser.set('main', 'Vendor' + str(i), myDir) setStaticInfo( cfgParser, tech, vendor ) # Call a method to set static info for the config file cfgParser.set('main', 'VendorCount', str(i)) ziph = zipfile.ZipFile( path + os.path.sep + tech + 'SINFO_' + ('MULTIVENDOR' if i > 1 else vendor) + '_' + re.sub('[-]', '', str(datetime.datetime.today())[:10]) + '_' + re.sub('[:.]', '', str(datetime.datetime.now().time())[:12]) + '_TMO-' + marketAbbr + '_Result.zip', 'w', zipfile.ZIP_DEFLATED) for dirs, subdirs, files in os.walk( string.replace(dirName[:string.find(dirName, region)], 'RAWFILES', 'SITEFILES')): for f in files: # Getting the right Site' + os.path.sep + 'Atoll file for the market if string.find(f, '-' + market + '.csv') > -1: fileNm = string.upper( string.replace(f, '-' + market, '')) fileNm = fileNm[:string. find(fileNm, '.')] + string.lower( fileNm[string.find(fileNm, '.'):]) # print( 'LINE 249: ' + dirs + os.path.sep + f + ' - [' + fileNm + ']' ) ziph.write(dirs + os.path.sep + f, fileNm) ''' for dirs, subdirs, files in os.walk( path + os.path.sep + 'TOKENs' ): for f in files: # Getting the right Token file for the market if string.find( f, market + '_' + tech + 'SINFO_Token.txt' ) > -1: ziph.write( dirs + os.path.sep + f, 'Token.txt' ) #NEED TO BE CHANGED print( 'LINE 257: ' + dirs + os.path.sep + f ) ''' if os.path.exists(path + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + marketAbbr + '_' + tech + 'SINFO_Token.txt'): ziph.write( path + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + marketAbbr + '_' + tech + 'SINFO_Token.txt', 'Token.txt') elif os.path.exists(path + os.path.sep + 'TOKENs' + os.path.sep + market + '_' + tech + 'SINFO_Token.txt'): ziph.write( path + os.path.sep + 'TOKENs' + os.path.sep + market + '_' + tech + 'SINFO_Token.txt', 'Token.txt') if depth == 5: i = 0 for myDir in subdirList: i += 1 cfgParser.set('bsc_' + myArr[len(myArr) - 1], 'bsc' + str(i), myDir) cfgParser.set('bsc_' + myArr[len(myArr) - 1], 'Num', str(i)) if depth == 6: for fname in fileList: ziph.write( os.path.join(dirName, fname), os.path.join(myArr[len(myArr) - 4], myArr[len(myArr) - 3], myArr[len(myArr) - 2], myArr[len(myArr) - 1], fname)) cfgParser.write(cfgFile) cfgFile.close() ziph.write(path + os.path.sep + 'Market.ini', 'Market.ini') ziph.close() # remove lock utilities.removeDir(os.path.sep + 'tmp' + os.path.sep + curr_py_filename + '.lock') return 0
def main(path, lZone, module): # create lock curr_py_path = os.path.realpath( __file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split( curr_py_path) # current folder and file - abs path try: os.makedirs(os.path.sep + 'tmp' + os.path.sep + curr_py_filename + '.lock') except OSError: return 1 for dirName, subdirList, fileList in os.walk(path): #myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) if depth == 0: for fname in fileList: if string.find(fname, module) > -1 and string.find( fname, '.zip') > -1: try: myFile = dirName + os.path.sep + fname os.rename(myFile, myFile) zip_ref = zipfile.ZipFile(myFile, 'r') for f in zip_ref.namelist(): # The if-statement below handles incoming zip file that were zipped from Windows OS # else-statement is for default extraction ''' if f.count(os.path.sep) == 0: myDir = lZone + os.path.sep + os.path.split(string.replace( f, '\\', os.path.sep ))[0] xFile = os.path.split(string.replace( f, '\\', os.path.sep ))[1] try: os.makedirs( myDir ) os.chmod( myDir, 0o777 ) except OSError: pass zip_ref.extract( f, myDir ) #print( myDir + os.path.sep + f + ' ---> ' + myDir + os.path.sep + xFile ) os.rename( myDir + os.path.sep + f, myDir + os.path.sep + xFile ) ''' if string.find(f, 'db_load_status') > -1: zip_ref.extract(f, lZone) if string.find(f, 'Token') > -1: zip_ref.extract(f, lZone) # Read Token file retToken = utilities.getTokenInfo( lZone + os.path.sep + f, False) break zip_ref.close() try: retToken except NameError: continue else: print('Moving file: ' + myFile + ' ---> ' + lZone + os.path.sep + fname) os.rename(myFile, lZone + os.path.sep + fname) cmd = curr_py_dir + os.path.sep + 'bin' + os.path.sep + 'ttssinfo_db.sh \"{}\" \"{}\" \"{}\" \"{}\" \"{}\" \"{}\" \"{}\"'.format( lZone, curr_py_dir + os.path.sep + '..' + os.path.sep + 'archive', string.lower( retToken['dbSchema']), retToken['dbHost'], '3306', retToken['dbUser'], retToken['dbPwd']) print(cmd) ret = subprocessShellExecute(cmd) exec_str = 'python ' + curr_py_dir + os.path.sep + 'db2csv.py -H {} -P {} -d {} -u {} -p {} -l \"{}\" \"{}db2csv_{}_{}_sinfolookup.ini\" \"{}\" 2>&1'.format( retToken['dbHost'], '3306', retToken['dbSchema'], retToken['dbUser'], retToken['dbPwd'], "db2csv.log", curr_py_dir + os.path.sep, string.lower(retToken['techToken']), string.lower(retToken['carr']), lZone) print(exec_str) print "[%s] Dumping lookup files...\n%s\n" % ( time.strftime("%Y-%m-%d %H:%M:%S"), exec_str) sys.stdout.flush( ) # flush buffer before executing next line retcode = os.system(exec_str) if (retcode == 0): print "[%s] Dumping lookup files completed.\n" % ( time.strftime("%Y-%m-%d %H:%M:%S")) if retToken['techToken'] == 'LTE': lteplanfile = 's_info_' + retToken[ 'techToken'] + '_' + retToken[ 'mktToken'] + '_' + retToken[ 'carr'] + '.zip' ziph = zipfile.ZipFile( lZone + os.path.sep + lteplanfile, 'w', zipfile.ZIP_DEFLATED) zip_ref = zipfile.ZipFile( lZone + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + 'archive' + os.path.sep + fname, 'r') zip_ref.extract('Token.txt', lZone) zip_ref.close() for dir, subdirlist, filelist in os.walk( lZone): for f in filelist: if string.find(f, 'Lookup') > -1: #cmd = 'mv -f ' + lZone + os.path.sep + f + ' ' + lZone + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + 'LandingZone' + os.path.sep + retToken['carr'] + os.path.sep + retToken['mktToken'] + '_' + f #print( cmd ) #ret = subprocessShellExecute( cmd ) os.rename( lZone + os.path.sep + f, lZone + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + 'LandingZone' + os.path.sep + retToken['carr'] + os.path.sep + retToken['mktToken'] + '_' + f) else: if retToken[ 'techToken'] == 'LTE' and string.find( f, '.txt') > -1: ziph.write( lZone + os.path.sep + f, f) try: lteplanfile except NameError: continue else: os.rename( lZone + os.path.sep + lteplanfile, lZone + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + 'lteplan' + os.path.sep + 'mover1' + os.path.sep + lteplanfile) else: print "[%s] Dumping lookup files failed with return code %d\n" % ( time.strftime("%Y-%m-%d %H:%M:%S"), retcode) #cleanupAndExit(4, work_location) ''' if ret['ret']: runidList = ret['output'].split('\n') for rid in runidList: break # skip header if not rid == 'runid' and not rid == '': deleteDataSQL = 'call delete_planau_lte({});'.format(rid) deleteCmd = "mysql -h {} -u {} -p{} -D {} -e \"{}\"".format(retToken['dbHost'], retToken['dbUser'], retToken['dbPwd'], retToken['dbSchema'], deleteDataSQL) ret = subprocessShellExecute(deleteCmd) if ret['ret']: deleteDataSQL = 'delete from {} where runid={};'.format(retToken['dbAuditTable'], rid) deleteCmd = "mysql -h {} -u {} -p{} -D {} -e \"{}\"".format(retToken['dbHost'], retToken['dbUser'], retToken['dbPwd'], retToken['dbSchema'], deleteDataSQL) ret = subprocessShellExecute(deleteCmd) if not ret['ret']: bDeleteError = True break else: bDeleteError = True break else: bDeleteError = True ''' #break except: print '{} cannot be accessed.'.format(myFile) continue # remove lock utilities.removeDir(os.path.sep + 'tmp' + os.path.sep + curr_py_filename + '.lock') return 0
def main(path, lZone, module): # create lock curr_py_path = os.path.realpath(__file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split(curr_py_path) # current folder and file - abs path curr_py_dir_arr = curr_py_dir.split(os.path.sep) try: os.makedirs( os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock') ) except OSError: return 1 utilities.copyFile( os.path.join(curr_py_dir, 'MarketAbbr.cfg'), path ) ''' cfgParser = ConfigParser.ConfigParser() # The following will tell cfgParserParser to keep case-sensitive names cfgParser.optionxform = str ''' tech = '' region = '' market = '' vendor = '' oFile = '' mktCnt = 1 # NEED TO BE DELETED & MOVED' + os.path.sep + 'UNCOMMENT BELOW mAbbrFile = ConfigParser.ConfigParser() mAbbrFile.read( os.path.join(path, 'MarketAbbr.cfg') ) # Crawl through MarketAbbr.cfg to find markets need to be combined based on mAbbrFile & include TEMPLATE folder for dirName, subdirList, fileList in os.walk( path ): myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) #print ( 'LINE 166 -- ' + '[' + str(depth) + ']' + '[' + dirName + ']' ) if depth == 1: if (len(myArr[ len(myArr) - 1 ].split('-')) > 1): tech = myArr[ len(myArr) - 1 ].split('-')[1] else: tech = myArr[ len(myArr) - 1 ] #mktCnt = 1 ''' ''' if depth == 2: # RAWFILES & SITEFILES & TEMPLATE folders if myArr[ len(myArr) - 1 ] == 'SITEFILES': for market, marketAbbr in mAbbrFile.items( 'DEFAULT' ): if string.find( marketAbbr, '|' ) > -1: marketAbbr = marketAbbr[:string.find(marketAbbr, '|')] for f in fileList: mktName = string.upper(string.replace(market, '.', '')) if string.find( f, mktName + '.csv' ) > -1: # Getting the right Site/Atoll file for the market #print ( 'LINE MarketMapping - [{}] [{}] [{}]'.format( mktName, marketAbbr, f ) ) tFile = os.path.join( dirName, string.replace(f, mktName, marketAbbr) ) #print( 'LINE tFile -- [{}] - [{}]'.format( mktName, tFile ) ) if utilities.fileExist( tFile ): #print( cmd = ( 'awk \'FNR > 1\' {} >> {}'.format( os.path.join(dirName, f), tFile ) ) else: #print( cmd = ( 'cat {} > {}'.format( os.path.join(dirName, f), tFile ) ) ret = utilities.subprocessShellExecute( cmd ) if depth == 3: region = myArr[ len(myArr) - 1 ] regionPath = dirName if depth == 4: market = myArr[ len(myArr) - 1 ] try: marketAbbr = mAbbrFile.get( 'DEFAULT', market ) if string.find( marketAbbr, '|' ) > -1: isPartOfBiggerMkt = True marketAbbr = marketAbbr[:string.find( marketAbbr, '|' )] combinedMktPath = os.path.join(regionPath, marketAbbr) else: isPartOfBiggerMkt = False combinedMktPath = dirName except (ConfigParser.NoOptionError): marketAbbr = market if depth == 5: # Vendor Level vendor = myArr[ len(myArr) - 1 ] if isPartOfBiggerMkt: for subdir in subdirList: try: os.makedirs( #print( 'LINE 204 -- ' + os.path.join(combinedMktPath, vendor, market+'-'+subdir) ) except OSError: dummy = 1 #print ( 'LINE Copy subdir -- ' + cmd = ( 'cp -p -r {} {}'\ .format( os.path.join(dirName, subdir, '*'), os.path.join(combinedMktPath, vendor, market+'-'+subdir) ) ) ret = utilities.subprocessShellExecute( cmd ) #print ( 'LINE Remove folder -- ' + cmd = ( 'rm -rf {}'\ .format( os.path.join(regionPath, market) ) ) ret = utilities.subprocessShellExecute( cmd ) #break #print ( 'LINE Copy templates -- ' + cmd = ( 'cp -p -r {} {}'\ .format( os.path.join(curr_py_dir, 'TEMPLATE_' + tech + '_' + vendor), os.path.join(combinedMktPath, vendor) ) ) ret = utilities.subprocessShellExecute( cmd ) ''' ''' # Vendor & Tech specifics here for dirName, subdirList, fileList in os.walk( path ): myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) #print ( 'LINE 165 -- ' + '[' + str(depth) + ']' + '[' + dirName + ']' ) #break if depth == 1: if (len(myArr[ len(myArr) - 1 ].split('-')) > 1): tech = myArr[ len(myArr) - 1 ].split('-')[1] else: tech = myArr[ len(myArr) - 1 ] #mktCnt = 1 ''' if depth == 2: # RAWFILES & SITEFILES & TEMPLATE folders ''' if depth == 3: region = myArr[ len(myArr) - 1 ] if depth == 4: market = myArr[ len(myArr) - 1 ] try: marketAbbr = mAbbrFile.get( 'DEFAULT', market ) if string.find( marketAbbr, ':' ) > -1: marketAbbr = marketAbbr[:string.find(marketAbbr, ':')] except (ConfigParser.NoOptionError): marketAbbr = market if mktCnt != 1: cfgParser.write( cfgFile ) cfgFile.close() ziph.write( os.path.join(path, 'Market.ini'), 'Market.ini' ) ziph.close() #cfgFile = open(path + os.path.sep + 'Market' + str(mktCnt) + '.ini', 'w') cfgFile = open(os.path.join(path, 'Market.ini'), 'w') mktCnt += 1 cfgParser = ConfigParser.ConfigParser() # The following will tell cfgParserParser to keep case-sensitive names cfgParser.optionxform = str print( 'Found directory: %s %s' % (tech + '-' + region + '-' + market + '-' + myArr[ len(myArr) - 1 ], str(depth)) ) #print( str( datetime.datetime.today() ) + ' ' + str( datetime.datetime.now().time() ) ) try: # add the settings to the structure of the file, and lets write it out... # Note to self: for DEFAULT section name, no need to add the section, it is by default there cfgParser.set( 'DEFAULT', 'Num', 0 ) cfgParser.set( 'DEFAULT', "Count", 0 ) cfgParser.set( 'DEFAULT', 'Tech', '' ) cfgParser.set( 'DEFAULT', 'Region', '' ) cfgParser.set( 'DEFAULT', 'Market', '' ) cfgParser.set( 'DEFAULT', 'VendorCount', 0 ) cfgParser.set( 'DEFAULT', 'VendorN', '' ) cfgParser.add_section( 'main' ) cfgParser.set( 'main', 'Company', 'TMO' ) cfgParser.set( 'main', 'Tech', tech ) cfgParser.set( 'main', 'Region', region ) cfgParser.set( 'main', 'Market', market ) cfgParser.set( 'main', 'VendorCount', '' ) except (ConfigParser.DuplicateSectionError): dummy = 1 i = 0 for myDir in subdirList: i+=1 vendor = myDir try: cfgParser.add_section( 'checklist_' + vendor ) cfgParser.add_section( 'bypasslist_' + vendor ) cfgParser.add_section( 'filelist_' + vendor ) cfgParser.add_section( 'bsc_' + vendor ) except (ConfigParser.DuplicateSectionError): dummy = 1 cfgParser.set( 'bsc_' + vendor, 'Num', '' ) cfgParser.set( 'main', 'Vendor' + str(i), myDir ) setStaticInfo( cfgParser, tech, vendor ) # Call a method to set static info for the config file cfgParser.set( 'main', 'VendorCount', str(i) ) ziph = zipfile.ZipFile( os.path.join((lZone if lZone != '' else path), tech + 'SINFO_' + ('MULTIVENDOR' if i>1 else vendor) + '_' + re.sub( '[-]', '', str( datetime.datetime.today() )[:10] ) + '_' + re.sub( '[:.]', '', str( datetime.datetime.now().time() )[:12] ) + '_TMO-' + marketAbbr + '_Result.zip'), 'w', zipfile.ZIP_DEFLATED ) for dirs, subdirs, files in os.walk( string.replace( dirName[:string.find( dirName, region )], 'RAWFILES', 'SITEFILES' ) ): for f in files: # Getting the right Site/Atoll file for the market if string.find( f, '-' + string.replace(market, '.', '') + '.csv' ) > -1: fileNm = string.upper( string.replace( f, '-' + string.replace(market, '.', ''), '' ) ) fileNm = fileNm[:string.find(fileNm, '.')] + string.lower( fileNm[string.find(fileNm, '.'):] ) print( 'LINE Site file: ' + os.path.join(dirs, f + ' - [' + fileNm + ']') ) ziph.write( os.path.join(dirs, f), fileNm ) ''' for dirs, subdirs, files in os.walk( path + os.path.sep + 'TOKENs' ): for f in files: # Getting the right Token file for the market if string.find( f, market + '_' + tech + 'SINFO_Token.txt' ) > -1: ziph.write( dirs + os.path.sep + f, 'Token.txt' ) #NEED TO BE CHANGED print( 'LINE 257: ' + dirs + os.path.sep + f ) ''' tokenFile = os.path.join(path, '..', '..', marketAbbr + '_' + tech + 'SINFO_Token.txt') if os.path.exists( tokenFile ): ziph.write( tokenFile, 'Token.txt' ) else: tokenFile = os.path.join(path, 'TOKENs', market + '_' + tech + 'SINFO_Token.txt') if os.path.exists( tokenFile ): ziph.write( tokenFile, 'Token.txt' ) if depth == 5: i = 0 for myDir in subdirList: i+=1 cfgParser.set( 'bsc_' + myArr[ len(myArr) - 1 ], 'bsc' + str(i), myDir ) cfgParser.set( 'bsc_' + myArr[ len(myArr) - 1 ], 'Num', str(i) ) if depth == 6: for fname in fileList: ziph.write( os.path.join( dirName, fname ), os.path.join( myArr[ len(myArr) - 4 ], myArr[ len(myArr) - 3 ], myArr[ len(myArr) - 2 ], myArr[ len(myArr) - 1 ], fname ) ) try: cfgFile except NameError: dummy = 1 else: cfgParser.write( cfgFile ) cfgFile.close() ziph.write( os.path.join(path, 'Market.ini'), 'Market.ini' ) ziph.close() # remove lock utilities.removeDir( os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock') ) return 0
def main(path, lZone, module): # create lock curr_py_path = os.path.realpath( __file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split( curr_py_path) # current folder and file - abs path curr_py_dir_arr = curr_py_dir.split(os.path.sep) try: os.makedirs(os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock')) except OSError: return 1 for dirName, subdirList, fileList in os.walk(path): #myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) if depth == 0: for fname in fileList: if string.find(fname, module) > -1 and string.find( fname, '.zip') > -1: try: myFile = dirName + os.path.sep + fname os.rename(myFile, myFile) zip_ref = zipfile.ZipFile(myFile, 'r') for f in zip_ref.namelist(): # The if-statement below handles incoming zip file that were zipped from Windows OS # else-statement is for default extraction ''' if f.count(os.path.sep) == 0: myDir = lZone + os.path.sep + os.path.split(string.replace( f, '\\', os.path.sep ))[0] xFile = os.path.split(string.replace( f, '\\', os.path.sep ))[1] try: os.makedirs( myDir ) os.chmod( myDir, 0o777 ) except OSError: pass zip_ref.extract( f, myDir ) #print( myDir + os.path.sep + f + ' ---> ' + myDir + os.path.sep + xFile ) os.rename( myDir + os.path.sep + f, myDir + os.path.sep + xFile ) ''' if string.find(f, 'Token') > -1: zip_ref.extract(f, lZone) # Read Token file retToken = utilities.getTokenInfo( lZone + os.path.sep + f, False) break zip_ref.close() try: retToken except NameError: continue else: recDate = utilities.getNowDateTimeString() val = utilities.dbStatusUpdater( 'new', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], '') val = utilities.dbStatusUpdater( 'insert', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'NEW') print('Moving file: ' + myFile + ' ---> ' + lZone + os.path.sep + fname) os.rename(myFile, lZone + os.path.sep + fname) #cmd = 'python ' + curr_py_dir + os.path.sep + 'bin/sinfo_loader.py -H {} -u {} -p{} -P {} --dbname {} \"\" \"\" \"\" \"\" >> output.log 2>&1'.format('InterimDB', retToken['dbUser'], retToken['dbPwd'], '3306', string.lower(retToken['dbSchema']) + '_temp') cmd = 'python ' + curr_py_dir + os.path.sep + 'bin/sinfo_loader.py -H {} -u {} -p{} -P {} --dbname {} \"\" \"\" {} \"\" >> {}output.log 2>&1'\ .format('interim-db-01', 'root', 'tts1234', '3306', string.lower(retToken['dbSchema']) + '_temp', curr_py_dir + os.path.sep + '..' + os.path.sep + 'archive/', curr_py_dir + os.path.sep) # .format('interim-db-01', 'root', 'tts1234', '3306', string.lower(retToken['dbSchema']) + '_temp', os.path.join(curr_py_dir, '..', 'archive'), curr_py_dir + os.path.sep) ret = subprocessShellExecute(cmd) print(cmd) val = utilities.dbStatusUpdater( 'update', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'QUEUED') ''' if ret['ret']: runidList = ret['output'].split('\n') for rid in runidList: break # skip header if not rid == 'runid' and not rid == '': deleteDataSQL = 'call delete_planau_lte({});'.format(rid) deleteCmd = "mysql -h {} -u {} -p{} -D {} -e \"{}\"".format(retToken['dbHost'], retToken['dbUser'], retToken['dbPwd'], retToken['dbSchema'], deleteDataSQL) ret = subprocessShellExecute(deleteCmd) if ret['ret']: deleteDataSQL = 'delete from {} where runid={};'.format(retToken['dbAuditTable'], rid) deleteCmd = "mysql -h {} -u {} -p{} -D {} -e \"{}\"".format(retToken['dbHost'], retToken['dbUser'], retToken['dbPwd'], retToken['dbSchema'], deleteDataSQL) ret = subprocessShellExecute(deleteCmd) if not ret['ret']: bDeleteError = True break else: bDeleteError = True break else: bDeleteError = True ''' #break except: print '{} cannot be accessed.'.format(myFile) continue ret = subprocessShellExecute('rm -f ' + lZone + os.path.sep + '*') # remove lock utilities.removeDir(os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock')) return 0
def main(path, lZone, module): # create lock curr_py_path = os.path.realpath( __file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split( curr_py_path) # current folder and file - abs path curr_py_dir_arr = curr_py_dir.split(os.path.sep) try: os.makedirs(os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock')) except OSError: return 1 for dirName, subdirList, fileList in os.walk(path): #myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) if depth == 0: for fname in fileList: if string.find(fname, module) > -1 and string.find( fname, '.zip') > -1: try: myFile = dirName + os.path.sep + fname os.rename(myFile, myFile) zip_ref = zipfile.ZipFile(myFile, 'r') ret = subprocessShellExecute('rm -f ' + lZone + os.path.sep + '*') for f in zip_ref.namelist(): if string.find(f, 'db_load_status') > -1: zip_ref.extract(f, lZone) if string.find(f, 'Token') > -1: zip_ref.extract(f, lZone) # Read Token file retToken = utilities.getTokenInfo( lZone + os.path.sep + f, False) break zip_ref.close() try: retToken except NameError: continue else: recDate = utilities.getNowDateTimeString() val = utilities.dbStatusUpdater( 'update', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'LOADING') #, '[DateOfData-' + ) print('Moving file: ' + myFile + ' ---> ' + lZone + os.path.sep + fname) os.rename(myFile, lZone + os.path.sep + fname) cmd = curr_py_dir + os.path.sep + 'bin' + os.path.sep + 'ttssinfo_db.sh \"{}\" \"{}\" \"{}\" \"{}\" \"{}\" \"{}\" \"{}\"'.format( lZone, curr_py_dir + os.path.sep + '..' + os.path.sep + 'archive', string.lower( retToken['dbSchema']), retToken['dbHost'], '3306', retToken['dbUser'], retToken['dbPwd']) print(cmd) ret = subprocessShellExecute(cmd) exec_str = 'python ' + curr_py_dir + os.path.sep + 'db2csv.py -H {} -P {} -d {} -u {} -p {} -l \"{}\" \"{}db2csv_{}_{}_sinfolookup.ini\" \"{}\" 2>&1'.format( retToken['dbHost'], '3306', retToken['dbSchema'], retToken['dbUser'], retToken['dbPwd'], "db2csv.log", curr_py_dir + os.path.sep, string.lower(retToken['techToken']), string.lower(retToken['carr']), lZone) print(exec_str) print "[%s] Dumping lookup files...\n%s\n" % ( time.strftime("%Y-%m-%d %H:%M:%S"), exec_str) sys.stdout.flush( ) # flush buffer before executing next line #retcode = os.system(exec_str) ret = subprocessShellExecute(exec_str) #if (retcode == 0): if ret['ret']: val = utilities.dbStatusUpdater( 'update', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'DONE') print "[%s] Dumping lookup files completed.\n" % ( time.strftime("%Y-%m-%d %H:%M:%S")) if retToken['techToken'] == 'LTE': lteplanfile = os.path.join( lZone, 's_info_' + retToken['techToken'] + '_' + retToken['mktToken'] + '_' + retToken['carr'] + '.zip') ziph = zipfile.ZipFile( lteplanfile, 'w', zipfile.ZIP_DEFLATED) zip_ref = zipfile.ZipFile( lZone + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + 'archive' + os.path.sep + fname, 'r') zip_ref.extract('Token.txt', lZone) zip_ref.close() for dir, subdirlist, filelist in os.walk( lZone): for f in filelist: if string.find(f, 'Lookup') > -1: os.rename( lZone + os.path.sep + f, lZone + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + '..' + os.path.sep + 'LandingZone' + os.path.sep + retToken['carr'] + os.path.sep + retToken['mktToken'] + '_' + f) else: if retToken[ 'techToken'] == 'LTE' and string.find( f, '.txt') > -1: ziph.write( lZone + os.path.sep + f, f) try: lteplanfile except NameError: continue else: ziph.close() # Copy zip file to web server, for AdHOC LTEPLAN adhoclteplanfile = os.path.join( lZone, 's_info_' + retToken['techToken'] + '.zip') cmd = 'cp -p {} {}'.format( lteplanfile, adhoclteplanfile) print('LINE Creating adhoclteplanfile: ' + cmd) ret = subprocessShellExecute(cmd) mainschema = 'optpcs_{}'.format( retToken['carr']) getwebserveripquery = "select web_ipaddress From markets where suffix = '{}'".format( retToken['mktToken']) getwebserveripcmd = "mysql -h {} -u {} -p{} -D {} -e \"{}\""\ .format(retToken['dbHost'], retToken['dbUser'], retToken['dbPwd'], mainschema , getwebserveripquery) ret = subprocessShellExecute( getwebserveripcmd) webip = '' if ret['ret']: out = ret['output'] try: webip = out.split('\n')[1].strip( '\n').strip('\r') except: print "wrong machineip information: {}".format( out) pass print webip if webip == '': val = utilities.dbStatusUpdater( 'warning', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'WARNING: no web ip present') websinfopath = '/var/www/html/{}/{}/resources/sinfo'.format( string.lower(retToken['carr']), string.lower(retToken['mktToken'])) print websinfopath # hard code test ret = utilities.checkRemoteDirExist( webip, 'imnosrf', 'tts1234', websinfopath, True) if not ret['ret']: print 'create remote dir failed: {}'.format( ret) val = utilities.dbStatusUpdater( 'warning', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'WARNING: create remote dir failed' ) ret = utilities.copyFileToRemote( webip, 'imnosrf', 'tts1234', adhoclteplanfile, websinfopath) if not ret['ret']: print 'copy sinfo to webserver failed' val = utilities.dbStatusUpdater( 'warning', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'WARNING: copy sinfo to web failed' ) cmd = "python {}ho_dump.py {} {} {} {} {} {} {} {} {}"\ .format( curr_py_dir+os.path.sep, retToken['carr'], retToken['mktToken'] ,recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], retToken['techToken'], 3 ,retToken['dbHost'], retToken['dbUser'], retToken['dbPwd'], '3306' ) print(cmd) ret = subprocessShellExecute(cmd) if not ret['ret']: val = utilities.dbStatusUpdater( 'warning', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'WARNING: ho_dump.py failed - API submission error - ' + ret['msg']) pprint(ret) if 'apiret' in ret: apiretObj = json.loads(ret['apiret']) if not apiretObj['success']: val = utilities.dbStatusUpdater( 'warning', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'WARNING: ho_dump.py failed - ' + ret['apiret']['msg']) cmd = "python {}clusterrecovery.py {} {} {} {} {} {}"\ .format( curr_py_dir+os.path.sep, retToken['carr'], retToken['mktToken'] ,retToken['dbHost'], retToken['dbUser'], retToken['dbPwd'], '3306' ) print(cmd) ret = subprocessShellExecute(cmd) if not ret['ret']: val = utilities.dbStatusUpdater( 'warning', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'WARNING: clusterrecovery.py failed' ) lteplanfileFINAL = os.path.join( os.path.sep + curr_py_dir_arr[1], curr_py_dir_arr[2], 'lteplan', 'mover1', string.replace( os.path.split(lteplanfile)[1], '.zip', '')) print('Moving file: ' + lteplanfile + ' ---> ' + lteplanfileFINAL + '.zip') try: os.rename(lteplanfile, lteplanfileFINAL + '.tmp') os.rename(lteplanfileFINAL + '.tmp', lteplanfileFINAL + '.zip') except OSError: print 'Renaming {} to {} failed.'.format( lteplanfile, lteplanfileFINAL + '.tmp') val = utilities.dbStatusUpdater( 'warning', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'WARNING: plan-audit submission failed' ) continue else: val = utilities.dbStatusUpdater( 'update', retToken, fname, recDate[:4] + '-' + recDate[4:6] + '-' + recDate[6:8], 'FAILED', 'Dumping lookup files failed with return code ' ) print "[%s] Dumping lookup files failed with return code %d\n" % ( time.strftime("%Y-%m-%d %H:%M:%S"), retcode) ret = subprocessShellExecute('rm -f ' + lZone + os.path.sep + '*') #break except OSError: print '{} cannot be accessed.'.format(myFile) continue # remove lock utilities.removeDir(os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock')) return 0
def main(path, lZone, module): # create lock curr_py_path = os.path.realpath(__file__) # current running file - abs path curr_py_dir, curr_py_filename = os.path.split(curr_py_path) # current folder and file - abs path curr_py_dir_arr = curr_py_dir.split(os.path.sep) try: os.makedirs( os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock') ) except OSError: return 1 for dirName, subdirList, fileList in os.walk( path ): #myArr = dirName.split(os.path.sep) depth = dirName.count(os.path.sep) - path.count(os.path.sep) if depth == 0: for fname in fileList: if string.find( fname, module ) > -1: fnameWithPath = os.path.join( dirName, fname ) print ( 'LINE File Found: {}'.format(fnameWithPath) ) try: os.rename( fnameWithPath, fnameWithPath ) except OSError: print '{} cannot be accessed.'.format(fname) continue tech = fname.split('_')[1] print ( 'LINE TECH: {}'.format(tech) ) if string.find( fname, 'ATOLL' ) > -1: #print ( cmd = ( ' python {} N {} {} \'\' \'\' '.format( os.path.join(curr_py_dir, 'ttssinfoEDWSplitter.py') , fnameWithPath, os.path.join(lZone, 'TMO_'+tech+'_ERIC-'+tech, 'SITEFILES') ) ) ret = utilities.subprocessShellExecute( cmd ) #print ( cmd = ( ' python {} N {} {} \'\' \'\' '.format( os.path.join(curr_py_dir, 'ttssinfoEDWSplitter.py') , fnameWithPath, os.path.join(lZone, 'TMO_'+tech+'_NOKIA-'+tech, 'SITEFILES') ) ) ret = utilities.subprocessShellExecute( cmd ) else: vendor = fname.split('_')[2] #print ( cmd = ( ' python {} Y {} {} NATIONAL {} '.format( os.path.join(curr_py_dir, 'ttssinfoEDWSplitter.py') , fnameWithPath, os.path.join(lZone, 'TMO_'+tech+'_'+vendor+'-'+tech, 'RAWFILES') , ('ERICSSON' if vendor == 'ERIC' else vendor) ) ) ret = utilities.subprocessShellExecute( cmd ) utilities.touch( os.path.join(curr_py_dir, 'TMO_'+tech+'_'+vendor+'-'+tech+'.edw') ) print ( 'LINE Archiving File: {} to {}'.format(fnameWithPath, fnameWithPath.replace( 'data', 'archive' )) ) try: os.rename( fnameWithPath, fnameWithPath.replace( 'data', 'archive' ) ) except OSError: print '{} cannot be accessed.'.format(fname) continue ''' ''' break; # remove lock utilities.removeDir( os.path.sep + os.path.join( 'tmp', curr_py_dir_arr[len(curr_py_dir_arr) - 1] + '-' + curr_py_filename + '.lock') ) return 0