def create_main_log_file(self):
path = self.logs_dir_path + 'xegtor.log'
file = open(path ,'w')
file.close()
# create log header
logger = Logger(log_file=path)
logger.add_log_header()
def __check_for_update(self) -> None:
'''
Check if any new updates are available.
'''
self.core_framework_location = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../../../'))
self.last_update_file_location = self.core_framework_location + '/source/core/python/update_logs/last_update_date.log'
self.versions_file_location = self.core_framework_location + '/source/core/python/update_logs/versions.json'
if not os.path.isfile(self.last_update_file_location):
with open(self.last_update_file_location, 'w'):
Logger().log('>>> Created empty last_updated.log')
if not os.path.isfile(self.versions_file_location):
with open(self.versions_file_location, 'w'):
Logger().log('>>> Created empty versions.json')
with open(self.last_update_file_location) as last_update_date:
last_update_date = last_update_date.read()
if last_update_date == '' or last_update_date != datetime.now(
).strftime('%b %d %Y'):
Logger().log('>>> Updating.')
if self.__update():
Logger().log('>>> Update was successful.')
else:
Logger().log('>>> Update date is not reached yet.')
def start( self ):
for index, news in enumerate( self.news_list, start=0 ):
try:
if news['status'] == 'pending':
news_content = self.download_news( news )
if news_content:
self.news_list[ index ]['status'] = 'completed'
self.news.append( news_content )
log.success('[ {nid} ] Dados salvos com sucesso!'.format(nid=news['id']))
print()
print()
else:
error_message = 'Não foi possível fazer o parse dos dados.'
log.error( error_message )
self.errors.append( error_message )
self.news_list[ index ]['errors'].append( error_message )
else:
log.warning('Dados já adquiridos [ {nid} ]'.format(nid=news['id']))
except Exception as error:
log.error('Erro ao baixar a notícia [ {nid} ]'.format(nid=news['id']))
log.error(error)
pass
finally:
helper.create_file( filename=self.dump_file, content=self.news, format='json', mode='w')
helper.create_file( filename=self.news_json_file, content=self.news_list, format='json', mode='w')
def __init__(self,logfile,debugmode):
global encode_logger
if debugmode == True:
encode_logger = Logger(logfile,logging.DEBUG,logging.INFO)
else:
encode_logger = Logger(logfile,logging.INFO,logging.INFO)
encode_logger.debug('Loading Encode Module')
def __init__(self, logfile, srcarray, srcuser, srcpasswd, srcscope):
global navi_logger
global array
global user
global password
global scope
navi_logger = Logger(logfile, logging.DEBUG, logging.INFO)
navi_logger.debug('Loading NaviCLI Module')
array = srcarray
user = srcuser
password = srcpasswd
scope = srcscope
def validate_env_variables(self) -> None:
try:
os.environ['SUDO_UID']
except KeyError:
Logger().log('Please start the application using "sudo -E".')
for env_variable in ENVIRONMENTAL_VARIABLES:
try:
os.environ[env_variable]
except KeyError:
Logger().log('Environmental variable {} is not set.'.format(
env_variable))
def download(type, filename, nid, url):
if type == 'image':
try:
response = requests.get(url, stream=True)
with open(filename, 'wb') as image:
shutil.copyfileobj(response.raw, image)
log.success('Imagem baixada com sucesso [{url}]'.format(url=url))
return True
except Exception as error:
log.error(error)
pass
def create_file(filename, content, mode='a+', encoding='utf-8', format=None): os.chdir(sys.path[0]) try: if format == 'json': content = json.dumps(content, indent=4, sort_keys=True) _file = open(filename, mode=mode, encoding=encoding) _file.write(content) _file.close() except Exception as error: log.error( error ) pass
class MonthToIntegerConverter(object):
def __init__(self):
self.logger = Logger()
self.months = [
'NoneToMakeNumberNotStupid', 'Jan', 'Feb', 'Mar', 'Apr', 'May',
'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'
]
def get_month(self, month):
try:
return self.months.index(month)
except Exception as e:
self.logger.error(
'Error converting month string to integer value: {}'.format(e))
class Applications(object):
def __init__(self, user, password):
self.logger = Logger()
self.xml_parser = xml_parser.XmlParser()
self.user = user
self.password = password
return
def get_apps(self):
apps = []
try:
self.logger.info('Attempting to get applications!!!')
resp = requests.post(endpoints.REQUEST_APPS,
auth=HTTPBasicAuth(self.user, self.password))
if resp.status_code == 200:
response = self.xml_parser.parse_xml(resp.text)
for item in response:
if item.attrib['app_id'] is not None and not item.attrib[
'app_id'] in apps:
apps.append({
'app_id': item.attrib['app_id'],
'app_name': item.attrib['app_name'],
'sandboxes': []
})
self.logger.info('Successfully retrieved app list!')
else:
self.logger.error('Authentication Issue: {}'.format(
resp.status_code))
except Exception as e:
self.logger.exception('Error retrieving app list: {}'.format(e))
return apps
def run_openface(configs, dataset):
text_file = open("./logs/bp4d_failure_20200724193802.txt", "r")
lines = text_file.readlines()
lines = [i.split(':')[3].strip(' ') for i in lines]
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S')
d = configs['paths']['input']
logger = Logger(f'./logs/{dataset}_success_{st}.txt',
f'./logs/{dataset}_failure_{st}.txt')
for dir, subdir, files in os.walk(d):
if not subdir:
if configs['params']['type'] == 'frames':
subject = dir.split('/')[-2]
task = dir.split('/')[-1]
fname = f"{subject}_{task}"
if fname in lines:
print(
f" extracting for Subject {dir.split('/')[-2]} task {dir.split('/')[-1]}..."
)
status, msg = extract_data(configs, dir)
if status == -1:
logger.log_failure(
dir.split('/')[-2], 'frames',
dir.split('/')[-1], msg)
print("failed!")
else:
logger.log_success(
dir.split('/')[-2], 'frames',
dir.split('/')[-1])
print("done!")
else:
continue
elif configs['params']['type'] == 'videos':
for file in files:
if os.path.splitext(file)[-1] == configs['params']['ext']:
subject = os.path.join(dir, file)
print(
f"extracting for Subject {subject.split('/')[-1]}...."
)
status, msg = extract_data(configs, subject)
if status == -1:
logger.log_failure(
dir.split('/')[-1], 'videos', msg)
print("failed!")
else:
logger.log_success(dir.split('/')[-1], 'videos')
print("done!")
print("openface extraction complete!")
def __init__(self,logfile,debugmode,global_XMS_IP,global_XMS_USER,global_XMS_PASS):
global rest_logger
global XMS_IP
global XMS_USERID
global XMS_PASS
if debugmode == True:
rest_logger = Logger(logfile,logging.DEBUG,logging.INFO)
else:
rest_logger = Logger(logfile,logging.INFO,logging.INFO)
rest_logger.debug('Loading Restful Module')
XMS_IP = global_XMS_IP
XMS_USERID = global_XMS_USER
XMS_PASS = global_XMS_PASS
class LCS(object):
"""main class"""
logger = Logger()
op_sys = OperatingSystem()
networking = Networking()
services = Services()
printers = Printers()
environments = Environments()
def __init__(self):
self.main()
def main(self):
"""
main function for starting up scanner
"""
self.logger.normal_output("Running linux security check")
self.run()
def run(self):
"""
default run method
"""
self.op_sys.run()
self.networking.run()
self.services.run()
self.environments.run()
class ExecuteCommand(object):
"""
Command execution object
"""
logger = Logger()
def __init__(self):
return
def execute_command(self, command_text):
"""
Execute a command using subprocess.check_output
"""
try:
return check_output(command_text.split(" "))
except (OSError, CalledProcessError) as error:
self.logger.error("Error processing command: " + command_text + " " + error.output)
return
def thing(self):
"""
test
"""
return self
def start_thread(mode, db_name, qid, task, info, tq_rank, log_dir=None):
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.read('config.ini')
db = Database(config.get('database', 'user'), config.get('database', 'pw'), config.get('database', 'host'), db_name, config.get('database', 'cache_dir'), timeout=config.get('database', 'timeout'), buffer_pool_size=config.get('database', 'buffer_pool_size'))
parser = SQLParser(db_name, config.get('parser', 'cache_dir'))
# only load aig if info includes range
aig = None
if (mode == 'greedybb' or mode == 'greedyfirst') and info == 'range':
aig = AIG(db, os.path.join(config.get('aig', 'dir'), db_name + '.aig'))
data = json.loads(task)
task_cleaned = {
'cqs': {},
'ans': []
}
for cqid, cq in data['cqs'].items():
task_cleaned['cqs'][int(cqid)] = cq
for cqid in data['ans']:
task_cleaned['ans'].append(int(cqid))
task = task_cleaned
if log_dir:
log_path = os.path.join(log_dir, str(qid) + '.log')
with Logger(log_path):
return run_task(mode, db, parser, qid, task, info, aig, tq_rank)
else:
return run_task(mode, db, parser, qid, task, info, aig, tq_rank)
def set_image( news, index, link ):
images_file = 'data/images.json'
images = helper.read_file( images_file, format='json' ) if os.path.isfile( images_file ) else []
try:
images.append({
'catalog': news['catalog'],
'notice': news['id'],
'downloaded': False,
'original_path': link,
'new_path': set_image_link( news, index, link )
})
helper.create_file(images_file, images, mode='w', format='json')
log.success('Imagem adicionada para a lista de downloads [ {image_link} ]'.format(image_link=set_image_link( news, index, link )))
except Exception as error:
log.error( error )
def read_file(filename, format=None, mode='r', encoding='utf-8'): os.chdir(sys.path[0]) if not os.path.isfile(filename): return None else: try: _file = open( filename, mode=mode, encoding=encoding ) _content = _file.read() _file.close() if format == 'json': return json.loads( _content ) else: return _content except Exception as error: log.error( error.args[0] ) pass
def __init__(self,logfile,srcarray,srcuser,srcpasswd,srcscope):
global navi_logger
global array
global user
global password
global scope
navi_logger = Logger(
logfile,
logging.DEBUG,
logging.INFO
)
navi_logger.debug(
'Loading NaviCLI Module'
)
array = srcarray
user = srcuser
password = srcpasswd
scope = srcscope
def __init__(self):
self.logger = Logger()
user = os.getenv('VC_USER')
password = os.getenv('VC_PASS')
if not self.should_run(user, password):
self.logger.error('User or password is not defined, exiting...')
sys.exit()
self.sandboxes = Sandboxes(user, password)
self.applications = Applications(user, password)
self.builds = Builds(user, password)
self.spreadsheet_creator = SpreadsheetCreator()
apps = self.applications.get_apps()
self.get_app_sandboxes(apps)
self.get_app_sandbox_builds(apps)
self.get_builds(apps)
self.create_spreadsheet(apps)
self.send_message()
def main(self) -> None:
try:
if len(sys.argv) == 1:
Logger().log('File requires at least one console argument.')
else:
Logger().log('Checking for updates...')
self.__check_for_update()
if sys.argv[1].lower() == 'gui':
if sys.argv[2].lower() == '--normal':
gui_elements.show_gui(False)
else:
gui_elements.show_gui(True)
elif sys.argv[1].lower() == 'nogui':
usb_detector = USBHotplugDetector()
usb_detector.start()
except KeyboardInterrupt:
Logger().log('Keyboard interrupt detected.')
def checkLog(connector):
# (↓) [-get forgotten biases-]
biases = Logger().getLog(LogTypes.FORGOTTEN_BIASES)
# (↓) [-delete all forgotten biases-]
for biasId in biases:
# (↓) [-delete forgotten bias-]
try:
connector.manager(actionType=ActionTypes.DELETE_BIAS,
biasId=biasId)
except:
checkLog(connector)
def get_value(fields, field_id, warn=False, prod_fields=None):
prod_fields = prod_fields or fields
if field_id in fields:
return fields[field_id]
else:
if warn:
warning_text = prod_fields["ARTNR"]
warning_text += ": Kein Wert für das Feld '"
warning_text += field_id
warning_text += "'. Das Feld in der Tabelle bleibt leer."
Logger().log(warning_text)
return ""
def get_welcome_text(prod_fields, ilugg_fields):
max_placeholders = 5
welcome_text = ""
if not "ARTWELCOMESTATE" in prod_fields:
Logger().log("[WARNUNG] {} hat kein ARTWELCOMESTATE Feld".format(
prod_fields["ARTNR"]))
return welcome_text
welcome_state = prod_fields["ARTWELCOMESTATE"]
if welcome_state == "0":
if not "WELCOMETHISTEXT" in prod_fields:
Logger().log(
"[WARNUNG] {} mit ARTWELCOMESTATE 0 hat kein WELCOMETHISTEXT Feld"
.format(prod_fields["ARTNR"]))
return welcome_text
welcome_text = prod_fields["WELCOMETHISTEXT"]
if welcome_state == "1":
index = 1
while index <= max_placeholders:
welcome_text = welcome_text + placeholder(index)
index += 1
if welcome_state == "2":
if not "WELCOMETEXT" in prod_fields:
Logger().log(
"[WARNUNG] {} mit ARTWELCOMESTATE 2 hat kein WELCOMETEXT Feld".
format(prod_fields["ARTNR"]))
return welcome_text
welcome_text = prod_fields["WELCOMETEXT"]
index = 1
while index <= max_placeholders:
welcome_text = welcome_text.replace(placeholder(index),
replacement(ilugg_fields, index))
index += 1
return welcome_text
def __init__(self, logfile, debugmode):
global encode_logger
if debugmode == True:
encode_logger = Logger(logfile, logging.DEBUG, logging.INFO)
else:
encode_logger = Logger(logfile, logging.INFO, logging.INFO)
encode_logger.debug('Loading Encode Module')
class SpreadsheetCreator(object):
def __init__(self):
self.have_data = False
self.logger = Logger()
self.filename = '{}VeracodeSandboxResults-{}.xlsx'.format(
os.getenv('VC_ATTACHMENT_BASE_PATH'), date.today())
return
def write_spreadsheet(self, apps):
try:
workbook = xlsxwriter.Workbook(self.filename)
worksheet = workbook.add_worksheet('Aggregated Build Data')
bold = workbook.add_format({'bold': True})
worksheet.write('A1', 'User', bold)
worksheet.write('B1', 'Total Builds', bold)
row = 1
col = 0
data = {}
if len(apps) > 0:
for app in apps:
for sandbox in app['sandboxes']:
for build in sandbox['builds']:
if not build['submitter'] in data:
data[build['submitter']] = 0
data[build['submitter']] += 1
if (len(data) > 0):
self.have_data = True
for user, number_of_builds in data.items():
worksheet.write(row, col, user)
worksheet.write(row, col + 1, number_of_builds)
row += 1
else:
raise 'Unable to create spreadsheet, no apps found in data'
workbook.close()
except Exception as e:
self.logger.exception('Error creating spreadsheet: {}'.format(e))
def get_factor(prod_fields, prod_field, ilugg_fields, ilugg_field):
prod_definition = prod_fields[prod_field]
ilugg_definition = ilugg_fields[ilugg_field]
factor_category = prod_definition.split(":")[0]
factor_definitions = ilugg_definition.split("§")
factor = None
for factor_definition in factor_definitions:
if factor_definition.startswith("{}:".format(factor_category)):
factor = factor_definition.split(":")[1]
break
if factor == None:
logger = Logger()
factor = prod_definition
if ":" in factor:
factor = factor.split(":")[1]
log_text = "{}: Faktor zur Preisberechnung".format(
prod_fields[ARTICLE_NUMBER])
log_text += " konnte nicht bestimmt werden."
log_text += " {} in PROD ist '{}',".format(prod_field, prod_definition)
log_text += " {} in ILUGG ist '{}'.".format(ilugg_field,
ilugg_definition)
log_text += " {} wird als Faktor angenommen.".format(factor)
logger.log(log_text)
return get_number(factor)
class Environments(object):
"""
Environment object
"""
logger = Logger()
executor = ExecuteCommand()
def __init__(self):
self.env = {}
def run(self):
"""
generic run method
"""
self.get_environment()
def get_environment(self):
"""
get system environment variables
"""
self.logger.normal_output("Grabbing environment variables")
self.logger.normal_output("Grabbing profile")
self.env.update(
{"profile": self.executor.execute_command("cat /etc/profile")})
self.logger.normal_output("Grabbing bashrc")
self.env.update(
{"bashrc": self.executor.execute_command("cat /etc/bashrc")})
self.logger.normal_output("Grabbing bash profile")
self.env.update({
"bashprofile":
self.executor.execute_command("cat ~/.bash_profile")
})
self.logger.normal_output("Grabbing bashrc")
self.env.update(
{"bashrc": self.executor.execute_command("cat ~/.bashrc")})
self.logger.normal_output("Grabbing logout")
self.env.update({
"bash logout":
self.executor.execute_command("cat ~/.bash_logout")
})
self.logger.normal_output("Grabbing env")
self.env.update({"env": self.executor.execute_command("env")})
self.logger.normal_output("Grabbing set")
self.env.update({"set": self.executor.execute_command("set")})
def download_news( self, news ):
init_crawling = '= Iniciando crawling, alvo: [ {nid} ] {link}'.format(nid=news['id'], link=os.path.basename( news['link'] ))
print()
log.success( '=' * len( init_crawling ) )
log.success( init_crawling )
log.success( '=' * len( init_crawling ) )
print()
request = requests.get( news['link'] )
document = BeautifulSoup( request.text, 'html.parser' )
if request.status_code == 200:
return parser.parse_news( news, document )
else:
error_message = 'Erro ao acessar a página: Status {status_code}'.format(status_code=request.status_code)
self.errors.append( error_message )
log.error( error_message )
def parse_download(download_content, download_field = None, article = None):
download_parts = download_content.split("][")
if (len(download_parts) < 3):
warning_text = "[ACHTUNG] Unbekanntes Download-Format "
if download_field != None and article != None:
warning_text += "von "
warning_text += download_field
warning_text += " in "
warning_text += article
else:
warning_text += "({})".format(download_content)
warning_text += ". Der Download wird übersprungen."
Logger().log(warning_text)
return None
return {
"path": build_download_path(download_parts[0]).strip(),
"product": download_parts[1].strip(),
"type": download_parts[2].strip()
}
class FileWriter(object):
"""
filewriter object
"""
logger = Logger()
def __init__(self, filename):
self.filename = filename
def write_output(self, command_output):
"""
output
"""
try:
with open(self.filename, "w") as text_file:
for output in command_output:
text_file.write("Command: {0}\nOutput: {1}\n\n".format(output, str(command_output[output]).rstrip()))
except OSError as error:
self.logger.error("Error writing output file{0}".format(error.message))
def __init__( self ):
super( Images, self ).__init__()
self.images_file = 'data/images.json'
self.images_folder = 'data/news/'
self.dump_file = 'data/news/dump.json'
if os.path.isfile( self.images_file ):
images = helper.read_file( self.images_file, format='json' )
for index, image in enumerate(images, start=0):
try:
if not image['downloaded']:
path = 'data/{image_path}'.format(image_path=image['new_path'].replace('https://static.weg.net/', ''))
filename = os.path.basename( path )
folder = path.split('/')
folder.pop()
folder = '/'.join( folder )
base_url = 'http://www.weg.net'
download_url = image['original_path']
if not os.path.isdir( folder ):
os.makedirs(folder, exist_ok=True)
if not download_url.startswith('http'):
download_url = '{base_url}/{path}'.format(base_url=base_url, path=download_url)
if helper.download(type='image', filename=path, nid=index, url=download_url):
images[ index ]['downloaded'] = True
log.success('Imagem baixada com sucesso [ {path} ]'.format(path=path))
else:
log.warning('Imagem já baixada [ {url} ]'.format(url=image['new_path']))
except Exception as error:
log.error( error )
finally:
helper.create_file(self.images_file, images, mode='w', format='json')
else:
log.error('[!] Dump de imagens não existe')
def __init__( self ):
super( Data, self ).__init__()
self.news_list_file = 'data/notices.list'
self.news_json_file = 'data/notices.json'
self.dump_file = 'data/dump.json'
self.proccess = os.getpid()
self.errors = []
self.news_id_length = 4
init_message = 'Iniciando processo: {proccess}'.format(proccess=self.proccess)
log.success( '=' * len( init_message ) )
log.success( init_message )
log.success( '=' * len( init_message ) )
print()
def __init__(self, parent=None):
super(MyWin, self).__init__(parent)
QWidget.__init__(self, parent)
self.ui = Ui_IPSca()
self.ui.setupUi(self)
self.setWindowIcon(QIcon('ipsca.jpg'))
self.logger = Logger()
self.Scan = None
self.IoTOnly = config.IoTOnly
self.dead_counter = 0
self.successful_counter = 0
self.alive_counter = 0
self.report = {}
self.ui.scan_btn.clicked.connect(self.click_scan)
self.ui.import_btn.clicked.connect(self.file_open)
self.ui.stop_btn.clicked.connect(self.click_stop)
self.ui.shuffle_btn.clicked.connect(self.shuffle_hosts)
self.ui.iot_chbx.stateChanged.connect(self.iot_only_change)
self.ui.brute_chbx.stateChanged.connect(self.brute_enable_change)
self.ui.submit_export.clicked.connect(self.export)
self.scan = QThread()
class Printers(object):
"""
Printer checking object
"""
logger = Logger()
executor = ExecuteCommand()
def __init__(self):
self.printers = {}
def run(self):
"""
default run method
"""
self.logger.normal_output("Running Printers")
self.get_printers()
def get_printers(self):
"""
get any printers
"""
self.logger.normal_output("Grabbing printers")
self.printers.update(
{"printers lpstat": self.executor.execute_command("lpstat -a")})
def __init__(self,
logfile,
debugmode,
global_XMS_IP,
global_XMS_USER,
global_XMS_PASS,
clus_name=None):
global rest_logger
global XMS_IP
global XMS_USERID
global XMS_PASS
self.clus_name = clus_name
if debugmode == True:
rest_logger = Logger(logfile, logging.DEBUG, logging.INFO)
else:
rest_logger = Logger(logfile, logging.INFO, logging.INFO)
rest_logger.debug('Loading Restful Module')
XMS_IP = global_XMS_IP
XMS_USERID = global_XMS_USER
XMS_PASS = global_XMS_PASS
def ScriptSet(log,array,user,passwd,scope,options):
scriptset_logger = Logger(log,logging.DEBUG,logging.INFO)
scriptset_logger.debug('Starting ScriptSet module.')
for x in xrange(len(array)):
cli = NaviCLI(log,array[x],user[x],passwd[x],scope[x])
var_array = array[x]
var_user=user[x]
var_passwd = passwd[x]
var_scope = scope[x]
bol_Analyzer = True
mod = 0
scriptset_logger.info('************************************************')
scriptset_logger.info('Getting Current Settings for '+var_array+' :')
SerialNo,SNerrcode = cli._GetSerialNumber()
if SNerrcode != '0':
scriptset_logger.critical(SerialNo)
else:
scriptset_logger.info('Serial Number:\t'+SerialNo)
NARInterval,NIerrcode = cli._GetNarInterval()
SPcollectstat,SPstaterrcode = cli._GetSPCollectStatus()
scriptset_logger.info('SPCollect AutoExecution:\t'+SPcollectstat)
if SPcollectstat != 'Enabled':
SPcollectAEstat,SPcollectAEerrcode = cli._SetSPcollectAE()
scriptset_logger.info('Enabling SPCollect AutoExecution....')
scriptset_logger.info('Archive Interval:\t'+NARInterval)
RTInterval,RTerrcode = cli._GetRealtimeInterval()
if RTerrcode != '0':
bol_Analyzer = False
scriptset_logger.info('**** Analyzer is not installed ****')
else:
scriptset_logger.info('Real Time Interval:\t'+RTInterval)
LogDays,LDerrcode = cli._GetLogPeriod()
scriptset_logger.info('Current Logging Period:\t'+LogDays)
if bol_Analyzer is False:
scriptset_logger.info('**** Skipping Periodic Archive Setting, Analyzer is not installed ****')
else:
PeriodicArchive,PAerrcode = cli._GetPeriodicArchive()
scriptset_logger.info('Periodic Archiving:\t'+PeriodicArchive)
if options['--ni'] != None:
if NARInterval == options['--ni']:
scriptset_logger.info('NAR Polling Interval is correctly set to:\t' + NARInterval)
else:
NARInterval = options['--ni']
scriptset_logger.info('****Modifying the Archive Poll interval to:\t' + NARInterval)
mod = mod + 1
if LogDays != 'nonstop':
if options['--ns'] == True:
LogDays = 'nonstop'
scriptset_logger.info('****Logging Period is being modified to:\t' + LogDays)
mod = mod + 1
elif LogDays != '7':
LogDays = '7'
scriptset_logger.info('****Logging Period is being modified to:\t' + LogDays)
mod = mod + 1
else:
scriptset_logger.info('Logging Period is correctly set to:\t' + LogDays)
else:
scriptset_logger.info('Logging Period is correctly set to:\t' + LogDays)
if bol_Analyzer is True:
if PeriodicArchive != 'Yes':
PeriodicArchive = 'Yes'
scriptset_logger.info('****Periodic Archiving is Being Enabled')
mod = mod + 1
else:
scriptset_logger.info('Periodic Archiving is correctly set to:\t' +PeriodicArchive)
else:
scriptset_logger.info('Skipping Periodic Archiving Modification - No Analyzer')
NaviStat = cli._GetAnalyzerStatus()
if mod > 0:
if 'Stopped' in NaviStat:
scriptset_logger.debug('*****Analyzer Logging is NOT Running *****')
else:
StopLogStat,SLSerrcode = cli._StopLogging()
time.sleep(10)
if 'stopped' in StopLogStat:
scriptset_logger.info('*****Analyzer Logging is Already Stopped *****')
if bol_Analyzer is True:
SetLogsStat,SetLogserrcode = cli._SetNaviAnalyzerOptionsLicensed(NARInterval,LogDays)
if SetLogserrcode != '0':
scriptset_logger.debug('*****Error Setting Analyzer Logging and Archiving Options *****')
else:
SetLogsStat,SetLogserrcode = cli._SetNaviAnalyzerOptions(NARInterval,LogDays)
if SetLogserrcode != '0':
scriptset_logger.debug('*****Error Setting Analyzer Logging and Archiving Options *****')
StartLogStat,Starterrcode = cli._StartLogging()
if Starterrcode != '0':
time.sleep(5)
scriptset_logger.debug('*****Error Starting Analyzer Logging *****')
NaviStat = cli._GetAnalyzerStatus()
if 'Stopped' in NaviStat:
scriptset_logger.debug('*****Analyzer Logging is NOT Running *****')
time.sleep(5)
StartLogStat,Starterrcode = cli._StartLogging()
if Starterrcode != '0':
scriptset_logger.info('*****Error Starting Analyzer Logging *****')
NARInterval, NIerrcode = cli._GetNarInterval()
if bol_Analyzer is True:
RTInterval,RTerrcode = cli._GetRealtimeInterval()
PeriodicArchive,PAerrcode = cli._GetPeriodicArchive()
else:
RTInterval = ''
PeriodicArchive = ''
LogDays, LDerrcode = cli._GetLogPeriod()
scriptset_logger.info('\tSerial Number:\t' + SerialNo)
scriptset_logger.info('\tNAR Interval:\t' + NARInterval)
scriptset_logger.info('\tRealTime Interval:\t' + RTInterval)
scriptset_logger.info('\tLog Days:\t' + LogDays)
scriptset_logger.info('\tPeriodic Archive:\t' + PeriodicArchive)
else:
scriptset_logger.info('No Need to Change Analyzer Options. Skipping....')
NaviStat = cli._GetAnalyzerStatus()
if 'Stopped' in NaviStat:
scriptset_logger.debug('*****Analyzer Logging is NOT Running *****')
StartLogStat,Starterrcode = cli._StartLogging()
if Starterrcode != '0':
scriptset_logger.debug('*****Error Starting Analyzer Logging *****')
time.sleep(5)
NaviStat = cli._GetAnalyzerStatus()
if 'Stopped' in NaviStat:
scriptset_logger.debug('*****Analyzer Logging is NOT Running *****')
time.sleep(5)
StartLogStat,Starterrcode = cli._StartLogging()
if Starterrcode != '0':
scriptset_logger.info('*****Error Starting Analyzer Logging *****')
time.sleep(5)
NaviStat = cli._GetAnalyzerStatus()
scriptset_logger.info(var_array + ' Analyzer Status = ' + NaviStat)
if 'Stopped' in NaviStat:
scriptset_logger.info('*****Analyzer Logging is NOT Running *****')
scriptset_logger.debug('Exiting def_ScriptSet module.')
scriptset_logger.info('Script is finished setting options.')
def ScriptGet(log,array,user,passwd,scope,options):
scriptget_logger = Logger(log,logging.DEBUG,logging.INFO)
scriptget_logger.debug('Starting def_ScriptGet module.')
timenow = datetime.datetime.now().hour + datetime.datetime.now().minute + datetime.datetime.now().second + datetime.datetime.now().microsecond
for x in xrange(len(array)):
cli = NaviCLI(log,array[x],user[x],passwd[x],scope[x])
var_array = array[x]
var_user=user[x]
var_passwd = passwd[x]
var_scope = scope[x]
bool_32plus = False
scriptget_logger.info('************************************************')
scriptget_logger.info('Getting Serial Number for ' + var_array + ' ....')
SerialNo, SNerror = cli._GetSerialNumber()
if SNerror != '0':
scriptget_logger.critical(SerialNo)
else:
scriptget_logger.info('Getting OE Version for ' + var_array + ' ....')
Version, Verror = cli._GetVersion()
scriptget_logger.info(' Running OE Version ' + Version)
if '7.32.' in Version:
bool_32plus = True
if '7.33.' in Version:
bool_32plus = True
outPath = ''.join([var_cwd,'\\',SerialNo])
if not os.path.exists(outPath):
scriptget_logger.info('Creating Local Directory ' + outPath + ' ....')
os.makedirs(outPath)
if bool_32plus == True:
skew_outpath = outPath+'_SkewData'
scriptget_logger.info('Creating Directory for Skew Data ' + skew_outpath + ' ....')
os.makedirs(skew_outpath)
else:
if os.path.isdir(outPath):
scriptget_logger.info('Found Existing Folder: ' + outPath)
outPath = outPath +'_'+str(date.today())+'_'+str(timenow)
if not os.path.exists(outPath):
scriptget_logger.info('Creating Local Directory ' + outPath + ' ....')
os.makedirs(outPath)
if bool_32plus == True:
skew_outpath = outPath+'_SkewData'
scriptget_logger.info('Creating Directory for Skew Data ' + skew_outpath + ' ....')
os.makedirs(skew_outpath)
scriptget_logger.info('Using ' + outPath + ' to save files for ' + SerialNo)
scriptget_logger.info('Retrieving SPcollect for '+SerialNo +' ....')
SPColStat = cli._GetSPCollectList(outPath)
scriptget_logger.debug(SPColStat)
l,narstring,oldnar,ol = cli._GetNARList2(SerialNo,options['--nnf'])
scriptget_logger.info('Retrieving ' + str(l) + ' NAR/NAZ files from ' + SerialNo + ' ....')
stat = cli._GetNarFiles(narstring,outPath)
scriptget_logger.info(stat)
if bool_32plus == True:
scriptget_logger.info('Retrieving skew files from ' + SerialNo + ' ....')
stat = cli._GetArrayconfigXML(skew_outpath)
scriptget_logger.debug(stat)
stat = cli._GetFASTData(skew_outpath)
scriptget_logger.debug(stat)
os.chdir(var_cwd)
scriptget_logger.info('**** File Retrieval Complete for ' + SerialNo + ' ****')
scriptget_logger.info('**** Creating zip of Data for ' + SerialNo + ' ****')
shutil.make_archive(outPath, 'zip', outPath)
if bool_32plus == True:
shutil.make_archive(skew_outpath, 'zip', skew_outpath)
scriptget_logger.info('**** Cleaning Up Local Directory ****')
shutil.rmtree(outPath, ignore_errors=True)
if bool_32plus == True:
shutil.rmtree(skew_outpath, ignore_errors=True)
if options['--c'] == True:
scriptget_logger.info('**** Cleaning Up '+str(ol)+' NAR/NAZ files from '+SerialNo+' ****')
cleanupstat = cli._CleanupOldNar(oldnar)
scriptget_logger.debug(cleanupstat)
scriptget_logger.debug('Exiting def_ScriptGet module.')
scriptget_logger.debug('Script is finished retrieving files.')
# coding: utf-8
import csv
import os
import numpy as np
import pandas as pd
from modules.logger import Logger, DEBUG, INFO, WARNING, ERROR, CRITICAL
from modules.exceptions import ColumnNotFoundException
logfile = os.path.join(Logger.get_logfile_dir(__file__), '..', 'logs', 'datareader.log')
_ = Logger.create(__file__, logfile, DEBUG)
_.addHandler(Logger.create_file_handler(logfile, DEBUG))
_.addHandler(Logger.create_stream_handler(WARNING))
class DataReader(object):
"""
Reads the CSV data file then cleans it
"""
def __init__(self, fname, delim=","):
self.delim = delim
self.header = []
self.data = []
self.rows = 0
self.cols = 0
_.debug("Reading %s" % fname)
with open(fname, 'r') as rf:
def OnlyCleanupOldNar(log, array, user, password, scope, options):
onlycleannarfiles_logger = Logger(log, logging.DEBUG, logging.INFO)
onlycleannarfiles_logger.debug('Staring def_OnlyCleanupOldNar module.')
for x in xrange(len(array)):
cli = NaviCLI(log, array[x], user[x], passwd[x], scope[x])
var_array = array[x]
var_user = user[x]
var_passwd = passwd[x]
var_scope = scope[x]
onlycleannarfiles_logger.info(
'************************************************')
onlycleannarfiles_logger.info('Getting Serial Number for ' +
var_array + ' ....')
## SerialNo,SNerror = def_GetSerialNumber(var_array,var_user,var_passwd,var_scope)
SerialNo, SNerror = cli._GetSerialNumber()
if SNerror != '0':
onlycleannarfiles_logger.info(SerialNo)
else:
## l,narstring,oldnar,ol = def_GetNARList2(SerialNo,var_array,var_user,var_passwd,var_scope)
l, narstring, oldnar, ol = cli._GetNARList2(
SerialNo, options['--nnf'])
onlycleannarfiles_logger.info(
'**** Cleaning Up ' + str(ol) +
' NAR/NAZ files, older than 7 days, from ' + SerialNo +
' ****')
## def_CleanupOldNar(var_array,var_user,var_passwd,var_scope,oldnar)
cli._CleanupOldNar(oldnar)
def get_content( news, content ):
if not content[0]: return ''
allowed_images_extension = ['.jpeg', '.jpg', '.png', '.gif', '.bmp', '.tif']
document = BeautifulSoup( content[0].encode('utf-8'), 'html.parser' )
to_remove = ['comparison', 'bgdark', 'bglight', 'default', 'clr', 'novaJanela']
link = news['link']
catalog = news['catalog']
nid = news['id']
for item in to_remove:
if document.select('.{selector}'.format(selector=item)):
for element in document.select('.{selector}'.format(selector=item)):
index = element['class'].index( item )
del element['class'][ index ]
if document.select('.center'):
for center in document.select('.center'):
center['class'] = 'text-center'
if document.select('p'):
paragraphs = document.select('p')
for paragraph in paragraphs:
for content in paragraph.contents:
if content == '\xa0' or not content:
paragraph.decompose()
if document.select('table'):
tables = document.select('table')
tablefilename = 'logs/weg/tables.list'
link = link if isinstance( link, str ) else link.attrs['href']
table_log = '[ {nid} ]: {link}\n'.format(link=link, nid=nid)
for table in tables:
to_remove = ['cellpadding', 'border', 'cellspacing', 'width', 'height']
responsive = document.new_tag('div')
responsive['class'] = 'table-responsive'
table.wrap( responsive )
table['class'].append('table table-bordered table-hover')
for item in to_remove:
del table[ item ]
if os.path.isfile( tablefilename ):
content = helper.read_file( tablefilename )
if link not in content:
helper.create_file(tablefilename, table_log)
else:
log.warning('Tabela já adicionada para a lista [ {url} ]'.format(url=link))
else:
helper.create_file(tablefilename, table_log)
log.success('Log de tabelas criado.')
if document.select('a'):
for index, link in enumerate( document.select('a'), start=0 ):
if 'href' in link.attrs:
filename, file_extension = os.path.splitext( link.attrs['href'] )
if link.attrs['href'] == 'javascript:void();':
link.attrs['href'] = '#{nid}'.format(nid=news['id'])
link.attrs['data-prevent-default'] = 'true'
if file_extension in allowed_images_extension:
set_image( news, index, link.attrs['href'] )
link.attrs['href'] = set_image_link( news, index, link.attrs['href'] )
if document.select('img'):
for index, image in enumerate( document.select('img'), start=0 ):
filename, file_extension = os.path.splitext( image.attrs['src'] )
responsive = True
if file_extension in allowed_images_extension:
set_image( news, index, image.attrs['src'] )
image.attrs['src'] = set_image_link( news, index, image.attrs['src'] )
# for parent in image.parents:
# if 'class' in parent.attrs:
# if 'coluna6' in parent.attrs['class']:
# responsive = False
# if responsive:
# if 'class' in image.attrs:
# image.attrs['class'].append('img-responsive')
# else:
# image.attrs['class'] = 'img-responsive'
if document.select('.coluna6'):
columns = document.select('.coluna6')
for column in columns:
column['class'] = 'xtt-gallery pull-right'
if document.select('ul'):
for ul in document.select('ul'):
ul['class'] = 'xtt-list-style'
for li in ul.select('> li'):
span = document.new_tag('span')
span.string = li.contents[0]
li.string = ''
li.append( span )
return str( document ).strip()
class App(object):
def __init__(self):
self.logger = Logger()
user = os.getenv('VC_USER')
password = os.getenv('VC_PASS')
if not self.should_run(user, password):
self.logger.error('User or password is not defined, exiting...')
sys.exit()
self.sandboxes = Sandboxes(user, password)
self.applications = Applications(user, password)
self.builds = Builds(user, password)
self.spreadsheet_creator = SpreadsheetCreator()
apps = self.applications.get_apps()
self.get_app_sandboxes(apps)
self.get_app_sandbox_builds(apps)
self.get_builds(apps)
self.create_spreadsheet(apps)
self.send_message()
def should_run(self, user, password):
return user != None and password != None and os.getenv(
'VC_ATTACHMENT_BASE_PATH') != None and os.getenv(
'VC_ADMIN_ADDRESS') != None and os.getenv(
'VC_SMTP') != None and os.getenv(
'VC_SEND_ADDRESS') != None and os.getenv(
'VC_RECIPIENTS') != None
def get_app_sandboxes(self, apps):
for app in apps:
app['sandboxes'] = self.sandboxes.get_sandboxes(app['app_id'])
def get_app_sandbox_builds(self, apps):
for app in apps:
for sandbox in app['sandboxes']:
sandbox['builds'] = self.builds.get_builds(
app['app_id'], sandbox['sandbox_id'])
def get_builds(self, apps):
for app in apps:
for sandbox in app['sandboxes']:
for build in sandbox['builds']:
build['submitter'] = self.builds.get_build(
app['app_id'], build['build_id'])
def create_spreadsheet(self, apps):
self.spreadsheet_creator.write_spreadsheet(apps)
def send_message(self):
if self.spreadsheet_creator.have_data:
Mailer(
os.getenv('VC_SEND_ADDRESS'), os.getenv('VC_RECIPIENTS'),
constants.MESSAGE_SUBJECT, os.getenv('VC_SMTP'), 25,
constants.BUILDS_FOUND_MESSAGE_TEXT.format(
os.getenv('VC_ADMIN_ADDRESS')),
self.spreadsheet_creator.filename)
else:
Mailer(
os.getenv('VC_SEND_ADDRESS'), os.getenv('VC_RECIPIENTS'),
constants.MESSAGE_SUBJECT, os.getenv('VC_SMTP'), 25,
constants.BUILDS_NOT_FOUND_MESSAGE_TEXT.format(
os.getenv('VC_ADMIN_ADDRESS')))
class Services(object):
"""
Object for service enumeration
"""
logger = Logger()
executor = ExecuteCommand()
def __init__(self):
self.services = {}
self.services_as_root = {}
self.installed_applications = {}
self.plaintext_passwords = {}
self.scheduled_jobs = {}
self.service_configurations = {}
def run(self):
"""
generic run method
"""
self.logger.normal_output("Running Service Checks")
self.get_services()
self.get_services_as_root()
self.get_installed_applications()
self.get_for_plaintext_passwords()
self.get_scheduled_jobs()
self.get_service_configurations()
def get_services(self):
"""
get services
"""
self.logger.normal_output("Running services")
self.services.update(
{"ps aux": self.executor.execute_command("ps aux")})
self.services.update(
{"ps aux -ef": self.executor.execute_command("ps -ef")})
# self.services.update({"top": self.executor.execute_command("top")})
self.services.update({
"cat /etc/services":
self.executor.execute_command("cat /etc/services")
})
def get_services_as_root(self):
"""
get root services
"""
self.logger.normal_output("Running services as root")
self.services_as_root.update({
"ps aux |grep root":
self.executor.execute_command("ps aux | grep root")
})
self.services_as_root.update({
"ps -ef | grep root":
self.executor.execute_command("ps -ef | grep root")
})
def get_installed_applications(self):
"""
get installed applications
"""
self.logger.normal_output("Running installed apps")
self.installed_applications.update({
"ls -alh /usr/bin":
self.executor.execute_command("ls -alh /usr/bin/")
})
self.installed_applications.update({
"ls -alh /sbin/":
self.executor.execute_command("ls -alh /sbin/")
})
self.installed_applications.update(
{"dpkg -l": self.executor.execute_command("dpkg -l")})
self.installed_applications.update(
{"rpm -qa": self.executor.execute_command("rpm -qa")})
self.installed_applications.update({
"ls -alh /var/cache/apt/archives0":
self.executor.execute_command("ls -alh /var/cache/apt/archivesO")
})
self.installed_applications.update({
"ls -alh /var/cache/yum":
self.executor.execute_command("ls -alh /var/cache/yum/")
})
def get_service_configurations(self):
"""
get service configurations
"""
self.logger.normal_output("Running service configs")
self.service_configurations.update(
{"syslog": self.executor.execute_command("cat /etc/syslog.conf")})
self.service_configurations.update(
{"chhtp": self.executor.execute_command("cat /etc/chttp.conf")})
self.service_configurations.update({
"lighthttpd":
self.executor.execute_command("cat /etc/lighttpd.conf")
})
self.service_configurations.update({
"cupsd":
self.executor.execute_command("cat /etc/cups/cupsd.conf")
})
self.service_configurations.update(
{"inetd": self.executor.execute_command("cat /etc/inetd.conf")})
self.service_configurations.update({
"apache2":
self.executor.execute_command("cat /etc/apache2/apache2.conf")
})
self.service_configurations.update(
{"my.conf": self.executor.execute_command("cat /etc/my.conf")})
self.service_configurations.update({
"httpd":
self.executor.execute_command("cat /etc/httpd/conf/httpd.conf")
})
self.service_configurations.update({
"lamphttpd":
self.executor.execute_command("cat /opt/lampp/etc/httpd.conf")
})
self.service_configurations.update({
"/etc/":
self.executor.execute_command("ls -aRl /etc/ | awk '$1 ~ /^.*r.*/")
})
def get_scheduled_jobs(self):
"""
get scheduled jobs
"""
self.logger.normal_output("Running scheduled jobs")
self.scheduled_jobs.update(
{"crontab -l": self.executor.execute_command("crontab -l")})
self.scheduled_jobs.update({
"ls alh cron":
self.executor.execute_command("ls -alh /var/spool/cron")
})
self.scheduled_jobs.update({
"/etc cron":
self.executor.execute_command("ls -al /etc/ | grep cron")
})
self.scheduled_jobs.update({
"ls -al /etc/cron*":
self.executor.execute_command("ls -al /etc/cron*")
})
self.scheduled_jobs.update(
{"etc/cron": self.executor.execute_command("cat /etc/cron*")})
self.scheduled_jobs.update({
"etc/at.allow":
self.executor.execute_command("cat /etc/at.allow")
})
self.scheduled_jobs.update(
{"etc/at.deny": self.executor.execute_command("cat /etc/at.deny")})
self.scheduled_jobs.update({
"etc/cron.allow":
self.executor.execute_command("cat /etc/cron.allow")
})
self.scheduled_jobs.update({
"etc/cron.deny":
self.executor.execute_command("cat /etc/cron.deny")
})
self.scheduled_jobs.update(
{"crontab": self.executor.execute_command("cat /etc/crontab")})
self.scheduled_jobs.update(
{"acrontab": self.executor.execute_command("cat /etc/anacrontab")})
self.scheduled_jobs.update({
"rootcrontab":
self.executor.execute_command("cat /var/spool/cron/crontabs/root")
})
def get_for_plaintext_passwords(self):
"""
check for different config files containing passwords
"""
self.logger.normal_output("Running plaintext passwords and users")
self.plaintext_passwords.update({
"grep for user":
self.executor.execute_command("grep -i user /*")
})
self.plaintext_passwords.update({
"grep for pass":
self.executor.execute_command("grep -i pass /*")
})
self.plaintext_passwords.update({
"grep -c 5":
self.executor.execute_command("grep -C 5 \"password\" /*")
})
self.plaintext_passwords.update({
"find php $password":
self.executor.execute_command(
"find . -name \"*.php\" -print0 | xargs -0 grep -i -n \"var $password\"" # Joomla
)
})
from modules.logger import Logger
logger = Logger()
def grouping(value, format_option):
try:
numeric_value = float(value.replace(",", "."))
matching_threshold = None
group_thresholds = format_option["thresholds"]
largest_threshold = max(group_thresholds)
digits = len(str(largest_threshold))
for threshold in group_thresholds:
if numeric_value <= threshold:
matching_threshold = threshold
break
if matching_threshold != None:
indicator = "bis"
else:
indicator = "über"
matching_threshold = largest_threshold
unit = format_option["unit"]
return "{} {}{}".format(indicator,
str(matching_threshold).zfill(digits), unit)
except:
logger.log(
"Der Wert '{}' kann nicht gruppiert werden, da er nicht numerisch ist."
.format(value))
return value
log.error(error)
pass
finally:
helper.create_file( filename=self.dump_file, content=self.news, format='json', mode='w')
helper.create_file( filename=self.news_json_file, content=self.news_list, format='json', mode='w')
if __name__ == '__main__':
scrapper = Scrapper()
try:
scrapper.start()
except Exception as error:
print()
error_message = 'Erro ao iniciar processo: {proccess}'.format(proccess=scrapper.proccess)
log.error('=' * len( error_message ))
log.error( error_message )
log.error(error)
log.error('=' * len( error_message ))
print()
finally:
finished_with_errors = 'Finalizado com {errors} erro{suffix}'.format(errors=len( scrapper.errors ), suffix='s' if len( scrapper.errors ) > 1 else '')
finished_without_errors = 'Finalizado sem erros'
if scrapper.errors:
print()
log.warning( '=' * len( finished_with_errors ) )
log.warning( finished_with_errors )
else:
print()
log.success( '=' * len( finished_without_errors ) )
def __init__(self):
self.have_data = False
self.logger = Logger()
self.filename = '{}VeracodeSandboxResults-{}.xlsx'.format(
os.getenv('VC_ATTACHMENT_BASE_PATH'), date.today())
return
def main():
log = options['--l']
if options['--debug'] == True:
main_logger = Logger(
log,
logging.DEBUG,
logging.INFO
)
else:
main_logger = Logger(
log,
logging.INFO,
logging.INFO
)
rd_csv = ReadCSV(
log,
options
)
if options['--set'] == True:
main_logger.debug(
'Set argument is set'
)
ScriptSet(
log,
rd_csv.host,
rd_csv.uname,
rd_csv.passwd,
rd_csv.scope,
options
)
if options['--get'] == True:
main_logger.debug(
'Get argument is set'
)
ScriptGet(
log,
rd_csv.host,
rd_csv.uname,
rd_csv.passwd,
rd_csv.scope,
options
)
if options['--c'] == True:
main_logger.debug(
'Cleanup argument is set with the Get argument'
)
if options['--get'] == False:
main_logger.debug(
'Cleanup argument is set by itself'
)
OnlyCleanupOldNar(
log,
rd_csv.host,
rd_csv.uname,
rd_csv.passwd,
rd_csv.scope,
options
)
main_logger.debug(
'Exiting.'
)
main_logger.info(
'Script Complete.'
)
sys.exit(0)
# pip install MySQL-python sqlalchemy
import sys
import time
import ConfigParser
from modules.config_reader import ConfigReader
from modules.stock import Stock
from modules.mysqlclient import MySQLClient
from modules.logger import Logger
CONF_FILE_PATH = 'config/stock.conf'
if __name__ == '__main__':
log = Logger.get_instance()
log.info('------------------ START TO FETCH STOCK HISTORY ------------------')
try:
conf = ConfigReader(ConfigParser.ConfigParser(), CONF_FILE_PATH)
except Exception as e:
print e
log.error('Failed to read config file: %s', e)
sys.exit(1)
db = MySQLClient(conf.user, conf.password, conf.host, conf.dbname)
db.create_tables()
Stock(None, from_date=conf.from_date, db=db, log=log)
def __init__(self,logfile,options):
global options_logger
if options['--debug'] == True:
options_logger = Logger(logfile,logging.DEBUG,logging.INFO)
else:
options_logger = Logger(logfile,logging.INFO,logging.INFO)
options_logger.debug('Loading Options Module')
encoder = Encode(options['--l'],options['--debug'])
if options['--encode'] is True:
options_logger.info('Encoding user id and password')
encode_user = encoder._encodeuser(options['XMS_USER'])
encode_pass = encoder._encodepass(options['XMS_PASS'])
print ''
print 'Encoded User ID = ' + encode_user
print 'Encoded Password = '******''
options_logger.info('Use the above, encoded, user id and password with the --e option')
options_logger.info('to execute the tool without using the plain text usename and password')
sys.exit(0)
elif options['--e']:
options_logger.debug('Using an encoded username and password')
XMS_USER = encoder._decodeuser(options['XMS_USER'])
XMS_PASS = encoder._decodepass(options['XMS_PASS'])
else:
options_logger.debug('Username and password are not encoded')
XMS_USER = options['XMS_USER']
XMS_PASS = options['XMS_PASS']
self.XMS_USER = XMS_USER
self.XMS_PASS = XMS_PASS
if options['--schedule'] == 'hourly':
options_logger.debug('Using the hourly schedule')
self.schedule = 'hourly'
elif options['--schedule'] == 'daily':
options_logger.debug('Using the daily schedule')
self.schedule = 'daily'
elif options['--schedule'] == 'weekly':
options_logger.debug('Using the weekly schedule')
self.schedule = 'weekly'
else:
options_logger.critical('No schedule, or incorrect option Specified. Exiting...')
sys.exit(1)
SnapFolder = '_Snapshots'
self.var_snap_tgt_folder = options['--tf'] ## this should be an optional variable, if the customer wants to organize snapshots under a folder hierarchy
if self.var_snap_tgt_folder == None:
options_logger.debug('No snapshot target folder specified')
self.var_snap_tgt_folder =''
self.snap_tgt_folder = SnapFolder
options_logger.debug('Using '+self.snap_tgt_folder+' as the snapshot target')
else:
self.snap_tgt_folder = self.var_snap_tgt_folder+'/'+SnapFolder
options_logger.debug('Using '+self.snap_tgt_folder+' as the snapshot target')
def __init__(self,logfile,options):
global csv_logger
csvfile = options['--csv']
csv_logger = Logger(logfile,logging.DEBUG,logging.INFO)
csv_logger.debug('Loading ReadCSV Module')
csv_logger.debug(csvfile)
arr_host = []
arr_uname = []
arr_passwd = []
arr_scope = []
result = _FileCheck(csvfile)
if result is 0:
if options['LDAP_USER'] != None:
f = open(csvfile, 'r')
r = csv.reader(f)
for row in _skip_blank(r):
arr_host.append(row[0])
arr_uname.append(options['LDAP_USER'])
arr_passwd.append(options['LDAP_PASS'])
arr_scope.append('2')
else:
try:
f = open(csvfile, 'r')
r = csv.reader(f)
for row in _skip_blank(r):
arr_host.append(row[0])
arr_uname.append(row[1])
arr_passwd.append(row[2])
arr_scope.append(row[3])
except:
csv_logger.critical('There was an error reading the username and/or password info, is it specified in the csv?')
csv_logger.critical('exiting...')
sys.exit(1)
self.uname = arr_uname
self.passwd = arr_passwd
self.host = arr_host
self.scope = arr_scope
csv_logger.debug('Exiting ReadCSV module.')
else:
sys.exit(1)
def main():
log = options['--l']
if options['--debug'] == True:
main_logger = Logger(
options['--l'],
logging.DEBUG,
logging.INFO
)
else:
main_logger = Logger(
options['--l'],
logging.INFO,
logging.INFO
)
main_logger.info(
'Running XtremIO Snap Script'
)
main_options = Options(
log,
options
)
XMS_IP = options['XMS_IP']
XMS_USER = main_options.XMS_USER
XMS_PASS = main_options.XMS_PASS
snap_tgt_folder = main_options.snap_tgt_folder
parent_folder_id = main_options.var_snap_tgt_folder
num_snaps = options['--n']
snap_src = options['--snap']
SnapFolder = '_Snapshots'
bool_create_folder = True ## will change to False if the /_Snapshots folder already exists
rest = Restful(
log,
options['--debug'],
XMS_IP,
XMS_USER,
XMS_PASS
)
folder_list = rest._get(
'/api/json/types/volume-folders'
).json()['folders']
for folder_list_rs in folder_list:
if folder_list_rs['name'] == '/'+ snap_tgt_folder:
bool_create_folder = False
main_logger.info(
'The target snapshot folder, '+folder_list_rs['name']+' already exists.'
)
if bool_create_folder is True:
cf_payload = '{\
"caption": \"'+SnapFolder+'\" , \
"parent-folder-id": \"/'+main_options.var_snap_tgt_folder+'\" \
}'
cf_resp = rest._post(
'/api/json/types/volume-folders',
cf_payload
)
if cf_resp.status_code == 201:
main_logger.warn(
'Created folder: '+main_options.snap_tgt_folder
)
if options['--f'] ==True:
newsnapsuffix = '.folder.'+main_options.schedule+'.'
folder_vol_list = rest._get(
'/api/json/types/volume-folders/?name=/'+snap_src
).json()['content']['direct-list']
arr_folder_vol_list_component = []
for folder_vol_list_rs in folder_vol_list:
if '/_Snapshots' in folder_vol_list_rs[1]:
pass
else:
arr_folder_vol_list_component.append(folder_vol_list_rs[1])
main_logger.info(
'Will retain '+num_snaps+' snapshots for volume: '+folder_vol_list_rs [1]
)
vol_snap_list = rest._get(
'/api/json/types/volumes/?name='+folder_vol_list_rs [1]
).json()['content']['dest-snap-list']##<--Initial list of snapshots
arr_vol_snap_list_component = []
for vol_snap_list_rs in vol_snap_list:
if newsnapsuffix in vol_snap_list_rs[1]:
arr_vol_snap_list_component.append(vol_snap_list_rs[1])
arr_vol_snap_list_component.sort(reverse=True)
for y in range(len(arr_vol_snap_list_component)): ##<--shifting the suffix of each matchin snap by 1
if newsnapsuffix in arr_vol_snap_list_component[y]:
list_snapname = []
try:
list_snapname = arr_vol_snap_list_component[y].split('.',3)
except:
pass
rename_to =list_snapname[0]+newsnapsuffix+str(y+1)
rename_payload = '{"vol-name": \"'+rename_to+'\"}'
rename_resp = rest._put(
'/api/json/types/volumes/?name='+arr_vol_snap_list_component[y],
rename_payload
)
if rename_resp.status_code == 200:
main_logger.info(
'Snapshot: '+arr_vol_snap_list_component[y]+' was renamed to '+rename_to
)
timestamp = datetime.datetime.now()
timestamp = timestamp.isoformat()
arr_timestamp = timestamp.split('.',2) ##<--stripping the microseconds from the timestamp for aesthetics
fullsuffix = '_'+arr_timestamp[0]+newsnapsuffix+'0'
fs_payload = '{\
"source-folder-id": \"/'+snap_src+'\" , \
"suffix": \"'+fullsuffix+'\" ,\
"folder-id": \"/'+snap_tgt_folder+'\" \
}'
vol_snap = rest._post(
'/api/json/types/snapshots',
fs_payload
)
folder_vol_list = rest._get(
'/api/json/types/volume-folders/?name=/'+snap_src
).json()['content']['direct-list']
arr_folder_vol_list_component = []
for folder_vol_list_rs in folder_vol_list:
if '/_Snapshots' in folder_vol_list_rs[1]:
pass
else:
vol_snap_list = rest._get(
'/api/json/types/volumes/?name='+folder_vol_list_rs [1]
).json()['content']['dest-snap-list']##<--Refresh the snap list
arr_vol_snap_list_component = []
for vol_snap_list_rs in vol_snap_list:
if newsnapsuffix in vol_snap_list_rs[1]:
arr_vol_snap_list_component.append(vol_snap_list_rs[1])
arr_vol_snap_list_component.sort(reverse=False)
for x in xrange(len(arr_vol_snap_list_component)-(int(num_snaps))):
if newsnapsuffix in arr_vol_snap_list_component[x]:
main_logger.debug(
str(x)+': '+ arr_vol_snap_list_component[x]
)
get_snap_details = rest._get(
'/api/json/types/snapshots/?name='+arr_vol_snap_list_component[x]
)
arr_ancestor_vol_id = get_snap_details.json()['content']['ancestor-vol-id']
snap_parent_name = arr_ancestor_vol_id[1]
snap_creation_time = get_snap_details.json()['content']['creation-time']
snap_space_consumed = get_snap_details.json()['content']['logical-space-in-use']
arr_snap_lun_mapping = get_snap_details.json()['content']['lun-mapping-list']
main_logger.warn(
'Parent Volume of '+arr_vol_snap_list_component[x]+' = '+snap_parent_name
)
main_logger.warn(
'Snapshot: ' +arr_vol_snap_list_component[x]
)
main_logger.warn(
' Snap was created on '+snap_creation_time
)
main_logger.warn(
' Snap is using '+ str((float(snap_space_consumed)/1024)/1024)+' GB'
)
arr_lun_mapping_component = []
if len(arr_snap_lun_mapping) > 0:##<--checking to see if an active LUN mapping exists
for rs in arr_snap_lun_mapping:
arr_lun_mapping_component = [[y] for y in rs[0]]
arr_lun_mapping_component =str(arr_lun_mapping_component[1])
arr_lun_mapping_component = arr_lun_mapping_component.replace('[u\'','')
arr_lun_mapping_component = arr_lun_mapping_component.replace('\']','')
main_logger.critical(
'Snapshot: '+arr_vol_snap_list_component[x]+' is currently mapped to '+arr_lun_mapping_component+', it will not be deleted.'
)
else:
main_logger.warn(
' No hosts mapped to '+arr_vol_snap_list_component[x]+', it will be deleted.'
)
delete_status = rest._delete(
'/api/json/types/volumes/?name='+arr_vol_snap_list_component[x]
)
elif options['--v'] ==True:
main_logger.info(
'Will retain '+num_snaps+' snapshots for volume: '+snap_src
)
newsnapsuffix = '.'+main_options.schedule+'.'
vol_snap_list = rest._get(
'/api/json/types/volumes/?name='+snap_src
).json()['content']['dest-snap-list']
arr_vol_snap_list_component = []
for vol_snap_list_rs in vol_snap_list:
if newsnapsuffix in vol_snap_list_rs[1]:
if '.folder.' in vol_snap_list_rs[1]:
pass
else:
arr_vol_snap_list_component.append(vol_snap_list_rs[1])
arr_vol_snap_list_component.sort(reverse=True)
for y in range(len(arr_vol_snap_list_component)): ##<--shifting the suffix of each matchin snap by 1
if newsnapsuffix in arr_vol_snap_list_component[y]:
list_snapname = []
try:
list_snapname = arr_vol_snap_list_component[y].split('.',3)
except:
pass
rename_to =list_snapname[0]+newsnapsuffix+str(y+1)
rename_payload = '{"vol-name": \"'+rename_to+'\"}'
rename_resp = rest._put(
'/api/json/types/volumes/?name='+arr_vol_snap_list_component[y],
rename_payload
)
if rename_resp.status_code == 200:
main_logger.info(
'Snapshot: '+arr_vol_snap_list_component[y]+' was renamed to '+rename_to
)
timestamp = datetime.datetime.now()
timestamp = timestamp.isoformat()
arr_timestamp = timestamp.split('.',2) ##<--stripping the microseconds from the timestamp for aesthetics
fullsuffix = '_'+arr_timestamp[0]+newsnapsuffix
newsnap = snap_src+fullsuffix+'0'##<--sets the newly created snapshot to always be .0
vol_snap_payload = '{\
"ancestor-vol-id": \"'+snap_src+'\" , \
"snap-vol-name": \"'+newsnap+'\" ,\
"folder-id": \"/'+snap_tgt_folder+'\" \
}'
vol_snap_resp = rest._post(
'/api/json/types/snapshots',
vol_snap_payload
)
vol_snap_list = rest._get(
'/api/json/types/volumes/?name='+snap_src
).json()['content']['dest-snap-list']
arr_vol_snap_list_component = []
for vol_snap_list_rs in vol_snap_list:
if newsnapsuffix in vol_snap_list_rs[1]:
if '.folder.' in vol_snap_list_rs[1]:
pass
else:
arr_vol_snap_list_component.append(vol_snap_list_rs[1])
arr_vol_snap_list_component.sort(reverse=False)
for x in xrange(len(arr_vol_snap_list_component)-(int(num_snaps))):
if newsnapsuffix in arr_vol_snap_list_component[x]:
main_logger.debug(str(x)+': '+ arr_vol_snap_list_component[x])
get_snap_details = rest._get(
'/api/json/types/snapshots/?name='+arr_vol_snap_list_component[x]
)
arr_ancestor_vol_id = get_snap_details.json()['content']['ancestor-vol-id']
snap_parent_name = arr_ancestor_vol_id[1]
snap_creation_time = get_snap_details.json()['content']['creation-time']
snap_space_consumed = get_snap_details.json()['content']['logical-space-in-use']
arr_snap_lun_mapping = get_snap_details.json()['content']['lun-mapping-list']
main_logger.warn(
'Parent Volume of '+arr_vol_snap_list_component[x]+' = '+snap_parent_name
)
main_logger.warn(
'Snapshot: '+arr_vol_snap_list_component[x]
)
main_logger.warn(
' Snap was created on '+snap_creation_time
)
main_logger.warn(
' Snap is using '+ str((float(snap_space_consumed)/1024)/1024)+' GB'
)
arr_lun_mapping_component = []
if len(arr_snap_lun_mapping) > 0:##<--checking to see if an active LUN mapping exists
for rs in arr_snap_lun_mapping:
arr_lun_mapping_component = [[y] for y in rs[0]]
arr_lun_mapping_component =str(arr_lun_mapping_component[1])
arr_lun_mapping_component = arr_lun_mapping_component.replace('[u\'','')
arr_lun_mapping_component = arr_lun_mapping_component.replace('\']','')
main_logger.critical(
'Snapshot '+arr_vol_snap_list_component[x]+' is currently mapped to '+arr_lun_mapping_component+', it will not be deleted.'
)
else:
main_logger.warn(
' No hosts mapped to '+arr_vol_snap_list_component[x]+', it will be deleted.'
)
delete_status = rest._delete(
'/api/json/types/volumes/?name='+arr_vol_snap_list_component[x]
)
else:
print 'NO FOLDER OR VOLUME OPTION SPECIFIED'
sys.exit(1)
main_logger.info('Complete!')
sys.exit(0)