def get_routine(self, workspace, profile):
# get options depend on workspace
options_path = current_path + \
'/storages/{0}/options.json'.format(workspace)
commands_path = current_path + '/storages/commands.json'
self.options = utils.reading_json(options_path)
if not self.options:
return None
self.commands = utils.reading_json(commands_path)
raw_routine = {}
for key, value in self.commands.items():
raw_routine[key] = self.commands[key].get(profile)
routines = {}
for module, cmds in raw_routine.items():
routines[module] = []
if cmds:
for item in cmds:
real_item = {}
for k, v in item.items():
real_item[k] = utils.replace_argument(self.options, v)
routines[module].append(real_item)
return routines
def post(self):
current_path = os.path.dirname(os.path.realpath(__file__))
# global options
data = Configurations.parser.parse_args()
options = data['options']
utils.just_write(current_path + '/storages/options.json', options, is_json=True)
if options.get('FORCE') == "False":
old_log = options['WORKSPACE'] + '/log.json'
if utils.not_empty_file(old_log) and utils.reading_json(old_log):
utils.print_info(
"It's already done. use '-f' options to force rerun the module")
raw_activities = utils.reading_json(
options['WORKSPACE'] + '/log.json')
utils.just_write(current_path + '/storages/activities.json',
raw_activities, is_json=True)
return options
utils.print_info("Cleasning activities log")
#Create skeleton activities
commands = utils.reading_json(current_path + '/storages/commands.json')
raw_activities = {}
for k,v in commands.items():
raw_activities[k] = []
utils.just_write(current_path + '/storages/activities.json',
raw_activities, is_json=True)
return options
def get(self, workspace):
#
# @TODO potential LFI here
#
ws_name = os.path.basename(os.path.normpath(workspace))
options_path = str(
BASE_DIR.joinpath('storages/{0}/options.json'.format(ws_name)))
self.options = utils.reading_json(options_path)
# looking for log file if options file not found
if not self.options:
ws_json = str(
str(Path.home().joinpath(
'.osmedeus/workspaces/{0}/{0}.json'.format(ws_name))))
return utils.reading_json(ws_json)
if not self.options:
return {'error': 'Log file not found'}
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + "/{0}/{0}.json".format(
ws_name)
if os.path.isfile(ws_json):
return utils.reading_json(ws_json)
return 'Custom 404 here', 404
def get(self, workspace):
#
# @TODO potential LFI here
#
ws_name = os.path.basename(os.path.normpath(workspace))
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + "/{0}/{0}.json".format(ws_name)
if os.path.isfile(ws_json):
utils.reading_json(ws_json)
return utils.reading_json(ws_json)
return 'Custom 404 here', 404
def post(self):
# global options
data = Configurations.parser.parse_args()
options = data['options']
# @TODO add another authen level when settings things from remote
# check if credentials is the same on the config file or not
if not self.verify(options):
return {"error": "Can't not verify to setup config"}
# write each workspace seprated folder
ws_name = utils.get_workspace(options)
utils.make_directory(current_path + '/storages/{0}/'.format(ws_name))
if not os.path.isdir(current_path + '/storages/{0}/'.format(ws_name)):
return {
"error":
"Can not create workspace directory with name {0} ".format(
ws_name)
}
activities_path = current_path + '/storages/{0}/activities.json'.format(
ws_name)
options_path = current_path + '/storages/{0}/options.json'.format(
ws_name)
# consider this is settings db
utils.just_write(options_path, options, is_json=True)
if options.get('FORCE') == "False":
old_log = options['WORKSPACE'] + '/log.json'
if utils.not_empty_file(old_log) and utils.reading_json(old_log):
utils.print_info(
"It's already done. use '-f' options to force rerun the module"
)
raw_activities = utils.reading_json(options['WORKSPACE'] +
'/log.json')
utils.just_write(activities_path, raw_activities, is_json=True)
return options
utils.print_info("Cleaning activities log")
# Create skeleton activities based on commands.json
commands = utils.reading_json(current_path + '/storages/commands.json')
raw_activities = {}
for k, v in commands.items():
raw_activities[k] = []
utils.just_write(activities_path, raw_activities, is_json=True)
return options
def get(self, workspace):
#
# @TODO potential LFI here
#
# get specific module
module = request.args.get('module')
ws_name = os.path.basename(os.path.normpath(workspace))
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + \
"/{0}/log.json".format(ws_name)
if os.path.isfile(ws_json):
raw_logs = utils.reading_json(ws_json)
log = raw_logs
for key in raw_logs.keys():
for i in range(len(raw_logs[key])):
log[key][i]['std_path'] = utils.replace_argument(self.options, raw_logs[key][i].get(
'std_path')).replace(self.options['WORKSPACES'], '')
log[key][i]['output_path'] = utils.replace_argument(self.options, raw_logs[key][i].get(
'output_path')).replace(self.options['WORKSPACES'], '')
if module:
cmds = log.get(module)
return {'commands': cmds}
else:
return log
return 'Custom 404 here', 404
def get(self, workspace):
#
# @TODO potential LFI here
#
ws_name = os.path.basename(os.path.normpath(workspace))
options_path = str(
BASE_DIR.joinpath('storages/{0}/options.json'.format(ws_name)))
self.options = utils.reading_json(options_path)
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + "/{0}/{0}.json".format(
ws_name)
if os.path.isfile(ws_json):
utils.reading_json(ws_json)
return utils.reading_json(ws_json)
return 'Custom 404 here', 404
def verify_file(self, filename):
option_files = glob.glob(str(BASE_DIR) + '/storages/**/options.json',
recursive=True)
# loop though all options avalible
for option in option_files:
json_option = utils.reading_json(option)
stdout_path = json_option.get('WORKSPACES') + "/" + filename
if utils.not_empty_file(stdout_path):
return json_option.get('WORKSPACES'), os.path.normpath(
filename)
# get real path
p = Path(filename)
ws = p.parts[0]
if ws != utils.url_encode(ws):
# just replace the first one
filename_encode = filename.replace(ws, utils.url_encode(ws), 1)
stdout_path_encode = json_option.get(
'WORKSPACES') + filename_encode
if utils.not_empty_file(stdout_path_encode):
return json_option.get('WORKSPACES'), os.path.normpath(
filename_encode)
return False, False
def get(self, workspace):
#
# @TODO potential LFI here
#
# get specific module
module = request.args.get('module')
ws_name = os.path.basename(os.path.normpath(workspace))
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + \
"/{0}/log.json".format(ws_name)
if os.path.isfile(ws_json):
raw_logs = utils.reading_json(ws_json)
log = raw_logs
for key in raw_logs.keys():
for i in range(len(raw_logs[key])):
log[key][i]['std_path'] = utils.replace_argument(
self.options,
raw_logs[key][i].get('std_path')).replace(
self.options['WORKSPACES'], '')
log[key][i]['output_path'] = utils.replace_argument(
self.options,
raw_logs[key][i].get('output_path')).replace(
self.options['WORKSPACES'], '')
if module:
cmds = log.get(module)
return {'commands': cmds}
else:
return log
return 'Custom 404 here', 404
def post(self, workspace):
ws_name = utils.get_workspace(workspace=workspace)
options_path = current_path + \
'/storages/{0}/options.json'.format(ws_name)
self.options = utils.reading_json(options_path)
module = request.args.get('module')
ws_name = os.path.basename(os.path.normpath(workspace))
ws_name_encode = utils.url_encode(ws_name)
utils.print_debug(ws_name)
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + "/{0}/log.json".format(
ws_name)
raw_logs = utils.reading_json(ws_json)
elif ws_name_encode in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + "/{0}/log.json".format(
utils.url_encode(ws_name))
# utils.print_debug(ws_json_encode)
raw_logs = utils.reading_json(ws_json)
if raw_logs:
all_commands = []
for k in raw_logs.keys():
for item in raw_logs[k]:
cmd_item = item
cmd_item["module"] = k
cmd_item['std_path'] = utils.replace_argument(
self.options, item.get('std_path')).replace(
self.options['WORKSPACES'], '')
cmd_item['output_path'] = utils.replace_argument(
self.options, item.get('output_path')).replace(
self.options['WORKSPACES'], '')
cmd_item["module"] = k
all_commands.append(cmd_item)
return {"commands": all_commands}
else:
return {
"error":
"Not found logs file for {0} workspace".format(ws_name)
}
def get(self):
# prevent reading secret from config file
secret_things = ['USERNAME','PASSWORD', 'BOT_TOKEN', 'GITHUB_API_KEY']
options = utils.reading_json(current_path + '/storages/options.json')
for item in secret_things:
del options[item]
return options
def get_activities(self, workspace):
ws_name = utils.get_workspace(workspace=workspace)
activities_path = current_path + \
'/storages/{0}/activities.json'.format(ws_name)
self.activities = utils.reading_json(activities_path)
if not self.activities:
return False
return True
def get_options(self, username, password):
option_files = glob.glob(current_path + '/storages/**/options.json',
recursive=True)
# loop though all options avalible
for option in option_files:
json_option = utils.reading_json(option)
if username == json_option.get('USERNAME'):
if password == json_option.get('PASSWORD'):
return True
return False
def get(self, workspace):
# prevent reading secret from config file though API
ws_name = utils.get_workspace(workspace=workspace)
options_path = current_path + \
'/storages/{0}/options.json'.format(ws_name)
secret_things = ['USERNAME', 'PASSWORD', 'BOT_TOKEN', 'GITHUB_API_KEY']
options = utils.reading_json(options_path)
for item in secret_things:
del options[item]
return options
def get(self, workspace):
# get options depend on workspace
ws_name = utils.get_workspace(workspace=workspace)
options_path = str(
BASE_DIR.joinpath('storages/{0}/options.json'.format(ws_name)))
self.options = utils.reading_json(options_path)
module = request.args.get('module')
ws_name = os.path.basename(os.path.normpath(workspace))
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + \
"/{0}/log.json".format(ws_name)
if os.path.isfile(ws_json):
raw_logs = utils.reading_json(ws_json)
log = raw_logs
for key in raw_logs.keys():
for i in range(len(raw_logs[key])):
log[key][i]['std_path'] = utils.replace_argument(
self.options,
raw_logs[key][i].get('std_path')).replace(
self.options['WORKSPACES'], '')
log[key][i]['output_path'] = utils.replace_argument(
self.options,
raw_logs[key][i].get('output_path')).replace(
self.options['WORKSPACES'], '')
if module:
cmds = log.get(module)
return {'commands': cmds}
else:
return log
return 'Custom 404 here', 404
def post(self):
# global options
data = Authentication.parser.parse_args()
username = data['username']
password = data['password']
current_path = os.path.dirname(os.path.realpath(__file__))
options = utils.reading_json(current_path + '/storages/options.json')
if username == options.get('USERNAME'):
if password == options.get('PASSWORD'):
expires = datetime.timedelta(days=365)
token = create_access_token(username, expires_delta=expires)
return {'access_token': token}
return {'error': "Credentials Incorrect"}
def post(self):
# global options
data = Authentication.parser.parse_args()
username = data['username']
password = data['password']
current_path = os.path.dirname(os.path.realpath(__file__))
options = utils.reading_json(current_path + '/storages/options.json')
if username == options.get('USERNAME'):
if password == options.get('PASSWORD'):
expires = datetime.timedelta(days=365)
token = create_access_token(username, expires_delta=expires)
return {'access_token': token}
return {'error': "Credentials Incorrect"}
def get(self):
# get all options file availiable
option_files = glob.glob(current_path + '/storages/**/options.json',
recursive=True)
if not option_files:
return {'error': 'No worksapce avaliable'}
ws = []
for options in option_files:
json_options = utils.reading_json(options)
ws.extend([
ws for ws in os.listdir(json_options['WORKSPACES'])
if ws[0] != '.'
])
return {'workspaces': list(set(ws))}
def post(self, workspace):
module = request.args.get('module')
ws_name = os.path.basename(os.path.normpath(workspace))
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + \
"/{0}/log.json".format(ws_name)
raw_logs = utils.reading_json(ws_json)
all_commands = []
for k in raw_logs.keys():
for item in raw_logs[k]:
cmd_item = item
cmd_item["module"] = k
cmd_item['std_path'] = utils.replace_argument(
self.options, item.get('std_path')).replace(self.options['WORKSPACES'], '')
cmd_item['output_path'] = utils.replace_argument(
self.options, item.get('output_path')).replace(self.options['WORKSPACES'], '')
cmd_item["module"] = k
all_commands.append(cmd_item)
return {"commands": all_commands}
def post(self):
current_path = os.path.dirname(os.path.realpath(__file__))
# global options
data = Configurations.parser.parse_args()
options = data['options']
utils.just_write(current_path + '/storages/options.json',
options,
is_json=True)
utils.print_info("Cleasning activities log")
#Create skeleton activities
commands = utils.reading_json(current_path + '/storages/commands.json')
raw_activities = {}
for k, v in commands.items():
raw_activities[k] = []
utils.just_write(current_path + '/storages/activities.json',
raw_activities,
is_json=True)
return options
def post(self, workspace=None):
# global options
data = Authentication.parser.parse_args()
username = data['username']
password = data['password']
# if no workspace specific
if not workspace:
if self.get_options(username, password):
# cause we don't have real db so it's really hard to manage JWT
# just change the secret if you want to revoke old token
expires = datetime.timedelta(days=365)
token = create_access_token(username, expires_delta=expires)
return {'access_token': token}
else:
return {'error': "Credentials Incorrect"}
elif workspace == 'None':
pass
current_path = os.path.dirname(os.path.realpath(__file__))
options_path = current_path + \
'/storages/{0}/options.json'.format(workspace)
if not utils.not_empty_file(options_path):
return {'error': "Workspace not found"}
options = utils.reading_json(options_path)
if username == options.get('USERNAME'):
if password == options.get('PASSWORD'):
# cause we don't have real db so it's really hard to manage JWT
# just change the secret if you want to revoke old token
expires = datetime.timedelta(days=365)
token = create_access_token(username, expires_delta=expires)
return {'access_token': token}
return {'error': "Credentials Incorrect"}
def get(self):
# get all options file availiable
option_files = glob.glob(str(BASE_DIR) + '/storages/**/options.json',
recursive=True)
if not option_files:
return {'error': 'No worksapce avaliable'}
ws = []
try:
for options in option_files:
json_options = utils.reading_json(options)
if json_options:
ws.extend([
ws for ws in os.listdir(json_options['WORKSPACES'])
if ws[0] != '.'
])
except Exception:
# @TODO get config from flask app
# loading default config path if some exeption happend
options = utils.just_read_config()
ws = os.listdir(options.get('WORKSPACES'))
return {'workspaces': list(set(ws))}
def post(self, workspace):
module = request.args.get('module')
ws_name = os.path.basename(os.path.normpath(workspace))
if ws_name in os.listdir(self.options['WORKSPACES']):
ws_json = self.options['WORKSPACES'] + \
"/{0}/log.json".format(ws_name)
raw_logs = utils.reading_json(ws_json)
all_commands = []
for k in raw_logs.keys():
for item in raw_logs[k]:
cmd_item = item
cmd_item["module"] = k
cmd_item['std_path'] = utils.replace_argument(
self.options, item.get('std_path')).replace(
self.options['WORKSPACES'], '')
cmd_item['output_path'] = utils.replace_argument(
self.options, item.get('output_path')).replace(
self.options['WORKSPACES'], '')
cmd_item["module"] = k
all_commands.append(cmd_item)
return {"commands": all_commands}
def get(self, workspace):
# module = request.args.get('module')
ws_name = os.path.basename(os.path.normpath(workspace))
options_path = str(BASE_DIR.joinpath('storages/{0}/options.json'.format(ws_name)))
self.options = utils.reading_json(options_path)
if not self.options:
return {"error": "Workspace {0} not found".format(ws_name)}
# get commands
commands_path = str(BASE_DIR.joinpath('commands.json'))
self.commands = utils.reading_json(commands_path)
# change to current workspace instead of get from running target
self.options['WORKSPACE'] = self.options['WORKSPACES'] + ws_name
self.options['OUTPUT'] = ws_name
final_reports = []
# reports = {}
for key in self.commands.keys():
final_reports.append({
"module": key,
"reports": []
})
for k in self.commands.keys():
if "report" in self.commands[k].keys():
report = utils.replace_argument(self.options, self.commands[k].get("report"))
# print(report)
if type(report) == str:
if utils.not_empty_file(report):
report_path = report.replace(
self.options.get('WORKSPACE'), ws_name)
report_item = {
"path": report_path,
"type": "html",
}
for i in range(len(final_reports)):
if final_reports[i].get('module') == k:
final_reports[i]["reports"].append(
report_item)
# final_reports[k]["reports"].append(report_item)
elif type(report) == list:
for item in report:
report_path = utils.replace_argument(self.options, item.get("path"))
if utils.not_empty_file(report_path):
report_path = report_path.replace(
self.options.get('WORKSPACE'), ws_name)
report_item = {
"path": report_path,
"type": item.get("type"),
}
for i in range(len(final_reports)):
if final_reports[i].get('module') == k:
final_reports[i]["reports"].append(report_item)
# just clean up
clean_reports = []
for i in range(len(final_reports)):
if final_reports[i].get('reports'):
clean_reports.append(final_reports[i])
return {'reports': clean_reports}
def __init__(self, **kwargs):
self.options = utils.reading_json(current_path +
'/storages/options.json')
def custom_static(filename):
options = utils.reading_json(current_path + '/rest/storages/options.json')
return send_from_directory(options['WORKSPACES'], filename)
def __init__(self, **kwargs):
self.options = utils.reading_json(current_path + '/storages/options.json')
def post(self):
data = Cmd.parser.parse_args()
cmd = data['cmd']
std_path = data['std_path']
output_path = data['output_path']
module = data['module']
nolog = data['nolog']
activity = {
'cmd': cmd,
'std_path': std_path,
'output_path': output_path,
'status': 'Running'
}
if nolog == 'False':
activities = utils.reading_json(current_path + '/storages/activities.json')
if activities.get(module):
activities[module].append(activity)
else:
activities[module] = [activity]
utils.just_write(current_path + '/storages/activities.json',
activities, is_json=True)
slack.slack_noti('log', self.options, mess={
'title': "{0} | {1} | Execute".format(self.options['TARGET'], module),
'content': '```{0}```'.format(cmd),
})
utils.print_info("Execute: {0} ".format(cmd))
stdout = execute.run(cmd)
# just ignore for testing purpose
# stdout = "<< stdoutput >> << {0} >>".format(cmd)
utils.check_output(output_path)
if nolog == 'False':
# change status of log
activities = utils.reading_json(current_path + '/storages/activities.json')
for item in activities[module]:
if item['cmd'] == cmd:
if stdout is None:
item['status'] = 'Error'
else:
item['status'] = 'Done'
try:
if std_path != '':
utils.just_write(std_path, stdout)
slack.slack_file('std', self.options, mess={
'title': "{0} | {1} | std".format(self.options['TARGET'], module),
'filename': '{0}'.format(std_path),
})
if output_path != '':
slack.slack_file('verbose-report', self.options, mess={
'channel': self.options['VERBOSE_REPORT_CHANNEL'],
'filename': output_path
})
except:
pass
utils.just_write(current_path + '/storages/activities.json', activities, is_json=True)
return jsonify(status="200", output_path=output_path)
def post(self, workspace):
ws_name = utils.get_workspace(workspace=workspace)
options_path = current_path + \
'/storages/{0}/options.json'.format(ws_name)
self.options = utils.reading_json(options_path)
data = Cmd.parser.parse_args()
cmd = data['cmd']
std_path = data['std_path']
output_path = data['output_path']
module = data['module']
nolog = data['nolog']
activity = {
'cmd': cmd,
'std_path': std_path,
'output_path': output_path,
'status': 'Running'
}
if nolog == 'False':
activities_path = current_path + '/storages/{0}/activities.json'.format(
ws_name)
# activities = utils.reading_json(activities_path)
activities = utils.reading_json(activities_path)
if activities.get(module):
activities[module].append(activity)
else:
activities[module] = [activity]
utils.just_write(activities_path, activities, is_json=True)
slack.slack_noti('log',
self.options,
mess={
'title':
"{0} | {1} | Execute".format(
self.options['TARGET'], module),
'content':
'```{0}```'.format(cmd),
})
utils.print_info("Execute: {0} ".format(cmd))
stdout = execute.run(cmd)
utils.check_output(output_path)
# just ignore for testing purpose
# stdout = "<< stdoutput >> << {0} >>".format(cmd)
if nolog == 'False':
# change status of log
activities = utils.reading_json(activities_path)
for item in activities[module]:
if item['cmd'] == cmd:
if stdout is None:
item['status'] = 'Error'
else:
item['status'] = 'Done'
try:
if std_path != '':
utils.just_write(std_path, stdout)
slack.slack_file(
'std',
self.options,
mess={
'title':
"{0} | {1} | std".format(
self.options['TARGET'], module),
'filename':
'{0}'.format(std_path),
})
if output_path != '':
slack.slack_file(
'verbose-report',
self.options,
mess={
'channel':
self.options['VERBOSE_REPORT_CHANNEL'],
'filename':
output_path
})
except:
pass
utils.just_write(activities_path, activities, is_json=True)
return jsonify(status="200", output_path=output_path)
def custom_static(filename):
options = utils.reading_json(current_path + '/rest/storages/options.json')
return send_from_directory(options['WORKSPACES'], filename)
def get(self):
# print(current_path)
options = utils.reading_json(current_path + '/storages/options.json')
return options
def __init__(self, **kwargs):
self.activities = utils.reading_json(current_path + '/storages/activities.json')