def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
if not (1 <= args.trial[0] <= last_trial_id
and 1 <= args.trial[1] <= last_trial_id):
utils.print_msg('inexistent trial id', True)
sys.exit(1)
trial_before = persistence.load_trial(args.trial[0]).fetchone()
modules_before = persistence.load_dependencies()
environment_before = persistence.load('environment_attr',
['name', 'value'],
trial_id=trial_before['id'])
trial_after = persistence.load_trial(args.trial[1]).fetchone()
modules_after = persistence.load_dependencies()
environment_after = persistence.load('environment_attr', ['name, value'],
trial_id=trial_after['id'])
diff_trials(trial_before, trial_after)
if args.modules:
diff_modules(set(modules_before.fetchall()),
set(modules_after.fetchall()))
if args.environment:
diff_environment(set(environment_before.fetchall()),
set(environment_after.fetchall()))
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
trial_id = args.trial if args.trial != None else last_trial_id
if not 1 <= trial_id <= last_trial_id:
utils.print_msg('inexistent trial id', True)
sys.exit(1)
print_trial(persistence.load_trial(trial_id).fetchone())
if args.modules:
print_modules(persistence.load_dependencies())
if args.function_defs:
print_function_defs(persistence.load('function_def',
trial_id=trial_id))
if args.environment:
environment = {
attr['name']: attr['value']
for attr in persistence.load('environment_attr', trial_id=trial_id)
}
utils.print_map(
'this trial has been executed under the following environment conditions',
environment)
if args.function_activations:
print_function_activations(
persistence.load('function_activation',
caller_id=None,
trial_id=trial_id).fetchone())
if args.file_accesses:
print_file_accesses(persistence.load('file_access', trial_id=trial_id))
def export_facts(trial_id): # TODO: export remaining data (now focusing only on activation and file access)
result = []
result.append('%\n% FACT: activation(id, name, start, finish, caller_activation_id).\n%\n')
result.append(':- dynamic activation/5.')
for activation in persistence.load('function_activation', trial_id = trial_id):
activation = dict(activation)
activation['name'] = str(activation['name'])
activation['start'] = timestamp(activation['start'])
activation['finish'] = timestamp(activation['finish'])
if not activation['caller_id']:
activation['caller_id'] = 'nil'
result.append('activation({id}, {name!r}, {start:-f}, {finish:-f}, {caller_id}).'.format(**activation))
result.append('\n%\n% FACT: access(id, name, mode, content_hash_before, content_hash_after, timestamp, activation_id).\n%\n')
result.append(':- dynamic access/7.')
for access in persistence.load('file_access', trial_id = trial_id):
access = dict(access)
access['name'] = str(access['name'])
access['mode'] = str(access['mode'])
access['buffering'] = str(access['buffering'])
access['content_hash_before'] = str(access['content_hash_before'])
access['content_hash_after'] = str(access['content_hash_after'])
access['timestamp'] = timestamp(access['timestamp'])
result.append('access({id}, {name!r}, {mode!r}, {content_hash_before!r}, {content_hash_after!r}, {timestamp:-f}, {function_activation_id}).'.format(**access))
return '\n'.join(result)
def print_function_activations(function_activation):
utils.print_msg('this trial has the following function activation graph:',
True)
for inner_function_activation in persistence.load(
'function_activation', caller_id=function_activation['id']):
print_function_activation(inner_function_activation)
def execute(args):
persistence.connect_existing(os.getcwd())
print_msg('trials available in the provenance store:', True)
for trial in persistence.load('trial'):
text = ' Trial {id}: {script} {arguments}'.format(**trial)
indent = text.index(': ') + 2
print(text)
print('{indent}with code hash {code_hash}'.format(indent = ' ' * indent, **trial))
print('{indent}ran from {start} to {finish}'.format(indent = ' ' * indent, **trial))
def convert_end_date():
job_postings = load()
altered_job_postings = []
for post in job_postings:
post = list(post)
post[Columns.end_date] = convert_end_date_to_sortable_format(
post[Columns.end_date])
altered_job_postings += [tuple(post)]
save(altered_job_postings)
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
if not (1 <= args.trial[0] <= last_trial_id and 1 <= args.trial[1] <= last_trial_id):
utils.print_msg('inexistent trial id', True)
sys.exit(1)
trial_before = persistence.load_trial(args.trial[0]).fetchone()
modules_before = persistence.load_dependencies()
environment_before = persistence.load('environment_attr', ['name', 'value'], trial_id = trial_before['id'])
trial_after = persistence.load_trial(args.trial[1]).fetchone()
modules_after = persistence.load_dependencies()
environment_after = persistence.load('environment_attr', ['name, value'], trial_id = trial_after['id'])
diff_trials(trial_before, trial_after)
if args.modules:
diff_modules(set(modules_before.fetchall()), set(modules_after.fetchall()))
if args.environment:
diff_environment(set(environment_before.fetchall()), set(environment_after.fetchall()))
def print_file_accesses(file_accesses):
utils.print_msg('this trial accessed the following files:', True)
output = []
for file_access in file_accesses:
stack = []
function_activation = persistence.load(
'function_activation',
id=file_access['function_activation_id']).fetchone()
while function_activation:
function_name = function_activation['name']
function_activation = persistence.load(
'function_activation',
id=function_activation['caller_id']).fetchone()
if function_activation:
stack.insert(0, function_name)
if not stack or stack[-1] != 'open':
stack.append(' ... -> open')
output.append(
' Name: {name}\n Mode: {mode}\n Buffering: {buffering}\n Content hash before: {content_hash_before}\n Content hash after: {content_hash_after}\n Timestamp: {timestamp}\n Function: {stack}'
.format(stack=' -> '.join(stack), **file_access))
print('\n\n'.join(output))
def print_function_defs(function_defs):
utils.print_msg('this trial has the following functions:', True)
output = []
for function_def in function_defs:
objects = {'GLOBAL': [], 'ARGUMENT': [], 'FUNCTION_CALL': []}
for obj in persistence.load('object',
function_def_id=function_def['id']):
objects[obj['type']].append(obj['name'])
output.append(
' Name: {name}\n Arguments: {arguments}\n Globals: {globals}\n Function calls: {calls}\n Code hash: {code_hash}'
.format(arguments=', '.join(objects['ARGUMENT']),
globals=', '.join(objects['GLOBAL']),
calls=', '.join(objects['FUNCTION_CALL']),
**function_def))
print('\n\n'.join(output))
def print_function_activation(function_activation, level=1):
object_values = {'GLOBAL': [], 'ARGUMENT': []}
for obj in persistence.load(
'object_value', function_activation_id=function_activation['id']):
object_values[obj['type']].append('{} = {}'.format(
obj['name'], obj['value']))
text = '{indent}{line}: {name} ({start} - {finish})'.format(
indent=' ' * level, **function_activation)
indent = text.index(': ') + 2
print(text)
if object_values['ARGUMENT']:
print('{indent}Arguments: {arguments}'.format(
indent=' ' * indent,
arguments=', '.join(object_values['ARGUMENT'])))
if object_values['GLOBAL']:
print('{indent}Globals: {globals}'.format(
indent=' ' * indent, globals=', '.join(object_values['GLOBAL'])))
if function_activation['return']:
print('{indent}Return value: {ret}'.format(
indent=' ' * indent, ret=function_activation['return']))
for inner_function_activation in persistence.load(
'function_activation', caller_id=function_activation['id']):
print_function_activation(inner_function_activation, level + 1)
def export_facts(
trial_id
): # TODO: export remaining data (now focusing only on activation and file access)
result = []
result.append(
'%\n% FACT: activation(id, name, start, finish, caller_activation_id).\n%\n'
)
result.append(':- dynamic activation/5.')
for activation in persistence.load('function_activation',
trial_id=trial_id):
activation = dict(activation)
activation['name'] = str(activation['name'])
activation['start'] = timestamp(activation['start'])
activation['finish'] = timestamp(activation['finish'])
if not activation['caller_id']:
activation['caller_id'] = 'nil'
result.append(
'activation({id}, {name!r}, {start:-f}, {finish:-f}, {caller_id}).'
.format(**activation))
result.append(
'\n%\n% FACT: access(id, name, mode, content_hash_before, content_hash_after, timestamp, activation_id).\n%\n'
)
result.append(':- dynamic access/7.')
for access in persistence.load('file_access', trial_id=trial_id):
access = dict(access)
access['name'] = str(access['name'])
access['mode'] = str(access['mode'])
access['buffering'] = str(access['buffering'])
access['content_hash_before'] = str(access['content_hash_before'])
access['content_hash_after'] = str(access['content_hash_after'])
access['timestamp'] = timestamp(access['timestamp'])
result.append(
'access({id}, {name!r}, {mode!r}, {content_hash_before!r}, {content_hash_after!r}, {timestamp:-f}, {function_activation_id}).'
.format(**access))
return '\n'.join(result)
def index():
return render_template('index.html', adverts=load(), c=Columns)
GITLAB_ADDRESS = 'http://gitlab.dibr.lanit'
# Don't put your telegram token to git
with open('telegram_token', 'r') as token_file:
TELEGRAM_TOKEN = token_file.read()
PROJECT_STATE_FILENAME = 'projects_state.json'
CHAT_IDS_FILENAME = 'chat_ids.json'
DEBUG_PROJECT = 'OSP/pipeline-sandbox'
messages = Queue(1000)
# last state of all projects
projects_state = {}
projects_state = persistence.load(PROJECT_STATE_FILENAME)
# ids of chats to notify people
chat_ids = []
chat_ids = persistence.load(CHAT_IDS_FILENAME)
def get_project_state(gl, gl_project):
project_name = gl_project.path_with_namespace
pipeline_states = {}
project_state = ProjectState(project_name, pipeline_states)
gl_pipelines = gl_project.pipelines.list()
for gl_pipeline in gl_pipelines:
ref = gl_pipeline.ref
# ignoring pipelines without ref and pipelines with pending statuses
def run(self):
parser = OptionParser("%prog [options] [url_or_domain]")
parser.add_option('-l', '--length', type='int', default=10, help='length of generated password (%default)')
parser.add_option('--ask', action='store_true', default=False, help='(ignored)')
parser.add_option('--no-ask', dest='ask', action='store_false', help='(ignored)')
parser.add_option('--save', action='store_true', default=False, help='(ignored)')
parser.add_option('--no-save', dest='save', action='store_false', help='(ignored)')
parser.add_option('--notify', action='store_true', default=True, help='Notify on completion (default is --notify)')
parser.add_option('-q', '--no-notify', dest='notify', action='store_false')
parser.add_option('-r', '--remember', action='store_true', default=True, help='remember on completion (default is --remember)')
parser.add_option('--hint', help='add a hint for this domain (implies --remember)')
parser.add_option('--no-remember', dest='remember', action='store_false')
parser.add_option('--forget', action='store_true', default=False, help='forget this domain from ~/.config/supergenpass/domains (undo a previous --remember)')
parser.add_option('--domains', dest='list_domains', action='store_true', default=False, help='list all remembered domains')
parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='more information')
parser.add_option('-p', '--print', dest='print_password', action='store_true', default=None, help='just print generated password')
opts, args = parser.parse_args()
self.opts = opts
if len(args) > 1:
parser.print_help()
sys.exit(1)
else:
url = args[0] if args else None
store = persistence.load()
if opts.list_domains:
print('\n'.join(store.list_domains()))
return
if not url:
url = self.do(guess_url)
if not url:
url = ui.get_input('Enter domain / URL: ')
domain = url_to_domain(url)
info("Using Domain: %s" % (domain,))
if opts.forget:
store.forget(domain)
elif opts.hint is not None:
store.remember(domain, opts.hint)
else:
if opts.print_password is None and not sys.stdout.isatty():
if opts.verbose:
info("sgp: assuming --print since stdout is not a TTY")
opts.print_password = True
current_hint = store.get_hint(domain)
if current_hint:
info("(Hint: %s)" % (current_hint,))
pass_ = ui.get_password('Enter master password: '******'%s'" % (opts.length, domain))
try:
save_clipboard(generated_pass)
info(" (password saved to the clipboard)")
except (StandardError, ImportError):
if opts.verbose:
import traceback
traceback.print_exc()
info("could not save clipboard. your password is: %s" % (generated_pass))
if opts.notify:
self.do(notify, domain)
if opts.remember:
store.remember(domain)
store.save()
def test_save_load(self):
key = "test_key"
value = 42
persistence.save(self.temp_file, key, value)
result = persistence.load(self.temp_file, key)
self.assertEqual(value, result)
def test_save_load_org(self):
key = "test_key"
value = org.Organism(3)
persistence.save(self.temp_file, key, value)
result = persistence.load(self.temp_file, key)
self.assertEqual(value, result)
def test_load_not_found(self):
with self.assertRaises(KeyError):
bad_result = persistence.load(self.temp_file, "fake_key")
import persistence
_FILE_NAME = "data.json"
# Load from the save file.
_high_scores = persistence.load(_FILE_NAME)
def get():
return _high_scores
def push(score):
# Add this score, and remove the smallest.
_high_scores.append(score)
_high_scores.remove(min(_high_scores))
# Order the scores.
_high_scores.sort()
_high_scores.reverse()
# Update save file
persistence.store(_FILE_NAME, _high_scores)