def reload(self):
"""Reload the workers
.. :quickref: Module; Reload workers
Requires the `manage_modules` permission.
Returns "ok".
"""
for repository in Repository.get_collection().find():
dispatcher.update_modules(Repository(repository))
updates = Internals(
get_or_404(Internals.get_collection(), name="updates"))
updates.update_value("last_update", time())
flash(
'Workers will reload once they are done with their current tasks',
'success')
return redirect('ok', url_for('ModulesView:index'))
class Repository(MongoDict):
collection_name = 'repositories'
def __init__(self, values={}):
keyfile = os.path.join(FAME_ROOT, "conf", "id_rsa")
self['ssh_cmd'] = "ssh -o StrictHostKeyChecking=no -i {}".format(
keyfile)
MongoDict.__init__(self, values)
def delete(self):
# First, remove modules from database
for module in ModuleInfo.find():
if module['path'].startswith('fame.modules.{}.'.format(
self['name'])):
module.delete()
# Then, delete the files
try:
rmtree(self.path())
except:
pass
# Finally, delete record of repository
MongoDict.delete(self)
def path(self):
return os.path.join(FAME_ROOT, 'fame', 'modules', self['name'])
def clone(self):
clone_repository.apply_async((self['_id'], ), queue='updates')
#clone_repository(self['_id'])
def do_clone(self):
print "[+] Cloning '{}'".format(self['name'])
try:
if self['private']:
Repo.clone_from(self['address'],
self.path(),
env=dict(GIT_SSH_COMMAND=self['ssh_cmd']))
else:
Repo.clone_from(self['address'], self.path())
dispatcher.update_modules(self)
self.update_value('status', 'active')
except Exception, e:
self['status'] = 'error'
self[
'error_msg'] = 'Could not clone repository, probably due to authentication issues.\n{}'.format(
e)
self.save()
internals = Internals.get(name="updates")
internals.update_value("last_update", time())
def do_clone(self):
print(("[+] Cloning '{}'".format(self['name'])))
try:
if self['private']:
Repo.clone_from(self['address'], self.path(), env=dict(GIT_SSH_COMMAND=self['ssh_cmd']))
else:
Repo.clone_from(self['address'], self.path())
dispatcher.update_modules(self)
self.update_value('status', 'active')
except Exception as e:
self['status'] = 'error'
self['error_msg'] = 'Could not clone repository, probably due to authentication issues.\n{}'.format(e)
self.save()
internals = Internals.get(name="updates")
internals.update_value("last_update", time())
def _restore_vm(self):
if self.always_ready:
vms = Internals.get(name='virtual_machines')
if self.name not in vms[self.vm_record]:
vms.update_value([self.vm_record, self.name], 1)
else:
vms.update_value([self.vm_record, self.name], vms[self.vm_record][self.name] + 1)
if vms[self.vm_record][self.name] >= self.restore_after:
self.should_restore = True
if self.should_restore:
self._vm.restore()
vms.update_value([self.vm_record, self.name], 0)
else:
self._vm.stop()
def do_clone(self, path=None):
print "[+] Cloning '{}'".format(self['name'])
try:
if self['private']:
Repo.clone_from(self['address'],
path or self.path(),
env=dict(GIT_SSH_COMMAND=self['ssh_cmd']))
else:
Repo.clone_from(self['address'], path or self.path())
internals = Internals.get(name="updates")
internals.update_value("last_update", time())
return True
except Exception, e:
self['status'] = 'error'
self[
'error_msg'] = 'Could not clone repository, probably due to authentication issues.\n{}'.format(
e) # noqa
self.save()
return False
def _release_lock(self):
vms = Internals.get(name='virtual_machines')
vms.update_value([self.vm_record, 'locked'], False)
def _acquire_lock(self):
LOCK_TIMEOUT = timedelta(minutes=120)
WAIT_STEP = 15
vms = Internals.get(name='virtual_machines')
if vms is None:
vms = Internals({'name': 'virtual_machines'})
vms.save()
locked_vm = False
while not locked_vm:
for i, label in enumerate(self.labels):
self._use_vm(i)
last_locked = "{}.last_locked".format(self.vm_record)
if vms.update_value([self.vm_record, 'locked'], True):
vms.update_value([self.vm_record, 'last_locked'],
datetime.now())
locked_vm = True
break
expired_date = datetime.now() - LOCK_TIMEOUT
if vms._update({'$set': {
last_locked: datetime.now()
}}, {last_locked: {
'$lt': expired_date
}}):
vms.update_value([self.vm_record, 'locked'], True)
locked_vm = True
break
if not locked_vm:
sleep(WAIT_STEP)
def configure(self, id):
"""Configure a module.
.. :quickref: Module; Configure a module
Requires the `manage_modules` permission.
For each configuration available, you should set the value in a form
parameter named ``config_NAME``. For boolean values, any value not ``0``
or ``False`` is considered to be ``True``.
If the setting should be an option (be available per analysis), you have
to set ``config_NAME_option`` to any value but ``0`` or ``False``.
If successful, will return the module in ``module``.
Otherwise, errors will be available in ``errors``.
:param id: id of the named configuration.
:form acts_on: comma-delimited list of FAME types this module can act on
(only for Processing modules).
:form triggered_by: comma-delimited list of triggers (only for Processing
modules).
:form queue: name of the queue to use for this module (only for Processing
modules).
"""
module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id))
if request.method == "POST":
if module['type'] == 'Processing':
if 'acts_on' in request.form:
module.update_setting_value(
'acts_on', request.form.get('acts_on', ''))
if 'triggered_by' in request.form:
module.update_setting_value(
'triggered_by', request.form.get('triggered_by', ''))
if 'queue' in request.form:
new_queue = request.form.get('queue')
if module['queue'] == '':
flash('queue cannot be empty', 'danger')
return validation_error()
else:
if module['queue'] != new_queue:
module.update_setting_value('queue', new_queue)
updates = Internals(
get_or_404(Internals.get_collection(),
name="updates"))
updates.update_value("last_update", time())
flash(
'Workers will reload once they are done with their current tasks',
'success')
errors = update_config(module['config'],
options=(module['type'] == 'Processing'))
if errors is not None:
return errors
module.save()
dispatcher.reload()
return redirect({'module': clean_modules(module)},
url_for('ModulesView:index'))
else:
return render({'module': clean_modules(module)},
'modules/module_configuration.html')
def do_pull(self):
print "[+] Pulling '{}'".format(self['name'])
try:
repo = Repo(self.path())
if self['private']:
with repo.git.custom_environment(
GIT_SSH_COMMAND=self['ssh_cmd']):
repo.remotes.origin.pull()
else:
repo.remotes.origin.pull()
# Make sure we delete orphan .pyc files
for root, dirs, files in os.walk(self.path()):
for f in files:
f = os.path.join(root, f)
if f.endswith(".pyc") and not os.path.exists(f[:-1]):
print "Deleting orphan file '{}'".format(f)
os.remove(f)
dispatcher.update_modules(self)
self.update_value('status', 'active')
except Exception, e:
self['status'] = 'error'
self['error_msg'] = 'Could not update repository.\n{}'.format(e)
self.save()
updates = Internals.get(name="updates")
updates.update_value("last_update", time())
def create_internals():
updates = Internals.get(name='updates')
if updates is None:
updates = Internals({'name': 'updates', 'last_update': time()})
updates.save()
def _acquire_lock(self):
LOCK_TIMEOUT = timedelta(minutes=120)
WAIT_STEP = 15
vms = Internals.get(name='virtual_machines')
if vms is None:
vms = Internals({'name': 'virtual_machines'})
vms.save()
last_locked = "{}.last_locked".format(self.vm_record)
while True:
if vms.update_value([self.vm_record, 'locked'], True):
vms.update_value([self.vm_record, 'last_locked'],
datetime.now())
break
expired_date = datetime.now() - LOCK_TIMEOUT
if vms._update({'$set': {
last_locked: datetime.now()
}}, {last_locked: {
'$lt': expired_date
}}):
vms.update_value([self.vm_record, 'locked'], True)
break
sleep(WAIT_STEP)
