def _snitch(self, session):
"""Orchestrates the creation of the environment.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
"""
# Load saved git data
try:
filename = os.path.join(self._basedir(), 'gitrepos.json')
with open(filename, 'r') as f:
gitdata = json.loads(f.read())
except IOError:
logger.info('No data for git could be found.')
return
# Let model compute environment identity
env = EnvironmentEntity(
account_number=gitdata['environment']['account_number'],
name=gitdata['environment']['name'])
identity = env.identity
# Try to locate environment by identity
env = EnvironmentEntity.find(session, identity)
if env is None:
logger.warning('Unable to locate environment {}.'.format(identity))
return
# Iterate over each git repo
gitrepos = []
for gitdict in gitdata.get('data', []):
gitrepo = self._update_gitrepo(session, env, gitdict)
gitrepos.append(gitrepo)
# Update edges
env.gitrepos.update(session, gitrepos, self.time_in_ms)
def _snitch(self, session):
"""Update the kernel modules subgraph.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
"""
env = EnvironmentEntity(uuid=self.run.environment_uuid)
for hostname, filename in self._find_host_tuples(self.file_pattern):
kms = []
# Find host in graph, continue if host not found.
host = HostEntity(hostname=hostname, environment=env.identity)
host = HostEntity.find(session, host.identity)
if host is None:
logger.warning(
'Unable to locate host entity {}'.format(hostname))
continue
# Read data from file
km_data = self.run.get_object(filename).get('data', {})
# Iterate over package maps
for km_name, km_dict in km_data.items():
km = self._update_kernel_module(session, host, km_name,
km_dict)
if km is not None:
kms.append(km)
host.kernelmodules.update(session, kms, self.time_in_ms)
def _snitch(self, session):
"""Orchestrates the creation of the environment.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
"""
env = EnvironmentEntity(
account_number=self.run.environment_account_number,
name=self.run.environment_name
)
for hostname, filename in self._find_host_tuples(self.file_pattern):
virtualenvs = []
host = HostEntity(hostname=hostname, environment=env.identity)
host = HostEntity.find(session, host.identity)
if host is None:
logger.warning(
'Unable to locate host entity {}'.format(hostname)
)
continue
with open(filename, 'r') as f:
pipdict = json.loads(f.read())
pipdict = pipdict.get('data', {})
for path, pkglist in pipdict.items():
virtualenv = self._update_virtualenv(
session,
host,
path,
pkglist
)
virtualenvs.append(virtualenv)
host.virtualenvs.update(session, virtualenvs, self.time_in_ms)
def _update_environment(self, session):
"""Create the environment from settings.
Creates the environment in graph.
:param session: Neo4j driver session.
:type session: neo4j.v1.session.BoltSession
:returns: Environment object
:rtype: HostEntity
"""
env = EnvironmentEntity(
account_number=self.run.environment_account_number,
name=self.run.environment_name
)
env.update(session, self.time_in_ms)
return env
def _snitch(self, session):
"""Orchestrates the creation of the environment.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
"""
# Load saved git data
try:
gitdata = self.run.get_object('gitrepos.json')
except IOError:
logger.info('No data for git could be found.')
return
# Try to locate environment by identity
uuid = self.run.environment_uuid
env = EnvironmentEntity.find(session, uuid)
if env is None:
logger.warning('Unable to locate environment {}.'.format(uuid))
return
# Iterate over each git repo
gitrepos = []
for gitdict in gitdata.get('data', []):
gitrepo = self._update_gitrepo(session, env, gitdict)
gitrepos.append(gitrepo)
# Update edges
env.gitrepos.update(session, gitrepos, self.time_in_ms)
def _snitch(self, session):
"""Update the apt part of the graph..
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
"""
env = EnvironmentEntity(
account_number=self.run.environment_account_number,
name=self.run.environment_name)
for hostname, filename in self._find_host_tuples(self.file_pattern):
aptpkgs = []
# Find host in graph, continue if host not found.
host = HostEntity(hostname=hostname, environment=env.identity)
host = HostEntity.find(session, host.identity)
if host is None:
logger.warning(
'Unable to locate host entity {}'.format(hostname))
continue
# Read data from file
with open(filename, 'r') as f:
aptdata = json.loads(f.read())
aptlist = aptdata.get('data', [])
# Iterate over package maps
for aptdict in aptlist:
aptpkg = self._update_apt_package(session, aptdict)
if aptpkg is not None:
aptpkgs.append(aptpkg)
host.aptpackages.update(session, aptpkgs, self.time_in_ms)
def _snitch(self, session):
"""Orchestrates the updating of the hosts.
Will first create/update any host entities.
Will then version edges from environment to each host.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
"""
env = EnvironmentEntity(
account_number=self.run.environment_account_number,
name=self.run.environment_name)
hosts = []
# Update each host entity
for host_tuple in self._find_host_tuples(self.file_pattern):
host = self._host_from_tuple(session, env, host_tuple)
hosts.append(host)
# Return early if no hosts found
if not hosts:
return
# Update edges from environment to each host.
env.hosts.update(session, hosts, self.time_in_ms)
def _update_environment(self, session):
"""Create the environment from settings.
Creates the environment in graph.
:param session: Neo4j driver session.
:type session: neo4j.v1.session.BoltSession
:returns: Environment object
:rtype: HostEntity
"""
env = EnvironmentEntity(
uuid=self.run.environment_uuid,
account_number=self.run.environment_account_number,
name=self.run.environment_name,
last_sync=self.time_in_ms,
status='na'
)
# TODO - Update status to something real in the future.
env.update(session, self.time_in_ms)
return env
def _snitch(self, session):
"""Orchestrates the creation of the environment.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
"""
# Load saved git data
try:
uservars_dict = self.run.get_object('uservars.json')
except IOError:
logger.info('No data for uservars could be found.')
return
# Try to find the parent environment.
env = EnvironmentEntity(uuid=self.run.environment_uuid)
identity = env.identity
env = EnvironmentEntity.find(session, identity)
if env is None:
logger.warning(
'Unable to locate environment {}.'.format(identity)
)
return
# Iterate over each uservariable
uservars = []
for key, val in uservars_dict.get('data', {}).items():
if isinstance(val, dict) or isinstance(val, list):
val = json.dumps(val, sort_keys=True)
uservar = UservarEntity(
environment=env.identity,
name=key,
value=val
)
uservar.update(session, self.time_in_ms)
uservars.append(uservar)
# Update edges
env.uservars.update(session, uservars, self.time_in_ms)
def find_environment(session, uuid):
"""Find an environment by uuid.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
:param uuid: UUID of the environment
:type uuid: str
:returns: Environment entity
:rtype: EnvironmentEntity
"""
env = EnvironmentEntity.find(session, uuid)
if env is None:
raise EnvironmentNotFoundError(uuid)
return env
def sync_single(path, key=None):
"""Used to sync a single discrete set of run data.
:param path: Path to run data
:type path: str
:param key: Encryption/Decryption base 64 encoded key
:type key: str
"""
with DriverContext() as driver:
run = runs.Run(path, key=key)
env = EnvironmentEntity(uuid=run.environment_uuid,
name=run.environment_name,
account_number=run.environment_account_number)
with lock_environment(driver, env):
sync_run(driver, run)
def _update_host(self, session, hostname, filename):
"""Update configuration files for a host.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
:param hostname: Name of the host
:type hostname: str
:param filename: Name of file
:type filename: str
"""
# Extract config and environment data.
with open(filename, 'r') as f:
configdata = json.loads(f.read())
envdict = configdata.get('environment', {})
env = EnvironmentEntity(
account_number=envdict.get('account_number'),
name=envdict.get('name'))
configdata = configdata.get('data', {})
# Find parent host object - return early if not exists.
host = HostEntity(hostname=hostname, environment=env.identity)
host = HostEntity.find(session, host.identity)
if host is None:
logger.warning('Unable to locate host {}'.format(hostname))
return
# Iterate over configration files in the host's directory
configfiles = []
for filename, contents in configdata.items():
_, name = os.path.split(filename)
md5 = hashlib.md5()
md5.update(contents.encode('utf-8'))
md5 = md5.hexdigest()
# Update configfile node
configfile = ConfigfileEntity(path=filename,
host=host.identity,
md5=md5,
contents=contents,
name=name)
configfile.update(session, self.time_in_ms)
configfiles.append(configfile)
# Update host -> configfile relationships.
host.configfiles.update(session, configfiles, self.time_in_ms)
def _update_host(self, session, hostname, filename):
"""Update configuredinterfaces for a host.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
:param hostname: Name of the host
:type hostname: str
:param filename: Name of file
:type filename: str
"""
# Extract config and environment data.
env = EnvironmentEntity(uuid=self.run.environment_uuid)
with open(filename, 'r') as f:
data = json.loads(f.read())
configdata = data.get('data', {})
# Find parent host object - return early if not exists.
host = HostEntity(hostname=hostname, environment=env.identity)
host = HostEntity.find(session, host.identity)
if host is None:
logger.warning('Unable to locate host {}'.format(hostname))
return
# Iterate over configration files in the host's directory
interfaces = []
for device, metadata in configdata.items():
interfacekwargs = {
'device': device,
'host': host.identity
}
for key, val in metadata.items():
if '-' in key:
# neo4j properties don't allow -
# dns-nameservers, offline-sg
key = key.replace('-', '_')
interfacekwargs[key] = val
interface = ConfiguredInterfaceEntity(**interfacekwargs)
interface.update(session, self.time_in_ms)
interfaces.append(interface)
# Update host -> configuredinterfaces relationships.
host.configuredinterfaces.update(session, interfaces, self.time_in_ms)
def check_run_time(driver, run):
"""Prevent a run from updating an environment.
Protects an environment with newer data from a run with older data.
:param driver: Neo4J database driver instance
:type driver: neo4j.v1.GraphDatabase.driver
:param run: Date run instance
:type run: cloud_snitch.runs.Run
"""
# Check to see if run data is new
with driver.session() as session:
e = EnvironmentEntity.find(session, run.environment_uuid)
# If the environment exists, check its last update
if e is not None:
last_update = utils.utcdatetime(e.last_update(session) or 0)
logger.debug("Comparing {} to {}".format(run.completed,
last_update))
if run.completed <= last_update:
raise RunContainsOldDataError(run, last_update)
def _update_host(self, session, hostname, filename):
"""Update configuration files for a host.
:param session: neo4j driver session
:type session: neo4j.v1.session.BoltSession
:param hostname: Name of the host
:type hostname: str
:param filename: Name of file
:type filename: str
"""
# Extract config and environment data.
env = EnvironmentEntity(uuid=self.run.environment_uuid)
with open(filename, 'r') as f:
configdata = json.loads(f.read())
configdata = configdata.get('data', {})
# Find parent host object - return early if not exists.
host = HostEntity(hostname=hostname, environment=env.identity)
host = HostEntity.find(session, host.identity)
if host is None:
logger.warning('Unable to locate host {}'.format(hostname))
return
# Iterate over configration files in the host's directory
configfiles = []
for filename, metadata in configdata.items():
_, name = os.path.split(filename)
# Update configfile node
configfile = ConfigfileEntity(path=filename,
host=host.identity,
md5=metadata.get('md5'),
contents=metadata.get('contents'),
is_binary=metadata.get('is_binary'),
name=name)
configfile.update(session, self.time_in_ms)
configfiles.append(configfile)
# Update host -> configfile relationships.
host.configfiles.update(session, configfiles, self.time_in_ms)
def sync_paths(paths):
"""Sync all runs indicated by paths.
:param paths: list of paths indicating runs.
:type paths: list
"""
# Start a neo4j driver context.
with DriverContext() as driver:
for path in paths:
run = runs.Run(path)
# Try to acquire environment lock.
# @TODO - Implement wait until timeout loop.
try:
env = EnvironmentEntity(
uuid=run.environment_uuid,
name=run.environment_name,
account_number=run.environment_account_number
)
with lock_environment(driver, env):
sync_run(driver, run)
except EnvironmentLockedError as e:
logger.error(e)