def __iter__(self):
while not self.__stop:
try:
# TODO: Connection timeout is too long.
self.__response = requests.get(self.__url,
params=self.__params,
stream=True,
auth=get_auth_info())
for raw_line in self.__response.iter_lines():
line = self.__sanitize(raw_line)
if not line:
continue
change = self.__parse_change(line)
if not change:
continue
yield change
except Exception, e:
import traceback
logger.debug(traceback.format_exc())
# Close everything but keep retrying
self.stop()
self.__stop = True
logger.warning(
u"Lost connection to CouchDB. Retrying in 5 seconds...")
time.sleep(5)
logger.info(u"Retrying...")
def process_db_change(change):
if change.created:
logger.info(u'Workspace {} was created'.format(change.db_name))
process_new_workspace(change.db_name)
elif change.deleted:
logger.info(u'Workspace {} was deleted'.format(change.db_name))
process_delete_workspace(change.db_name)
def stop_server():
if not daemonize.stop_server():
# Exists with an error if it couldn't close the server
return False
else:
logger.info("Faraday Server stopped successfully")
return True
def import_on_new_database(db_conn, couchdb_conn):
if db_conn.exists():
raise RuntimeError('Database {} already exists'.format(
db_conn.db_name))
logger.info(u'Creating database for workspace {}'.format(db_conn.db_name))
_create_and_import_db(db_conn, couchdb_conn)
def __get_modified_entity(self, change):
try:
metadata = self.database.session.query(server.models.EntityMetadata)\
.filter(server.models.EntityMetadata.couchdb_id == change.doc_id)\
.one_or_none()
except MultipleResultsFound:
logger.warning(u'Multiple entities were found for doc {}.'\
'Ignoring change'.format(change.doc_id))
return None
if metadata is not None:
# Obtain the proper table on which to perform the entity operation
entity_cls = server.models.FaradayEntity.get_entity_class_from_type(
metadata.document_type)
entity = self.database.session.query(entity_cls)\
.join(server.models.EntityMetadata)\
.filter(server.models.EntityMetadata.couchdb_id == change.doc_id)\
.one()
return entity
else:
logger.info(u'Doc {} was not found in the database'.format(
change.doc_id))
return None
def main():
args = parse_arguments()
if args.stop:
sys.exit(0 if stop_server() else 1)
if not args.no_setup:
setup_environment()
import_workspaces()
if is_server_running():
sys.exit(1)
if args.debug:
server.utils.logger.set_logging_level(server.config.DEBUG)
if args.start:
# Starts a new process on background with --ignore-setup
# and without --start nor --stop
devnull = open('/dev/null', 'w')
params = [
'/usr/bin/env', 'python2.7',
os.path.join(server.config.FARADAY_BASE, __file__), '--no-setup'
]
if args.ssl: params.append('--ssl')
if args.debug: params.append('--debug')
logger.info('Faraday Server is running as a daemon')
subprocess.Popen(params, stdout=devnull, stderr=devnull)
else:
run_server(args)
def __get_modified_entity(self, change):
try:
metadata = self.database.session.query(server.models.EntityMetadata)\
.filter(server.models.EntityMetadata.couchdb_id == change.doc_id)\
.one_or_none()
except MultipleResultsFound:
logger.warning(u'Multiple entities were found for doc {}.'\
'Ignoring change'.format(change.doc_id))
return None
if metadata is not None:
# Obtain the proper table on which to perform the entity operation
entity_cls = server.models.FaradayEntity.get_entity_class_from_type(
metadata.document_type)
entity = self.database.session.query(entity_cls)\
.join(server.models.EntityMetadata)\
.filter(server.models.EntityMetadata.couchdb_id == change.doc_id)\
.one()
return entity
else:
logger.info(u'Doc {} was not found in the database'.format(change.doc_id))
return None
def stop_server():
if not daemonize.stop_server():
# Exists with an error if it couldn't close the server
return False
else:
logger.info("Faraday Server stopped successfully")
return True
def __iter__(self):
while not self.__stop:
try:
# TODO: Connection timeout is too long.
self.__response = requests.get(
self.__url, params=self.__params,
stream=True, auth=get_auth_info())
for raw_line in self.__response.iter_lines():
line = self.__sanitize(raw_line)
if not line:
continue
change = self.__parse_change(line)
if not change:
continue
yield change
except Exception, e:
import traceback
logger.debug(traceback.format_exc())
# Close everything but keep retrying
self.stop()
self.__stop = True
logger.warning(u"Lost connection to CouchDB. Retrying in 5 seconds...")
time.sleep(5)
logger.info(u"Retrying...")
def process_db_change(change):
if change.created:
logger.info(u'Workspace {} was created'.format(change.db_name))
process_new_workspace(change.db_name)
elif change.deleted:
logger.info(u'Workspace {} was deleted'.format(change.db_name))
process_delete_workspace(change.db_name)
def main():
args = parse_arguments()
if args.stop:
sys.exit(0 if stop_server() else 1)
if not args.no_setup:
setup_environment()
import_workspaces()
if is_server_running():
sys.exit(1)
if args.debug:
server.utils.logger.set_logging_level(server.config.DEBUG)
if args.start:
# Starts a new process on background with --ignore-setup
# and without --start nor --stop
devnull = open('/dev/null', 'w')
params = ['/usr/bin/env', 'python2.7', os.path.join(server.config.FARADAY_BASE, __file__), '--no-setup']
if args.ssl: params.append('--ssl')
if args.debug: params.append('--debug')
logger.info('Faraday Server is running as a daemon')
subprocess.Popen(params, stdout=devnull, stderr=devnull)
else:
run_server(args)
def process_new_workspace(ws_name):
if ws_name in workspace:
logger.info(
u"Workspace {} was already migrated. Ignoring change.".format(
ws_name))
else:
setup_workspace(ws_name)
def add_entity_from_doc(self, document):
"""
ISSUES:
* Other entities related to this new document may be not already
include into the database (ie: these documents are added on future
changes)
"""
entity = server.models.FaradayEntity.parse(document)
if entity is None:
return False
entity.add_relationships_from_db(self.__db_conn.session)
self.__db_conn.session.add(entity)
try:
self.__db_conn.session.commit()
logger.info(u'New {} ({}) was added in Workspace {}'.format(
entity.entity_metadata.document_type,
getattr(entity, 'name', '<no-name>'), self.__db_conn.db_name))
except IntegrityError, e:
# For now, we silently rollback because it is an excepted
# scenario when we create documents from the server and its
# change notification arrives
self.__db_conn.session.rollback()
return False
def install_packages(packages):
if ask_to_install(packages):
logger.info("Dependencies installed. Please launch Faraday Server again")
sys.exit(0)
else:
logger.error("Dependencies not met")
sys.exit(1)
def add_entity_from_doc(self, document):
"""
ISSUES:
* Other entities related to this new document may be not already
include into the database (ie: these documents are added on future
changes)
"""
entity = server.models.FaradayEntity.parse(document)
if entity is None:
return False
entity.add_relationships_from_db(self.__db_conn.session)
self.__db_conn.session.add(entity)
try:
self.__db_conn.session.commit()
logger.info(u'New {} ({}) was added in Workspace {}'.format(
entity.entity_metadata.document_type,
getattr(entity, 'name', '<no-name>'),
self.__db_conn.db_name))
except IntegrityError, e:
# For now, we silently rollback because it is an excepted
# scenario when we create documents from the server and its
# change notification arrives
self.__db_conn.session.rollback()
return False
def __process_delete_workspace(self, ws_name):
if ws_name not in self.__workspaces:
logger.info(u"Workspace {} doesn't exist. Ignoring change.".format(
ws_name))
else:
logger.info(
u"Deleting workspace {} from Faraday Server".format(ws_name))
self.__delete_workspace(ws_name)
def __init__(self, enable_ssl=False):
logger.info('Starting web server at {}://{}:{}/'.format(
'https' if enable_ssl else 'http',
server.config.faraday_server.bind_address,
server.config.faraday_server.port))
self.__ssl_enabled = enable_ssl
self.__config_server()
self.__build_server_tree()
def __init__(self, enable_ssl=False):
logger.info('Starting web server at {}://{}:{}/'.format(
'https' if enable_ssl else 'http',
server.config.faraday_server.bind_address,
server.config.faraday_server.port))
self.__ssl_enabled = enable_ssl
self.__config_server()
self.__build_server_tree()
def install_packages(packages):
if ask_to_install(packages):
logger.info(
"Dependencies installed. Please launch Faraday Server again")
sys.exit(0)
else:
logger.error("Dependencies not met")
sys.exit(1)
def __process_new_workspace(self, ws_name):
if ws_name in self.__workspaces:
logger.info(u"Workspace {} already exists. Ignoring change.".format(ws_name))
elif not server.couchdb.server_has_access_to(ws_name):
logger.error(u"Unauthorized access to CouchDB for Workspace {}. Make sure faraday-server's"\
" configuration file has CouchDB admin's credentials set".format(ws_name))
else:
self.__create_and_import_workspace(ws_name)
def reimport_on_database(db_conn, couchdb_conn):
""" WARNING: Make sure to do all necessary verifications on
the database you are working on. If the database exists then
it will truncate and lose all data previously stored there"""
if not db_conn.exists():
raise RuntimeError('Database {} does not exist'.format(db_conn.db_name))
logger.info(u'Importing workspace {} again'.format(db_conn.db_name))
_truncate_and_import_db(db_conn, couchdb_conn)
def process_delete_workspace(ws_name):
if ws_name not in workspace:
logger.info(
u"Workspace {} wasn't migrated at startup. Ignoring change.".
format(ws_name))
else:
logger.info(
u"Deleting workspace {} from Faraday Server".format(ws_name))
delete_workspace(ws_name)
def run_server(args):
import server.web
server.database.initialize()
server.app.setup()
web_server = server.web.WebServer(enable_ssl=args.ssl)
daemonize.create_pid_file()
logger.info('Faraday Server is ready')
web_server.run()
def __process_new_workspace(self, ws_name):
if ws_name in self.__workspaces:
logger.info(
u"Workspace {} already exists. Ignoring change.".format(
ws_name))
elif not server.couchdb.server_has_access_to(ws_name):
logger.error(u"Unauthorized access to CouchDB for Workspace {}. Make sure faraday-server's"\
" configuration file has CouchDB admin's credentials set".format(ws_name))
else:
self.__create_and_import_workspace(ws_name)
def check_database_integrity(self):
if not self.was_migration_successful():
logger.info(u"Workspace {} wasn't migrated successfully. Remigrating workspace...".format(
self.__workspace))
self.remigrate_database()
elif self.get_schema_version() != server.models.SCHEMA_VERSION:
logger.info(u"Workspace {} has an old schema version ({} != {}). Remigrating workspace...".format(
self.__workspace, self.get_schema_version(), server.models.SCHEMA_VERSION))
self.remigrate_database()
def run_server(args):
import server.web
server.database.initialize()
server.app.setup()
web_server = server.web.WebServer(enable_ssl=args.ssl)
daemonize.create_pid_file()
logger.info('Faraday Server is ready')
web_server.run()
def is_integrous(self):
if not self.__db_conf.was_migration_successful():
logger.info(u"Workspace {} wasn't migrated successfully".format(self.db_name))
return False
elif self.__db_conf.get_schema_version() != server.models.SCHEMA_VERSION:
logger.info(u"Workspace {} has an old schema version ({} != {})".format(
self.db_name, self.__db_conf.get_schema_version(), server.models.SCHEMA_VERSION))
return False
return True
def import_workspaces():
couchdb_server_conn, workspaces_list = _open_couchdb_conn()
for workspace_name in workspaces_list:
logger.info(u'Setting up workspace {}'.format(workspace_name))
if not server.couchdb.server_has_access_to(workspace_name):
logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\
" configuration file has CouchDB admin's credentials set")
sys.exit(1)
import_workspace_into_database(workspace_name, couchdb_server_conn=couchdb_server_conn)
def __process_update(self, change):
"""
ISSUES:
* Updated relationships are not taken into account. Status: TODO
"""
entity = self.__get_modified_entity(change)
if entity is not None:
entity.update_from_document(change.doc)
entity.entity_metadata.update_from_document(change.doc)
self.database.session.commit()
logger.info(u'A {} ({}) was updated'.format(
entity.entity_metadata.document_type, entity.name))
def __process_update(self, change):
"""
ISSUES:
* Updated relationships are not taken into account. Status: TODO
"""
entity = self.__get_modified_entity(change)
if entity is not None:
entity.update_from_document(change.doc)
entity.entity_metadata.update_from_document(change.doc)
self.database.session.commit()
logger.info(u'A {} ({}) was updated'.format(
entity.entity_metadata.document_type, entity.name))
def check_database_integrity(self):
if not self.was_migration_successful():
logger.info(
u"Workspace {} wasn't migrated successfully. Remigrating workspace..."
.format(self.__workspace))
self.remigrate_database()
elif self.get_schema_version() != server.models.SCHEMA_VERSION:
logger.info(
u"Workspace {} has an old schema version ({} != {}). Remigrating workspace..."
.format(self.__workspace, self.get_schema_version(),
server.models.SCHEMA_VERSION))
self.remigrate_database()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--ssl', action='store_true', help='enable HTTPS')
parser.add_argument('--debug',
action='store_true',
help='run Faraday Server in debug mode')
parser.add_argument('--start',
action='store_true',
help='run Faraday Server in background')
parser.add_argument('--stop',
action='store_true',
help='stop Faraday Server')
parser.add_argument('--nodeps',
action='store_true',
help='Skip dependency check')
parser.add_argument('--no-setup',
action='store_true',
help=argparse.SUPPRESS)
args = parser.parse_args()
if args.debug:
server.utils.logger.set_logging_level(server.config.DEBUG)
if args.stop:
sys.exit(0 if stop_server() else 1)
if is_server_running():
sys.exit(1)
# Overwrites config option if SSL is set by argument
if args.ssl:
server.config.ssl.set('enabled', 'true')
if not args.no_setup:
setup_environment(not args.nodeps)
import_workspaces()
if args.start:
# Starts a new process on background with --ignore-setup
# and without --start nor --stop
devnull = open('/dev/null', 'w')
params = [
'/usr/bin/env', 'python2.7',
os.path.join(server.config.FARADAY_BASE, __file__), '--no-setup'
]
if args.ssl: params.append('--ssl')
if args.debug: params.append('--debug')
logger.info('Faraday Server is running as a daemon')
subprocess.Popen(params, stdout=devnull, stderr=devnull)
else:
run_server(args)
def setup_environment(check_deps=False):
# Configuration files generation
server.config.copy_default_config_to_local()
if check_deps:
# Check dependencies
installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies(
requirements_file=server.config.REQUIREMENTS_FILE)
logger.info("Checking dependencies...")
if conflict_deps:
logger.info("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"")
if missing_deps:
install_deps = query_yes_no("Do you want to install them?", default="no")
if install_deps:
dependencies.install_packages(missing_deps)
logger.info("Dependencies installed. Please launch Faraday Server again.")
sys.exit(0)
else:
logger.error("Dependencies not met. Please refer to the documentation in order to install them. [%s]",
", ".join(missing_deps))
logger.info("Dependencies met")
# Web configuration file generation
server.config.gen_web_config()
def __process_add(self, change):
"""
ISSUES:
* Other entities related to this new document may be not already
include into the database (ie: these documents are added on future
changes)
"""
entity = server.models.FaradayEntity.parse(change.doc)
if entity is not None:
entity.add_relationships_from_db(self.database.session)
self.database.session.add(entity)
self.database.session.commit()
logger.info(u'New {} ({}) was added'.format(
entity.entity_metadata.document_type, entity.name))
def __process_add(self, change):
"""
ISSUES:
* Other entities related to this new document may be not already
include into the database (ie: these documents are added on future
changes)
"""
entity = server.models.FaradayEntity.parse(change.doc)
if entity is not None:
entity.add_relationships_from_db(self.database.session)
self.database.session.add(entity)
self.database.session.commit()
logger.info(u'New {} ({}) was added'.format(
entity.entity_metadata.document_type, entity.name))
def setup_environment(check_deps=False):
# Configuration files generation
server.config.copy_default_config_to_local()
if check_deps:
# Check dependencies
installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies(
requirements_file=server.config.REQUIREMENTS_FILE)
logger.info("Checking dependencies...")
if conflict_deps:
logger.info("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"")
if missing_deps:
install_deps = query_yes_no("Do you want to install them?", default="no")
if install_deps:
dependencies.install_packages(missing_deps)
logger.info("Dependencies installed. Please launch Faraday Server again.")
sys.exit(0)
else:
logger.error("Dependencies not met. Please refer to the documentation in order to install them. [%s]",
", ".join(missing_deps))
logger.info("Dependencies met")
# Web configuration file generation
server.config.gen_web_config()
def is_integrous(self):
if not self.__db_conf.was_migration_successful():
logger.info(u"Workspace {} wasn't migrated successfully".format(
self.db_name))
return False
elif self.__db_conf.get_schema_version(
) != server.models.SCHEMA_VERSION:
logger.info(
u"Workspace {} has an old schema version ({} != {})".format(
self.db_name, self.__db_conf.get_schema_version(),
server.models.SCHEMA_VERSION))
return False
return True
def __build_websockets_resource(self):
websocket_port = int(server.config.faraday_server.websocket_port)
url = '{0}:{1}'.format(self.__bind_address, websocket_port)
if self.__ssl_enabled:
url = 'wss://' + url
else:
url = 'ws://' + url
# logger.info(u"Websocket listening at {url}".format(url=url))
logger.info(
'Starting websocket server at port {0} with bind address {1}. '
'SSL {2}'.format(websocket_port, self.__bind_address,
self.__ssl_enabled))
factory = WorkspaceServerFactory(url=url)
factory.protocol = BroadcastServerProtocol
return factory
def __get_modified_entity(self, document_id):
metadata = self.get_document_metadata(document_id)
if metadata is None:
logger.info(u'Doc {} was not found in the database'.format(document_id))
return None
# Obtain the proper table on which to perform the entity operation
entity_cls = server.models.FaradayEntity.get_entity_class_from_type(
metadata.document_type)
# TODO(mrocha): Add error handling here when no or more than one entities where found.
entity = self.__db_conn.session.query(entity_cls)\
.join(server.models.EntityMetadata)\
.filter(server.models.EntityMetadata.couchdb_id == document_id)\
.one()
return entity
def __process_del(self, change):
"""
ISSUES:
* Delete child entities. Have not found cases where this is a problem. So far,
clients are deleting all CouchDBs documents properly, and if they don't, the
DBs still are consistent. Maybe use SQLAlchemy's cascades if this become a
problem. Status: Somewhat OK
* Doc ID maps to multiple elements. This could happen since the ID is a hash
based in a few entity's properties which can be replicated. Status: TODO
"""
entity = self.__get_modified_entity(change)
if entity is not None:
self.database.session.delete(entity)
self.database.session.commit()
logger.info(u'A {} ({}) was deleted'.format(
entity.entity_metadata.document_type, entity.name))
def __process_del(self, change):
"""
ISSUES:
* Delete child entities. Have not found cases where this is a problem. So far,
clients are deleting all CouchDBs documents properly, and if they don't, the
DBs still are consistent. Maybe use SQLAlchemy's cascades if this become a
problem. Status: Somewhat OK
* Doc ID maps to multiple elements. This could happen since the ID is a hash
based in a few entity's properties which can be replicated. Status: TODO
"""
entity = self.__get_modified_entity(change)
if entity is not None:
self.database.session.delete(entity)
self.database.session.commit()
logger.info(u'A {} ({}) was deleted'.format(
entity.entity_metadata.document_type, entity.name))
def __get_modified_entity(self, document_id):
metadata = self.get_document_metadata(document_id)
if metadata is None:
logger.info(
u'Doc {} was not found in the database'.format(document_id))
return None
# Obtain the proper table on which to perform the entity operation
entity_cls = server.models.FaradayEntity.get_entity_class_from_type(
metadata.document_type)
# TODO(mrocha): Add error handling here when no or more than one entities where found.
entity = self.__db_conn.session.query(entity_cls)\
.join(server.models.EntityMetadata)\
.filter(server.models.EntityMetadata.couchdb_id == document_id)\
.one()
return entity
def main():
os.chdir(FARADAY_BASE)
parser = argparse.ArgumentParser()
parser.add_argument('--ssl', action='store_true', help='enable HTTPS')
parser.add_argument('--debug', action='store_true', help='run Faraday Server in debug mode')
parser.add_argument('--start', action='store_true', help='run Faraday Server in background')
parser.add_argument('--stop', action='store_true', help='stop Faraday Server')
parser.add_argument('--nodeps', action='store_true', help='Skip dependency check')
parser.add_argument('--no-setup', action='store_true', help=argparse.SUPPRESS)
f = open(server.config.VERSION_FILE)
f_version = f.read().strip()
parser.add_argument('-v', '--version', action='version',
version='Faraday v{version}'.format(version=f_version))
args = parser.parse_args()
if args.debug:
server.utils.logger.set_logging_level(server.config.DEBUG)
if args.stop:
sys.exit(0 if stop_server() else 1)
if is_server_running():
sys.exit(1)
# Overwrites config option if SSL is set by argument
if args.ssl:
server.config.ssl.set('enabled', 'true')
if not args.no_setup:
setup_environment(not args.nodeps)
import_workspaces()
if args.start:
# Starts a new process on background with --ignore-setup
# and without --start nor --stop
devnull = open('/dev/null', 'w')
params = ['/usr/bin/env', 'python2.7', os.path.join(server.config.FARADAY_BASE, __file__), '--no-setup']
if args.ssl: params.append('--ssl')
if args.debug: params.append('--debug')
logger.info('Faraday Server is running as a daemon')
subprocess.Popen(params, stdout=devnull, stderr=devnull)
else:
run_server(args)
def update_entity_from_doc(self, document):
"""
ISSUES:
* Updated relationships are not taken into account. Status: TODO
"""
entity = self.__get_modified_entity(document.get('_id'))
if entity is not None:
entity.update_from_document(document)
entity.entity_metadata.update_from_document(document)
self.__db_conn.session.commit()
logger.info(u'A {} ({}) was updated in Workspace {}'.format(
entity.entity_metadata.document_type,
getattr(entity, 'name', '<no-name>'),
self.__db_conn.db_name))
return True
logger.debug(u'Document ({}) was not present in database to update'.format(document.get('_id')))
return False
def __build_websockets_resource(self):
websocket_port = int(server.config.faraday_server.websocket_port)
url = '{0}:{1}'.format(self.__bind_address, websocket_port)
if self.__ssl_enabled:
url = 'wss://' + url
else:
url = 'ws://' + url
# logger.info(u"Websocket listening at {url}".format(url=url))
logger.info('Starting websocket server at port {0} with bind address {1}. '
'SSL {2}'.format(
websocket_port,
self.__bind_address,
self.__ssl_enabled
))
factory = WorkspaceServerFactory(url=url)
factory.protocol = BroadcastServerProtocol
return factory
def run(self):
def signal_handler(*args):
logger.info('Received SIGTERM, shutting down.')
logger.info("Stopping threads, please wait...")
# teardown()
self.raw_report_processor.stop()
site = twisted.web.server.Site(self.__root_resource)
if self.__ssl_enabled:
ssl_context = self.__load_ssl_certs()
self.__listen_func = functools.partial(reactor.listenSSL,
contextFactory=ssl_context)
else:
self.__listen_func = reactor.listenTCP
try:
self.install_signal()
# start threads and processes
self.raw_report_processor = RawReportProcessor()
self.raw_report_processor.start()
# web and static content
self.__listen_func(self.__listen_port,
site,
interface=self.__bind_address)
# websockets
try:
listenWS(self.__build_websockets_resource(),
interface=self.__bind_address)
except:
logger.warn(
'Could not start websockets, address already open. This is ok is you wan to run multiple instances.'
)
logger.info('Faraday Server is ready')
reactor.addSystemEventTrigger('before', 'shutdown', signal_handler)
reactor.run()
except error.CannotListenError as e:
logger.error(str(e))
sys.exit(1)
except Exception as e:
logger.error(
'Something went wrong when trying to setup the Web UI')
logger.exception(e)
sys.exit(1)
def update_entity_from_doc(self, document):
"""
ISSUES:
* Updated relationships are not taken into account. Status: TODO
"""
entity = self.__get_modified_entity(document.get('_id'))
if entity is not None:
entity.update_from_document(document)
entity.entity_metadata.update_from_document(document)
self.__db_conn.session.commit()
logger.info(u'A {} ({}) was updated in Workspace {}'.format(
entity.entity_metadata.document_type,
getattr(entity, 'name', '<no-name>'), self.__db_conn.db_name))
return True
logger.debug(
u'Document ({}) was not present in database to update'.format(
document.get('_id')))
return False
def __iter__(self):
while not self.__stop:
try:
# TODO: Connection timeout is too long.
self.__response = requests.get(self.__url,
params=self.__params,
stream=True,
auth=get_auth_info())
for raw_line in self.__response.iter_lines():
if self.__stop:
break
line = self.__sanitize(raw_line)
if not line:
continue
change = self.__parse_change(line)
if not change:
continue
yield change
except Exception, e:
# On workspace deletion, requests will probably
# fail to perform the request or the connection
# will be closed. Check if this was intentional
# by checking on the __stop flag.
if self.__stop:
break
import traceback
logger.debug(traceback.format_exc())
# Close everything but keep retrying
self.stop()
self.__stop = False
logger.warning(
u"Lost connection to CouchDB. Retrying in 3 seconds...")
time.sleep(3)
logger.info(u"Retrying...")
def restart_server(args_port):
devnull = open('/dev/null', 'w')
if args_port:
ports = [args_port]
else:
ports = daemonize.get_ports_running()
if not ports:
logger.error('Faraday Server is not running')
sys.exit(1)
for port in ports:
stop_server(port)
params = ['/usr/bin/env', 'python2.7',\
os.path.join(server.config.FARADAY_BASE, __file__), '--no-setup', '--port', str(port)]
logger.info('Restarting Faraday Server...')
subprocess.Popen(params, stdout=devnull, stderr=devnull)
logger.info('Faraday Server is running as a daemon in port {}'.format(port))
def restart_server(args_port):
devnull = open('/dev/null', 'w')
if args_port:
ports = [args_port]
else:
ports = daemonize.get_ports_running()
if not ports:
logger.error('Faraday Server is not running')
sys.exit(1)
for port in ports:
stop_server(port)
params = ['/usr/bin/env', 'python2.7',\
os.path.join(server.config.FARADAY_BASE, __file__), '--no-setup', '--port', str(port)]
logger.info('Restarting Faraday Server...')
subprocess.Popen(params, stdout=devnull, stderr=devnull)
logger.info('Faraday Server is running as a daemon in port {}'.format(port))
def import_from_couchdb(self):
total_amount = self.couchdb.get_total_amount_of_documents()
processed_docs, progress = 0, 0
should_flush_changes = False
host_entities = {}
def flush_changes():
host_entities.clear()
self.database.session.commit()
self.database.session.expunge_all()
for doc in self.couchdb.get_documents(per_request=1000):
processed_docs = processed_docs + 1
current_progress = (processed_docs * 100) / total_amount
if current_progress > progress:
self.__show_progress(
u' * Importing {} from CouchDB'.format(self.__workspace),
progress)
progress = current_progress
should_flush_changes = True
entity = server.models.FaradayEntity.parse(doc.get('doc'))
if entity is not None:
if isinstance(entity,
server.models.Host) and should_flush_changes:
flush_changes()
should_flush_changes = False
try:
entity.add_relationships_from_dict(host_entities)
except server.models.EntityNotFound as e:
logger.warning(
u"Ignoring {} entity ({}) because its parent wasn't found"
.format(entity.entity_metadata.document_type,
entity.entity_metadata.couchdb_id))
else:
host_entities[doc.get('key')] = entity
self.database.session.add(entity)
logger.info(u'{} importation done!'.format(self.__workspace))
flush_changes()
def run(self):
def signal_handler(*args):
logger.info("Stopping threads, please wait...")
# teardown()
self.raw_report_processor.stop()
reactor.stop()
site = twisted.web.server.Site(self.__root_resource)
if self.__ssl_enabled:
ssl_context = self.__load_ssl_certs()
self.__listen_func = functools.partial(
reactor.listenSSL,
contextFactory=ssl_context)
else:
self.__listen_func = reactor.listenTCP
try:
self.install_signal()
# start threads and processes
self.raw_report_processor = RawReportProcessor()
self.raw_report_processor.start()
# web and static content
self.__listen_func(
self.__listen_port, site,
interface=self.__bind_address)
# websockets
try:
listenWS(self.__build_websockets_resource(), interface=self.__bind_address)
except :
logger.warn('Could not start websockets, address already open. This is ok is you wan to run multiple instances.')
logger.info('Faraday Server is ready')
reactor.addSystemEventTrigger('before', 'shutdown', signal_handler)
reactor.run()
except error.CannotListenError as e:
logger.error(str(e))
sys.exit(1)
except Exception as e:
logger.error('Something went wrong when trying to setup the Web UI')
logger.exception(e)
sys.exit(1)
def __iter__(self):
while not self.__stop:
try:
# TODO: Connection timeout is too long.
self.__response = requests.get(
self.__url, params=self.__params,
stream=True, auth=get_auth_info())
for raw_line in self.__response.iter_lines():
if self.__stop:
break
line = self.__sanitize(raw_line)
if not line:
continue
change = self.__parse_change(line)
if not change:
continue
yield change
except Exception, e:
# On workspace deletion, requests will probably
# fail to perform the request or the connection
# will be closed. Check if this was intentional
# by checking on the __stop flag.
if self.__stop:
break
import traceback
logger.debug(traceback.format_exc())
# Close everything but keep retrying
self.stop()
self.__stop = False
logger.warning(u"Lost connection to CouchDB. Retrying in 3 seconds...")
time.sleep(3)
logger.info(u"Retrying...")
def create_database(self):
logger.info(u'Creating database for workspace {}'.format(self.__workspace))
self.database.create()
self.database.open_session()
try:
# Add metadata information to database
self.set_last_seq(self.couchdb.get_last_seq())
self.set_migration_status(False)
self.set_schema_version()
self.import_from_couchdb()
# Reaching this far without errors means a successful migration
self.set_migration_status(True)
except Exception, e:
import traceback
logger.debug(traceback.format_exc())
logger.error(u'Error while importing workspace {}: {!s}'.format(self.__workspace, e))
self.delete()
raise e
def delete_entity_from_doc_id(self, document_id):
"""
ISSUES:
* Delete child entities. Have not found cases where this is a problem. So far,
clients are deleting all CouchDBs documents properly, and if they don't, the
DBs still are consistent. Maybe use SQLAlchemy's cascades if this become a
problem. Status: Somewhat OK
* Doc ID maps to multiple elements. This could happen since the ID is a hash
based in a few entity's properties which can be replicated. Status: TODO
"""
entity = self.__get_modified_entity(document_id)
if entity is not None:
self.__db_conn.session.delete(entity)
self.__db_conn.session.commit()
logger.info(u'A {} ({}) was deleted in Workspace {}'.format(
entity.entity_metadata.document_type,
getattr(entity, 'name', '<no-name>'),
self.__db_conn.db_name))
return True
logger.debug(u'Document ({}) was not present in database to delete'.format(document_id))
return False
def import_from_couchdb(self):
total_amount = self.couchdb.get_total_amount_of_documents()
processed_docs, progress = 0, 0
should_flush_changes = False
host_entities = {}
def flush_changes():
host_entities.clear()
self.database.session.commit()
self.database.session.expunge_all()
for doc in self.couchdb.get_documents(per_request=1000):
processed_docs = processed_docs + 1
current_progress = (processed_docs * 100) / total_amount
if current_progress > progress:
self.__show_progress(u' * Importing {} from CouchDB'.format(
self.__workspace), progress)
progress = current_progress
should_flush_changes = True
entity = server.models.FaradayEntity.parse(doc.get('doc'))
if entity is not None:
if isinstance(entity, server.models.Host) and should_flush_changes:
flush_changes()
should_flush_changes = False
try:
entity.add_relationships_from_dict(host_entities)
except server.models.EntityNotFound as e:
logger.warning(u"Ignoring {} entity ({}) because its parent wasn't found".format(
entity.entity_metadata.document_type, entity.entity_metadata.couchdb_id))
else:
host_entities[doc.get('key')] = entity
self.database.session.add(entity)
logger.info(u'{} importation done!'.format(self.__workspace))
flush_changes()
def import_on_new_database(db_conn, couchdb_conn):
if db_conn.exists():
raise RuntimeError('Database {} already exists'.format(db_conn.db_name))
logger.info(u'Creating database for workspace {}'.format(db_conn.db_name))
_create_and_import_db(db_conn, couchdb_conn)
def main():
os.chdir(FARADAY_BASE)
check_alembic_version()
check_postgresql()
parser = argparse.ArgumentParser()
parser.add_argument('--ssl', action='store_true', help='enable HTTPS')
parser.add_argument('--debug', action='store_true', help='run Faraday Server in debug mode')
parser.add_argument('--start', action='store_true', help='run Faraday Server in background')
parser.add_argument('--stop', action='store_true', help='stop Faraday Server')
parser.add_argument('--restart', action='store_true', help='Restart Faraday Server')
parser.add_argument('--nodeps', action='store_true', help='Skip dependency check')
parser.add_argument('--no-setup', action='store_true', help=argparse.SUPPRESS)
parser.add_argument('--port', help='Overides server.ini port configuration')
parser.add_argument('--websocket_port', help='Overides server.ini websocket port configuration')
parser.add_argument('--bind_address', help='Overides server.ini bind_address configuration')
f = open(server.config.VERSION_FILE)
f_version = f.read().strip()
parser.add_argument('-v', '--version', action='version',
version='Faraday v{version}'.format(version=f_version))
args = parser.parse_args()
setUpLogger(args.debug)
if args.debug:
server.utils.logger.set_logging_level(server.config.DEBUG)
if args.restart:
restart_server(args.port)
sys.exit()
if args.stop:
if args.port:
sys.exit(0 if stop_server(args.port) else 1)
else:
ports = daemonize.get_ports_running()
if not ports:
logger.info('Faraday Server is not running')
exit_code = 0
for port in ports:
exit_code += 0 if stop_server(port) else 1
sys.exit(exit_code)
else:
if not args.port:
args.port = '5985'
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((args.bind_address or server.config.faraday_server.bind_address, int(args.port or server.config.faraday_server.port)))
if is_server_running(args.port) and result == 0:
sys.exit(1)
if result == 0:
logger.error("Faraday Server port in use. Check your processes and run the server again...")
sys.exit(1)
# Overwrites config option if SSL is set by argument
if args.ssl:
server.config.ssl.enabled = 'true'
if not args.no_setup:
setup_environment(not args.nodeps)
if args.port:
server.config.faraday_server.port = args.port
if args.bind_address:
server.config.faraday_server.bind_address = args.bind_address
if args.websocket_port:
server.config.faraday_server.websocket_port = args.websocket_port
if args.start:
# Starts a new process on background with --ignore-setup
# and without --start nor --stop
devnull = open('/dev/null', 'w')
params = ['/usr/bin/env', 'python2.7', os.path.join(server.config.FARADAY_BASE, __file__), '--no-setup']
arg_dict = vars(args)
for arg in arg_dict:
if arg not in ["start", "stop"] and arg_dict[arg]:
params.append('--'+arg)
if arg_dict[arg] != True:
params.append(arg_dict[arg])
logger.info('Faraday Server is running as a daemon')
subprocess.Popen(params, stdout=devnull, stderr=devnull)
elif not args.start:
run_server(args)
