def __init__(self):
super(SaveReports, self).__init__()
save_unknown_components = str2bool(config.get(
"save-reports.save_unknown_components", "false"))
basedir_keys = ["ureport.directory", "report.spooldirectory"]
self.load_config_to_self("basedir", basedir_keys, "/var/spool/faf/")
self.load_config_to_self("create_components",
["ureport.createcomponents"],
save_unknown_components, callback=str2bool)
# Instance of 'SaveReports' has no 'basedir' member
# pylint: disable-msg=E1101
self.dir_report = paths["reports"]
self.dir_report_incoming = paths["reports_incoming"]
self.dir_report_saved = paths["reports_saved"]
self.dir_report_deferred = paths["reports_deferred"]
self.dir_attach = paths["attachments"]
self.dir_attach_incoming = paths["attachments_incoming"]
self.dir_attach_saved = paths["attachments_saved"]
self.dir_attach_deferred = paths["attachments_deferred"]
try:
ensure_dirs([self.dir_report_incoming, self.dir_report_saved,
self.dir_report_deferred, self.dir_attach_incoming,
self.dir_attach_saved, self.dir_attach_deferred])
except FafError as ex:
self.log_error("Required directories can't be created: {0}"
.format(str(ex)))
raise
def unpack_rpm_to_tmp(path, prefix="faf"):
"""
Unpack RPM package to a temp directory. The directory is either specified
in storage.tmpdir config option or use the system default temp directory.
"""
tmpdir = config.get("storage.tmpdir", None)
result = tempfile.mkdtemp(prefix=prefix, dir=tmpdir)
for dirname in ["bin", "lib", "lib64", "sbin"]:
os.makedirs(os.path.join(result, "usr", dirname))
os.symlink(os.path.join("usr", dirname), os.path.join(result, dirname))
rpm2cpio = Popen(["rpm2cpio", path], stdout=PIPE, stderr=PIPE)
cpio = Popen(["cpio", "-id", "--quiet"],
stdin=rpm2cpio.stdout, stderr=PIPE, cwd=result)
# do not check rpm2cpio exitcode as there may be a bug for large files
# https://bugzilla.redhat.com/show_bug.cgi?id=790396
rpm2cpio.wait()
if cpio.wait() != 0:
shutil.rmtree(result)
raise FafError("Failed to unpack RPM '{0}'".format(path))
return result
def unpack_rpm_to_tmp(path, prefix="faf"):
"""
Unpack RPM package to a temp directory. The directory is either specified
in storage.tmpdir config option or use the system default temp directory.
"""
tmpdir = config.get("storage.tmpdir", None)
result = tempfile.mkdtemp(prefix=prefix, dir=tmpdir)
for dirname in ["bin", "lib", "lib64", "sbin"]:
os.makedirs(os.path.join(result, "usr", dirname))
os.symlink(os.path.join("usr", dirname), os.path.join(result, dirname))
rpm2cpio = Popen(["rpm2cpio", path], stdout=PIPE, stderr=PIPE)
cpio = Popen(["cpio", "-id", "--quiet"],
stdin=rpm2cpio.stdout,
stderr=PIPE,
cwd=result)
# do not check rpm2cpio exitcode as there may be a bug for large files
# https://bugzilla.redhat.com/show_bug.cgi?id=790396
rpm2cpio.wait()
if cpio.wait() != 0:
shutil.rmtree(result)
raise FafError("Failed to unpack RPM '{0}'".format(path))
return result
def unpack_rpm_to_tmp(path, prefix="faf"):
"""
Unpack RPM package to a temp directory. The directory is either specified
in storage.tmpdir config option or use the system default temp directory.
"""
tmpdir = config.get("storage.tmpdir", None)
result = tempfile.mkdtemp(prefix=prefix, dir=tmpdir)
for dirname in ["bin", "lib", "lib64", "sbin"]:
os.makedirs(os.path.join(result, "usr", dirname))
os.symlink(os.path.join("usr", dirname), os.path.join(result, dirname))
rpm2cpio = Popen(["rpm2cpio", path], stdout=PIPE, stderr=PIPE)
cpio = Popen(["cpio", "-id", "--quiet"],
stdin=rpm2cpio.stdout,
stderr=PIPE,
cwd=result)
#FIXME: false positive by pylint # pylint: disable=fixme
rpm2cpio.stdout.close() # pylint: disable=no-member
try:
# generous timeout of 15 minutes (kernel unpacking)
cpio.communicate(timeout=900)
except TimeoutExpired:
cpio.kill()
cpio.communicate()
finally:
if cpio.returncode != 0:
shutil.rmtree(result)
raise FafError("Failed to unpack RPM '{0}'".format(path))
return result
def _get_env_or_config(conf_s, env_s, default):
found = os.environ.get(env_s, None)
if not found:
found = config.get(conf_s, None)
if not found:
found = default
return found
def server_name():
"""
Return web server root URL if applicable
"""
if webfaf_installed():
return config.get("hub.server_name", None)
logging.warning("Unable to get web server name, webfaf not available")
return None
def server_name() -> Optional[str]:
"""
Return web server root URL if applicable
"""
if webfaf_installed():
return config.get("hub.server_name", None)
logging.warning("Unable to get web server name, webfaf not available")
return None
def server_url():
"""
Return web server root URL if applicable
"""
if webfaf_installed():
return config.get("hub.url", None)
logging.warning("Unable to get web server URL, webfaf not available")
return None
def server_url():
"""
Return web server root URL if applicable
"""
if webfaf_installed():
return config.get("hub2.url", None)
else:
logging.warn("Unable to get web server URL, webfaf not available")
return None
def require_https() -> bool:
"""
Return if web server requires https (default true)
"""
if webfaf_installed():
from pyfaf.utils.parse import str2bool
return str2bool(config.get("hub.require_https", "true"))
logging.warning("Unable to get require https option, webfaf not available")
return True
def require_https():
"""
Return if web server requires https (default true)
"""
if webfaf_installed():
from pyfaf.utils.parse import str2bool
return str2bool(config.get("hub.require_https", "true"))
logging.warning("Unable to get require https option, webfaf not available")
return True
def get_temp_dir(subdir=None):
"""
Get the temp directory path. If storage.tmpdir exists, it will be
considered temp root, otherwise system default will be used.
Temp directory is user-specific - to be able to run tools under different
users. If `subdir` is specified, the appropriate subdirectory is returned.
"""
basetmp = config.get("storage.tmpdir", tempfile.gettempdir())
username = pwd.getpwuid(os.getuid()).pw_name
# no functional reason, just avoid 'faf-faf'
if username == "faf":
userdir = "faf"
else:
userdir = "faf-{0}".format(username)
if subdir is None:
return os.path.join(basetmp, userdir)
return os.path.join(basetmp, userdir, subdir)
import bunch
import datetime
import logging
from celery import Celery
from celery.signals import task_postrun
from pyfaf.actions import actions
from pyfaf.config import config
from pyfaf.storage import DatabaseFactory, TaskResult
from pyfaf.utils.contextmanager import captured_output_combined
celery_app = Celery("pyfaf_tasks",
broker=config.get("celery_tasks.broker", ""),
backend=config.get("celery_tasks.backend", ""))
db_factory = DatabaseFactory(autocommit=True)
class ActionError(Exception):
def __init__(self, output):
self.output = output
Exception.__init__(self, output)
def __str__(self):
return self.output
@task_postrun.connect
def task_postrun_handler(signal, sender, **named):
db = db_factory.get_database()
from pyfaf.storage import (OpSysComponent,
Report,
ReportBacktrace,
ReportBtThread,
ReportBtFrame,
ReportHash,
SymbolSource)
from pyfaf.config import config
from webfaf.webfaf_main import db # pylint: disable=wrong-import-order
url_prefix = "/symbol_transfer"
symbol_transfer = Blueprint("symbol_transfer", __name__)
symbol_transfer_auth_key = config.get("symbol_transfer.auth_key", False)
def process_symbol(build_id, path, offset, problem_type, create_symbol_auth_key) -> Tuple[Dict[str, Any], int]:
db_ssource = (db.session.query(SymbolSource)
.filter(SymbolSource.build_id == build_id)
.filter(SymbolSource.path == path)
.filter(SymbolSource.offset == offset)
.first())
if db_ssource is None:
if (create_symbol_auth_key
and symbol_transfer_auth_key
and create_symbol_auth_key == symbol_transfer_auth_key
and problem_type in ("kerneloops", "core")):
# We need to attach our symbols to a dummy report in order to set
AUTHENTICATION_BACKENDS = (
'django_openid_auth.auth.OpenIDBackend',
'django.contrib.auth.backends.ModelBackend',
)
# required to work around
# Exception Value: <openid.yadis.manager.YadisServiceManager object at
# 0x7fd2f43b2250> is not JSON serializable
# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
OPENID_CREATE_USERS = True
OPENID_UPDATE_DETAILS_FROM_SREG = True
OPENID_ENABLED = str2bool(config.get("openid.enabled", ""))
LOGIN_URL = '/openid/login/'
LOGIN_REDIRECT_URL = '/'
DAJAXICE_MEDIA_PREFIX = 'faf/dajaxice'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'formatters': {
)
AUTHENTICATION_BACKENDS = (
'django_openid_auth.auth.OpenIDBackend',
'django.contrib.auth.backends.ModelBackend',
)
# required to work around
# Exception Value: <openid.yadis.manager.YadisServiceManager object at
# 0x7fd2f43b2250> is not JSON serializable
# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
OPENID_CREATE_USERS = True
OPENID_UPDATE_DETAILS_FROM_SREG = True
OPENID_ENABLED = str2bool(config.get("openid.enabled", ""))
LOGIN_URL = '/openid/login/'
LOGIN_REDIRECT_URL = '/'
DAJAXICE_MEDIA_PREFIX = 'faf/dajaxice'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'formatters': {
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# faf is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with faf. If not, see <http://www.gnu.org/licenses/>.
from pyfaf.config import config
from pyfaf.utils.parse import str2bool
fedmsg_name = config.get("fedmsg.name", "fedora-infrastructure")
fedmsg_environment = config.get("fedmsg.environment", "dev")
notify_reports = str2bool(config.get("fedmsg.realtime_reports", "false"))
notify_problems = str2bool(config.get("fedmsg.realtime_problems", "false"))
if notify_reports or notify_problems:
from sqlalchemy import event
from . import Report
import fedmsg
from pyfaf.utils import web
from pyfaf.common import log
logger = log.getChildLogger(__name__)
levels = tuple(10**n for n in range(7))
fedmsg.init(name=fedmsg_name, environment=fedmsg_environment)
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# faf is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with faf. If not, see <http://www.gnu.org/licenses/>.
from pyfaf.config import config
from pyfaf.utils.parse import str2bool
notify_reports = str2bool(config.get("fedmsg.realtime_reports", "false"))
notify_problems = str2bool(config.get("fedmsg.realtime_problems", "false"))
# pylint: disable=ungrouped-imports
if notify_reports or notify_problems:
from sqlalchemy import event
from fedora_messaging.api import publish
from fedora_messaging.exceptions import ConnectionException, PublishReturned
from . import Report
from faf_schema.schema import FafReportMessage, FafProblemMessage
from pyfaf.utils import web
from pyfaf.common import log
logger = log.getChild(__name__)
levels = tuple(10**n for n in range(7))
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# faf is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with faf. If not, see <http://www.gnu.org/licenses/>.
from pyfaf.config import config
from pyfaf.utils.parse import str2bool
fedmsg_name = config.get("fedmsg.name", "fedora-infrastructure")
fedmsg_environment = config.get("fedmsg.environment", "dev")
notify_reports = str2bool(config.get("fedmsg.realtime_reports", "false"))
notify_problems = str2bool(config.get("fedmsg.realtime_problems", "false"))
if notify_reports or notify_problems:
from sqlalchemy import event
from . import Report
import fedmsg
from pyfaf.utils import web
from pyfaf.common import log
logger = log.getChildLogger(__name__)
levels = tuple(10**n for n in range(7))
fedmsg.init(name=fedmsg_name, environment=fedmsg_environment)
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# faf is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with faf. If not, see <http://www.gnu.org/licenses/>.
from pyfaf.config import config
from pyfaf.utils.parse import str2bool
notify_reports = str2bool(config.get("fedmsg.realtime_reports", "false"))
notify_problems = str2bool(config.get("fedmsg.realtime_problems", "false"))
# pylint: disable=ungrouped-imports
if notify_reports or notify_problems:
from sqlalchemy import event
from fedora_messaging.api import publish
from fedora_messaging.exceptions import ConnectionException, PublishReturned
from . import Report
from faf_schema.schema import FafReportMessage, FafProblemMessage
from pyfaf.utils import web
from pyfaf.common import log
logger = log.getChildLogger(__name__)
levels = tuple(10**n for n in range(7))
import bunch
import datetime
import logging
from celery import Celery
from celery.signals import task_postrun
from pyfaf.actions import actions
from pyfaf.config import config
from pyfaf.storage import DatabaseFactory, TaskResult
from pyfaf.utils.contextmanager import captured_output_combined
celery_app = Celery("pyfaf_tasks",
broker=config.get("celery_tasks.broker", ""),
backend=config.get("celery_tasks.backend", ""))
db_factory = DatabaseFactory(autocommit=True)
class ActionError(Exception):
def __init__(self, output):
self.output = output
Exception.__init__(self, output)
def __str__(self):
return self.output
@task_postrun.connect
def task_postrun_handler(signal, sender, **named):
db = db_factory.get_database()
tr = TaskResult()
from pyfaf.storage import (OpSysComponent,
Report,
ReportBacktrace,
ReportBtThread,
ReportBtFrame,
ReportHash,
SymbolSource)
from pyfaf.config import config
from webfaf_main import db
url_prefix = "/symbol_transfer"
symbol_transfer = Blueprint("symbol_transfer", __name__)
symbol_transfer_auth_key = config.get("symbol_transfer.auth_key", False)
def process_symbol(build_id, path, offset, problem_type, create_symbol_auth_key):
db_ssource = (db.session.query(SymbolSource)
.filter(SymbolSource.build_id == build_id)
.filter(SymbolSource.path == path)
.filter(SymbolSource.offset == offset)
.first())
if db_ssource is None:
if (create_symbol_auth_key
and symbol_transfer_auth_key
and create_symbol_auth_key == symbol_transfer_auth_key
and problem_type in ("kerneloops", "core")):
# We need to attach our symbols to a dummy report in order to set
