def create_db(self, schema_file=None): """ Create the database for this source, using given SQL schema file. If schema file is not given, defaults to "template_schema/project.sql.tmpl". """ import MySQLdb if schema_file is None: schema_file = path("model", "sql", "template_schema", "project.sql.tmpl") filterwarnings('ignore', category=MySQLdb.Warning) with connection.cursor() as cursor: cursor.execute("CREATE DATABASE IF NOT EXISTS {0}".format(self.name)) cursor.execute("USE {0}".format(self.name)) try: with open(schema_file) as f: # set the engine to use sql = f.read() statement_list = sql.split(";") for statement in statement_list: cursor.execute(statement) finally: cursor.execute("USE {0}".format( settings.DATABASES['default']['NAME'] )) resetwarnings()
def create_db(self, schema_file=None): """ Create the database for this source, using given SQL schema file. If schema file is not given, defaults to "template_schema/project.sql.tmpl". """ import MySQLdb if schema_file is None: schema_file = path("model", "sql", "template_schema", "project.sql.tmpl") filterwarnings('ignore', category=MySQLdb.Warning) with connection.cursor() as cursor: cursor.execute("CREATE DATABASE IF NOT EXISTS {0}".format( self.name)) cursor.execute("USE {0}".format(self.name)) try: with open(schema_file) as f: # set the engine to use sql = f.read() statement_list = sql.split(";") for statement in statement_list: cursor.execute(statement) finally: cursor.execute("USE {0}".format( settings.DATABASES['default']['NAME'])) resetwarnings()
class OAuthCredentials(): credentials = {} param_keys = set([ 'oauth_body_hash', 'oauth_signature', 'oauth_consumer_key', 'oauth_nonce', 'oauth_timestamp', 'oauth_signature_method', 'oauth_version', 'oauth_token', 'user' ]) credentials_file = path('etl', 'data', 'credentials.json') @classmethod def get_parameters(cls, query_params): parameters = {} for key in cls.param_keys: parameters[key] = query_params.get(key, None) return parameters @classmethod def set_credentials(cls, credentials={}): # Only get the credentials once if not cls.credentials and not credentials: credentials_string = os.environ.get('TREEHERDER_CREDENTIALS', None) if credentials_string: credentials = json.loads(credentials_string) else: try: with open(cls.credentials_file) as f: credentials_string = f.read() credentials = json.loads(credentials_string) except IOError: msg = ( 'Credentials file not found at {0}.' ' Try running `manage.py export_project_credentials`' ' to generate them').format(cls.credentials_file) logger.error(msg) except Exception as e: logger.error(e) raise e cls.credentials = credentials @classmethod def get_credentials(cls, project): return copy.deepcopy(cls.credentials.get(project, {})) @classmethod def get_consumer_secret(cls, project): return copy.deepcopy(cls.credentials.get(project, {}))
def create_db(self, schema_file=None): """ Create the database for this source, using given SQL schema file. If schema file is not given, defaults to "template_schema/schema_<contenttype>.sql.tmpl". Assumes that the database server at ``self.host`` is accessible, and that ``DATABASE_USER`` (identified by ``DATABASE_PASSWORD`` exists on it and has permissions to create databases. """ from django.conf import settings import MySQLdb DB_USER = settings.DATABASES["default"]["USER"] DB_PASS = settings.DATABASES["default"]["PASSWORD"] if self.type.lower().startswith("mysql-"): engine = self.type[len("mysql-") :] elif self.type.lower() == "mysql": engine = "InnoDB" else: raise NotImplementedError("Currently only MySQL data source is supported.") if schema_file is None: schema_file = path("model", "sql", "template_schema", "project_{0}_1.sql.tmpl".format(self.contenttype)) conn = MySQLdb.connect(host=self.host, user=DB_USER, passwd=DB_PASS) filterwarnings("ignore", category=MySQLdb.Warning) cur = conn.cursor() cur.execute("CREATE DATABASE IF NOT EXISTS {0}".format(self.name)) conn.close() resetwarnings() # MySQLdb provides no way to execute an entire SQL file in bulk, so we # have to shell out to the commandline client. with open(schema_file) as f: # set the engine to use sql = f.read().format(engine=engine) args = ["mysql", "--host={0}".format(self.host), "--user={0}".format(DB_USER)] if DB_PASS: args.append("--password={0}".format(DB_PASS)) args.append(self.name) proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) (output, _) = proc.communicate(sql) if proc.returncode: raise IOError( "Unable to set up schema for datasource {0}: " "mysql returned code {1}, output follows:\n\n{2}".format(self.key, proc.returncode, output) )
def create_db(self, schema_file=None): """ Create the database for this source, using given SQL schema file. If schema file is not given, defaults to "template_schema/schema_<contenttype>.sql.tmpl". Assumes that the database server at ``self.host`` is accessible, and that ``DATABASE_USER`` (identified by ``DATABASE_PASSWORD`` exists on it and has permissions to create databases. """ import MySQLdb if self.type.lower().startswith("mysql-"): engine = self.type[len("mysql-"):] elif self.type.lower() == "mysql": engine = "InnoDB" else: raise NotImplementedError( "Currently only MySQL data source is supported.") if schema_file is None: schema_file = path( "model", "sql", "template_schema", "project_{}.sql.tmpl".format(self.contenttype), ) filterwarnings('ignore', category=MySQLdb.Warning) with connection.cursor() as cursor: cursor.execute("CREATE DATABASE IF NOT EXISTS {0}".format( self.name)) cursor.execute("USE {0}".format(self.name)) try: with open(schema_file) as f: # set the engine to use sql = f.read().format(engine=engine) statement_list = sql.split(";") for statement in statement_list: cursor.execute(statement) finally: cursor.execute("USE {0}".format( settings.TREEHERDER_DATABASE_NAME)) resetwarnings()
def create_db(self, schema_file=None): """ Create the database for this source, using given SQL schema file. If schema file is not given, defaults to "template_schema/schema_<contenttype>.sql.tmpl". Assumes that the database server at ``self.host`` is accessible, and that ``DATABASE_USER`` (identified by ``DATABASE_PASSWORD`` exists on it and has permissions to create databases. """ import MySQLdb if self.type.lower().startswith("mysql-"): engine = self.type[len("mysql-"):] elif self.type.lower() == "mysql": engine = "InnoDB" else: raise NotImplementedError( "Currently only MySQL data source is supported.") if schema_file is None: schema_file = path( "model", "sql", "template_schema", "project_{}.sql.tmpl".format(self.contenttype), ) filterwarnings('ignore', category=MySQLdb.Warning) with connection.cursor() as cursor: cursor.execute("CREATE DATABASE IF NOT EXISTS {0}".format(self.name)) cursor.execute("USE {0}".format(self.name)) try: with open(schema_file) as f: # set the engine to use sql = f.read().format(engine=engine) statement_list = sql.split(";") for statement in statement_list: cursor.execute(statement) finally: cursor.execute("USE {0}".format( settings.TREEHERDER_DATABASE_NAME )) resetwarnings()
# Default to retaining data for ~4 months. DATA_CYCLE_DAYS = env.int("DATA_CYCLE_DAYS", default=120) # Make this unique, and don't share it with anybody. SECRET_KEY = env("TREEHERDER_DJANGO_SECRET_KEY") ROOT_URLCONF = "treeherder.config.urls" WSGI_APPLICATION = 'treeherder.config.wsgi.application' TIME_ZONE = "UTC" USE_I18N = False USE_L10N = True SERVE_MINIFIED_UI = env.bool("SERVE_MINIFIED_UI", default=False) # Files in this directory will be served by WhiteNoise at the site root. WHITENOISE_ROOT = path("..", "dist" if SERVE_MINIFIED_UI else "ui") STATIC_ROOT = path("static") STATIC_URL = "/static/" MEDIA_ROOT = path("media") MEDIA_URL = "/media/" # Default minimum regression threshold for perfherder is 2% (otherwise # e.g. the build size tests will alert on every commit) PERFHERDER_REGRESSION_THRESHOLD = 2 # Various settings for treeherder's t-test "sliding window" alert algorithm PERFHERDER_ALERTS_MIN_BACK_WINDOW = 12 PERFHERDER_ALERTS_MAX_BACK_WINDOW = 24 PERFHERDER_ALERTS_FORE_WINDOW = 12
ADMINS = [] # TBD MANAGERS = ADMINS SITE_ID = 1 ROOT_URLCONF = "treeherder.webapp.urls" WSGI_APPLICATION = 'treeherder.webapp.wsgi.application' TIME_ZONE = "America/Los_Angeles" LANGUAGE_CODE = "en-us" USE_I18N = False USE_L10N = True USE_TZ = False SERVE_MINIFIED_UI = os.environ.get("SERVE_MINIFIED_UI") == "True" WHITENOISE_ROOT = path("..", "dist" if SERVE_MINIFIED_UI else "ui") STATIC_ROOT = path("webapp", "static") STATIC_URL = "/static/" MEDIA_ROOT = path("webapp", "media") MEDIA_URL = "/media/" # Additional locations of static files STATICFILES_DIRS = [] STATICFILES_FINDERS = [ "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", # "django.contrib.staticfiles.finders.DefaultStorageFinder", ]
class Command(BaseCommand): help = "Init master database and call syncdb" option_list = BaseCommand.option_list + ( make_option( '--noinput', action='store_false', dest='interactive', default=True, help='Tells Django to NOT prompt the user for input of any kind.'), make_option( '--engine', action='store', dest='engine', default='InnoDB', help='Define the db engine to use.', ), make_option( '--template-path', action='store', dest='template_path', default=path('model', 'sql', 'template_schema'), help='Directory containing the sql templates', ), make_option( '--skip-fixtures', action='store_true', dest='skip_fixtures', default=False, help='Tell this command to NOT load initial fixtures', ), ) def handle(self, *args, **options): interactive = options['interactive'] if interactive: confirm = input("""You have requested an init of the database. This will IRREVERSIBLY DESTROY all data currently in the %r database, and return each table to the state it was in after syncdb. Are you sure you want to do this? Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME']) else: confirm = 'yes' if confirm == 'yes': for sql_file in ( 'treeherder.sql.tmpl', 'treeherder_reference_1.sql.tmpl', ): with open(os.path.join(options['template_path'], sql_file)) as f: sql = f.read() cursor = connection.cursor() try: rendered_sql = sql.format(engine=options['engine']) cursor.execute(rendered_sql) except Exception as e: print "Error on sql execution:{0}".format(e) finally: cursor.close() print "Sql files executed successfully." # safely apply all migrations call_command("migrate") # load initial fixtures for reference data # the order of this list of fixtures is important # to avoid integrity errors if not options['skip_fixtures']: call_command('load_initial_data')
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, you can obtain one at http://mozilla.org/MPL/2.0/. # Django settings for webapp project. import os import sys from datetime import timedelta from kombu import Exchange, Queue from treeherder import path # Insure the vendor libraries are added to the python path # in production, and before the site-packages entry. sys.path.insert(1, path('..', 'vendor')) # These settings can all be optionally set via env vars, or in local.py: TREEHERDER_DATABASE_NAME = os.environ.get("TREEHERDER_DATABASE_NAME", "") TREEHERDER_DATABASE_USER = os.environ.get("TREEHERDER_DATABASE_USER", "") TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", "") TREEHERDER_DATABASE_HOST = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost") TREEHERDER_RO_DATABASE_USER = os.environ.get("TREEHERDER_RO_DATABASE_USER", TREEHERDER_DATABASE_USER) TREEHERDER_RO_DATABASE_PASSWORD = os.environ.get("TREEHERDER_RO_DATABASE_PASSWORD", TREEHERDER_DATABASE_PASSWORD) TREEHERDER_RO_DATABASE_HOST = os.environ.get("TREEHERDER_RO_DATABASE_HOST", TREEHERDER_DATABASE_HOST) TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "") TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder") DEBUG = os.environ.get("TREEHERDER_DEBUG", False)
ADMINS = [] # TBD MANAGERS = ADMINS SITE_ID = 1 ROOT_URLCONF = "treeherder.webapp.urls" WSGI_APPLICATION = 'treeherder.webapp.wsgi.application' TIME_ZONE = "America/Los_Angeles" LANGUAGE_CODE = "en-us" USE_I18N = False USE_L10N = True USE_TZ = False STATIC_ROOT = path("webapp", "static") STATIC_URL = "/static/" MEDIA_ROOT = path("webapp", "media") MEDIA_URL = "/media/" # Additional locations of static files STATICFILES_DIRS = [] STATICFILES_FINDERS = [ "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", # "django.contrib.staticfiles.finders.DefaultStorageFinder", ] TEMPLATE_LOADERS = [ "django.template.loaders.filesystem.Loader",
# cycling data DATA_CYCLE_CHUNK_SIZE = env.int("DATA_CYCLE_CHUNK_SIZE", default=100) DATA_CYCLE_SLEEP_TIME = env.int("DATA_CYCLE_SLEEP_TIME", default=0) # Make this unique, and don't share it with anybody. SECRET_KEY = env("TREEHERDER_DJANGO_SECRET_KEY") ROOT_URLCONF = "treeherder.config.urls" WSGI_APPLICATION = 'treeherder.config.wsgi.application' TIME_ZONE = "UTC" USE_I18N = False USE_L10N = True # Files in this directory will be served by WhiteNoise at the site root. WHITENOISE_ROOT = path("..", "dist") STATIC_ROOT = path("static") STATIC_URL = "/static/" # Default minimum regression threshold for perfherder is 2% (otherwise # e.g. the build size tests will alert on every commit) PERFHERDER_REGRESSION_THRESHOLD = 2 # Various settings for treeherder's t-test "sliding window" alert algorithm PERFHERDER_ALERTS_MIN_BACK_WINDOW = 12 PERFHERDER_ALERTS_MAX_BACK_WINDOW = 24 PERFHERDER_ALERTS_FORE_WINDOW = 12 # Only generate alerts for data newer than this time in seconds in perfherder PERFHERDER_ALERTS_MAX_AGE = timedelta(weeks=2)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, you can obtain one at http://mozilla.org/MPL/2.0/. # Django settings for webapp project. import os import sys from datetime import timedelta from treeherder import path # Insure the vendor libraries are added to the python path # in production sys.path.append(path('..', 'vendor')) # These settings can all be optionally set via env vars, or in local.py: TREEHERDER_DATABASE_NAME = os.environ.get("TREEHERDER_DATABASE_NAME", "") TREEHERDER_DATABASE_USER = os.environ.get("TREEHERDER_DATABASE_USER", "") TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", "") TREEHERDER_DATABASE_HOST = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost") TREEHERDER_DATABASE_PORT = os.environ.get("TREEHERDER_DATABASE_PORT", "") TREEHERDER_RO_DATABASE_USER = os.environ.get("TREEHERDER_RO_DATABASE_USER", TREEHERDER_DATABASE_USER) TREEHERDER_RO_DATABASE_PASSWORD = os.environ.get( "TREEHERDER_RO_DATABASE_PASSWORD", TREEHERDER_DATABASE_PASSWORD) TREEHERDER_RO_DATABASE_HOST = os.environ.get("TREEHERDER_RO_DATABASE_HOST", TREEHERDER_DATABASE_HOST)
ADMINS = [] # TBD MANAGERS = ADMINS SITE_ID = 1 ROOT_URLCONF = "treeherder.webapp.urls" WSGI_APPLICATION = 'treeherder.webapp.wsgi.application' TIME_ZONE = "America/Los_Angeles" LANGUAGE_CODE = "en-us" USE_I18N = False USE_L10N = True USE_TZ = False STATIC_ROOT = path("webapp", "static") STATIC_URL = "/static/" MEDIA_ROOT = path("webapp", "media") MEDIA_URL = "/media/" # Additional locations of static files STATICFILES_DIRS = [] STATICFILES_FINDERS = [ "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", #"django.contrib.staticfiles.finders.DefaultStorageFinder", ] TEMPLATE_LOADERS = [ "django.template.loaders.filesystem.Loader",
# cycling data DATA_CYCLE_CHUNK_SIZE = env.int("DATA_CYCLE_CHUNK_SIZE", default=100) DATA_CYCLE_SLEEP_TIME = env.int("DATA_CYCLE_SLEEP_TIME", default=0) # Make this unique, and don't share it with anybody. SECRET_KEY = env("TREEHERDER_DJANGO_SECRET_KEY") ROOT_URLCONF = "treeherder.config.urls" WSGI_APPLICATION = 'treeherder.config.wsgi.application' TIME_ZONE = "UTC" USE_I18N = False USE_L10N = True # Files in this directory will be served by WhiteNoise at the site root. WHITENOISE_ROOT = path("..", "dist") STATIC_ROOT = path("static") STATIC_URL = "/static/" MEDIA_ROOT = path("media") MEDIA_URL = "/media/" # Default minimum regression threshold for perfherder is 2% (otherwise # e.g. the build size tests will alert on every commit) PERFHERDER_REGRESSION_THRESHOLD = 2 # Various settings for treeherder's t-test "sliding window" alert algorithm PERFHERDER_ALERTS_MIN_BACK_WINDOW = 12 PERFHERDER_ALERTS_MAX_BACK_WINDOW = 24 PERFHERDER_ALERTS_FORE_WINDOW = 12
def create_db(self, schema_file=None): """ Create the database for this source, using given SQL schema file. If schema file is not given, defaults to "template_schema/schema_<contenttype>.sql.tmpl". Assumes that the database server at ``self.host`` is accessible, and that ``DATABASE_USER`` (identified by ``DATABASE_PASSWORD`` exists on it and has permissions to create databases. """ from django.conf import settings import MySQLdb DB_USER = settings.DATABASES["default"]["USER"] DB_PASS = settings.DATABASES["default"]["PASSWORD"] if self.type.lower().startswith("mysql-"): engine = self.type[len("mysql-"):] elif self.type.lower() == "mysql": engine = "InnoDB" else: raise NotImplementedError( "Currently only MySQL data source is supported.") if schema_file is None: schema_file = path( "model", "sql", "template_schema", "project_{0}_1.sql.tmpl".format(self.contenttype), ) conn = MySQLdb.connect( host=self.host, user=DB_USER, passwd=DB_PASS, ) filterwarnings('ignore', category=MySQLdb.Warning) cur = conn.cursor() cur.execute("CREATE DATABASE IF NOT EXISTS {0}".format(self.name)) conn.close() resetwarnings() # MySQLdb provides no way to execute an entire SQL file in bulk, so we # have to shell out to the commandline client. with open(schema_file) as f: # set the engine to use sql = f.read().format(engine=engine) args = [ "mysql", "--host={0}".format(self.host), "--user={0}".format(DB_USER), ] if DB_PASS: args.append("--password={0}".format(DB_PASS)) args.append(self.name) proc = subprocess.Popen( args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) (output, _) = proc.communicate(sql) if proc.returncode: raise IOError( "Unable to set up schema for datasource {0}: " "mysql returned code {1}, output follows:\n\n{2}".format( self.key, proc.returncode, output))
DATA_CYCLE_DAYS = env.int("DATA_CYCLE_DAYS", default=120) # Make this unique, and don't share it with anybody. SECRET_KEY = env("TREEHERDER_DJANGO_SECRET_KEY") SITE_ID = 1 ROOT_URLCONF = "treeherder.config.urls" WSGI_APPLICATION = 'treeherder.config.wsgi.application' TIME_ZONE = "America/Los_Angeles" USE_I18N = False USE_L10N = True SERVE_MINIFIED_UI = env.bool("SERVE_MINIFIED_UI", default=False) # Files in this directory will be served by WhiteNoise at the site root. WHITENOISE_ROOT = path("..", "dist" if SERVE_MINIFIED_UI else "ui") STATIC_ROOT = path("static") STATIC_URL = "/static/" MEDIA_ROOT = path("media") MEDIA_URL = "/media/" # Default minimum regression threshold for perfherder is 2% (otherwise # e.g. the build size tests will alert on every commit) PERFHERDER_REGRESSION_THRESHOLD = 2 # Various settings for treeherder's t-test "sliding window" alert algorithm PERFHERDER_ALERTS_MIN_BACK_WINDOW = 12 PERFHERDER_ALERTS_MAX_BACK_WINDOW = 24 PERFHERDER_ALERTS_FORE_WINDOW = 12